taking the difference between adjacent documents in mongoDB - javascript

How do I take the difference between adjacent records in mongoDB using javascript? For example, if I have the following three documents in a collection:
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z")
}
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z")
}
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z")
}
I want to take the difference in the "time" field between adjacent values to get:
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z"),
"time_difference" : null
}
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z"),
"time_difference" : 1
}
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z"),
"time_difference" : 3
}
Any ideas on how to do this efficiently in javascript/mongoDB? Thanks.

I don't know whether this was true when the question was asked seven years ago, but this can be solved completely within the aggregation framework. Assuming the collection name is AdjacentDocument, the following aggregation will get the results you're looking for:
db.AdjacentDocument.aggregate(
{$sort: {time: 1}},
{$group: {_id: 0, document: {$push: '$$ROOT'}}},
{$project: {documentAndPrevTime: {$zip: {inputs: ['$document', {$concatArrays: [[null], '$document.time']}]}}}},
{$unwind: {path: '$documentAndPrevTime'}},
{$replaceWith: {$mergeObjects: [{$arrayElemAt: ['$documentAndPrevTime', 0]}, {prevTime: {$arrayElemAt: ['$documentAndPrevTime', 1]}}]}},
{$set: {time_difference: {$trunc: [{$divide: [{$subtract: ['$time', '$prevTime']}, 1000]}]}}},
{$unset: 'prevTime'}
);
Aggregation pipeline walkthrough
First, the documents are sorted from oldest to newest. They are grouped into a single document with the documents stored in an ordered array field:
{$sort: {time: 1}},
{$group: {_id: 0, document: {$push: '$$ROOT'}}}
/*
{
"_id" : 0,
"document" : [
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z")
},
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z")
},
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z")
}
]
}
*/
Next, the previous times are zipped into the document array, creating an array of [document, previousTime]:
{$project: {documentAndPrevTime: {$zip: {inputs: ['$document', {$concatArrays: [[null], '$document.time']}]}}}}
/*
{
"_id" : 0,
"documentAndPrevTime" : [
[
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z")
},
null
],
[
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z")
},
ISODate("2013-02-13T15:45:41.148Z")
],
[
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z")
},
ISODate("2013-02-13T15:45:42.148Z")
]
]
}
*/
Next, the document & time array is unwound, creating a document for each of the initial documents:
{$unwind: {path: '$documentAndPrevTime'}}
/*
{
"_id" : 0,
"documentAndPrevTime" : [
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z")
},
null
]
}
{
"_id" : 0,
"documentAndPrevTime" : [
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z")
},
ISODate("2013-02-13T15:45:41.148Z")
]
}
{
"_id" : 0,
"documentAndPrevTime" : [
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z")
},
ISODate("2013-02-13T15:45:42.148Z")
]
}
*/
Next, we replace the document with the value of the document array element, merged with previous time element (using null if it's the initial index):
{$replaceWith: {$mergeObjects: [{$arrayElemAt: ['$documentAndPrevTime', 0]}, {prevTime: {$arrayElemAt: ['$documentAndPrevTime', 1]}}]}}
/*
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z"),
"prevTime" : null
}
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z"),
"prevTime" : ISODate("2013-02-13T15:45:41.148Z")
}
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z"),
"prevTime" : ISODate("2013-02-13T15:45:42.148Z")
}
*/
Finally, we update the document by setting the time_difference to the difference of the two time fields, and removing the temporary prevTime field. Since the difference between two dates is in milliseconds and your example uses seconds, we calculate the seconds by dividing by 1000 and truncating.
{$set: {time_difference: {$trunc: [{$divide: [{$subtract: ['$time', '$prevTime']}, 1000]}]}}},
{$unset: 'prevTime'}
/*
{
"_id" : ObjectId("50ed90a55502684f440001ac"),
"time" : ISODate("2013-02-13T15:45:41.148Z"),
"time_difference" : null
}
{
"_id" : ObjectId("50ed90a55502684f440001ad"),
"time" : ISODate("2013-02-13T15:45:42.148Z"),
"time_difference" : 1
}
{
"_id" : ObjectId("50ed90a55502684f440001ae"),
"time" : ISODate("2013-02-13T15:45:45.148Z"),
"time_difference" : 3
}
*/

The one thing you will want to make sure of here is that you have a sort on the query you wish to use to garnish your records. If no sort is used it will actually use find order, which is not $natural order.
Find order can differ between queries so if you run the query twice within the period of 2 minutes you might find that they don't return the same order. It does seem however that your query would be logically sorted on tiem_difference.
It should also by noted that this is not possible through normal querying. I also do not see an easy way doing this through the aggregation framework.
So already it seems the next plausible method is either using multiple queries or client side processing. Client side processing is probably the better here using a function like the one defined by #Marlon above.

One thing, I want to clear you. Unlike MYSQL, MongoDB is not give gurantee to the position. I mean, MongoDB will give you different sort at different time. So compare adjacent document may give different result, on every reading.
If you are fine with that and you want to compare then try with MongoDB's MapReduce http://docs.mongodb.org/manual/applications/map-reduce/

Assuming those 3 objects are coming through in an array, you could do something like the below:
var prevTime;
var currentTime;
for(var i = 0; i < records.length; i++)
{
currentTime = new Date(records[i].time).getTime();
records[i].time_difference = currentTime - prevTime;
prevTime = currentTime;
}
Of course you'll need to swap bits out to make it use the records from mongo.
If you need to do any more complex date calculations, I highly suggest checking out datejs (which you can get a node wrapper for if you want).

Related

MongoDb - Do i need an index on a second field when using deleteOne

I have a simple collection events
{
"_id" : ObjectId("5e2a9bb9dcb646448f9409b3"),
"year" : 2020,
"employee_id" : "5e1afe5ab7bad92b20365476",
"event" : ["Holidays"],
"total" : 21,
"used" : 1
}
and i don't want to be able to delete the documents that have used field grater than 0.
I use this
db.collection('events').deleteOne({_id: ObjectId("5e2a9bb9dcb646448f9409b3"), used: 0});
Do i need to set an index on used field if i already use _id?
Thanks
Not necessary. MongoDB already uses _id (unique index created by MongoDB) to delete documents.
db.collection('events').find({_id: ObjectId("5e2a9bb9dcb646448f9409b3"), used: 0}).explain();
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"used" : {
"$eq" : 0.0
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : []
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_id" : [
"[ObjectId('5e2a9bb9dcb646448f9409b3'), ObjectId('5e2a9bb9dcb646448f9409b3')]"
]
}
}
}

MongoDB: Get all nearby with filter

In MongoDB, I have models of User, Token, and Boost.
A user can have one or more tokens and one or more boosts.
Token has a 2dsphere location field.
And Boost has startTime and stopTime Date fields.
A user is said to have an active boost if Date.now() is greater than boost.startTime() and less than boost.stopTime().
What Mongo aggregation can I write to fetch me all the tokens near a particular location that belong to users with at least one active boost?
Based on your question, I have created a mock data
token collection:
{
"_id" : ObjectId("5b97541c6af22cc65216ffd8"),
"userid" : "5b9753726af22cc65216ffd6",
"location" : {
"longitude" : 80.250875,
"latitude" : 13.052519
}
},
{
"_id" : ObjectId("5b97543a6af22cc65216ffd9"),
"userid" : "5b97537e6af22cc65216ffd7",
"location" : {
"longitude" : 80.249995,
"latitude" : 13.051819
}
}
boost collection :
{
"_id" : ObjectId("5b9754796af22cc65216ffda"),
"startTime" : ISODate("2018-09-11T05:36:57.149Z"),
"stopTime" : ISODate("2018-09-11T05:36:57.149Z"),
"userid" : "5b9753726af22cc65216ffd6"
},
{
"_id" : ObjectId("5b9754b46af22cc65216ffdb"),
"startTime" : ISODate("2018-10-08T18:30:00.000Z"),
"stopTime" : ISODate("2018-10-08T18:30:00.000Z"),
"userid" : "5b97537e6af22cc65216ffd7"
}
Users collection :
{
"_id" : ObjectId("5b9753726af22cc65216ffd6"),
"userName" : "user111"
},
{
"_id" : ObjectId("5b97537e6af22cc65216ffd7"),
"userName" : "user222"
}
The aggregate query to fetch all the tokens near a particular location that belong to users with at least one active boost is:
db.token.aggregate([
{
"$geoNear": {
"near": { type: "Point", coordinates: [80.248797,13.050599] },
"distanceField": "location",
"maxDistance": 1000,
"includeLocs": "location",
"spherical": true
}
},
{"$lookup" : {"from":"boost",
"localField" : "userid",
"foreignField" : "userid",
"as" : "boostDocs"
}},
{"$unwind" : "$boostDocs"},
{"$match" : {"$and":[{"boostDocs.startTime":{"$lte":new Date("11/09/2018")}},{"boostDocs.stopTime":{"$gte":new Date("10/09/2018")}}]}}
])
Notice that query to match the location is at the top of the query as $geoNear will only work if its the first stage of the aggregation pipeline.
The Date that I've used for comparison is just to check if my query works. You can specify your date or Date.now() as per your requirement.

Merge arrays and group to produce a count for each combined array value

I have a dataset like this:
{
"_id" : ObjectId("5a4c6fb6993a721b3479a27e"),
"score" : 8.3,
"page" : "message",
"lastmodified" : ISODate("2018-01-03T06:49:19.232Z"),
"createdate" : ISODate("2018-01-03T05:52:54.446Z"),
"slug" : [
"#APPLE"
],
"__v" : 0
},
{
"_id" : ObjectId("5a4c6fb6993a721b3479a27e"),
"score" : 9.3,
"page" : "#BANANA",
"lastmodified" : ISODate("2018-01-03T06:49:19.232Z"),
"createdate" : ISODate("2018-01-03T05:52:54.446Z"),
"slug" : [
"#APPLE"
],
"__v" : 0
}
{
"_id" : ObjectId("5a4c6fb6993a721b3479a27e"),
"score" : 5.3,
"page" : "#BANANA",
"lastmodified" : ISODate("2018-01-03T06:49:19.232Z"),
"createdate" : ISODate("2018-01-03T05:52:54.446Z"),
"slug" : [
"#BANANA"
],
"__v" : 0
}
Now I want to calculate the sum of score according to my Filter Like this:
#APPLE: 8.3+9.3 = 17.6 i.e #APPLE: 17.6,
#BANANA: 9.3+5.3 = 14.6 i.e #BANANA: 14.6
So for this I have to pick only last 1 hour data rather than picking the whole database
. So my query is like this
var newTime = new Date();
newTime.setHours( newTime.getHours() - 1 );
db.Test.find({"lastmodified":{$gt: newTime}})
so By this I can get only last 1 hour value. Now I am confuse that how i can do sum with filter. I also attached filter query i.e
db.Test.find({"lastmodified":{$gt: newTime}}, {$or: [{slug: {$in: ['#APPLE']}}, {page: '#APPLE'}]})
But it does not give anything. any help is appreciated
Try this aggregate query...
db.tests.aggregate([{
"$unwind": "$slug"
},
{
"$group": {
"_id": "$slug",
"totalScore": {
"$sum": "$score"
}
}
}
]);
Result:
{
"_id" : "#BANANA",
"totalScore" : 5.3
}
{
"_id" : "#APPLE",
"totalScore" : 17.6
}

MongoDB Range Issue

I'm using nodejs and mongodb.
I want to search docs between a num range but the function always give me number that outside the range. For example, this is my function and I want to get the results of the docs that they has a field size with the numbers between 1 to 1200:
db.collection(example).find({
size: {
"$gte": 1,
"$lte": 1200
}
}).toArray(function(err, results) {
db.close();
console.log("results=" + results);
});
the doc in the Database:
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d2"), "size" : -1 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d3"), "size" : 100 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d4"), "size" : 800 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d5"), "size" : 1999 }
the result of the query should be:
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d3"), "size" : 100 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d4"), "size" : 800 }
but the query result is:
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d2"), "size" : -1 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d3"), "size" : 100 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d4"), "size" : 800 }
{ "_id" : ObjectId("56659a492b9eaad2d9e6d4d5"), "size" : 1999 }
I believe you need to use a $and clause on your size conditions. It might look something like this:
db.collection(example).find( {
$and: [
{ size: { $gte: 1 } },
{ size: { $lte: 1200 } }
] } ).toarray...
See the examples in the mongo docs here.
Edit: Actually, an implicit and was fine. As noted, the actual problem here was size values were stored as strings, not ints.

Mongodb-native (node.js): Query Date range

I'm using node 0.10.21 and mongodb-native (aka require('mongodb')).
The problem I'm having is, that I cannot aggregate a timeseries collection AND use match to select a certain time frame:
var start = new Date(); //just demo, the start date is actually lower than NOW()
var end = new Date(); // is usally NOW() in my queries
$match : [{'$match' : {'date' : {'$gte' : start, '$lte' : end}}}]
The the generated query (JSON.stringify(query);) looks like this:
{"$match":{"date":{"$gte":"2013-11-09T23:00:00.000Z","$lte":"2013-11-12T05:00:00.000Z"}}}
Of course, I've read the docs at http://mongodb.github.io/node-mongodb-native/api-articles/nodekoarticle1.html#mongo-db-data-types and the docs say:
Date maps directly to a Javascript Date
Obviously not, or I'm overlooking something. I've seen similar questions regarding this issue, but none of them provide a solution.
In MongoDB shell everything works fine using {date : {'$gte' : new Date(2013,10,25)}} - so the collection itself is full of valid data that I should be able to query using node. I've also written a PHP-Script doing the same type of query and it works fine. Hence I suspect I either misread the docs, or the mongodb-native driver does not map JavaScript's Date() to MongoDB's Date().
Can someone please tell me how to query for a date range?
Update1: more Code
Here's a bit more of the code I'm using, as requested by JohnnyHK. I've stripped out the code that builds the actual aggregation values - those work as expected, only the $match part is giving me headaches.
var start = new Date(start_date);
var end = new Date(end_date);
console.log(start,end);
//outputs: 2013-11-10 0:00 2013-11-12 6:00
var ops = [{'$match' : {'date' : { '$gte' : start, '$lte' : end} }}, {'$group' : group_values}];
console.log(JSON.stringify(ops));
//outputs: [{"$match":{"date":{"$gte":"2013-11-09T23:00:00.000Z","$lte":"2013-11-12T05:00:00.000Z"}}},{"$group":{"used":{"$avg":"$used"},"system":{"$avg":"$system"},"iowait":{"$avg":"$iowait"},"_id":{"group_id":{"$subtract":[{"$divide":["$timestamp",3600]},{"$mod":[{"$divide":["$timestamp",3600]},1]}]}}}}]
collection.aggregate(ops,function(err, results) {
console.log('mongo results');
console.log(arguments);
//output: mongo results
//{ '0': null, '1': [] }
});
Update2
Sample Data:
{ "date" : ISODate("2013-10-22T13:52:16Z"), "timestamp" : 1382449936, "used" : 743.7768451188878, "system" : 855.7432519109785, "iowait" : 0, "max" : 400, "_id" : ObjectId("5266831aacce8ec133bb11af") }
{ "date" : ISODate("2013-10-22T13:52:06Z"), "timestamp" : 1382449926, "used" : 758.1516303260652, "system" : 840.5681136227246, "iowait" : 0, "max" : 400, "_id" : ObjectId("5266831aacce8ec133bb11ae") }
{ "date" : ISODate("2013-10-22T13:51:56Z"), "timestamp" : 1382449916, "used" : 826.0163785417379, "system" : 765.362035769591, "iowait" : 1.9952086438206234, "max" : 400, "_id" : ObjectId("52668306acce8ec133bb10f7") }
{ "date" : ISODate("2013-10-22T13:51:46Z"), "timestamp" : 1382449906, "used" : 1079.9727415609912, "system" : 526.4513588679281, "iowait" : 0, "max" : 400, "_id" : ObjectId("52668306acce8ec133bb10f6") }
{ "date" : ISODate("2013-10-22T13:51:36Z"), "timestamp" : 1382449896, "used" : 1327.1878627467097, "system" : 273.2917036211522, "iowait" : 0, "max" : 400, "_id" : ObjectId("526682f2acce8ec133bb1043") }

Categories