I have a collection of timestamps which record what actions are performed by users at which time. For now, the collection consists of only two actions start and end. There can only be a single end action, while there can be multiple start actions per user.
Now I want a generate a list of users where the time difference between the last start action and the end action is - for example - less than a minute.
The simplified documents in my collection timestamps look like this:
document #1
{
id: 123,
user: "user1",
type: "start",
date: 2019-09-10
}
document #2
{
id: 234,
user: "user1",
type: "end",
date: 2019-09-11
}
Now the result I want should look like this:
{
id: null,
list: ["user1, user2"]
}
The field list should contain every user, where the time difference between the start and end action is less than a minute.
I am having trouble combining the documents which contain the start and end attribute. I was trying to combine them into documents that looks like this:
{
id: 345
user: "user1"
date_start: 2019-09-10
date_end: 2019-09-11
}
I don't know where to start with the aggregation pipeline and how to split and combine the different types of timestamps. Furthermore, I still need to add a field that contains the difference between both dates.
The following query can get us the expected output:
db.collection.aggregate([
{
$sort:{
"date":-1
}
},
{
$group:{
"_id":{
"id":"$id",
"type":"$type"
},
"id":{
$first:"$id"
},
"user":{
$first:"$user"
},
"type":{
$first:"$type"
},
"date":{
$first:"$date"
}
}
},
{
$group:{
"_id":"$id",
"user":{
$first:"$user"
},
"info":{
$push:{
"k":"$type",
"v":"$date"
}
}
}
},
{
$addFields:{
"info":{
$arrayToObject:"$info"
}
}
},
{
$match:{
$expr:{
$lt:[
{
$subtract:[
{
$toDate:"$info.end"
},
{
$toDate:"$info.start"
}
]
},
60000
]
}
}
},
{
$group:{
"_id":null,
"users":{
$push:"$user"
}
}
},
{
$project:{
"_id":0
}
}
]).pretty()
Data set:
{
"_id" : ObjectId("5d77a117bd4e75c58d598214"),
"id" : 123,
"user" : "user1",
"type" : "start",
"date" : "2019-09-10T13:01:14.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d598215"),
"id" : 123,
"user" : "user1",
"type" : "start",
"date" : "2019-09-10T13:04:14.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d598216"),
"id" : 123,
"user" : "user1",
"type" : "start",
"date" : "2019-09-10T13:09:02.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d598217"),
"id" : 123,
"user" : "user1",
"type" : "end",
"date" : "2019-09-10T13:09:14.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d598218"),
"id" : 234,
"user" : "user2",
"type" : "start",
"date" : "2019-09-10T13:02:02.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d598219"),
"id" : 234,
"user" : "user2",
"type" : "end",
"date" : "2019-09-10T13:09:14.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d59821a"),
"id" : 345,
"user" : "user3",
"type" : "start",
"date" : "2019-09-10T13:08:55.242Z"
}
{
"_id" : ObjectId("5d77a117bd4e75c58d59821b"),
"id" : 345,
"user" : "user3",
"type" : "end",
"date" : "2019-09-10T13:09:14.242Z"
}
Output:
{ "users" : [ "user3", "user1" ] }
Query analysis:
Stage I: Sorting the documents in descending order of the date
Stage II: Grouping on [id, type] and picking the first date for
each type i.e. the latest date for each type
Stage III: Grouping only on id and pushing the type and associated date into an array as key-value pairs
Stage IV: Converting the array of key-value pairs into an object
Stage V: Filtering documents which has the difference between end and start date less than 60000 ms. (milliseconds equivalent of 1 minute)
Stage VI: Pushing all filtered names into an array
Related
I tried to create an API for filtering the products by sending an array of objects as filters.
this is my Product schema:
const mongoose = require("mongoose");
const { s, rs, rn, rref, ref } = require("../utils/mongo");
let schema = new mongoose.Schema(
{
user: rref("user"),
name: rs,
description: s,
images: [s],
price: rn,
category: ref("category"),
filters: [
{
parent: ref("filter"),
value: s,
name: s,
},
],
subFilter: [
{
parent: s,
value: s,
title: s,
},
],
},
{ timestamps: true }
);
module.exports = mongoose.model("product", schema);
and this one is what I want to send as body to the API
{
category: '62445c3d922d127512867245'
filters: [
{ name: 'filter name 1', value: '62445c3d922d127512861236' },
{ name: 'filter name 2', value: '62445c3d922d127512861458' },
.....
]
}
as you see I want to filter my products based on category Id and an array of filter objects. I tried to write this query but it return an empty array.
this is my query:
filter: async (req, res) => {
try {
const { category, filters } = req.body;
const products = await Product.find({
category,
filters: {
$in: filters,
},
});
res.status(200).json(products);
} catch (err) {
res.status(500).json(err);
}
},
what stored on db
{
"_id" : ObjectId("62643acf19636d7db1804cb3"),
"images" : [
"image-1650735823476۸.jpg"
],
"user" : ObjectId("622606af0f40cb8ea37383dc"),
"name" : "شیر توپی 2 اینچ کلاس 150 پیشگام",
"description" : " برند پیشگام با مدارک و تاییدیه ",
"price" : NumberInt(5000000),
"category" : ObjectId("62445c4d922d127512867246"),
"filters" : [
{
"_id" : ObjectId("62643acf19636d7db1804cb4"),
"parent" : ObjectId("6264307f19636d7db1804b77"),
"value" : "626430bb19636d7db1804b78",
"name" : "Valve Type"
},
{
"_id" : ObjectId("62643acf19636d7db1804cb5"),
"parent" : ObjectId("6264319819636d7db1804b7b"),
"value" : "6264319819636d7db1804b7e",
"name" : "Body Type"
},
{
"_id" : ObjectId("62643acf19636d7db1804cb6"),
"parent" : ObjectId("626431ef19636d7db1804b82"),
"value" : "626431ef19636d7db1804b83",
"name" : "Bore Type"
},
{
"_id" : ObjectId("62643acf19636d7db1804cb7"),
"parent" : ObjectId("6264328519636d7db1804b85"),
"value" : "6264328519636d7db1804b86",
"name" : "Material Type"
},
{
"_id" : ObjectId("62643acf19636d7db1804cb8"),
"parent" : ObjectId("626435de19636d7db1804c10"),
"value" : "626439b619636d7db1804ca7",
"name" : "Trim Material"
},
{
"_id" : ObjectId("62643acf19636d7db1804cb9"),
"parent" : ObjectId("6264367919636d7db1804c17"),
"value" : "6264367919636d7db1804c18",
"name" : "End Conection"
},
{
"_id" : ObjectId("62643acf19636d7db1804cba"),
"parent" : ObjectId("626436a719636d7db1804c1f"),
"value" : "6264378119636d7db1804c28",
"name" : "Size"
},
{
"_id" : ObjectId("62643acf19636d7db1804cbb"),
"parent" : ObjectId("6264389219636d7db1804c6d"),
"value" : "6264389219636d7db1804c6f",
"name" : "Class / Pressure"
}
],
"subFilter" : [
{
"_id" : ObjectId("62643acf19636d7db1804cbc"),
"parent" : "6264328519636d7db1804b85",
"value" : "626433b919636d7db1804b93",
"title" : "Body Material"
}
],
"createdAt" : ISODate("2022-04-23T17:43:43.421+0000"),
"updatedAt" : ISODate("2022-04-23T17:53:29.016+0000"),
"__v" : NumberInt(0)
}
Consider this shrunk down set of inputs that capture the essence of the question. The comments "give away" what we are going to try to find and why. We only show one value for category because matching on that is trivial and not the interesting part of the query.
[
{
"category" : ObjectId("62445c4d922d127512867246"),
"filters" : [
// Matching Valve/value; include this doc
{"name" : "Valve", "value" : "626430bb19636d7db1804b78"},
// ALSO match Body/value; include this doc (but needs only 1 match)
{"name" : "Body", "value" : "6264319819636d7db1804b7e"}
]
}
,{
"category" : ObjectId("62445c4d922d127512867246"),
"filters" : [
// Not target value for Valve name (..79 instead of ...78):
{"name" : "Valve", "value" : "626430bb19636d7db1804b79"},
// ...but correct value for Body, so include this doc
{"name" : "Body", "value" : "6264319819636d7db1804b7e"}
]
}
,{
"category" : ObjectId("62445c4d922d127512867246"),
// No matching Valve or Body so this whole doc is ignored.
"filters" : [
{"name" : "Valve", "value" : "626430bb19636d7db1804b79"},
{"name" : "Body", "value" : "6264319819636d7db1804b7f"}
]
}
,{
"category" : ObjectId("62445c4d922d127512867246"),
// Not even name matches so ignore this too:
"filters" : [
{"name" : "Pipe", "value" : "6264319819636d7db1804eee"}
]
}
]
Assume also we set up inputs coming from the API like this, in their native form i.e. strings NOT ObjectId:
var targ_cat = '62445c4d922d127512867246';
var any_one_of = [
{ name: 'Valve', value: '626430bb19636d7db1804b78' },
{ name: 'Body', value: '6264319819636d7db1804b7e'}
];
We will use $filter as our main function but to do so, we must convert the incoming material into a form required by $filter.
// Convert inbound array of any_one_of into a something designed to work
// in the $filter function by comparing each name/value entry in the
// filters field to the item presented in $$this, meaning take:
// { name: 'Valve', value: '626430bb19636d7db1804b78' },
// and turn it into:
// {$and: [ {$eq:['Valve','$$this.name']}, {$eq:['62643...','$$this.value']} ] }
// Since any one of the entries is considered a hit, we package it all
// into an $or wrapper, not $and.
var or_list = [];
any_one_of.forEach(function(f) {
or_list.push( {$and: [
{$eq:[f['name'], '$$this.name']},
{$eq:[f['value'], '$$this.value']}
]});
});
var or_expr = {$or: or_list};
Now we are ready to query mongoDB:
db.foo.aggregate([
// Get this out of the way quickly; note we must make a new ObjectId!
{$match: {'category': new ObjectId(targ_cat)}}
// The interesting part of the query:
,{$addFields: {filters: {$filter: {input: '$filters', cond: or_expr}}}}
// Only keep those items where $filter found at least one of the
// targets:
,{$match: {$expr: {$gt:[{$size: '$filters'},0]} }}
]);
These two lines of code simply output the same result, so what is the difference between them? I know I know, documentation... But I mean in this context. Thank you for your answers!
db.someData.find({$and: [{genre: {$eq: "action"}}, {genre: {$eq: "thriller"}}]}).pretty()
db.someData.find({genre: {$all: ["action", "thriller"]}}).pretty()
This is the collection in my mongodb database.
{
"_id" : ObjectId("5d19fe6080fc4d046d99d42b"),
"title" : "The Last Student Returns",
"meta" : {
"rating" : 9.5,
"aired" : 2018,
"runtime" : 100
},
"visitors" : 1300000,
"expectedVisitors" : 1550000,
"genre" : [
"thriller",
"drama",
"action"
]
}
{
"_id" : ObjectId("5d19fe6080fc4d046d99d42c"),
"title" : "Teach me if you can",
"meta" : {
"rating" : 8.5,
"aired" : 2014,
"runtime" : 90
},
"visitors" : 590378,
"expectedVisitors" : 500000,
"genre" : [
"action",
"thriller"
]
}
{
"_id" : ObjectId("5d19fe6080fc4d046d99d42d"),
"title" : "Supercharged Teaching",
"meta" : {
"rating" : 9.3,
"aired" : 2016,
"runtime" : 60
},
"visitors" : 370000,
"expectedVisitors" : 1000000,
"genre" : [
"thriller",
"action"
]
}
Interesting that you mentioned documentation since your exact question is actually answered there:
Behavior
Equivalent to $and Operation
The $all is equivalent to an $and operation of the specified values;
i.e. the following statement:
{ tags: { $all: [ "ssl" , "security" ] } }
is equivalent to:
{ $and: [ { tags: "ssl" }, { tags: "security" } ] }
But overall there are many ways to get the same result with mongo just like there are many ways to get the same exact result with JS etc.
I need get value of field without using $unwind
because $unwind and $group takes much longer time.
My document (looks like):
{
"_id" : ObjectId("512e28984815cbfcb21646a7"),
"providers" : [
{
"list" : [
{
"code" : "ATT",
"descr" : "Attending"
}
],
"Name" : "John Doe",
"prvdId" : "1"
},
{
"list" : [
{
"code" : "RFR",
"descr" : "Referring"
},
{
"code" : "TRT",
"descr" : "Treating"
}
],
"Name" : "Smith William",
"prvdId" : "2"
}
]
}
cond is if "code" : "TRT", than get "prvdId"
Expected result is
{"prvdId" : "2"}
Use $filter with $in to look for a match in nested array followed by $let with $arrayElemAt to output prvdId in 3.4.
db.col.aggregate([
{"$match":{"providers.list.code":"TRT"}},
{"$project":{
"_id":0,
"prvdId":{
"$let":{
"vars":{
"providersl":{
"$filter":{
"input":"$providers",
"as":"providerf",
"cond":{"$in":["TRT","$$providerf.list.code"]}
}
}
},
"in":{"$arrayElemAt":["$$providersl.prvdId",0]}
}
}
}}
])
According to above mentioned description as a solution expected result can be obtained by using $elemMatch operator used to search an array element into find operation.
db.collection.find({
providers: {
$elemMatch: {
list: {
$elemMatch: {
code: "TRT"
}
}
}
}
}, {
'providers.$.prvdId': 1
})
I am trying to create a dashboard where I show summaries of order data within the app. In this case I am simply wanting count the number of items in a given category in my Orders collection. My code so far looks like this:
Collection data
{
"_id" : "a6wHiXxyM5DwSAsfq",
"orderNumber" : 1234,
"createdAt" : "11/01/2016, 14:43:49",
"productsInOrder" : [
{
"category" : "ambient",
"item" : 50818,
"desc" : "Tasty Rubber Chicken",
"quantity" : "44",
"price" : "0.92",
"lineprice" : "40.48",
"_id" : "FFNxG8vujs6NGN69r"
},
{
"category" : "frozen",
"item" : 71390,
"desc" : "Generic Granite Fish",
"quantity" : "11",
"price" : "1.00",
"lineprice" : "11.00",
"_id" : "LcRtpyLxkWyh39kkB"
}
]
}
{
"_id" : "PdpywXCvfew7qojmA",
"orderNumber" : 1234,
"createdAt" : "11/01/2016, 14:44:15",
"productsInOrder" : [
{
"category" : "frozen",
"item" : 71390,
"desc" : "Generic Granite Fish",
"quantity" : "44",
"price" : "1.00",
"lineprice" : "44.00",
"_id" : "dAscx4R8pcBgbzoZs"
},
{
"category" : "frozen",
"item" : 66940,
"desc" : "Gorgeous Granite Bike",
"quantity" : "55",
"price" : "4.21",
"lineprice" : "231.55",
"_id" : "xm3mFRmPmmdPxjfP9"
},
{
"category" : "frozen",
"item" : 96029,
"desc" : "Gorgeous Plastic Fish",
"quantity" : "1234",
"price" : "4.39",
"lineprice" : "5417.26",
"_id" : "7u7SHnpTf7PWcrhGA"
}
]
}
{
"_id" : "xcHZ25qwvyDpDJtAZ",
"orderNumber" : 1234,
"createdAt" : "11/01/2016, 14:44:47",
"productsInOrder" : [
{
"category" : "frozen",
"item" : 31104,
"desc" : "Handcrafted Rubber Keyboard",
"quantity" : "11",
"price" : "4.78",
"lineprice" : "52.58",
"_id" : "LMMwbKFEgnCbgCt9c"
},
{
"category" : "frozen",
"item" : 77832,
"desc" : "Practical Rubber Shirt",
"quantity" : "21",
"price" : "0.62",
"lineprice" : "13.02",
"_id" : "63otkkXWGrTJkwEgX"
},
{
"category" : "frozen",
"item" : 66940,
"desc" : "Gorgeous Granite Bike",
"quantity" : "111",
"price" : "4.21",
"lineprice" : "467.31",
"_id" : "rbPSujey8CFeMPjza"
}
]
}
JS
So far I have tried:
Orders.find({ 'productsInOrder': ['ambient']}).count();
Orders.find({ productsInOrder: { category: 'ambient' }}).count();
Orders.find({ productsInOrder: { $all: [ 'frozen' ] }}).count();
I am having a hard time understanding Mongo queries when the data is nested in this manner. Please can you help point me in the right direction? Many thanks in advance.
* SOLUTION *
I have accomplished the desired result thanks to the contributions below. To make this work I created a method on the server as the query cannot be run on the client using an existing collection. This is done as follows:
Meteor.methods({
'byCategory': function() {
var result = Orders.aggregate([
{ "$unwind": "$productsInOrder" },
{
"$group": {
"_id": null,
"ambient_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "ambient" ] }, 1, 0 ]
}
},
"frozen_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "frozen" ] }, 1, 0 ]
}
},
"other_category_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "other_category" ] }, 1, 0 ]
}
}
}
}
]);
return result;
}
})
and then on the client:
Meteor.call('byCategory', function( error, result ) {
if( error ) {
console.log( error.reason );
} else {
console.log( result[0].ambient_count );
console.log( result[0].frozen_count );
etc....
}
});
Thanks and credit to #chridam and #Brett.
An alternative approach is to use the aggregation framework. Consider the following aggregation pipeline which as the first stage of the aggregation pipeline, the $unwind operator denormalizes the productsInOrder array to output for each input document, n documents where n is the number of array elements. The next pipeline stage has the $group operator which groups all the documents into a single document and stores the counts for each category with the help of the $sum and $cond operators.
In Meteor, you can then use meteorhacks:aggregate package to implement the aggregation:
Add to your app with
meteor add meteorhacks:aggregate
Note, this only works on server side and there is no oberserving support or reactivity built in. Then simply use .aggregate function like below.
var coll = new Mongo.Collection('orders');
var pipeline = [
{ "$unwind": "$productsInOrder" },
{
"$group": {
"_id": null,
"ambient_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "ambient" ] }, 1, 0 ]
}
},
"frozen_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "frozen" ] }, 1, 0 ]
}
},
"other_category_count": {
"$sum": {
"$cond": [ { "$eq": [ "$productsInOrder.category", "other_category" ] }, 1, 0 ]
}
}
}
}
];
var result = coll.aggregate(pipeline);
Running the same pipeline in mongo shell using the sample data will yield:
{
"result" : [
{
"_id" : null,
"ambient_count" : 1,
"frozen_count" : 7,
"other_category_count" : 0
}
],
"ok" : 1
}
You can access the native mongo collection and publish the aggregation results to the orders collection on the client side:
Meteor.publish('categoryCounts', function() {
var self = this,
db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
orders = db.collection("orders").aggregate(pipeline, // Need to wrap the callback so it gets called in a Fiber.
Meteor.bindEnvironment(
function(err, result) {
// Add each of the results to the subscription.
_.each(result, function(e) {
self.added("orders", e._id, e);
});
self.ready();
},
function(error) {
Meteor._debug( "Error doing aggregation: " + error);
}
)
);
});
If you don't want to do this within Meteor, you will need to use mongo aggregation. Minimongo doesn't include aggregation though, so you will need this package to accomplish it:
https://docs.mongodb.org/manual/core/aggregation-introduction/
I only tested this in mongo itself, so you will have to adapt it to the way that the aggregation package does it:
db.orders.aggregate([
{
$unwind: "$productsInOrder"
},
{
$match: {
"productsInOrder.category": "frozen"
}
},
{
$group: {
_id: null,
count: {
$sum: 1
}
}
}
]);
The first part is unwinding the collection. It will basically make an "order" entry for every instance of $productsInOrder. Once you have the array flattened out, we match on the category you care about; in this case, the "frozen" category. Next we group it up so we can count the number of documents returned. $group is simply constructing the final object that will be output from the query. You can modify this to be whatever you want, or you could group by productsInOrder.category and not even $match on "frozen".
How I can convert an array to an object in MongoDB?
For example, I want to convert this document:
{
"_id" : NumberLong(279),
"userAddressList" : [
{
"street" : "Street",
"house" : "House",
"building" : "Building",
"flat" : NumberLong(0),
"entrance" : NumberLong(0),
"floor" : NumberLong(0),
"intercom" : "Intercome"
}
],
}
to this:
{
"_id" : NumberLong(279),
"userAddressList" :
{
"street" : "Street",
"house" : "House",
"building" : "Building",
"flat" : NumberLong(0),
"entrance" : NumberLong(0),
"floor" : NumberLong(0),
"intercom" : "Intercome"
},
}
So I need to convert ""userAddressList" : [{..}]" to the ""userAddressList" : {..}".
For MongoDB 4.2 and newer
You could try the following query which uses the aggregation pipeline in the update:
db.collection.updateMany(
{},
[
{ '$addFields': {
'userAddressList': {
'$arrayElemAt': ['$userAddressList', 0]
}
} }
]
)
For older MongoDB versions:
db.collection.find().forEach(function(doc){
userAddressList = doc.userAddressList[0];
doc.userAddressList = userAddressList;
db.collection.save(doc);
})
or use the aggregation framework where you run the following pipeline
db.collection.aggregate([
{ "$addFields": {
"userAddressList": {
"$arrayElemAt": ["$userAddressList", 0]
}
} },
{ "$out": "collection" }
])
Note that this does not update your collection but replaces the existing one and does not change any indexes that existed on the previous collection. If the aggregation fails, the $out operation makes no changes to the pre-existing collection.