How to join three (multipe) collections with $lookup in mongodb?
Hi I am looking to join data from three collection
users collection:
[
{
_id:0,
name:"abc",
phone:999999999
},
{
_id:1,
name:"xyz",
phone:888888888
},
]
product collection:
[
{
_id:"p01",
name:"product-name",
price:1200
},
{
_id:"p02",
name:"product-name1",
price:100
}
]
productreviews collection:
[
{
_id:"pr0",
userId:0,
productId:"p01",
star:4
},
{
_id:"pr1",
userId:1,
productId:"p01",
star:3
}
]
mongodb query:
product.aggregate([
{
$lookup: {
from: "productreviews",
localField: "_id",
foreignField: "productId",
as: "review",
},
},
{
$lookup: {
from: "users",
localField: "review.userId",
foreignField: "_id",
as: "review.userInfo",
},
},
])
I am not able to get that output which i need.
How can i get this following output:
{
product: [
{
_id: "p01",
name: "product-name",
price: 1200,
review: [
{
_id: "pr0",
userId: 0,
productId: "p01",
star: 4,
"userInfo": {
name: "abc",
phone: 999999999
}
},
{
_id: "pr1",
userId: 1,
productId: "p01",
star: 3,
"userInfo": {
"name": "xyz",
"phone": 888888888,
}
},
]
},
{
_id: "p02",
name: "product-name1",
price: 100,
},
]
}
Any help appreciated!. Thank You...
db.product.aggregate([
{
$lookup: {
from: "review",
localField: "_id",
foreignField: "productId",
as: "review",
},
},
{
$lookup: {
from: "users",
localField: "review.userId", //See here
foreignField: "_id",
as: "review.userInfo",
},
},
])
Local field name is userId in the second lookup.
playground
EDIT:
To preserve reviews also
Add a unwind stage
db.product.aggregate([
{
$lookup: {
from: "review",
localField: "_id",
foreignField: "productId",
as: "review",
},
},
{
"$unwind": "$review"
},
{
$lookup: {
from: "users",
localField: "review.userId",
foreignField: "_id",
as: "review.userInfo",
},
},
])
Update
To keep the docs where there is no match, preserve null arrays at unwind stage as below
{
$unwind: {
path: "$review",
"preserveNullAndEmptyArrays": true
}
}
Related
I am new to MongoDB and mongoose.
I am trying to retrieve the entire array of object hierarchy from the database as a JSON.
By searching, I learned how to group nested lookup with 3 child levels without array, but the problem is I cannot find a way to deal with the nested arrays.
Here is what I am struggling with.
user:
[
{
_id: ObjectId("a1"),
username: "user1",
name: "name1"
},
{
_id: ObjectId("a2"),
username: "user2",
name: "name2"
},
{
_id: ObjectId("a3"),
username: "user2",
name: "name2"
},
...
]
reply:
[
{
_id: ObjectId("a"),
author: {ObjectId("a3")},
title: {"Reply1"};
body: {"Hello World!"}
},
{
_id: ObjectId("b"),
author: {ObjectId("a1")},
title: {"Reply2"};
body: {"Hello World!"}
},
{
_id: ObjectId("c"),
author: {ObjectId("a2")},
title: {"Reply3"};
body: {"Hello World!"}
},
{
_id: ObjectId("d"),
author: {ObjectId("a2")},
title: {"Reply4"};
body: {"Hello World!"}
}
...
]
post:
[
//First post
{
_id: ObjectId("0"),
title: 'post title',
author: {ObjectId("a1")},
reply: [{ObjectId("a")}, {ObjectId("b")}, ...],
},
//Second Post
{
_id: ObjectId("1"),
title: 'post title2',
author: {ObjectId("a2")},
reply: [{ObjectId("c")}, {ObjectId("d")}, ...],
},
...
]
Expected example:
[
//First post
{
_id: ObjectId("0"),
title: 'post title',
author: {
_id: ObjectId("a1"),
username: "user1",
name: "name1"
},
reply: [{
_id: ObjectId("a"),
author: {
_id: ObjectId("a3"),
username: "user2",
name: "name2"
},
title: {"Reply1"},
body: {"Hello World!"}
},
{
_id: ObjectId("b"),
author: {
_id: ObjectId("a1"),
username: "user1",
name: "name1"
},
title: {"Reply2"};
body: {"Hello World!"}
},
...
],
},
{
//Second post
},
...
]
Here is the code that I used.
posts = await Post.aggregate([{
$match: searchQuery
},
{
$lookup: {
from: 'users',
localField: 'author',
foreignField: '_id',
as: 'author'
}
},
{
$lookup: {
from: 'replies',
localField: 'reply',
foreignField: '_id',
as: 'reply'
}
},
{
$unwind: '$author',
},
{
$lookup: {
from: 'users',
localField: 'reply.author',
foreignField: '_id',
as: 'reply.author'
}
},
{
$project: {
title: 1,
author: 1,
reply: 1
}
},
]).exec();
How should I change the code if I want to get the expected example?
Here's one way to do it. Lots of "$lookup" for author of post, reply to the post, author ids of reply, and then author document matching _id.
db.post.aggregate([
{
"$lookup": {
"from": "user",
"localField": "author",
"foreignField": "_id",
"as": "author"
}
},
{
"$set": {
"author": {
"$first": "$author"
}
}
},
{
"$lookup": {
"from": "reply",
"localField": "reply",
"foreignField": "_id",
"as": "reply"
}
},
{
"$lookup": {
"from": "user",
"localField": "reply.author",
"foreignField": "_id",
"as": "replyAuthors"
}
},
{
"$project": {
"title": 1,
"author": 1,
"reply": {
"$map": {
"input": "$reply",
"as": "repObj",
"in": {
"$mergeObjects": [
"$$repObj",
{
"author": {
"$first": {
"$filter": {
"input": "$replyAuthors",
"as": "repA",
"cond": {
"$eq": [
"$$repA._id",
"$$repObj.author"
]
}
}
}
}
}
]
}
}
}
}
}
])
Try it on mongoplayground.net.
I am working with mongodb aggregate and i was able to write aggregate in mongo shell and test it and it worked fine. However when i tried to make it dynamic in Nodejs method and passed values from frontend it showed me nothing. The reason i think is with this new ObjectId(YOUR ID IN STRING TYPE). The moment i pass the aggregate to execute function it gets strigified and new ObjectId gets removed so then it does not get matched.
Here is my working aggregate that i wrote in mongo shell
db.ParcelStatus.aggregate([
{
$match: {
$or: [
{
"statusRepositoryId": new ObjectId("5dd7fa20dcfa9600152cc2d8")
},
{
"statusRepositoryId": new ObjectId("5dd7fa20dcfa9600152cc2dd")
},
{
"createdAt": {
"$gte": new Date("2020-05-01T18:59:59.001Z")
}
},
{
"createdAt": {
"$lte": new Date("2020-05-31T18:59:59.099Z")
}
}
]
}
},
{
"$lookup": {
"from": "Parcel",
"localField": "parcelId",
"foreignField": "_id",
"as": "parcel"
}
},
{
"$unwind": {
"path": "$parcel",
"preserveNullAndEmptyArrays": true
}
},
{
"$lookup": {
"from": "CustomerData",
"localField": "parcel.customerDataId",
"foreignField": "_id",
"as": "parcel.customerData"
}
},
{
"$unwind": "$parcel.customerData"
},
{
"$lookup": {
"from": "Customer",
"localField": "parcel.customerData.customerId",
"foreignField": "_id",
"as": "parcel.customerData.customer"
}
},
{
"$unwind": "$parcel.customerData.customer"
},
{
"$lookup": {
"from": "City",
"localField": "parcel.customerData.cityId",
"foreignField": "_id",
"as": "parcel.customerData.city"
}
},
{
"$unwind": "$parcel.customerData.city"
}
])
Now in nodejs here is how i am building it
let pipeline = [];
const matchObj = {
$match: { $or: [] },
};
filters.forEach((obj) => {
if (obj.key === "date") {
matchObj.$match.$or.push(
{ createdAt: { $gte: new Date(obj.values.from) } },
{ createdAt: { $lte: new Date(obj.values.to) } }
);
}
if (obj.key === "status_repository") {
if (
report.filters.find((x) => x.key === obj.key).selectionType === "single"
) {
matchObj.$match.$or.push({
statusRepositoryId: { $toObjectId: obj.values },
});
} else {
obj.values.forEach((id) => {
matchObj.$match.$or.push({ statusRepositoryId: { $toObjectId: id } });
});
}
}
});
pipeline.push(matchObj);
pipeline = [
...pipeline,
{
$lookup: {
from: "Parcel",
localField: "parcelId",
foreignField: "_id",
as: "parcel",
},
},
{
$unwind: {
path: "$parcel",
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: "CustomerData",
localField: "parcel.customerDataId",
foreignField: "_id",
as: "parcel.customerData",
},
},
{ $unwind: "$parcel.customerData" },
{
$lookup: {
from: "Customer",
localField: "parcel.customerData.customerId",
foreignField: "_id",
as: "parcel.customerData.customer",
},
},
{ $unwind: "$parcel.customerData.customer" },
{
$lookup: {
from: "City",
localField: "parcel.customerData.cityId",
foreignField: "_id",
as: "parcel.customerData.city",
},
},
{
$unwind: "$parcel.customerData.city",
},
];
and in nodejs this is how it shows up in console
db.ParcelStatus.aggregate([
{
"$match": {
"$or": [
{
"statusRepositoryId": "5dd7fa20dcfa9600152cc2d8"
},
{
"statusRepositoryId":"5dd7fa20dcfa9600152cc2dd"
},
{
"createdAt": {
"$gte": "2020-05-01T18:59:59.001Z"
}
},
{
"createdAt": {
"$lte": "2020-05-31T18:59:59.099Z"
}
}
]
}
},
{
"$lookup": {
"from": "Parcel",
"localField": "parcelId",
"foreignField": "_id",
"as": "parcel"
}
},
{
"$unwind": {
"path": "$parcel",
"preserveNullAndEmptyArrays": true
}
},
{
"$lookup": {
"from": "CustomerData",
"localField": "parcel.customerDataId",
"foreignField": "_id",
"as": "parcel.customerData"
}
},
{
"$unwind": "$parcel.customerData"
},
{
"$lookup": {
"from": "Customer",
"localField": "parcel.customerData.customerId",
"foreignField": "_id",
"as": "parcel.customerData.customer"
}
},
{
"$unwind": "$parcel.customerData.customer"
},
{
"$lookup": {
"from": "City",
"localField": "parcel.customerData.cityId",
"foreignField": "_id",
"as": "parcel.customerData.city"
}
},
{
"$unwind": "$parcel.customerData.city"
}
])
Notice the difference in nodejs result in $match,new Date(DATE) and in new ObjectId(ID). i would very much appreciate if you can tell me how can i fix this.
From the $match docs:
The $match query syntax is identical to the read operation query syntax; i.e. $match does not accept raw aggregation expressions. To include aggregation expression in $match, use a $expr query expression:
What are raw aggregation expressions?
Expressions can include field paths, literals, system variables, expression objects, and expression operators. Expressions can be nested.
And in our context $toObjectId is an aggregation expression operators which means we cannot use it in $match without using $expr, like so:
db.collection.aggregate([
{
$match: {
$expr: {
$eq: [
"$statusRepositoryId",
{
$toObjectId: "5dd7fa20dcfa9600152cc2d8"
}
]
}
}
}
])
Mongo Playground
Meaning you'll have to re-structure your query which could be quite annoying. But we do have a better solution, just import ObjectId from Mongo and cast the string to that while constructing the query:
if (obj.key === "status_repository") {
if (
report.filters.find((x) => x.key === obj.key).selectionType === "single"
) {
matchObj.$match.$or.push({
statusRepositoryId: new ObjectId(obj.values),
});
} else {
obj.values.forEach((id) => {
matchObj.$match.$or.push({ statusRepositoryId: new ObjectId(id) });
});
}
}
I have two different collections (example below) methods & items. As for now, I'm using pre 3.6 vanilla aggregation query for $lookup:
MongoPlayground Example
{
$lookup: {
from: "items",
localField: "reagents._id",
foreignField: "_id",
as: "reagent_items"
}
}
The problem is that if I am using it, I miss quantity field (from methods.reagents embedded) during $lookup stage from original collection. For now, I return quantity right after lookup but as I heard, Mongo introduced from 3.6 a new syntax for lookup queries, so the question is:
Can it solve my problem for receiving the following results:
{
"_id": 1,
"name": "Test",
"reagent_items": [ // <= the exact schema of what I need after lookup
{
"_id": 1,
"name": "ItemOne",
"other": "field",
"quantity": 2 //quantity field from original {array of objects} after lookup
},
{
"_id": 2,
"name": "ItemTwo",
"other": "field",
"quantity": 4 //quantity field from original {array of objects} after lookup
}
],
"reagents": [ //original reagents field here just for example, we could remove it
{
"_id": 1,
"quantity": 2
},
{
"_id": 2,
"quantity": 4
}
]
}
methods
{
"_id": 1,
"name": "Test",
"reagents": [
{
_id: 1,
quantity: 2
},
{
_id: 2,
quantity: 4
}
]
}
items
{
"_id": 1,
"name": "ItemOne",
"other": "field"
},
{
"_id": 2,
"name": "ItemTwo",
"other": "field"
}
Use $map along with $arrayElemAt to find corresponding reagent for each reagent_items and the apply $mergeObjects to get one object:
db.methods.aggregate([
{
$lookup: {
from: "items",
localField: "reagents._id",
foreignField: "_id",
as: "reagent_items"
}
},
{
$project: {
_id:1,
name: 1,
reagents: 1,
reagent_items: {
$map: {
input: "$reagent_items",
as: "ri",
in: {
$mergeObjects: [
"$$ri",
{
$arrayElemAt: [ { $filter: { input: "$reagents", cond: { $eq: [ "$$this._id", "$$ri._id" ] } } }, 0 ]
}
]
}
}
}
}
}
])
Mongo Playground
I'm trying to fetch all latest messages between User A and any other user.
I keep running into the error ,
The field name '$acknowledged' cannot be an operator name
Not sure what I'm doing wrong here. Mongo playground.
The expected output should be the latest message exchanged between user with id 5a934e000102030405000001, and any other user.
[
{
"from": ObjectId("5a934e000102030405000002"),
"to": ObjectId("5a934e000102030405000001"),
"acknowledged": true,
date: "2020-04-17T18:26:34.353+00:00"
},
{
"from": ObjectId("5a934e000102030405000001"),
"to": ObjectId("5a934e000102030405000003"),
"acknowledged": false,
date: "2020-04-17T18:26:31.353+00:00"
},
{
"from": ObjectId("5a934e000102030405000004"),
"to": ObjectId("5a934e000102030405000001"),
"acknowledged": false,
date: "2020-04-17T18:26:29.353+00:00"
},
]
You had a typo here:
$acknowledged: { acknowledged: {
$first: "$acknowledged", --> $first: "$acknowledged"
}
},
and
then: "$responseTo", --> then: "$to",
db.Message.aggregate([
{
$match: {
$or: [
{
from: {
$in: [
ObjectId("5a934e000102030405000001")
]
}
},
{
to: {
$in: [
ObjectId("5a934e000102030405000001")
]
}
}
]
}
},
{
$sort: {
date: -1
}
},
{
$group: {
_id: {
userConcerned: {
$cond: [
{
$in: [
"$to",
[
ObjectId("5a934e000102030405000001")
]
]
},
"$to",
"$from"
]
},
interlocutor: {
$cond: [
{
$in: [
"$to",
[
ObjectId("5a934e000102030405000001")
]
]
},
"$from",
"$to"
]
}
},
id: {
$first: "$_id"
},
from: {
$first: "$from"
},
acknowledged: {
$first: "$acknowledged"
},
to: {
$first: "$to"
},
date: {
$first: "$date"
}
}
},
{
$lookup: {
from: "User",
localField: "to",
foreignField: "_id",
as: "to"
}
},
{
$unwind: "$to"
},
{
$lookup: {
from: "User",
localField: "from",
foreignField: "_id",
as: "from"
}
},
{
$unwind: "$from"
},
{
$project: {
_id: 0,
date: 1,
acknowledged: 1,
from: "$from._id",
to: "$to._id"
}
}
])
MongoPlayground
This is an example of getList of Arrivals by 10 items per page:
router.get('/arrivals', isAuthenticated, async (request, response, next) => {
jsonPreProcessor.response = response;
const resPerPage = 10;
const page = request.query.page || 1;
Arrival.find({})
.populate({
path: 'product',
populate: {
path: 'type'
}
})
.select('-__v')
.skip((resPerPage * page) - resPerPage)
.limit(resPerPage).then(arrivals => {
Arrival.countDocuments({}).then(numberOfResults => {
return jsonPreProcessor.paginate(arrivals, page, Math.ceil(numberOfResults / resPerPage), numberOfResults);
}).catch(error => {
return jsonPreProcessor.error(error.message);
});
}).catch(error => {
return jsonPreProcessor.error(error.message);
});
});
And this is an output:
{
"time": "2020-01-16T10:11:22.588Z",
"message": "success",
"success": true,
"data": {
"list": [
{
"quantity": 1,
"discount": 0,
"_id": "5e0db80a37dd4437b4329960",
"product": {
"_id": "5e0cecaaa9a5cc2c7c62e379",
"title": "Ортопедический",
"type": {
"_id": "5ddcbc4685e53838dc564a44",
"title": "fsdkjhfs",
"createdAt": "2019-11-26T05:46:46.797Z",
"updatedAt": "2019-11-26T05:46:46.797Z",
"alt": "fsdkjhfs",
"__v": 0
},
...
"mode": "simple",
"createdAt": "2020-01-01T19:02:02.840Z",
"updatedAt": "2020-01-01T19:02:02.840Z",
"alt": "ortopedicheskij",
"__v": 0
},
...
"sellPrice": 6,
"minSellPrice": 0,
"createdAt": "2020-01-02T09:29:46.688Z",
"updatedAt": "2020-01-13T09:30:26.126Z"
},
... // And other 9 items
],
"currentPage": 1,
"pages": 2,
"numberOfResults": 16,
"incompleteResults": true
},
"type": null
}
All those are arrivals, and as you see arrivals have product. Some arrivals can have same product (by id)
My problem is to group arrivals which have same product. (sorted by first item of arrivals array) populate all objects.. and paginate. I use aggregate, and unfortunately I don't have any idea how to use it (first of all aggregate's output is random) with pagination and of course output is not what I need. I used this link as solving of my problem.
router.get('/arrivalls', isAuthenticated, async (request, response, next) => {
jsonPreProcessor.response = response;
Arrival.aggregate(
[
// This is not working (may be it's not working in array)
// {
// "$sort": {
// "createdAt": 1
// }
// },
{
"$group": {
"_id": "$product",
"arrivals": {
"$push": "$$ROOT"
}
}
},
// {
// "$lookup": {
// "from": "arrivals",
// "localField": "product",
// "foreignField": "_id",
// "as": "product"
// }
// },
// {
// "$unwind": {
// "path": "$arrivals"
// }
// }
]
).then(arrivals => {
// console.log(arrivals);
return jsonPreProcessor.success(arrivals);
}).catch(error => {
return jsonPreProcessor.error(error.message);
});
});
Output should be like:
"data": {
"list": [
{
"_id": "5e1d5dba611485397cfb0386",
"arrivals": [
{
"_id": "5e1d5e26611485397cfb0387",
"quantity": 6,
"discount": 0,
"product": {
"_id": "5e1d5dba611485397cfb0386",
... // etc
},
"sellPrice": 5000,
"minSellPrice": 4500,
"createdAt": "2020-01-14T06:22:30.366Z",
"updatedAt": "2020-01-14T09:14:13.824Z",
"__v": 0
},
{
"_id": "5e1ff4d15d059430e8405f94",
"quantity": 2,
"discount": 0,
"product": {
"_id": "5e1d5dba611485397cfb0386",
... // etc
},
"sellPrice": 7000,
"minSellPrice": 6000,
"comment": "",
"createdAt": "2020-01-16T05:29:53.907Z",
"updatedAt": "2020-01-16T05:29:53.907Z",
"__v": 0
}
]
},
{
"_id": "5e1d84884d387d2334a7e9d9",
"arrivals": [
{
// etc...
}
]
}
],
"currentPage": 1,
"pages": 2,
"numberOfResults": 16,
"incompleteResults": true
},
"type": null
}
I solved my problem.. And here is grouping, sorting, populating and paginating of list.
const resPerPage = 10;
const page = request.query.page || 1;
Arrival.aggregate(
[
{
$lookup: {
from: 'products',
localField: 'product',
foreignField: '_id',
as: 'product'
}
},
{
$unwind: {
path: "$product",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'branchdans',
localField: 'branch',
foreignField: '_id',
as: 'branch'
}
},
{
$unwind: {
path: "$branch",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'colors',
localField: 'color',
foreignField: '_id',
as: 'color'
}
},
{
$unwind: {
path: "$color",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'types',
localField: 'product.type',
foreignField: '_id',
as: 'product.type'
}
},
{
$unwind: {
path: "$product.type",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'subcategories',
localField: 'product.subcategory',
foreignField: '_id',
as: 'product.subcategory'
}
},
{
$unwind: {
path: "$product.subcategory",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'categories',
localField: 'product.subcategory.category',
foreignField: '_id',
as: 'product.subcategory.category'
}
},
{
$unwind: {
path: "$product.subcategory.category",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'countries',
localField: 'product.country',
foreignField: '_id',
as: 'product.country'
}
},
{
$unwind: {
path: "$product.country",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: 'manufacturers',
localField: 'product.manufacturer',
foreignField: '_id',
as: 'product.manufacturer'
}
},
{
$unwind: {
path: "$product.manufacturer",
preserveNullAndEmptyArrays: true
}
},
{
$group: {
"_id": "$product._id",
"sizes": {
"$first": "$product.sizes"
},
"title": {
"$first": "$product.title"
},
"type": {
"$first": "$product.type"
},
"subcategory": {
"$first": "$product.subcategory"
},
"country": {
"$first": "$product.country"
},
"manufacturer": {
"$first": "$product.manufacturer"
},
"description": {
"$first": "$product.description"
},
"comment": {
"$first": "$product.comment"
},
"mode": {
"$first": "$product.mode"
},
"createdAt": {
"$first": "$product.createdAt"
},
"updatedAt": {
"$first": "$product.updatedAt"
},
"alt": {
"$first": "$product.alt"
},
arrivals: {
$push: "$$ROOT"
},
"date": {
$last: "$createdAt"
}
},
},
{
$unset: "arrivals.product"
},
{
$sort: {
"date": 1
}
},
{
$skip: (resPerPage * page) - resPerPage
},
{
$limit: resPerPage
}
]
).then(arrivals => {
Arrival.aggregate([
{
$group: {
"_id": "$product",
arrivals: {
$push: "$$ROOT"
},
"date": {
"$last": "$createdAt"
}
},
},
{
$sort: {
"date": 1
}
},
{
$count: "numberOfResults"
}
]).then(countArrivals => {
if(countArrivals.length === 0){
return jsonPreProcessor.error('Ошибка при высчитывании прибытий товаров');
}
// Todo make more practise
return jsonPreProcessor.paginate(arrivals, page, Math.ceil(countArrivals[0].numberOfResults / resPerPage), countArrivals[0].numberOfResults);
}).catch(error => {
return jsonPreProcessor.error(error.message);
});
// return jsonPreProcessor.success(arrivals);
}).catch(error => {
return jsonPreProcessor.error(error.message);
});