I'm trying to add in a project pipeline where the values are greater than 100, the values are fields inside and object which are part of an array. I have something like this:
Database:
---Clients Collection---
client: {
_id: 1,
taxID: aldsfkjasdlñfk
// other stuff
}
---Invoices Collection---
invoice: {
_id: 1,
clientID: 1,
total: 50
},
invoice: {
_id: 2,
clientID: 1,
total: 150
},
invoice: {
_id: 3,
clientID: 1,
total: 200
}
AND THIS IS MY QUERY:
{
$lookup: {
from: 'invoices',
localField: '_id',
foreignField: 'client.id',
as: 'invoices'
}
},
{
$project: {
id: 1,
taxID: aldsfkjasdlñfk,
invoicesAmountGreaterThanOneHundred: {
$sum: {
$cond: { if: { $gte: ['$invoices.total', 100] }, then: '$invoices.total', else: 0 }
}
}
}
}
So the output should be:
{
_id: 1.
taxID: aldsfkjasdlñfk,
invoicesAmountGreaterThanOneHundred: 350
}
I'm using Mongo 3.6.3.
Also in the future I will add a "invoicesAmountLesserThanOneHundred", same method, but for lesser than 100 of course.
use $filter before $sum
db.client.aggregate([
{
$lookup: {
from: "invoices",
localField: "_id",
foreignField: "clientID",
as: "invoices"
}
},
{
$set: {
"invoices": {
"$filter": {
"input": "$invoices",
"as": "i",
"cond": { $gte: [ "$$i.total", 100 ] }
}
}
}
},
{
$project: {
id: 1,
taxID: 1,
invoicesAmountGreaterThanOneHundred: {
$sum: "$invoices.total"
}
}
}
])
mongoplayground
use $reduce
db.client.aggregate([
{
$lookup: {
from: "invoices",
localField: "_id",
foreignField: "clientID",
as: "invoices"
}
},
{
$set: {
"invoicesAmountGreaterThanOneHundred": {
$reduce: {
input: "$invoices",
initialValue: "",
in: {
$sum: [
"$$value",
{
$cond: {
if: { $gte: [ "$$this.total", 100 ] },
then: "$$this.total",
else: 0
}
}
]
}
}
}
}
}
])
mongoplayground
I would like to return this only if there are 2 by in the data array. The number of _id can be unlimited.
However, the code { $size: { data: 2 }, } does not work because I get $size is not allowed in this atlas tier error.
Expected return:
[
{
"_id": "Something1?",
"data": [
{
"by": "user1",
},
{
"by": "user2",
}
]
},
]
I want to include something like $size in the code, otherwise it will return the data even if there is only 1 by, or 3 by, or 0 by. I only want to return the data if there are 2 by.
What should I do? Full code without $size:
let x = await Answer.aggregate([
{
$match: {
$and: [
{
by: {
$in: [user.email, user2[0].email],
},
},
],
},
},
{
$group: {
_id: "$question",
data: {
$push: "$$ROOT",
},
},
},
{
$project: {
"data._id": 0,
"data.question": 0,
"data.__v": 0,
},
},
{ $sort: { "data.date": -1 } },
]);
Looks like your atlas tier doesn't support $size.
But you can have a field like count that increments by 1 when grouping:
db.collection.aggregate([
{
$group: {
_id: "$question",
data: {
$push: "$$ROOT",
},
count: {
$sum: 1
}
}
},
{
$match: {
count: 2
}
}
])
Try this in playground
Update
Finally, your aggregation should look like this:
[
{
$match: {
$and: [
{
by: {
$in: [user.email, user2[0].email],
},
},
],
},
},
{
$group: {
_id: "$question",
data: {
$push: "$$ROOT",
},
count: {
$sum: 1
}
},
},
{
$match: {
count: 2
}
},
{
$project: {
"data._id": 0,
"data.question": 0,
"data.__v": 0,
"count": 0
},
},
{ $sort: { "data.date": -1 } },
]
You can learn more about $sum here.
Presuming your model is called Employee:
Employee.find({ { "social_account.2": { "$exists": false }} },function(err,docs) {
})
As $exists asks for the 2 index of an array which means it has something in it.
The same applies to a maximum number:
Employee.find({ { "social_account.9": { "$exists": true}} },function(err,docs) {
})
For your perspective I think this should be your answer:
Employee.find({ { "data.2": { "$exists": false }} },function(err,docs) {
})
I have ten stations stored in the stations collection: Station A, Station B, Station C, Station D, Station E, Station F, Station G, Station H, Station I, Station J.
Right now, to create a count list of all inter-station rides between all possible pairs of stations, I do the following in my Node.js code (using Mongoose):
const stationCombinations = []
// get all stations from the stations collection
const stationIds = await Station.find({}, '_id name').lean().exec()
// list of all possible from & to combinations with their names
stationIds.forEach(fromStation => {
stationIds.forEach(toStation => {
stationCombinations.push({ fromStation, toStation })
})
})
const results = []
// loop through all station combinations
for (const stationCombination of stationCombinations) {
// create aggregation query promise
const data = Ride.aggregate([
{
$match: {
test: false,
state: 'completed',
duration: { $gt: 2 },
fromStation: mongoose.Types.ObjectId(stationCombination.fromStation._id),
toStation: mongoose.Types.ObjectId(stationCombination.toStation._id)
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
},
{
$addFields: {
fromStation: stationCombination.fromStation.name,
toStation: stationCombination.toStation.name
}
}
])
// push promise to array
results.push(data)
}
// run all aggregation queries
const stationData = await Promise.all(results)
// flatten nested/empty arrays and return
return stationData.flat()
Executing this function give me the result in this format:
[
{
"fromStation": "Station A",
"toStation": "Station A",
"count": 1196
},
{
"fromStation": "Station A",
"toStation": "Station B",
"count": 1
},
{
"fromStation": "Station A",
"toStation": "Station C",
"count": 173
},
]
And so on for all other combinations...
The query currently takes a lot of time to execute and I keep getting alerts from MongoDB Atlas about excessive load on the database server because of these queries. Surely there must be an optimized way to do something like this?
You need to use MongoDB native operations. You need to $group by fromStation and toStation and with $lookup join two collections.
Note: I assume you have MongoDB >=v3.6 and Station._id is ObjectId
db.ride.aggregate([
{
$match: {
test: false,
state: "completed",
duration: {
$gt: 2
}
}
},
{
$group: {
_id: {
fromStation: "$fromStation",
toStation: "$toStation"
},
count: {
$sum: 1
}
}
},
{
$lookup: {
from: "station",
let: {
fromStation: "$_id.fromStation",
toStation: "$_id.toStation"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
[
"$$fromStation",
"$$toStation"
]
]
}
}
}
],
as: "tmp"
}
},
{
$project: {
_id: 0,
fromStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.fromStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
toStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.toStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
count: 1
}
},
{
$sort: {
fromStation: 1,
toStation: 1
}
}
])
MongoPlayground
Not tested:
const data = Ride.aggregate([
{
$match: {
test: false,
state: 'completed',
duration: { $gt: 2 }
}
},
{
$group: {
_id: {
fromStation: "$fromStation",
toStation: "$toStation"
},
count: { $sum: 1 }
}
},
{
$lookup: {
from: "station",
let: {
fromStation: "$_id.fromStation",
toStation: "$_id.toStation"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
[
"$$fromStation",
"$$toStation"
]
]
}
}
}
],
as: "tmp"
}
},
{
$project: {
_id: 0,
fromStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.fromStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
toStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.toStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
count: 1
}
},
{
$sort: {
fromStation: 1,
toStation: 1
}
}
])
I have this endpoint, it's the initial endpoint when a customer is visiting the eshop:
export const getAllProductsByCategory = async (req, res, next) => {
const pageSize = parseInt(req.query.pageSize);
const sort = parseInt(req.query.sort);
const skip = parseInt(req.query.skip);
const { order, filters } = req.query;
const { brands, tags, pricesRange } = JSON.parse(filters);
try {
const aggregate = Product.aggregate();
aggregate.lookup({
from: 'categories',
localField: 'categories',
foreignField: '_id',
as: 'categories'
});
aggregate.match({
productType: 'product',
available: true,
categories: {
$elemMatch: {
url: req.params
}
}
});
aggregate.lookup({
from: 'tags',
let: { tags: '$tags' },
pipeline: [
{
$match: {
$expr: { $in: ['$_id', '$$tags'] }
}
},
{
$project: {
_id: 1,
name: 1,
slug: 1
}
}
],
as: 'tags'
});
aggregate.lookup({
from: 'brands',
let: { brand: '$brand' },
pipeline: [
{
$match: {
$expr: { $eq: ['$_id', '$$brand'] }
}
},
{
$project: {
_id: 1,
name: 1,
slug: 1
}
}
],
as: 'brand'
});
if (brands.length > 0) {
const filterBrands = brands.map((_id) => utils.toObjectId(_id));
aggregate.match({
$and: [{ brand: { $elemMatch: { _id: { $in: filterBrands } } } }]
});
}
if (tags.length > 0) {
const filterTags = tags.map((_id) => utils.toObjectId(_id));
aggregate.match({ tags: { $elemMatch: { _id: { $in: filterTags } } } });
}
if (pricesRange.length > 0 && pricesRange !== 'all') {
const filterPriceRange = pricesRange.map((_id) => utils.toObjectId(_id));
aggregate.match({
_id: { $in: filterPriceRange }
});
}
aggregate.facet({
tags: [
{ $unwind: { path: '$tags' } },
{ $group: { _id: '$tags', tag: { $first: '$tags' }, total: { $sum: 1 } } },
{
$group: {
_id: '$tag._id',
name: { $addToSet: '$tag.name' },
total: { $addToSet: '$total' }
}
},
{
$project: {
name: { $arrayElemAt: ['$name', 0] },
total: { $arrayElemAt: ['$total', 0] },
_id: 1
}
},
{ $sort: { total: -1 } }
],
brands: [
{ $unwind: { path: '$brand' } },
{
$group: {
_id: '$brand._id',
name: { $first: '$brand.name' },
slug: { $first: '$brand.slug' },
total: {
$sum: 1
}
}
},
{ $sort: { name: 1 } }
],
pricesRange: [
{
$bucket: {
groupBy: {
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
boundaries: [0, 20.01, 50.01],
default: 'other',
output: {
count: { $sum: 1 },
products: { $push: '$_id' }
}
}
}
],
products: [
{ $skip: (skip - 1) * pageSize },
{ $limit: pageSize },
{
$project: {
_id: 1,
images: 1,
onSale: 1,
price: 1,
quantity: 1,
slug: 1,
sale: 1,
sku: 1,
status: 1,
title: 1,
brand: 1,
tags: 1,
description: 1
}
},
{ $sort: { [order]: sort } }
],
total: [
{
$group: {
_id: null,
count: { $sum: 1 }
}
},
{
$project: {
count: 1,
_id: 0
}
}
]
});
aggregate.addFields({
total: {
$arrayElemAt: ['$total', 0]
}
});
const [response] = await aggregate.exec();
if (!response.total) {
response.total = 0;
}
res.status(httpStatus.OK);
return res.json(response);
} catch (error) {
console.log(error);
return next(error);
}
};
If no filters are applied all products matches the category requested with no problem.
My issue is when a customer selects a brand or tag, then the facet returns the products, but returns only one brand/tag (as it should be since the products filtered have only this brand).
What I must do in order to retain all brands/tags and let the user select more than one brand/tag? If customer selects a brand, then the tags should match the returned products tags and vice versa.
Is there a better way to implement tags stage in $facet since tags is an array and the desired output is: [{_id: 123, name: {label: 'test', value: 123]}]
The request is like:(1,2,3,4 represents _id)
http://locahost:3000/get-products/?filters={brands: [1, 2], tags: [3,4], pricesRange:[]}
Update
This is the products schema with tags and brands:
brand: {
ref: 'Brand',
type: Schema.Types.ObjectId
},
tags: [
{
ref: 'Tags',
type: Schema.Types.ObjectId
}
]
tags schema:
{
metaDescription: {
type: String
},
metaTitle: {
type: String
},
name: {
label: {
type: String,
index: true
},
value: {
type: Schema.Types.ObjectId
},
},
slug: {
type: String,
index: true
},
status: {
label: {
type: String
},
value: {
default: true,
type: Boolean
}
}
}
brands schema:
description: {
default: '',
type: String
},
name: {
required: true,
type: String,
unique: true
},
slug: {
type: String,
index: true
},
status: {
label: {
default: 'Active',
type: String
},
value: {
default: true,
type: Boolean
}
}
Scenario:
User visits store, selects a category and all matching products should return with matched brands, tags, priceRange & pagination.
Case 1:
User clicks a brand from checkbox, then the request returns matching products,tags & priceRanges and all brands of the selected category, not of matched products
Case 2:
User selects a brand like Case 1, but then decides to check a tag too, then the request should return all brands and tags again, but products matched against them.
Case 3:
User do not select brand but selects a tag only, the request should return all matching products that have that tag/tags and return the brands that matched the products returned.
Case 4:
Same as case 3, but user selects a brand after selecting a tag/tags, the request should return matching products, brands & tags.
In all cases pagination should return proper total, also priceRanges should match the returned results.
I hope it's clear now, I think I've not missed any other case. I could probably grey out/disable the tags/brands that do not match the response in the front end but I don't know if this is user friendly.
This is what I ended up with:
export const getAllProductsByCategory = async (req, res, next) => {
const pageSize = parseInt(req.query.pageSize);
const sort = parseInt(req.query.sort);
const skip = parseInt(req.query.skip);
const { order, filters } = req.query;
const { brands, tags, pricesRange } = JSON.parse(filters);
try {
const aggregate = Product.aggregate();
aggregate.lookup({
from: 'categories',
localField: 'categories',
foreignField: '_id',
as: 'categories'
});
aggregate.match({
productType: 'product',
available: true,
categories: {
$elemMatch: {
url: `/${JSON.stringify(req.params['0']).replace(/"/g, '')}`
}
}
});
aggregate.lookup({
from: 'tags',
let: { tags: '$tags' },
pipeline: [
{
$match: {
$expr: { $in: ['$_id', '$$tags'] }
}
},
{
$project: {
_id: 1,
name: 1,
slug: 1
}
}
],
as: 'tags'
});
aggregate.lookup({
from: 'brands',
let: { brand: '$brand' },
pipeline: [
{
$match: {
$expr: { $eq: ['$_id', '$$brand'] }
}
},
{
$project: {
_id: 1,
name: 1,
slug: 1
}
}
],
as: 'brand'
});
const filterBrands = brands.map((_id) => utils.toObjectId(_id));
const filterTags = tags.map((_id) => utils.toObjectId(_id));
const priceRanges = pricesRange ? pricesRange.match(/\d+/g).map(Number) : '';
aggregate.facet({
tags: [
{ $unwind: { path: '$brand' } },
{ $unwind: { path: '$tags' } },
{
$match: {
$expr: {
$and: [
filterBrands.length ? { $in: ['$brand._id', filterBrands] } : true
]
}
}
},
{ $group: { _id: '$tags', tag: { $first: '$tags' }, total: { $sum: 1 } } },
{
$group: {
_id: '$tag._id',
name: { $addToSet: '$tag.name' },
total: { $addToSet: '$total' }
}
},
{
$project: {
name: { $arrayElemAt: ['$name', 0] },
total: { $arrayElemAt: ['$total', 0] },
_id: 1
}
},
{ $sort: { name: 1 } }
],
brands: [
{ $unwind: { path: '$brand' } },
{ $unwind: { path: '$tags' } },
{
$match: {
$expr: {
$and: [
filterTags.length ? { $in: ['$tags._id', filterTags] } : true
]
}
}
},
{
$group: {
_id: '$brand._id',
name: { $first: '$brand.name' },
slug: { $first: '$brand.slug' },
total: {
$sum: 1
}
}
},
{ $sort: { name: 1 } }
],
products: [
{ $unwind: { path: '$brand', preserveNullAndEmptyArrays: true } },
{ $unwind: { path: '$tags', preserveNullAndEmptyArrays: true } },
{
$match: {
$expr: {
$and: [
filterBrands.length ? { $in: ['$brand._id', filterBrands] } : true,
filterTags.length ? { $in: ['$tags._id', filterTags] } : true,
pricesRange.length
? {
$and: [
{
$gte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
priceRanges[0]
]
},
{
$lte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
priceRanges[1]
]
}
]
}
: true
]
}
}
},
{ $skip: (skip - 1) * pageSize },
{ $limit: pageSize },
{
$project: {
_id: 1,
brand: 1,
description: 1,
images: 1,
onSale: 1,
price: 1,
quantity: 1,
sale: 1,
shipping: 1,
sku: 1,
skuThreshold: 1,
slug: 1,
status: 1,
stock: 1,
tags: 1,
title: 1
}
},
{ $sort: { [order]: sort } }
],
pricesRange: [
{ $unwind: { path: '$brand', preserveNullAndEmptyArrays: true } },
{ $unwind: { path: '$tags', preserveNullAndEmptyArrays: true } },
{
$match: {
$expr: {
$and: [
filterBrands.length ? { $in: ['$brand._id', filterBrands] } : true,
filterTags.length ? { $in: ['$tags._id', filterTags] } : true
]
}
}
},
{
$project: {
price: 1,
onSale: 1,
sale: 1,
range: {
$cond: [
{
$and: [
{
$gte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
0
]
},
{
$lte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
20
]
}
]
},
'0-20',
{
$cond: [
{
$and: [
{
$gte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
20
]
},
{
$lte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
50
]
}
]
},
'20-50',
'50+'
]
}
]
}
}
},
{
$group: {
_id: '$range',
count: { $sum: 1 }
}
},
{
$project: {
_id: 0,
range: '$_id',
count: 1
}
},
{ $unwind: { path: '$range', preserveNullAndEmptyArrays: true } },
{
$sort: {
range: 1
}
}
],
total: [
{ $unwind: { path: '$brand', preserveNullAndEmptyArrays: true } },
{ $unwind: { path: '$tags', preserveNullAndEmptyArrays: true } },
{
$match: {
$expr: {
$and: [
filterBrands.length ? { $in: ['$brand._id', filterBrands] } : true,
filterTags.length ? { $in: ['$tags._id', filterTags] } : true,
pricesRange.length
? {
$and: [
{
$gte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
priceRanges[0]
]
},
{
$lte: [
{
$cond: {
if: { $ne: ['$onSale.value', true] },
then: '$price',
else: '$sale.salePrice'
}
},
priceRanges[1]
]
}
]
}
: true
]
}
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
},
{
$project: {
count: 1,
_id: 0
}
}
]
});
aggregate.addFields({
total: {
$arrayElemAt: ['$total', 0]
}
});
const [response] = await aggregate.exec();
if (!response.total) {
response.total = 0;
}
res.status(httpStatus.OK);
return res.json(response);
} catch (error) {
console.log(error);
return next(error);
}
};