We Keep Coding

How to get record based upon two range of date and time in mongodb with Aggregation?

i'm trying to $match two $addFields dates (FromDate and ToDate) to
get record between given date and time but i am faceing issue with $match
two dates.
const Objdata = await Events.aggregate([
{
$lookup: {
from: "MeetingReminderSetting",
let: {
flag: false,
cid: "$ClientID",
esd: "$EventStartDtByTimeZone",
},
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ["$IsDeleted", "$$flag"] },
{ $eq: ["$ClientID", "$$cid"] },
],
},
},
},
],
as: "p1",
},
},
{
$unwind: {
path: "$p1",
preserveNullAndEmptyArrays: false,
},
},
{
$addFields: {
FromDate: {
$dateAdd: {
startDate: new Date("2022-01-01T00:00:00.000+00:00"),
unit: "hour",
amount: 24,
},
},
Todate: {
$dateAdd: {
startDate: {
$dateAdd: {
startDate: new Date("2022-01-01T00:00:00.000+00:00"),
unit: "hour",
amount: 24,
},
},
unit: "minute",
amount: 10,
},
},
},
},
//---issue facing------//
**{
$match: {
EventStartDtByTimeZone: {
$gte: "$FromDate",
$lte: "$Todate",
},
},
},**
//----------------//
]);
it working when i give static date in $gte and $lte.
but it not working when i apply date from >$addFields
i used multiple cases like i change date in $dateToString and than campare but it don't work

Meteor Collection Find Documents Based on Latest Date Time in an Array

How to get the latest documents from a collection using date time?
I have searched in SO for this specific problem, but couldn't find an example that is similar to my data structure. I have this kind of data structure:
[
{
stationId: 'xxxxx',
stationName: 'xxxx',
state: 'xxxx',
lat: 'xxxxx',
long: 'xx.xxxxx',
waterLevel: [
{
wlDateTime: '11/04/2022 11:30',
wlSeverity: 'Danger',
wlLevel: 7.5
},
{
wlDateTime: '11/04/2022 09:00',
wlSeverity: 'Danger',
wlLevel: 7.3
},
{
wlDateTime: '11/04/2022 03:00',
wlSeverity: 'Normal',
wlLevel: 5.2
}
],
rainfallData: [
{
rfDateTime: '11/04/2022 11:30',
rfSeverity: 'Heavy',
rfLevel: 21
},
{
rfDateTime: '11/04/2022 10:30',
rfSeverity: 'Heavy',
rfLevel: 21
},
{
rfDateTime: '11/04/2022 9:30',
rfSeverity: 'Heavy',
rfLevel: 21
}
]
}
]
The question is, how can I get documents that have wlDateTime equal today, with wlSeverity equal to Danger, but I just want the latest record from the waterLevel array. The same case with the rainfallDataarray i.e. to return with the latest reading for today.
Sample expected return will be like this:
[
{
stationId: 'xxxxx',
stationName: 'xxxx',
state: 'xxxx',
lat: 'xxxxx',
long: 'xx.xxxxx',
waterLevelData: [
{
wlDateTime: '11/04/2022 11:30', //latest data compared to the array
wlSeverity: 'Danger',
wlLevel: 7.5
}
],
rainfallData: [
{
rfDateTime: '11/04/2022 11:30', //latest data compared to the array
rfSeverity: 'Heavy',
rfLevel: 21
}
]
}
]
I've tried querying it like this:
Meteor.publish('Alerts', function(){
return AlertLatest.find({
'waterLevelData.wlSeverity':'Danger',
}, {
fields : {
'stationName' : 1,
'state' : 1,
'lat' : 1,
'long' : 1,
'waterLevelData.wlDateTime' : 1,
'waterLevelData.wlSeverity' : 1,
'waterLevelData.wlLevel' : 1,
'rainfallData.rfSeverity' : 1,
}},{sort: { 'waterLevelData.wlDateTime' : -1}});
})
but the query returned data that isn't how I wanted. Any help will be much appreciated.
UPDATE
I've tried the solution provided by #YuTing, which is using aggregate to customise the publication query. I went ahead and read a bit about Mongodb Aggregation, and found a Meteorjs community package (tunguska:reactive-aggregate) which simplifies the process.
This is the sample of a working aggregation so far:
Meteor.publish('PIBDataAlerts', function(){
const start = dayjs().startOf('day'); // set to 12:00 am today
const end = dayjs().endOf('day'); // set to 23:59 pm today
ReactiveAggregate(this, PIBLatest, [
{
$match: {
'stationStatus' : 'ON',
'waterLevelData': { //trying to get only today's docs
"$elemMatch" : {
"wlDateTime" : {
$gte: start.format() , $lt: end.format()
}
}
}
}
},
{
$set: {
waterLevelHFZ: {
$filter: {
input: "$waterLevelData",
as: "w",
cond: {
$and: [
{ $or : [
{ $eq: [ "$$w.wlSeverity", "Alert" ] },
{ $eq: [ "$$w.wlSeverity", "Warning" ] },
{ $eq: [ "$$w.wlSeverity", "Danger" ] },
]},
{ $eq: [ "$$w.wlDateTime", { $max: "$waterLevelData.wlDateTime" } ] }
],
}
}
},
rainfallDataHFZ: {
$filter: {
input: "$rainfallData",
as: "r",
cond: { $eq: [ "$$r.rfDateTime", { $max: "$rainfallData.rfDateTime" } ] }
}
}
}
},
{
$project : {
"stationId": 1,
"stationName" :1,
"state": 1,
"waterLevelHFZ": 1,
"rainfallDataHFZ": 1
}
}
]);
})
I'm struggling to get documents that only have the wlDateTime that equals today. I've tried a query in the $match but it returned empty array. If the $match is set to {}, it'll return all 1548 records even though the wlDateTime is not equals to today.

change your date string to date
filter the array to find the max one
db.collection.aggregate([
{
$match: {
$expr: {
$or: [
{
$ne: [
{
$filter: {
input: "$waterLevel",
as: "w",
cond: {
$eq: [
{
$dateTrunc: {
date: {
$dateFromString: {
dateString: "$$w.wlDateTime",
format: "%d/%m/%Y %H:%M"
}
},
unit: "day"
}
},
{
$dateTrunc: {
date: "$$NOW",
unit: "day"
}
}
]
}
}
},
[]
]
},
{
$ne: [
{
$filter: {
input: "$rainfallData",
as: "r",
cond: {
$eq: [
{
$dateTrunc: {
date: {
$dateFromString: {
dateString: "$$r.rfDateTime",
format: "%d/%m/%Y %H:%M"
}
},
unit: "day"
}
},
{
$dateTrunc: {
date: "$$NOW",
unit: "day"
}
}
]
}
}
},
[]
]
}
]
}
}
},
{
$set: {
waterLevel: {
$map: {
input: "$waterLevel",
as: "w",
in: {
$mergeObjects: [
"$$w",
{
wlDateTime: {
$dateFromString: {
dateString: "$$w.wlDateTime",
format: "%d/%m/%Y %H:%M"
}
}
}
]
}
}
},
rainfallData: {
$map: {
input: "$rainfallData",
as: "r",
in: {
$mergeObjects: [
"$$r",
{
rfDateTime: {
$dateFromString: {
dateString: "$$r.rfDateTime",
format: "%d/%m/%Y %H:%M"
}
}
}
]
}
}
}
}
},
{
$set: {
waterLevel: {
$filter: {
input: "$waterLevel",
as: "w",
cond: {
$and: [
{
$in: [
"$$w.wlSeverity",
[
"Alert",
"Warning",
"Danger"
]
]
},
{
$eq: [
"$$w.wlDateTime",
{
$max: "$waterLevel.wlDateTime"
}
]
},
{
$eq: [
{
$dateTrunc: {
date: "$$w.wlDateTime",
unit: "day"
}
},
{
$dateTrunc: {
date: "$$NOW",
unit: "day"
}
}
]
}
]
}
}
},
rainfallData: {
$filter: {
input: "$rainfallData",
as: "r",
cond: {
$and: [
{
$eq: [
"$$r.rfDateTime",
{
$max: "$rainfallData.rfDateTime"
}
]
},
{
$eq: [
{
$dateTrunc: {
date: "$$r.rfDateTime",
unit: "day"
}
},
{
$dateTrunc: {
date: "$$NOW",
unit: "day"
}
}
]
}
]
}
}
}
}
}
])
mongoplayground

I don't think you can sort by embedded document in an array field. It's not how mongodb works.

but I just want the latest
I you are only interested in the latest docs you can omit the sort and instead use a natural negative cursor:
Meteor.publish('Alerts', function(){
return AlertLatest.find({
'waterLevelData.wlSeverity':'Danger',
}, {
fields : {
'stationName' : 1,
'state' : 1,
'lat' : 1,
'long' : 1,
'waterLevelData.wlDateTime' : 1,
'waterLevelData.wlSeverity' : 1,
'waterLevelData.wlLevel' : 1,
'rainfallData.rfSeverity' : 1,
}},{ hint: { $natural: -1}});
})
It will start counting docs from the end, instead of the beginning.
https://docs.meteor.com/api/collections.html#Mongo-Collection-find

converting the datestring in a nested array of objects (MongoDB)

What I want to achieve is finding a specific document on that current month based on the provided date. The date is stored as a string, in order for me to compare the date I need to convert the date first. However I have trouble on converting the datestring in a nested array of objects.
My collections:
{
sections: [{
fields: [{
name: 'Date',
value: '2020-11-30T15:59:59.999Z' // this is string
},
{
name: 'Title',
value: 'My book'
},
{
name: 'Author',
value: 'Henry'
}
]
]
}
}
What I have tried:
1)
const existingReport = await Report.find({
$expr: {
$gte: [
{
$dateFromString: {
dateString: "$sections.field[0].value",
},
},
moment(payload.forPeriod).startOf("month").toDate(),
],
$lt: [
{
$dateFromString: {
dateString: "$sections.field[0].value",
},
},
moment(payload.forPeriod).endOf("month").toDate(),
],
},
});
const existingReport1 = await Report.aggregate([
{
$addFields: {
formattedData: {
$cond: {
if: {
$eq: ["$sections.field.value", "Date"],
},
then: {
$dateFromString: {
dateString: "$sections.field.value",
},
},
else: "$sections.field.value",
},
},
},
},
]);

You can simply do a $toDate with the help of 2 $reduce to iterate the sections and fields array.
db.collection.aggregate([
{
"$match": {
$expr: {
$eq: [
true,
{
"$reduce": {
"input": "$sections",
"initialValue": false,
"in": {
"$reduce": {
"input": "$$this.fields",
"initialValue": false,
"in": {
$or: [
"$$value",
{
$and: [
{
$gte: [
{
"$toDate": "$$this.value"
},
new Date("2020-11-01")
]
},
{
$lte: [
{
"$toDate": "$$this.value"
},
new Date("2020-11-30")
]
}
]
}
]
}
}
}
}
}
]
}
}
}
])
Here is the Mongo playground for your reference.

Optimize combinational MongoDB query in Node.js

I have ten stations stored in the stations collection: Station A, Station B, Station C, Station D, Station E, Station F, Station G, Station H, Station I, Station J.
Right now, to create a count list of all inter-station rides between all possible pairs of stations, I do the following in my Node.js code (using Mongoose):
const stationCombinations = []
// get all stations from the stations collection
const stationIds = await Station.find({}, '_id name').lean().exec()
// list of all possible from & to combinations with their names
stationIds.forEach(fromStation => {
stationIds.forEach(toStation => {
stationCombinations.push({ fromStation, toStation })
})
})
const results = []
// loop through all station combinations
for (const stationCombination of stationCombinations) {
// create aggregation query promise
const data = Ride.aggregate([
{
$match: {
test: false,
state: 'completed',
duration: { $gt: 2 },
fromStation: mongoose.Types.ObjectId(stationCombination.fromStation._id),
toStation: mongoose.Types.ObjectId(stationCombination.toStation._id)
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
},
{
$addFields: {
fromStation: stationCombination.fromStation.name,
toStation: stationCombination.toStation.name
}
}
])
// push promise to array
results.push(data)
}
// run all aggregation queries
const stationData = await Promise.all(results)
// flatten nested/empty arrays and return
return stationData.flat()
Executing this function give me the result in this format:
[
{
"fromStation": "Station A",
"toStation": "Station A",
"count": 1196
},
{
"fromStation": "Station A",
"toStation": "Station B",
"count": 1
},
{
"fromStation": "Station A",
"toStation": "Station C",
"count": 173
},
]
And so on for all other combinations...
The query currently takes a lot of time to execute and I keep getting alerts from MongoDB Atlas about excessive load on the database server because of these queries. Surely there must be an optimized way to do something like this?

You need to use MongoDB native operations. You need to $group by fromStation and toStation and with $lookup join two collections.
Note: I assume you have MongoDB >=v3.6 and Station._id is ObjectId
db.ride.aggregate([
{
$match: {
test: false,
state: "completed",
duration: {
$gt: 2
}
}
},
{
$group: {
_id: {
fromStation: "$fromStation",
toStation: "$toStation"
},
count: {
$sum: 1
}
}
},
{
$lookup: {
from: "station",
let: {
fromStation: "$_id.fromStation",
toStation: "$_id.toStation"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
[
"$$fromStation",
"$$toStation"
]
]
}
}
}
],
as: "tmp"
}
},
{
$project: {
_id: 0,
fromStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.fromStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
toStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.toStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
count: 1
}
},
{
$sort: {
fromStation: 1,
toStation: 1
}
}
])
MongoPlayground
Not tested:
const data = Ride.aggregate([
{
$match: {
test: false,
state: 'completed',
duration: { $gt: 2 }
}
},
{
$group: {
_id: {
fromStation: "$fromStation",
toStation: "$toStation"
},
count: { $sum: 1 }
}
},
{
$lookup: {
from: "station",
let: {
fromStation: "$_id.fromStation",
toStation: "$_id.toStation"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
[
"$$fromStation",
"$$toStation"
]
]
}
}
}
],
as: "tmp"
}
},
{
$project: {
_id: 0,
fromStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.fromStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
toStation: {
$reduce: {
input: "$tmp",
initialValue: "",
in: {
$cond: [
{
$eq: [
"$_id.toStation",
"$$this._id"
]
},
"$$this.name",
"$$value"
]
}
}
},
count: 1
}
},
{
$sort: {
fromStation: 1,
toStation: 1
}
}
])

mongo find only if every element match the condition

i want to find every room that are available in a given time-period by passing a start and a end date, every collection that fail to at least once in one of these two test needs to be entirely excluded from the find query.
for the moment, it looks like if at least one collection succeed to match one of these condition once, it is returned to me.
here is a sample of a collection
"resa": [
{
"_id": "5cf2a38372373620263c84f1",
"start": "2019-06-01T15:23:00.000Z",
"end": "2019-06-01T16:23:00.000Z"
},
{
"_id": "5cf2a3a772373620263c84f2",
"start": "2022-03-05T16:23:00.000Z",
"end": "2022-03-05T17:23:00.000Z"
}
]
and here is my attempt so far
$or: [
{ resa: { $all: [{ $elemMatch: { start: { $lt : req.query.start }, end: { $lt : req.query.start } } } ]} },
{ resa: { $all: [{ $elemMatch: { start: { $gt : req.query.end }, end: { $gt : req.query.end } } } ]} }
],
derivating from Mickl answer, i've tried it like so but it show me no results
Post.find({
capacity: { $gte : req.query.capacity },
$expr: {
$allElementsTrue: {
$map: {
input: "$resa",
in: {
$or: [
{
$and: [
{ $gte: [ "$$this.start", req.query.end ] },
{ $gte: [ "$$this.end", req.query.end ] }
]
},
{
$and: [
{ $te: [ "$$this.start", req.query.start ] },
{ $lte: [ "$$this.end", req.query.start ] }
]
},
{ resa: [] },
]
}
}
}
}
},
i've also tried to reverse the query by finding collection that are NOT matching condition that means they will not be available at the given period
resa: {
$elemMatch: {
$not: {
$or: [
{
$and: [
{ start: { $gte : req.query.start }},
{ start: { $lte : req.query.end } }]
},
{
$and: [
{ end: { $lte : req.query.end }},
{ end: { $gte : req.query.start } }]
},
],
},
},
},

I managed to find the solution :
resa: {
$not: {
$elemMatch: {
$or: [
{
$and: [
{ start: { $gte : req.query.start }},
{ start: { $lte : req.query.end } }]
},
{
$and: [
{ end: { $gte : req.query.start }},
{ end: { $lte : req.query.end } }]
},
{
$and: [
{ start: { $gte : req.query.start }},
{ end: { $lte : req.query.end } }]
},
{
$and: [
{ start: { $lte : req.query.start }},
{ end: { $gte : req.query.end } }]
},
],
},
},
},
the important part to understand is the assembly of
resa: {
$not: {
$elemMatch: {
$or: [
{
$and: [
{ x: y},
{ x: y }]
},
{
$and: [
{ x: y},
{ x: y }]
},
...
that is a bit confusing in the meaning : "i want to find all the collection that does not match {this and this} or {that and that}"