I am trying to make aggregation with a Parse server (back4app, Parse server v2.7.1) but while I am able to aggregate by using the fields I explicitly created in the mongoDb, I am unable to aggregate by using the fields 'createdAt' or 'updatedAt'.
As an example, if I invoke:
query.aggregate(pipeline)
With:
{
project: {
objectId: "$objectId",
instr: "$instructions"
}
};
I have an array of records like:
{instr: "1", objectId: "CNHAdpMD0U"}
If on the other side I use:
{
project: {
objectId: "$objectId",
date: "$createdAt"
}
};
I have just:
{objectId: "CNHAdpMD0U"}
Finally, the pipeline:
{
project: {
objectId: "$objectId",
dayOfYear: { $dayOfYear: "$createdAt" }
}
};
Gives "500 - internal server error", but I guess is due to the missing retrieval of "$createdAt".
it seems there is a fix on Parse Server about this. At the moment, you can use like example the cloud function below:
Parse.Cloud.define('yourFunctionName', (req, res) => {
var pipeline = [{
group: {
objectId: { day: { $dayOfMonth: "$_created_at" }, month: { $month: "$_created_at" }, year: { $year: "$_created_at" } },
count: { $sum: 1 } }
}
];
var query = new Parse.Query(Parse.User);
query.aggregate(pipeline, { useMasterKey: true })
.then(function(results) {
res.success(results);
})
.catch(function(error) {
res.error(error)
});
});
Related
I try to find all data in my collection with mongoose but I have some problems to understand.
Now I use
const mongoose = require('mongoose');
const CaseSchema = new mongoose.Schema({
szenario: {
type: String,
default: 'deprecated'
},
name: {
type: String,
default: 'test'
},
date: {
type: Date,
default: Date.now
}
});
const Case = mongoose.model('tests', CaseSchema);
module.exports = Case;
May idea of the call is:
Case.find().distinct(name).exec();
But how I can select it distinct for the newest date with mongoose?
To get distinct name with lastest date, you need perform MongoDB aggregation with $group operator:
Case.aggregate([
{
$sort: {
name: 1,
date: 1
}
},
{
$group: {
_id: "$name",
data: {
$last: {
date: "$date",
_id: "$_id"
}
}
}
},
{
$project: {
_id: "$data._id",
date: "$data.date",
name: "$_id"
}
}
]).exec((err, cases) => {
if (err) throw err;
console.log(cases);
});
MongoPlayground
on my project i have users that complete combinations (called sessions) of courses. the fact of playing a course is called an attempt. During the attempt they can close it and come back later (so we keep a timelog object).
I have a request from the client which needs to return for each session, the users (and their attempts) that have played whole or part of their session during a certain timeframe.
During a certain timeframe means that the client sends a begin and end date and we count a user for a specific session if:
- the first attempt has begun before the end of the timeframe => the started of the first timelog of the first < ending date
- the last attempt has been finished after the begining of the timeframe => the end of the last timelog of the last attempt > starting date
Here is an example of an attempt object (the only one we need to use here):
{
"_id" : ObjectId("5b9148650ab5f43b5e829a4b"),
"index" : 0,
"author" : ObjectId("5acde2646055980a84914b6b"),
"timelog" : [
{
"started" : ISODate("2018-09-06T15:31:49.163Z"),
"ended" : ISODate("2018-09-06T15:32:03.935Z")
},
...
],
"session" : ObjectId("5b911d31e58dc13ab7586f9b")}
My idea was to make an aggregate on the attempts, to group those using author and session as an _id for the $group stage, and to push all the attempts of the user for this particular session into an array userAttempts.
Then to make an $addField stage to retrieve the started field of the first timelog of the first attempt and the last ended of the last attempt.
And finally to $filter or $match using those new fields.
Here is my aggregate:
const newDate = new Date()
_db.attempts.aggregate([
{ $match: {
author: { $in: programSessionsData.users },
$or: [{ programSession: { $in: programSessionIds } }, { oldTryFor: { $in: programSessionIds } }],
globalTime: $ex,
timelog: $ex }
},
{
$group: {
_id: {
user: "$author",
programSession: "$programSession"
},
userAttempts: { $push: { attemptId: "$_id", lastTimelog: { $arrayElemAt: ["$timelog", -1] }, timelog: "$timelog" } }
}
},
{
$addFields: { begin: { $reduce: {
input: "$userAttempts",
initialValue: newDate,
in: {
$cond: {
if: { $lt: ["$$this.timelog.0.started", "$$value"] },
then: "$$this.timelog.0.started",
else: "$$value"
} }
} } }
}
I also tried this for the addFields stage:
{
$addFields: { begin: { $reduce: {
input: "$userAttempts",
initialValue: newDate,
in: { $min: ["$$this.timelog.0.started", "$$value] }
} } }
}
However everytime begin is an empty array.
I do not really know how i can extract those two date, or compare dates between them.
To Note: the end one is more difficult that is why i have to first extract lastTimelog. If you an other method i would gladly take it.
Also this code is on a node server so i cannot use ISODate. and the mongo version used is 3.6.3.
After playing with aggregate a bit i came up with 2 solutions:
Solution 1
_db.attempts.aggregate([
{ $match: {
query
},
{
$group: {
_id: {
user: "$author",
programSession: "$programSession"
},
userAttempts: { $push: { attemptId: "$_id", timelog: "$timelog" } }
}
}, {
$addFields: {
begin: { $reduce: {
input: "$userAttempts",
initialValue: newDate,
in: { $min: [{ $reduce: {
input: "$$this.timelog",
initialValue: newDate,
in: { $min: ["$$this.started", "$$value"] }
} }, "$$value"] }
} },
end: { $reduce: {
input: "$userAttempts",
initialValue: oldDate,
in: { $max: [{ $reduce: {
input: "$$this.timelog",
initialValue: oldDate,
in: { $max: ["$$this.ended", "$$value"] }
} }, "$$value"] }
} }
}
},
{
$match: {
begin: { $lt: req.body.ended },
end: { $gt: req.body.started }
}
}
], { allowDiskUse: true });
newDate is today and oldDate is an arbitrary date in the past.
I had to chain 2 reduce because "$$this.timelog.0.started" would always return nothing. Don't really know why though.
Solution 2
_db.attempts.aggregate([
{ $match: {
query
},
{
$addFields: {
firstTimelog: { $arrayElemAt: ["$timelog", 0] },
lastTimelog: { $arrayElemAt: ["$timelog", -1] }
}
},
{
$group: {
_id: {
user: "$author",
programSession: "$programSession"
},
begin: { $min: "$firstTimelog.started" },
end: { $max: "$lastTimelog.ended" },
userAttempts: { $push: { attemptId: "$_id", timelog: "$timelog"} }
}
},
{
$match: {
begin: { $lt: req.body.ended },
end: { $gt: req.body.started }
}
}
], { allowDiskUse: true });
This one is a lot more straight forward and seems simpler, but oddly enough, from my testing, Solution 1 is always quicker at least in the object distribution for my project.
I'm working on a node.js project to display some data using charts and tables on the front end.
I have the two following queries on my route:
atendimentos.find({})
.then(atendimentos => {
final = atendimentos.filter(atendimentos => atendimentos.status === 'F')
testeea = atendimentos.filter(atendimentos => atendimentos.status === 'EA')
res.render('home', {user: req.user, fin: final.length, ea: testeea.length});
//Funciona
console.log(final.length)
})
.catch(err => console.error(err));
atendimentos.aggregate([
{ $project:
{ _id: "$month",
year: {$year: "$date" },
month: { $month: "$date"},
amount: 1
}
},
{ $group:
{ _id: { year: "$year", month: ("$month")},
sum: { $sum: 1}
}
}]).exec(function(error, items){
if(error){return next(error);}
console.log(items);
});
EDIT 1:
So, the input data... I guess that I don't have any because I'm actually fetching everything from the database through my queries. The data that I expect, are the documents/object with the status F or EA which I'm rendering on my chart.
The database has around 8.5k documents, the F one returns 8041 documents and the EA returns 351, it is a simple array with the number that is returned using .length on my route. Those numbers are rendered on the chart.
Now, related to the aggregation part, I'm trying to make a table using the collection. I intend to show the number of support calls (atendimentos) per month. It's actually logging the correct data like this:
[ { _id: { year: 2018, month: 6 }, sum: 4005 },
{ _id: { year: 2018, month: 7 }, sum: 43 },
{ _id: { year: 2018, month: 5 }, sum: 3996 },
{ _id: { year: 2018, month: 4 }, sum: 434 } ]
And I want to use this data to render the table on my view.
END OF EDIT 1
EDIT 2
router.get('/home', isAuthenticated, async (req, res, next) => {
let final;
let testeea;
atendimentos.find({})
.then(atendimentos => {
final = atendimentos.filter(atendimentos => atendimentos.status === 'F')
testeea = atendimentos.filter(atendimentos => atendimentos.status === 'EA')
res.render('home', {user: req.user, fin: final.length, ea: testeea.length});
//Funciona
console.log(final.length)
})
.catch(err => console.error(err));
So, here's the route, the other part is just the aggregation query that I've tried to do and closing brackets. As you can see, I get the data and use Array.filter to filter the results fetched, using status = F or = EA.
It returns me the length of the array, so it counts the number of status with each letter. This number is rendered in the chart, because I'm sending it to the front end as fin: final.length and ea: testeea.length. No formatted data or something like that in here. It's okay this way.
Related to the aggregation part where it returns the calls per month, I want to use just the number of calls, month and year. In this part I expected the data like: [ { _id: { year: 2018, month: 6 }, sum: 4005 }
I wish I could fetch the data the same way as I've fetched the fin and ea, using .length to count and put it into the view.
END OF EDIT 2
Both are returning exactly what I need, the problem is that I can't just put the aggregation query before the find query and add items: items to the render method. I would like to know how do I do these queries to display the same that that I'm fetching on these two queries. Thanks in advance!
MongoDB Server 3.2 and below
You need to run two aggregate queries and merge the objects in the results. This can be done in a multiple ways but can show you the Promise way and the async/await approach.
1. Using Promises
router.get('/home', isAuthenticated, (req, res, next) => {
const counts = atendimentos.aggregate([
{ '$group': {
'_id': null,
'fin': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'F' ] }, 1, 0 ]
}
},
'ea': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'EA' ] }, 1, 0 ]
}
}
} }
]).exec();
const monthly = atendimentos.aggregate([
{ '$group': {
'_id': {
'year': { '$year': '$date' },
'month': { '$month': '$date' }
},
'sum': { '$sum': 1 }
} },
{ '$group': {
'_id': null,
'back': { '$push': '$$ROOT' }
} },
]).exec();
Promise.all([ counts, monthly ]).then(([ counts, monthly ]) => {
const statusData = counts[0];
const monthlyData = monthly[0];
const data = {...statusData, ...monthlyData, user: req.user};
console.log(JSON.stringify(data, null, 4));
res.render('home', data);
}).catch(err => next(err));
});
2. Using async/await
router.get('/home', isAuthenticated, async (req, res, next) => {
try {
const counts = await atendimentos.aggregate([
{ '$group': {
'_id': null,
'fin': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'F' ] }, 1, 0 ]
}
},
'ea': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'EA' ] }, 1, 0 ]
}
}
} }
]).exec();
const monthly = await atendimentos.aggregate([
{ '$group': {
'_id': {
'year': { '$year': '$date' },
'month': { '$month': '$date' }
},
'sum': { '$sum': 1 }
} },
{ '$group': {
'_id': null,
'back': { '$push': '$$ROOT' }
} },
]).exec();
const statusData = counts[0];
const monthlyData = monthly[0];
const data = {...statusData, ...monthlyData, user: req.user};
console.log(JSON.stringify(data, null, 4));
res.render('home', data);
} catch (err) {
next(err);
}
});
MongoDB Server 3.4.4 and above
The aggregation pipeline can also handle filtering, you just need to use the $facet pipeline step which is capable of processing multiple aggregation pipelines within a single stage on the same set of input documents. Each sub-pipeline has its own field in the output document where its results are stored as an array of documents.
Consider running the following pipeline:
atendimentos.aggregate([
{ '$facet': {
'counts': [
{ '$group': {
'_id': null,
'fin': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'F' ] }, 1, 0 ]
}
},
'ea': {
'$sum': {
'$cond': [ { '$eq': [ '$status', 'EA' ] }, 1, 0 ]
}
}
} }
],
'monthly': [
{ '$group': {
'_id': {
'year': { '$year': '$date' },
'month': { '$month': '$date' }
},
'sum': { '$sum': 1 }
} },
{ '$group': {
'_id': null,
'items': { '$push': '$$ROOT' }
} },
]
} },
{ '$replaceRoot': {
'newRoot': {
'$mergeObjects': {
'$concatArrays': ['$counts', '$monthly']
}
}
} }
]).exec((err, results) => {
const data = results[0];
console.log(data);
res.render('home', { user: req.user, ...data });
})
I'm trying to query my mLab database and get the average time between two dates. I'm matching the data by name and between two dates (trying to get data within a day). There is data in the DB that i within the dates I'm providing but I'm getting undefined back. I'm not sure what I'm doing wrong.
var dataSchema = mongoose.Schema({
name: String,
start: Date,
end: Date,
key: String
});
module.exports.GetAverageDataWithinRange = function(name, dates, callback) {
Data.aggregate([{
$match: {
name: name,
start: {
$gte: dates.startDate,
$lt: dates.endDate + 1
}
}
}, {
$group: {
_id: "$name",
average: {
$avg: {
$subtract: [{
$millisecond: "$end"
}, {
$millisecond: "$start"
}]
}
}
}
}], function(err, results) {
if (err) {
console.log(err);
} else {
callback(results);
console.log(results);
}
});
}
How can I rewrite this MongoDB query using the Aggregation Framework to return the average price for the following Model in between the supplied date range:
Model
var PriceSchema = new Schema({
price: {
type: Number,
required: true
},
date: {
type: Date,
required: true
}
};
Query
exports.getPriceAverage = function(req, res, next) {
var start = moment.utc('03-01-2012').startOf('day');
var end = moment.utc('03-01-2012').endOf('month')
// Aggregation Framework Query Here...
Price.find({ date: { $lt: end, $gt: start }}, function(err, priceAverage) {
// Return average price...
});
};
You mention using aggregation, but you're using the find function which would return all results to the client.
Instead, you need to use aggregate with $avg:
Price.aggregate([
{ $match: { date: { $lt: end, $gt: start } } },
{ $group: { _id: null, avgPrice: { $avg: '$price' } } }
], function(err, results){
// process the results (an array of JavaScript objects)
});