We Keep Coding

MongoDB group by date and select values

I'm relatively new to MongoDB and node.js. Here is what I'm trying to achieve:
I have a collection that contains several thousand documents in MongoDB. The documents look like this:
{
"date": "2020-02-24",
"iso_code": "USA",
"country": "USA",
"avg_temperature": "25F",
},
{
"date": "2020-02-25",
"iso_code": "USA",
"country": "USA",
"avg_temperature": "27F",
},
{
"date": "2020-02-24",
"iso_code": "CHN",
"country": "China",
"avg_temperature": "10C",
}
I want to avoid running costly calculation in node.js, so I would like to receive the temperatures grouped by date and retrieve the actual values for avg_temperature (so no $min or $max calculation). I expect the result to look something like this:
{
"date": "2020-02-24",
"avg_temperatures": {
"USA": "25F",
"China": "10C"
}
}
My ultimate goal is to create a chart that visualizes the data over a timeline for multiple countries. My idea is to simply iterate through the contents of the result and then trough the temperatures and display the data.
If someone has done something similar before, feel free to suggest a better alternative :)

apparently getting the result as you described will be a little complex. Instead, you can do it by making a simple aggregation pipeline like this
var pipeline = [
{
"$group" : {
"_id" : "$date",
"docs" : {
"$push" : {
"country" : "$country",
"temp" : "$avg_temperature"
}
}
}
}
];
For the above pipeline you will get the output like
db.stack.aggregate(pipeline).pretty()
{
"_id" : "2020-02-25",
"docs" : [
{
"country" : "USA",
"temp" : "27F"
}
]
}
{
"_id" : "2020-02-24",
"docs" : [
{
"country" : "USA",
"temp" : "25F"
},
{
"country" : "China",
"temp" : "10C"
}
]
}
I hope this will solve your problem!

You can do it with Aggregation framework:
db.collection.aggregate([
{
"$group": {
"_id": "$date",
"avg_temperatures": {
"$mergeObjects": {
"$arrayToObject": [
[
{
k: "$country",
v: "$avg_temperature"
}
]
]
}
}
}
}
])
Working example

To increase performance, first it would be good to create index for "date" field in your collection. By doing that, group by date in aggregation pipeline will be much faster. You can do that by adding this in your file where you have defined Schema for your collection. "Schema" is name of schema for your collection.
Schema.index({
date: 1,
});
Do not forget to rebuild indexes for specific collection. You can do that by typing this in mongo shell, where "collection" is name of your collection:
db.collection.reIndex()
And you need to add group stage in aggregation pipeline.
{
"$group" : {
"_id" : "$date",
"docs": { $push : "$$ROOT" },
}
}

ElasticSearch EJS query to fetch missing field

I have an existing ejs query as below:
let queryBody = ejs.Request()
.size(0)
.query(
ejs.BoolQuery()
.must(
ejs.RangeQuery('hour_time_stamp').gte(this.lastDeviceDate).lte(this.lastDeviceDate)
)
)
.agg(ejs.TermsAggregation('market_agg').field('market').order('sum', 'asc').size(50000)
.agg(ejs.SumAggregation('sum').field('num_devices'))
)
currently the field('market') returns the values where data for market is present. There is data in the database for missing values for market as well, which I need to access. How do I do that?
EDIT:
Values for market in ES is either null or field is missing. I wrote ES query to get all those fields but I am not able to incorporate an ejs query for the same. Any idea how this can be done?
{
"query": {
"bool": {
"should": [
{
"exists": {
"field": "market"
}
},
{
"bool": {
"must_not": [
{
"exists": {
"field": "market"
}
}
]
}
}
]
}
}
}

As per your problem you need a way to group the empty market fields too.
So for that you can use the "missing" value parameter. It defines how the values which are missing(as in your case) are grouped. So you query in json form will be modified like below :-
{
"query":
{
"must": [
"range": {
"hour_time_stamp": {
"gte": lastDeviceDate,
"lte": lastDeviceDate
}
}
]
},
"aggs": {
"market_agg" : {
"market": {
"missing": "empty_markets",
"order": { "sum": "asc" }
}
},
"sum_agg": {
"sum" : { "field" : "num_devices" }
}
}
}
Or in your code it could be done by adding missing parameter like this.
let queryBody = ejs.Request()
.size(0)
.query(
ejs.BoolQuery()
.must(
ejs.RangeQuery('hour_time_stamp').gte(this.lastDeviceDate).lte(this.lastDeviceDate)
)
)
.agg(ejs.TermsAggregation('market_agg').field('market').missing('empty_markets').order('sum', 'asc').size(50000)
.agg(ejs.SumAggregation('sum').field('num_devices'))
)

How to add within an array information using findOneAndUpdate without deleting information that was previously contained [duplicate]

I am working on an express js application where I need to update a nested array.
1) Schema :
//Creating a mongoose schema
var userSchema = mongoose.Schema({
_id: {type: String, required:true},
name: String,
sensors: [{
sensor_name: {type: String, required:true},
measurements: [{time: String}]
}] });
2)
Here is the code snippet and explanation is below:
router.route('/sensors_update/:_id/:sensor_name/')
.post(function (req, res) {
User.findOneAndUpdate({_id:req.body._id}, {$push: {"sensors" :
{"sensor_name" : req.body.sensor_name , "measurements.0.time": req.body.time } } },
{new:true},function(err, newSensor) {
if (err)
res.send(err);
res.send(newSensor)
}); });
I am able to successfully update a value to the measurements array using the findOneAndUpdate with push technique but I'm failing when I try to add multiple measurements to the sensors array.
Here is current json I get if I get when I post a second measurement to the sensors array :
{
"_id": "Manasa",
"name": "Manasa Sub",
"__v": 0,
"sensors": [
{
"sensor_name": "ras",
"_id": "57da0a4bf3884d1fb2234c74",
"measurements": [
{
"time": "8:00"
}
]
},
{
"sensor_name": "ras",
"_id": "57da0a68f3884d1fb2234c75",
"measurements": [
{
"time": "9:00"
}
]
}]}
But the right format I want is posting multiple measurements with the sensors array like this :
Right JSON format would be :
{
"_id" : "Manasa",
"name" : "Manasa Sub",
"sensors" : [
{
"sensor_name" : "ras",
"_id" : ObjectId("57da0a4bf3884d1fb2234c74"),
"measurements" : [
{
"time" : "8:00"
}
],
"measurements" : [
{
"time" : "9:00"
}
]
}],
"__v" : 0 }
Please suggest some ideas regarding this. Thanks in advance.

You might want to rethink your data model. As it is currently, you cannot accomplish what you want. The sensors field refers to an array. In the ideal document format that you have provided, you have a single object inside that array. Then inside that object, you have two fields with the exact same key. In a JSON object, or mongo document in this context, you can't have duplicate keys within the same object.
It's not clear exactly what you're looking for here, but perhaps it would be best to go for something like this:
{
"_id" : "Manasa",
"name" : "Manasa Sub",
"sensors" : [
{
"sensor_name" : "ras",
"_id" : ObjectId("57da0a4bf3884d1fb2234c74"),
"measurements" : [
{
"time" : "8:00"
},
{
"time" : "9:00"
}
]
},
{
// next sensor in the sensors array with similar format
"_id": "",
"name": "",
"measurements": []
}],
}
If this is what you want, then you can try this:
User.findOneAndUpdate(
{ _id:req.body._id "sensors.sensor_name": req.body.sensor_name },
{ $push: { "sensors.0.measurements": { "time": req.body.time } } }
);
And as a side note, if you're only ever going to store a single string in each object in the measurements array, you might want to just store the actual values instead of the whole object { time: "value" }. You might find the data easier to handle this way.

Instead of hardcoding the index of the array it is possible to use identifier and positional operator $.
Example:
User.findOneAndUpdate(
{ _id: "Manasa" },
{ $push: { "sensors.$[outer].measurements": { "time": req.body.time } } }
{ "arrayFilters:" [{"outer._id": ObjectId("57da0a4bf3884d1fb2234c74")}]
);
You may notice than instead of getting a first element of the array I specified which element of the sensors array I would like to update by providing its ObjectId.
Note that arrayFilters are passed as the third argument to the update query as an option.
You could now make "outer._id" dynamic by passing the ObjectId of the sensor like so: {"outer._id": req.body.sensorId}
In general, with the use of identifier, you can get to even deeper nested array elements by following the same procedure and adding more filters.
If there was a third level nesting you could then do something like:
User.findOneAndUpdate(
{ _id: "Manasa" },
{ $push: { "sensors.$[outer].measurements.$[inner].example": { "time": req.body.time } } }
{ "arrayFilters:" [{"outer._id": ObjectId("57da0a4bf3884d1fb2234c74"), {"inner._id": ObjectId("57da0a4bf3884d1fb2234c74"}}]
);
You can find more details here in the answer written by Neil Lunn.

refer ::: positional-all
--- conditions :: { other_conditions, 'array1.array2.field_to_be_checked': 'value' }
--- updateData ::: { $push : { 'array1.$[].array2.$[].array3' : 'value_to_be_pushed' } }

I've found a Json data using findById, how do I use it in my code?

I am creating an API that gets Patients data(id and name), Physicians data(id and name) and Appointments(id, phyId, patId, app_date) and displays the Patients appointed to a particular physician. I need to create a remote method in physician.js in such a way that I get related Appointment that has phyId and print the details of the Patients using the patId obtained from appointment.
I'm using loopback 3.
Refer this link for clear idea:
https://loopback.io/doc/en/lb3/HasManyThrough-relations.html
I have related models (Physicians, Patients) that are related by "hasMany" with each other "through" Appointment(another model) and Appointment is related to each of these by belongsTo, in my loopback application and i need to print the Patients of a particular Physician.
Patient data:
[
{
"id": 1,
"name": "Anna Mull"
},
{
"id": 2,
"name": "Paige Trner"
}
]
Physician data:
[
{
"id": 1,
"name": "Cardiologist"
}
]
Appointment data:
[
{
"id": 1,
"physicianId": 1,
"patientId": 1,
"appointmentDate": "2019-01-28T10:06:33.530Z"
},
{
"id": 2,
"physicianId": 1,
"patientId": 2,
"appointmentDate": "2019-01-28T10:06:33.530Z"
}
]
I know there is a method already available to query the Patients of a Physician, but I want to code it myself to learn and also print it in the following format.
My idea is to get all the Appointments having the specific phyId in it and find the patId in those appointment and store it in an array. I then use that array to get the patients from the Patient model. I managed to get the Patient details in a function, but I can only console.log(Patients) but I am not able to display it in the API response.
The following is the format i need it in. (EXPECTED OUTPUT in API response)
Physician:
{
"id": 1,
"name": "Cardiologist"
}
Patients:
[
{
"id": 1,
"name": "Anna Mull"
},
{
"id": 2,
"name": "Paige Trner"
}
]
or any similar format.
I've tried to the same and here is my code.
common/models/physician.js
'use strict';
var app = require('../../server/server');
module.exports = function (Physician) {
Physician.getDetails = function (phyid, cb) {
var Appointments = app.models.Appointment;
var Patient = app.models.Patient;
Physician.findById(phyid, function (err, Physician) {
Appointments.find({ where: { physicianId: phyid } }, function (err, Appointment) {
if (err) {
cb(null, "Errorrrrrrrr", "Errorrrrrr");
}
else {
var patients = [], i = 0;
var patobj= [];
for (i in Appointment) {
patients[i] = Appointment[i].patientId;
//console.log(patients);
Patient.findById(patients[i], function(err, Patients){
if(err){
cb("Error in patients", "--");
}
else{
patobj[i]=Patients;//doesnt have any effect
console.log(Patients);//prints in console
}
});
}
cb(null, Physician, patobj);//only Physician is printed, patobj is empty.
}
});
});
}
Physician.remoteMethod('getDetails', {
http: {
path:
'/:phyid/getDetails',
verb: 'get'
},
accepts: {
arg: 'phyid',
type: 'number'
},
returns: [{
arg: 'Physician',
type: 'Object'
}, {
arg: 'Patient',
type: 'Object'
}]
});
};
I am actually getting this in the API response:
{
"Physician": {
"id": 1,
"name": "Cardiologist"
},
"Patient": []
}
and this in the console:
D:\Project\Project1>node .
Web server listening at: http://localhost:3000
Browse your REST API at http://localhost:3000/explorer
{ name: 'Anna Mull', id: 1 }
{ name: 'Paige Trner', id: 2 }
How am I supposed to get the patient data to be printed in the API response?

You patients are empty because, finding Patients by Id is an asynchronous operation. But the for loop is synchronous. The loop finishes and calls the following line before any of the Patients are found.
cb(null, Physician, patobj);//only Physician is printed, patobj is empty.
You need to wait for all the patients to be found by using either Promise.all or async.each.

Parsing Exception error when using Terms in ElasticSearch

I'm getting an error on this elastic search for terms. The error message is
"[parsing_exception] [terms] unknown token [START_ARRAY] after [activeIds], with { line=1 & col=63 }"
Active Ids is an array of unique ids. It sort of looks like
const activeIds = [ '157621a1-d892-4f4b-80ca-14feddb837a0',
'd04c5c93-a22c-48c3-a3b0-c79a61bdd923',
'296d40d9-f316-4560-bbc9-001d6f46858b',
'2f8c6c37-588d-4d24-9e69-34b6dd7366c2',
'ba0508dd-0e76-4be8-8b6e-9e938ab4abed',
'ab076ed9-1dd5-4987-8842-15f1b995bc0d',
'ea6b0cff-a64f-4ce3-844e-b36d9f161e6f' ]
let items = await es.search({
"index": table,
"body": {
"from": 0, "size": 25,
"query": {
"terms" : {
"growerId" : {
activeIds
}
},
"bool": {
"must_not": [
{ "match":
{
"active": false
}
},
],
"must": [
{ "query_string" :
{
"query": searchQuery,
"fields": ["item_name"]
}
}
],
}
}
}
})
Appreciate the help!
Edit: Answering this question- "What's the expected result? Can you elaborate and share some sample data? – Nishant Saini 15 hours ago"
I'll try to elaborate a bit.
1) Overall I'm trying to retrieve items that belong to active users. There are 2 tables: user and items. So I'm initially running an ES that returns all the users that contain { active: true } from the user table
2) Running that ES returns an array of ids which I'm calling activeIds. The array looks like what I've already displayed in my example. So this works so far (let me know if you want to see the code for that, but if I'm getting an expected result then I don't think we need that now)
3) Now I want to search through the items table, and retrieve only the items that contain one of the active ids. So an item should look like:
4) expected result is retrieve an array of objects that match the growerId with one of the activeIds. So if I do a search query for "flowers", a single expected result should look like:
[ { _index: 'items-dev',
_type: 'items-dev_type',
_id: 'itemId=fc68dadf-21c8-43c2-98d2-cf574f71f06d',
_score: 11.397207,
_source:
{ itemId: 'fc68dadf-21c8-43c2-98d2-cf574f71f06d',
'#SequenceNumber': '522268700000000025760905838',
item_name: 'Flowers',
grower_name: 'Uhs',
image: '630b5d6e-566f-4d55-9d31-6421eb2cff87.jpg',
dev: true,
growerId: 'd04c5c93-a22c-48c3-a3b0-c79a61bdd923',
sold_out: true,
'#timestamp': '2018-12-20T16:09:38.742599',
quantity_type: 'Pounds',
active: true,
pending_inventory: 4,
initial_quantity: 5,
price: 10,
item_description: 'Field of flowers' } },
So here the growerId matches activeIds[1]
But if I do a search for "invisible", which is created by a an inactive user, I get:
[ { _index: 'items-dev',
_type: 'items-dev_type',
_id: 'itemId=15200473-93e1-477c-a1a7-0b67831f5351',
_score: 1,
_source:
{ itemId: '15200473-93e1-477c-a1a7-0b67831f5351',
'#SequenceNumber': '518241400000000004028805117',
item_name: 'Invisible too',
grower_name: 'Field of Greens',
image: '7f37d364-e768-451d-997f-8bb759343300.jpg',
dev: true,
growerId: 'f25040f4-3b8c-4306-9eb5-8b6c9ac58634',
sold_out: false,
'#timestamp': '2018-12-19T20:47:16.128934',
quantity_type: 'Pounds',
pending_inventory: 5,
initial_quantity: 5,
price: 122,
item_description: 'Add' } },
Now that growerId does not match any of the ids in activeIds.
5) Using the code you helped with, it's returning 0 items.
Let me know if you need more detail. I've been working on this for a bit too long :\

Terms query accept array of terms so the terms query should be defined as below:
"terms": {
"growerId": activeIds
}
You might face other errors as well after making the above correction. So below is full query which might help you:
{
"from": 0,
"size": 25,
"query": {
"bool": {
"must_not": [
{
"match": {
"active": false
}
}
],
"must": [
{
"query_string": {
"query": searchQuery,
"fields": [
"item_name"
]
}
},
{
"terms": {
"growerId": activeIds
}
}
]
}
}
}