I have a Mongo database which uses the following Schema:
const userSchema = {
first_login: {
type: Date,
default: Date.now
},
last_login: {
type: Date,
default: Date.now
},
provider_use_count: {
logins: {
type: Object,
required: true
}
},
total_login_count: {
type: Number,
required: true,
default: 0
},
login_hours: [
{
type: Number,
required: true
}
],
playlists: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Playlist'
}
],
genres: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Genre'
}
],
artists: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Artist'
}
],
auth_providers: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Provider'
}
],
platform_ids: [
{
type: Object,
required: true,
}
]
}
I am updating it with the following function, which is in a class:
async updateUser(user, playlists, currentProvider) {
return new Promise((resolve, reject) => {
// Add one to the use count for the current provider
user.provider_use_count.logins[currentProvider]++;
user.total_login_count++;
user.last_login = new Date();
user.login_hours.push((new Date()).getHours() + 1);
return user.save((err, doc) => {
if(err){
return reject(err);
} else {
console.log(doc);
return resolve(doc);
}
});
});
}
However, when I use the .save() function it is not storing the updated value for provider_use_count.logins. I have tried everything I can think of: using ++, copying the object and then incrementing it, getting a variable of the current count and then saving it, removing the required field from the Schema, using + 1, using += 1, using an array which houses an object that has a field for count and provider.
I am out of ideas as the save function clearly works -- It saves all other updates. I feel like this could be due to the Schema and it is something which I am either just overlooking or haven't experienced before.
EDIT:
The exact field which is not being saved is:
user.provider_use_count.logins[currentProvider]++;
Also, I am not relying on the console.log(doc) block, I have checked in Atlas and it seems that field just remains at 0 while all others update.
As I have not implemented any other auth providers so the field in question should be identical to total_login_count
An example of the user object which is passed to the function is as follows:
{
total_login_count: 7,
login_hours: [
20, 20, 20, 20,
21, 21, 20, 20
],
playlists: [],
genres: [],
artists: [],
auth_providers: [],
platform_ids: [ { provider: 'spotify', id: 'makingstuffs' } ],
_id: 5dadfe7c9a25cc0c6a4d387c,
first_login: 2019-10-21T18:52:44.814Z,
last_login: 2019-10-21T19:23:30.580Z,
provider_use_count: { logins: { spotify: 0 } },
__v: 7
}
The issue was with the fact that mongoose has issues with saving mixed data types as they are considered schema-less. This means that you have to make a call to person.markModified('name_of_the_update_field') prior to calling the .save() function.
My updateUser() function now looks as follows:
async updateUser(user, playlists, currentProvider) {
return new Promise((resolve, reject) => {
// Add one to the use count for the current provider
user.provider_use_count.logins[currentProvider]++;
user.markModified('provider_use_count');
user.total_login_count++;
user.last_login = new Date();
user.login_hours.push((new Date()).getHours());
return user.save((err, doc) => {
if(err)
return reject(err);
return resolve(doc);
});
});
}
I am trying to setup my patch api so that I can create a dynamic query to push, pull, and set data in my mongoose schema. I have plenty of values that I would change using set, but I also have an array of objects which would require me to call push when I need to insert and pull when I need to remove an item. I'm trying to find the best way to combine this into a dynamic structure.
Schema:
const StepSchema = new Schema({
position: {
type: Number,
required: true
},
name: {
type: String,
required: true
},
due_date: {
type: Date
},
status: [{
label: {
type: String,
enum: ['Inactive', 'In Progress', 'Flagged', 'Complete'],
default: 'Inactive'
},
user: {
type: Schema.Types.ObjectId,
ref: 'users',
},
date: {
type: Date
}
}],
comments: [{
user: {
type: Schema.Types.ObjectId,
ref: 'users',
required: true
},
body: {
type: String,
required: true
},
date: {
type: Date,
required: true
},
}],
});
Api:
router.patch('/',
async (req, res) => {
let setQuery = req.body;
let pushQuery = {};
let pullQuery = {};
//remove id from set query
delete setQuery.id;
//if there is a comment
if(req.body.comment){
pushQuery.comments = req.body.comment
}
//if I need to remove a comment
if(req.body.remove_comment){
pullQuery.comments = {_id: req.body.remove_comment.id}
}
//Push new status into array
if(req.body.status) {
pushQuery.status = {
label: req.body.status,
user: req.user._id,
date: new Date()
};
delete setQuery.status;
}
//update step
await Step.findByIdAndUpdate(req.body.id, {$set: setQuery, $push: pushQuery, $pull: pushQuery})
.then(step => {
if(!step){
errors.noflow = "There was a problem updating the step";
return res.status(400).json(errors);
}
res.json(step)
})
.catch(err => {
console.log(err);
res.status(404).json(err);
});
});
I've been getting the following error when trying to push a new status into my document:
operationTime: Timestamp { bsontype: 'Timestamp', low: 1, high_:
1560978288 }, ok: 0, errmsg: 'Updating the path \'status\' would
create a conflict at \'status\'', code: 40, codeName:
'ConflictingUpdateOperators', '$clusterTime': { clusterTime:
Timestamp { bsontype: 'Timestamp', low: 1, high_: 1560978288 },
signature: { hash: [Object], keyId: [Object] } },
Oh, you're doing that $set and $push on a status. Your pushQuery is trying to have status be an array on the document, and your setQuery wants to set it to whatever it was on the actual body (I'm guessing the same object.
A quickfix would be to remove it from the set object:
delete setQuery.status
A reasonable and stable way to do this would be to actually only take the things from req.body which you really want for each of the stages. Example:
const { position, name, dueDate, status, comment, remove_comment } = req.body;
const setQuery = { position, name, dueDate };
const pushQuery = { status, comments: comment };
// ...
That way your queries are not conflicting in any way.
I'm pretty new to Mongoose and MongoDB in general so I'm having a difficult time figuring out if something like this is possible:
Item = new Schema({
id: Schema.ObjectId,
dateCreated: { type: Date, default: Date.now },
title: { type: String, default: 'No Title' },
description: { type: String, default: 'No Description' },
tags: [ { type: Schema.ObjectId, ref: 'ItemTag' }]
});
ItemTag = new Schema({
id: Schema.ObjectId,
tagId: { type: Schema.ObjectId, ref: 'Tag' },
tagName: { type: String }
});
var query = Models.Item.find({});
query
.desc('dateCreated')
.populate('tags')
.where('tags.tagName').in(['funny', 'politics'])
.run(function(err, docs){
// docs is always empty
});
Is there a better way do this?
Edit
Apologies for any confusion. What I'm trying to do is get all Items that contain either the funny tag or politics tag.
Edit
Document without where clause:
[{
_id: 4fe90264e5caa33f04000012,
dislikes: 0,
likes: 0,
source: '/uploads/loldog.jpg',
comments: [],
tags: [{
itemId: 4fe90264e5caa33f04000012,
tagName: 'movies',
tagId: 4fe64219007e20e644000007,
_id: 4fe90270e5caa33f04000015,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
},
{
itemId: 4fe90264e5caa33f04000012,
tagName: 'funny',
tagId: 4fe64219007e20e644000002,
_id: 4fe90270e5caa33f04000017,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
}],
viewCount: 0,
rating: 0,
type: 'image',
description: null,
title: 'dogggg',
dateCreated: Tue, 26 Jun 2012 00:29:24 GMT
}, ... ]
With the where clause, I get an empty array.
With a modern MongoDB greater than 3.2 you can use $lookup as an alternate to .populate() in most cases. This also has the advantage of actually doing the join "on the server" as opposed to what .populate() does which is actually "multiple queries" to "emulate" a join.
So .populate() is not really a "join" in the sense of how a relational database does it. The $lookup operator on the other hand, actually does the work on the server, and is more or less analogous to a "LEFT JOIN":
Item.aggregate(
[
{ "$lookup": {
"from": ItemTags.collection.name,
"localField": "tags",
"foreignField": "_id",
"as": "tags"
}},
{ "$unwind": "$tags" },
{ "$match": { "tags.tagName": { "$in": [ "funny", "politics" ] } } },
{ "$group": {
"_id": "$_id",
"dateCreated": { "$first": "$dateCreated" },
"title": { "$first": "$title" },
"description": { "$first": "$description" },
"tags": { "$push": "$tags" }
}}
],
function(err, result) {
// "tags" is now filtered by condition and "joined"
}
)
N.B. The .collection.name here actually evaluates to the "string" that is the actual name of the MongoDB collection as assigned to the model. Since mongoose "pluralizes" collection names by default and $lookup needs the actual MongoDB collection name as an argument ( since it's a server operation ), then this is a handy trick to use in mongoose code, as opposed to "hard coding" the collection name directly.
Whilst we could also use $filter on arrays to remove the unwanted items, this is actually the most efficient form due to Aggregation Pipeline Optimization for the special condition of as $lookup followed by both an $unwind and a $match condition.
This actually results in the three pipeline stages being rolled into one:
{ "$lookup" : {
"from" : "itemtags",
"as" : "tags",
"localField" : "tags",
"foreignField" : "_id",
"unwinding" : {
"preserveNullAndEmptyArrays" : false
},
"matching" : {
"tagName" : {
"$in" : [
"funny",
"politics"
]
}
}
}}
This is highly optimal as the actual operation "filters the collection to join first", then it returns the results and "unwinds" the array. Both methods are employed so the results do not break the BSON limit of 16MB, which is a constraint that the client does not have.
The only problem is that it seems "counter-intuitive" in some ways, particularly when you want the results in an array, but that is what the $group is for here, as it reconstructs to the original document form.
It's also unfortunate that we simply cannot at this time actually write $lookup in the same eventual syntax the server uses. IMHO, this is an oversight to be corrected. But for now, simply using the sequence will work and is the most viable option with the best performance and scalability.
Addendum - MongoDB 3.6 and upwards
Though the pattern shown here is fairly optimized due to how the other stages get rolled into the $lookup, it does have one failing in that the "LEFT JOIN" which is normally inherent to both $lookup and the actions of populate() is negated by the "optimal" usage of $unwind here which does not preserve empty arrays. You can add the preserveNullAndEmptyArrays option, but this negates the "optimized" sequence described above and essentially leaves all three stages intact which would normally be combined in the optimization.
MongoDB 3.6 expands with a "more expressive" form of $lookup allowing a "sub-pipeline" expression. Which not only meets the goal of retaining the "LEFT JOIN" but still allows an optimal query to reduce results returned and with a much simplified syntax:
Item.aggregate([
{ "$lookup": {
"from": ItemTags.collection.name,
"let": { "tags": "$tags" },
"pipeline": [
{ "$match": {
"tags": { "$in": [ "politics", "funny" ] },
"$expr": { "$in": [ "$_id", "$$tags" ] }
}}
]
}}
])
The $expr used in order to match the declared "local" value with the "foreign" value is actually what MongoDB does "internally" now with the original $lookup syntax. By expressing in this form we can tailor the initial $match expression within the "sub-pipeline" ourselves.
In fact, as a true "aggregation pipeline" you can do just about anything you can do with an aggregation pipeline within this "sub-pipeline" expression, including "nesting" the levels of $lookup to other related collections.
Further usage is a bit beyond the scope of what the question here asks, but in relation to even "nested population" then the new usage pattern of $lookup allows this to be much the same, and a "lot" more powerful in it's full usage.
Working Example
The following gives an example using a static method on the model. Once that static method is implemented the call simply becomes:
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
Or enhancing to be a bit more modern even becomes:
let results = await Item.lookup({
path: 'tags',
query: { 'tagName' : { '$in': [ 'funny', 'politics' ] } }
})
Making it very similar to .populate() in structure, but it's actually doing the join on the server instead. For completeness, the usage here casts the returned data back to mongoose document instances at according to both the parent and child cases.
It's fairly trivial and easy to adapt or just use as is for most common cases.
N.B The use of async here is just for brevity of running the enclosed example. The actual implementation is free of this dependency.
const async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/looktest');
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt,callback) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
this.aggregate(pipeline,(err,result) => {
if (err) callback(err);
result = result.map(m => {
m[opt.path] = m[opt.path].map(r => rel(r));
return this(m);
});
callback(err,result);
});
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
function log(body) {
console.log(JSON.stringify(body, undefined, 2))
}
async.series(
[
// Clean data
(callback) => async.each(mongoose.models,(model,callback) =>
model.remove({},callback),callback),
// Create tags and items
(callback) =>
async.waterfall(
[
(callback) =>
ItemTag.create([{ "tagName": "movies" }, { "tagName": "funny" }],
callback),
(tags, callback) =>
Item.create({ "title": "Something","description": "An item",
"tags": tags },callback)
],
callback
),
// Query with our static
(callback) =>
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
],
(err,results) => {
if (err) throw err;
let result = results.pop();
log(result);
mongoose.disconnect();
}
)
Or a little more modern for Node 8.x and above with async/await and no additional dependencies:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [opt.path]: m[opt.path].map(r => rel(r)) })
));
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.create(
["movies", "funny"].map(tagName =>({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
const result = (await Item.lookup({
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
mongoose.disconnect();
} catch (e) {
console.error(e);
} finally {
process.exit()
}
})()
And from MongoDB 3.6 and upward, even without the $unwind and $group building:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
},{ timestamps: true });
itemSchema.statics.lookup = function({ path, query }) {
let rel =
mongoose.model(this.schema.path(path).caster.options.ref);
// MongoDB 3.6 and up $lookup with sub-pipeline
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": path,
"let": { [path]: `$${path}` },
"pipeline": [
{ "$match": {
...query,
"$expr": { "$in": [ "$_id", `$$${path}` ] }
}}
]
}}
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [path]: m[path].map(r => rel(r)) })
));
};
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.insertMany(
["movies", "funny"].map(tagName => ({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
let result = (await Item.lookup({
path: 'tags',
query: { 'tagName': { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
await mongoose.disconnect();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
what you are asking for isn't directly supported but can be achieved by adding another filter step after the query returns.
first, .populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } ) is definitely what you need to do to filter the tags documents. then, after the query returns you'll need to manually filter out documents that don't have any tags docs that matched the populate criteria. something like:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags.length;
})
// do stuff with docs
});
Try replacing
.populate('tags').where('tags.tagName').in(['funny', 'politics'])
by
.populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } )
Update: Please take a look at the comments - this answer does not correctly match to the question, but maybe it answers other questions of users which came across (I think that because of the upvotes) so I will not delete this "answer":
First: I know this question is really outdated, but I searched for exactly this problem and this SO post was the Google entry #1. So I implemented the docs.filter version (accepted answer) but as I read in the mongoose v4.6.0 docs we can now simply use:
Item.find({}).populate({
path: 'tags',
match: { tagName: { $in: ['funny', 'politics'] }}
}).exec((err, items) => {
console.log(items.tags)
// contains only tags where tagName is 'funny' or 'politics'
})
Hope this helps future search machine users.
After having the same problem myself recently, I've come up with the following solution:
First, find all ItemTags where tagName is either 'funny' or 'politics' and return an array of ItemTag _ids.
Then, find Items which contain all ItemTag _ids in the tags array
ItemTag
.find({ tagName : { $in : ['funny','politics'] } })
.lean()
.distinct('_id')
.exec((err, itemTagIds) => {
if (err) { console.error(err); }
Item.find({ tag: { $all: itemTagIds} }, (err, items) => {
console.log(items); // Items filtered by tagName
});
});
#aaronheckmann 's answer worked for me but I had to replace return doc.tags.length; to return doc.tags != null; because that field contain null if it doesn't match with the conditions written inside populate.
So the final code:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags != null;
})
// do stuff with docs
});
I am trying to make a query to find documents depending on another document in the same collection as below.
The first one finds the user and the second one finds the data by using that user data received. But I want to do it with one query like join in SQL
This is schema
var ConnectionSchema = new Schema({
socketId: {
type: String,
require: true
},
location: {
type: [Number],
index: '2dsphere'
},
user: { type: Schema.ObjectId, ref: "User" },
date: {
type: Date,
require: true,
default: new Date()
}
});
// queries
return mongoose.model("Connection").findOne({ user: userId }).populate("user").then(usr => {
return mongoose.model("Connection").find({
location: {
$near: {
$maxDistance: config.searchDistance,
$geometry: { type: Number, coordinates: usr.location }
}
},
user: { $ne: userId },
});
});
Is there any way to do that with a just single query?
Thanks.
yes there is a way you can do like this
return mongoose.model("Connection").findOne({ user: userId })
.populate("user" ,
match : {$and : [{location: {
$near: {
$maxDistance: config.searchDistance,
$geometry: { type: Number, coordinates: usr.location }
}
}},
{user: { $ne: userId }}]})
.then(usr => {
// perform your action
});
In my novice skill to Mongodb and Mongoose, I seem to be failing miserably at this fundamental task.
I have a bounded array of 10 elements. A user can only have 10 pets, so I figured to make it bounded with set fields and empty values was the best way.
The pets array values are blank at the time of creation, because the user can add pets as they go along. When I look in mongo console, the pets array is unbounded with no fields. I also can't add values to the array.
Mongoose Schema:
var userSchema = new Schema({
firstName: { type: String, required: true },
lastName: { type: String, required: true },
username: { type: String, required: true, unique: true },
location: String,
created_at: Date,
pets: [
{ "pet0": {} },
{ "pet1": {} },
{ "pet2": {} },
{ "pet3": {} },
{ "pet4": {} },
{ "pet5": {} },
{ "pet6": {} },
{ "pet7": {} },
{ "pet8": {} },
{ "pet9": {} }
]
});
mongodb:
{ "_id" : ObjectId("56a3e324bdebcf801c1ca224"), "firstName" : "bob", "lastName" : "smith", "username" : "bob123", "pets" : [ ], "__v" : 0 }
When modifying the array:
UserModel.findOne({ firstName: "bob" }, 'pets', function(err, user) {
user.pets[0] = { "name": "felix", "type": "cat" }
user.save(function(err) { console.log(err); console.log('saved')});
});
output:
Mongoose: users.findOne({ firstName: 'bob' }) { fields: { pets: 1 } }
null
/home/one/github/foo/node_modules/mongoose/lib/schema/documentarray.js:100
doc.validate({ __noPromise: true }, function(err) {
^
TypeError: undefined is not a function
at /home/one/github/foo/node_modules/mongoose/lib/schema/documentarray.js:100:11
at DocumentArray.SchemaType.doValidate (/home/one/github/foo/node_modules/mongoose/lib/schematype.js:654:22)
at DocumentArray.doValidate (/home/one/github/foo/node_modules/mongoose/lib/schema/documentarray.js:78:35)
at /home/one/github/foo/node_modules/mongoose/lib/document.js:1156:9
at process._tickCallback (node.js:355:11)
MongoDB allows you to limit the number of elements in an array. This feature has also been implemented in Mongoose as part of an .update query. The steps for adding an element to an array and limiting its size are as follows:
Push the element(s) into the array.
Slice the array.
This snippet of code explains how to do this using Mongoose:
UserModel.findOne({ firstName: "bob" }, function(err, user) {
UserModel.update(user, {
$push: {
pets: {
$each: [{ name: "felix", type: "cat" }],
$slice: -10
}
}
}, function(err, numAffected) {
if (err) console.log(err);
console.log('updated');
});
});