I'm using mongoose to connect to Mongo DB.
At first this is my schema:
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const TestSchema = new Schema({
story: String,
seenByUser: [String],
medicationStage: {
type: String,
enum: [
"no-medication",
"just-after-medication",
"towards-end-of-medication",
],
required: true,
},
});
// Compile model from schema
const TestModel = mongoose.model("Test", TestSchema);
module.exports = {
Test: TestModel,
};
Here you can see I have a field called seenByUser which is an array of strings that is an array of user names. I'm setting up a data aggregation pipeline where I want to see given a user name fetch me all the documents where this user Name does not occur in seenByUser array. grouped by medicationStage. I'm unable to create this pipeline. Please help me out.
If I've undesrtood correctly you can try this aggregation:
First $match where does not exists yourValue into the array seenByUser.
Then $group by medicationStage. This create a nested array (because $push add the whole array)
And $project to use $reduce and flat the array.
db.collection.aggregate({
"$match": {
"seenByUser": {
"$ne": yourValue
}
}
},
{
"$group": {
"_id": "$medicationStage",
"seenByUser": {
"$push": "$seenByUser"
}
}
},
{
"$project": {
"_id": 0,
"medicationStage": "$_id",
"seenByUser": {
"$reduce": {
"input": "$seenByUser",
"initialValue": [],
"in": {
"$concatArrays": [
"$$value",
"$$this"
]
}
}
}
}
})
Example here
I'm new to mongoDB, I am trying to achieve the following SQL query on it. but could not find anything useful so far. can anyone tell equivalent mongoose query
select * from interviews
inner join candidate on interviews.clientId = candidate._id
inner join billing on appointment._id = billing.appointmentId
where ('
interviews.status= "upcoming",
interviews.startTime= "2017-01-01",
candidate.clientAgeGroup= "adult",
candidate.candidatetatus= "new",
billing.paymentStatus= "paid"
')
what I got so far is following
const [result, err] = await of(Interview.find({ ...filterQuery }).limit(perPage)
.skip(perPage * page)
.sort({
startTime: 'asc'
})
.populate([{ path: 'candidateId', model: 'Candidate', select: 'firstName status avatar' },
{ path: 'billingId', model: 'Billing', select: "status" }]));
UPDATE
I have following name and export scheme
//interview.model.js => mongodb show name as interview
module.exports = mongoose.model('Interview', interviewSchema);
//candidate.model.js => mongodb show name as candidate
module.exports = mongoose.model('Candidate', candidateSchema);
You can use filter out objects included in resulting array using match but in the case if it couldn't find any, it would still return a null value. So in comparison this works similar to sql left join.
const [result, err] = await of(Interview.find({ ...filterQuery }).limit(perPage)
.skip(perPage * page)
.sort({
startTime: 'asc'
})
.populate([{ path: 'candidateId', model: 'Candidate', select: 'firstName status avatar', match: {clientAgeGroup: "adult", candidatetatus: "new"} },
{ path: 'billingId', model: 'Billing', select: "status", match: {paymentStatus: "paid"} }]));
Also see https://mongoosejs.com/docs/populate.html#query-conditions
If you need strictly a inner join then you can use mongodb aggregate pipeline.
Interview.aggregate([
{
"$match": {
status: "upcoming",
startTime: "2017-01-01",
}
},
{
'$lookup': {
'from': 'candidates', // this should be your collection name for candidates.
'localField': 'candidateId', // there should be an attribute named candidateId in interview model that refer to candidate collection
'foreignField': '_id',
'as': 'candidates'
}
}, {
'$match': {
'candidates.clientAgeGroup': "adult",
'candidates.candidatetatus': "new"
}
},
{
'$lookup': {
'from': 'billing', // this should be your collection name for billing.
'localField': 'billingId', // there should be an attribute named billingId in interview model that refer to billing collection
'foreignField': '_id',
'as': 'billing'
}
}, {
'$match': {
'billing.paymentStatus': "paid"
}
},
{ "$sort": { startTime: 1 } },
{ "$limit": perPage },
{ "$skip": perPage * page }
])
I'm creating a very basic functionality kanban board.
My board has 4 models so far:
User model
var userSchema = new Schema({
name: {
type: String,
required: true
}
})
module.exports = mongoose.model('User', userSchema)
Board model
var boardSchema = new Schema({
title: {
type: String,
required: true
},
lists: [ listSchema ]
members: [
{
type: Schema.Types.ObjectId,
ref: 'user'
}
]
});
module.exports = mongoose.model('Board', boardSchema)
List schema
let listSchema = new Schema({
title: {
type: String,
required: true
},
userCreated: {
type: Schema.Types.ObjectId,
required: true,
ref: 'user'
},
boardId: {
type: Schema.Types.ObjectId,
required: true,
ref: 'board'
},
sort: {
type: Number,
decimal: true,
required: true
}
})
module.exports = mongoose.model('List', listSchema)
Card schema
var cardSchema = new Schema({
title: {
type: String,
required: true
},
description: {
type: String
},
boardId: {
type: Schema.Types.ObjectId,
required: true,
ref: 'board'
},
listId: {
type: Schema.Types.ObjectId,
required: true,
ref: 'list'
},
members: [
{
type: Schema.Types.ObjectId,
ref: 'user'
}
],
sort: {
type: Number,
decimal: true,
required: true
}
})
module.exports = mongoose.model('Card', cardSchema)
What am I looking for?
My front-end is made with Vue.js and sortable.js drag and drop lib.
I want to find the best way to render board with lists (columns) and cards in them.
From what I understand, I should get my board first, by the users id in members array.
Then I have my board which has lists embedded already.
On second api request, I get all the cards by boardId.
My question is - how do I correctly put/render all the cards into each owns lists?
So in the end I want to have something like:
{
title: 'My board',
lists: [
{
title: 'My list',
_id: '35jj3j532jj'
cards: [
{
title: 'my card',
listId: '35jj3j532jj'
}
]
},
{
title: 'My list 2',
_id: '4gfdg5454dfg'
cards: [
{
title: 'my card 22',
listId: '4gfdg5454dfg'
},
{
title: 'my card 22',
listId: '4gfdg5454dfg'
}
]
}
]
members: [
'df76g7gh7gf86889gf989fdg'
]
}
What I've tried?
I've came up with only one thing so far, that is:
Two api calls in mounted hook - one to get the board with lists, second to get all cards.
Then I loop trough lists and loop trough cards and push each card into the list by id?
But this way it seems that my lists would need to have an empty array called cards: [] just for the front-end card-to-list sorting by id, seems somehow wrong.
Is this a good way? Or should I redesign my models and schemas and go with some other way? Any help would be appreciated!
The schema you've defined is pretty good, just one modifications though.
No need to have 'lists' in Board model, since it's already available in lists and also if you keep it in boards, then everytime a new list is added, you'll need to edit the board as well.
Here's how the flow would be.
Initially, when a user signs in, you'll need to show them the list of boards. This should be easy since you'll just do a find query with the user_id on the board collection.
Board.find({members: user_id}) // where user_id is the ID of the user
Now when a user clicks on a particular board, you can get the lists with the board_id, similar to the above query.
List.find({boardId: board_id}) // where board_id is the ID of the board
Similarly, you can get cards with the help of list_id and board_id.
Card.find({boardId: board_id, listId: list_id}) // where board_id is the ID of the board and listId is the Id of the list
Now, let's look at cases wherein you might need data from 2 or more collection at the same time. For example, when a user clicks on board, you not only need the lists in the board but also the cards in that board. In that case, you'll need to write an aggregation as such,
Board.aggregate([
// get boards which match a particular user_id
{
$match: {members: user_id}
},
// get lists that match the board_id
{
$lookup:
{
from: 'list',
localField: '_id',
foreignField: 'boardId',
as: 'lists'
}
}
])
This will return the boards, and in each board, there'll be an array of lists associated with that board. If a particular board doesn't have a list, then it'll have an empty array.
Similarly, if you want to add cards to the list and board, the aggregation query will be a bot more complex, as such,
Board.aggregate([
// get boards which match a particular user_id
{
$match: {members: user_id}
},
// get lists that match the board_id
{
$lookup:
{
from: 'list',
localField: '_id',
foreignField: 'boardId',
as: 'lists'
}
},
// get cards that match the board_id
{
$lookup:
{
from: 'card',
localField: '_id',
foreignField: 'boardId',
as: 'cards'
}
}
])
This will add an array of cards as well to the mix. Similarly, you can get cards of the lists as well.
A bit late to the answer but I think it'll help someone nevertheless. The problem you have could be solved using aggregation framework. While the other answer mentions a pretty good way, it still doesn't have the cards data embedded into it.
MongoDB docs show a way for nested aggregation queries. Nested Lookup
A similar approach could be used for your question.
Board.aggregate([
{
$match: { _id: mongoose.Types.ObjectId(boardId) },
},
{
$lookup: {
from: 'lists',
let: { boardId: '$_id' },
pipeline: [
{ $match: { $expr: { $eq: ['$boardId', '$$boardId'] } } },
{
$lookup: {
from: 'cards',
let: { listId: '$_id' },
pipeline: [{ $match: { $expr: { $eq: ['$listId', '$$listId'] } } }],
as: 'cards',
},
},
],
as: 'lists',
},
},
]);
This will include the cards as an array inside of every list.
I'm pretty new to Mongoose and MongoDB in general so I'm having a difficult time figuring out if something like this is possible:
Item = new Schema({
id: Schema.ObjectId,
dateCreated: { type: Date, default: Date.now },
title: { type: String, default: 'No Title' },
description: { type: String, default: 'No Description' },
tags: [ { type: Schema.ObjectId, ref: 'ItemTag' }]
});
ItemTag = new Schema({
id: Schema.ObjectId,
tagId: { type: Schema.ObjectId, ref: 'Tag' },
tagName: { type: String }
});
var query = Models.Item.find({});
query
.desc('dateCreated')
.populate('tags')
.where('tags.tagName').in(['funny', 'politics'])
.run(function(err, docs){
// docs is always empty
});
Is there a better way do this?
Edit
Apologies for any confusion. What I'm trying to do is get all Items that contain either the funny tag or politics tag.
Edit
Document without where clause:
[{
_id: 4fe90264e5caa33f04000012,
dislikes: 0,
likes: 0,
source: '/uploads/loldog.jpg',
comments: [],
tags: [{
itemId: 4fe90264e5caa33f04000012,
tagName: 'movies',
tagId: 4fe64219007e20e644000007,
_id: 4fe90270e5caa33f04000015,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
},
{
itemId: 4fe90264e5caa33f04000012,
tagName: 'funny',
tagId: 4fe64219007e20e644000002,
_id: 4fe90270e5caa33f04000017,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
}],
viewCount: 0,
rating: 0,
type: 'image',
description: null,
title: 'dogggg',
dateCreated: Tue, 26 Jun 2012 00:29:24 GMT
}, ... ]
With the where clause, I get an empty array.
With a modern MongoDB greater than 3.2 you can use $lookup as an alternate to .populate() in most cases. This also has the advantage of actually doing the join "on the server" as opposed to what .populate() does which is actually "multiple queries" to "emulate" a join.
So .populate() is not really a "join" in the sense of how a relational database does it. The $lookup operator on the other hand, actually does the work on the server, and is more or less analogous to a "LEFT JOIN":
Item.aggregate(
[
{ "$lookup": {
"from": ItemTags.collection.name,
"localField": "tags",
"foreignField": "_id",
"as": "tags"
}},
{ "$unwind": "$tags" },
{ "$match": { "tags.tagName": { "$in": [ "funny", "politics" ] } } },
{ "$group": {
"_id": "$_id",
"dateCreated": { "$first": "$dateCreated" },
"title": { "$first": "$title" },
"description": { "$first": "$description" },
"tags": { "$push": "$tags" }
}}
],
function(err, result) {
// "tags" is now filtered by condition and "joined"
}
)
N.B. The .collection.name here actually evaluates to the "string" that is the actual name of the MongoDB collection as assigned to the model. Since mongoose "pluralizes" collection names by default and $lookup needs the actual MongoDB collection name as an argument ( since it's a server operation ), then this is a handy trick to use in mongoose code, as opposed to "hard coding" the collection name directly.
Whilst we could also use $filter on arrays to remove the unwanted items, this is actually the most efficient form due to Aggregation Pipeline Optimization for the special condition of as $lookup followed by both an $unwind and a $match condition.
This actually results in the three pipeline stages being rolled into one:
{ "$lookup" : {
"from" : "itemtags",
"as" : "tags",
"localField" : "tags",
"foreignField" : "_id",
"unwinding" : {
"preserveNullAndEmptyArrays" : false
},
"matching" : {
"tagName" : {
"$in" : [
"funny",
"politics"
]
}
}
}}
This is highly optimal as the actual operation "filters the collection to join first", then it returns the results and "unwinds" the array. Both methods are employed so the results do not break the BSON limit of 16MB, which is a constraint that the client does not have.
The only problem is that it seems "counter-intuitive" in some ways, particularly when you want the results in an array, but that is what the $group is for here, as it reconstructs to the original document form.
It's also unfortunate that we simply cannot at this time actually write $lookup in the same eventual syntax the server uses. IMHO, this is an oversight to be corrected. But for now, simply using the sequence will work and is the most viable option with the best performance and scalability.
Addendum - MongoDB 3.6 and upwards
Though the pattern shown here is fairly optimized due to how the other stages get rolled into the $lookup, it does have one failing in that the "LEFT JOIN" which is normally inherent to both $lookup and the actions of populate() is negated by the "optimal" usage of $unwind here which does not preserve empty arrays. You can add the preserveNullAndEmptyArrays option, but this negates the "optimized" sequence described above and essentially leaves all three stages intact which would normally be combined in the optimization.
MongoDB 3.6 expands with a "more expressive" form of $lookup allowing a "sub-pipeline" expression. Which not only meets the goal of retaining the "LEFT JOIN" but still allows an optimal query to reduce results returned and with a much simplified syntax:
Item.aggregate([
{ "$lookup": {
"from": ItemTags.collection.name,
"let": { "tags": "$tags" },
"pipeline": [
{ "$match": {
"tags": { "$in": [ "politics", "funny" ] },
"$expr": { "$in": [ "$_id", "$$tags" ] }
}}
]
}}
])
The $expr used in order to match the declared "local" value with the "foreign" value is actually what MongoDB does "internally" now with the original $lookup syntax. By expressing in this form we can tailor the initial $match expression within the "sub-pipeline" ourselves.
In fact, as a true "aggregation pipeline" you can do just about anything you can do with an aggregation pipeline within this "sub-pipeline" expression, including "nesting" the levels of $lookup to other related collections.
Further usage is a bit beyond the scope of what the question here asks, but in relation to even "nested population" then the new usage pattern of $lookup allows this to be much the same, and a "lot" more powerful in it's full usage.
Working Example
The following gives an example using a static method on the model. Once that static method is implemented the call simply becomes:
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
Or enhancing to be a bit more modern even becomes:
let results = await Item.lookup({
path: 'tags',
query: { 'tagName' : { '$in': [ 'funny', 'politics' ] } }
})
Making it very similar to .populate() in structure, but it's actually doing the join on the server instead. For completeness, the usage here casts the returned data back to mongoose document instances at according to both the parent and child cases.
It's fairly trivial and easy to adapt or just use as is for most common cases.
N.B The use of async here is just for brevity of running the enclosed example. The actual implementation is free of this dependency.
const async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/looktest');
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt,callback) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
this.aggregate(pipeline,(err,result) => {
if (err) callback(err);
result = result.map(m => {
m[opt.path] = m[opt.path].map(r => rel(r));
return this(m);
});
callback(err,result);
});
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
function log(body) {
console.log(JSON.stringify(body, undefined, 2))
}
async.series(
[
// Clean data
(callback) => async.each(mongoose.models,(model,callback) =>
model.remove({},callback),callback),
// Create tags and items
(callback) =>
async.waterfall(
[
(callback) =>
ItemTag.create([{ "tagName": "movies" }, { "tagName": "funny" }],
callback),
(tags, callback) =>
Item.create({ "title": "Something","description": "An item",
"tags": tags },callback)
],
callback
),
// Query with our static
(callback) =>
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
],
(err,results) => {
if (err) throw err;
let result = results.pop();
log(result);
mongoose.disconnect();
}
)
Or a little more modern for Node 8.x and above with async/await and no additional dependencies:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [opt.path]: m[opt.path].map(r => rel(r)) })
));
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.create(
["movies", "funny"].map(tagName =>({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
const result = (await Item.lookup({
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
mongoose.disconnect();
} catch (e) {
console.error(e);
} finally {
process.exit()
}
})()
And from MongoDB 3.6 and upward, even without the $unwind and $group building:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
},{ timestamps: true });
itemSchema.statics.lookup = function({ path, query }) {
let rel =
mongoose.model(this.schema.path(path).caster.options.ref);
// MongoDB 3.6 and up $lookup with sub-pipeline
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": path,
"let": { [path]: `$${path}` },
"pipeline": [
{ "$match": {
...query,
"$expr": { "$in": [ "$_id", `$$${path}` ] }
}}
]
}}
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [path]: m[path].map(r => rel(r)) })
));
};
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.insertMany(
["movies", "funny"].map(tagName => ({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
let result = (await Item.lookup({
path: 'tags',
query: { 'tagName': { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
await mongoose.disconnect();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
what you are asking for isn't directly supported but can be achieved by adding another filter step after the query returns.
first, .populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } ) is definitely what you need to do to filter the tags documents. then, after the query returns you'll need to manually filter out documents that don't have any tags docs that matched the populate criteria. something like:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags.length;
})
// do stuff with docs
});
Try replacing
.populate('tags').where('tags.tagName').in(['funny', 'politics'])
by
.populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } )
Update: Please take a look at the comments - this answer does not correctly match to the question, but maybe it answers other questions of users which came across (I think that because of the upvotes) so I will not delete this "answer":
First: I know this question is really outdated, but I searched for exactly this problem and this SO post was the Google entry #1. So I implemented the docs.filter version (accepted answer) but as I read in the mongoose v4.6.0 docs we can now simply use:
Item.find({}).populate({
path: 'tags',
match: { tagName: { $in: ['funny', 'politics'] }}
}).exec((err, items) => {
console.log(items.tags)
// contains only tags where tagName is 'funny' or 'politics'
})
Hope this helps future search machine users.
After having the same problem myself recently, I've come up with the following solution:
First, find all ItemTags where tagName is either 'funny' or 'politics' and return an array of ItemTag _ids.
Then, find Items which contain all ItemTag _ids in the tags array
ItemTag
.find({ tagName : { $in : ['funny','politics'] } })
.lean()
.distinct('_id')
.exec((err, itemTagIds) => {
if (err) { console.error(err); }
Item.find({ tag: { $all: itemTagIds} }, (err, items) => {
console.log(items); // Items filtered by tagName
});
});
#aaronheckmann 's answer worked for me but I had to replace return doc.tags.length; to return doc.tags != null; because that field contain null if it doesn't match with the conditions written inside populate.
So the final code:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags != null;
})
// do stuff with docs
});