I want to insert 1500000 documents in MongoDB. First, I query a database and get a list of 15000 instructors from there and for each instructor I want to insert 100 courses by each of them.
I run two loops: first it loops through all instructors and secondly, in each iteration it will insert 100 docs for that id as in the code below:
const instructors = await Instructor.find();
//const insrtuctor contains 15000 instructor
instructors.forEach((insructor) => {
for(let i=0; i<=10; i++) {
const course = new Course({
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: insructor._id,
prise: Math.floor(Math.random()*11),
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
});
course.save().then(result => {
console.log(result._id);
Instructor.findByIdAndUpdate(insructor._id, { $push: { courses: course._id } })
.then(insructor => {
console.log(`Instructor Id : ${insructor._id} add Course : ${i} `);
}).catch(err => next(err));
console.log(`Instructor id: ${ insructor._id } add Course: ${i}`)
}).catch(err => console.log(err));
}
});
Here is my package.json file where I put something I found on the internet:
{
"scripts": {
"start": "nodemon app.js",
"fix-memory-limit": "cross-env LIMIT=2048 increase-memory-limit"
},
"devDependencies": {
"cross-env": "^5.2.0",
"faker": "^4.1.0",
"increase-memory-limit": "^1.0.6",
}
}
This is my course model definition
const mongoose = require('mongoose');
const Course = mongoose.model('courses', new mongoose.Schema({
title: {
type: String,
required: true,
minlength: 3
},
author: {
type: mongoose.Schema.Types.ObjectId,
ref: 'instructor'
},
description: {
type: String,
required: true,
minlength: 5
},
ratings: [{
user: {
type: mongoose.Schema.Types.ObjectId,
ref: 'users',
required: true,
unique: true
},
rating: {
type: Number,
required: true,
min: 0,
max: 5
},
description: {
type: String,
required: true,
minlength: 5
}
}],
tags: [String],
rating: {
type: Number,
min: 0,
default: 0
},
ratedBy: {
type: Number,
min: 0,
default: 0
},
prise: {
type: Number,
required: function() { this.isPublished },
min: 0
},
isPublished: {
type: Boolean,
default: false
}
}));
module.exports = Course;
For big amount of data You've to use cursors.
Idea is to process document asap as You get one from db.
Like You're asking db to give instructors and db sends back with small batches and You operate with that batch and process them until reach the end of all batches.
Otherwise await Instructor.find() will load all data to memory and populate that instances with mongoose methods that You don't need.
Even await Instructor.find().lean() will not give memory benefit.
Cursor is mongodb's feature when You do find on collection.
With mongoose it's accessible using: Instructor.collection.find({})
Watch this video.
Below I've written solution for batch processing data using cursor.
Add this somewhere inside the module:
const createCourseForInstructor = (instructor) => {
const data = {
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: instructor._id,
prise: Math.floor(Math.random()*11), // typo: "prise", must be: "price"
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
};
return Course.create(data);
}
const assignCourseToInstructor = (course, instructor) => {
const where = {_id: instructor._id};
const operation = {$push: {courses: course._id}};
return Instructor.collection.updateOne(where, operation, {upsert: false});
}
const processInstructor = async (instructor) => {
let courseIds = [];
for(let i = 0; i < 100; i++) {
try {
const course = await createCourseForInstructor(instructor)
await assignCourseToInstructor(course, instructor);
courseIds.push(course._id);
}
catch (error) {
console.error(error.message);
}
}
console.log(
'Created ', courseIds.length, 'courses for',
'Instructor:', instructor._id,
'Course ids:', courseIds
);
};
and in Your asynchronous block replace Your loop with:
const cursor = await Instructor.collection.find({}).batchSize(1000);
while(await cursor.hasNext()) {
const instructor = await cursor.next();
await processInstructor(instructor);
}
P.S. I'm using native collection.find and collection.updateOne for performance to avoid mongoose use extra heap for mongoose methods and fields on model instances.
BONUS:
Even if with this cursor solution Your code will get out of memory issue again, run Your code like in this example (define size in megabytes according server's ram):
nodemon --expose-gc --max_old_space_size=10240 app.js
The reason is that you are not awaiting the promises returned by save, and immediately continue with the next iterations of the for and forEach loops. This means you are launching a huge amount of (pending) save operations, which will indeed grow the memory usage by the mongodb library.
It would be better to wait for a save (and the chained findByIdAndUpdate) to resolve before continuing with the next iterations.
Since you are apparently in an async function context, you can use await for this, provided that you replace the forEach loop with a for loop (so that you remain in the same function context):
async function yourFunction() {
const instructors = await Instructor.find();
for (let instructor of instructors) { // Use `for` loop to allow for more `await`
for (let i=0; i<10; i++) { // You want 10 times, right?
const course = new Course({
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: instructor._id,
prise: Math.floor(Math.random()*11),
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
});
const result = await course.save();
console.log(result._id);
instructor = await Instructor.findByIdAndUpdate(instructor._id, { $push: { courses: course._id } });
console.log(`Instructor Id : ${instructor._id} add Course : ${i}`);
}
}
}
Now all the save operations are serialised: the next only starts when the previous has completed.
Note that I have not included the error handling you had: this should best be done with a catch call chained to the call of this async function.
Related
When I run the following code, I get the object along with the populated fields logged on the console.
Screenshot
But, the fields have not been populated in the books collection. Can someone please help me figure this out?
const bookSchema = new Schema({
title: String,
genre: { type: Schema.Types.ObjectId, ref: "genre" },
author: { type: Schema.Types.ObjectId, ref: "author" },
numberInStock: { type: Number, default: 0 },
rating: Number,
yearPublished: Number,
dateAdded: { type: Date, default: Date.now },
liked: { type: Boolean, default: false },
});
const genreSchema = new Schema({ name: String });
const authorSchema = new Schema({ name: String });
const Book = model("book", bookSchema);
const Genre = model("genre", genreSchema);
const Author = model("author", authorSchema);
const books = [
{
title: "Sapiens",
genre: "632873144b0bbfc10ae1942d",
author: "632873e706fe265eaee77de3",
numberInStock: 6,
rating: 4.4,
yearPublished: 2011,
},
];
async function saveBook(b) {
let book = new Book(b);
book
.save()
.then((result) => {
populateBook(result._id);
})
.catch((err) => console.log("Error: ", err));
}
function populateBook(id) {
Book.findById(id)
.populate("genre")
.populate("author")
.exec((err, book) => {
if (err) {
console.log("Error: ", err);
return;
}
console.log(book);
});
}
books.forEach((b) => {
saveBook(b);
});
That's how population works, it only stores references to other documents in the database. At query time, and if you ask for it (using .populate()), Mongoose will retrieve the referenced documents and insert them into the "parent" document.
If you want the referenced documents to be stored in the database, you can't use population but have to use subdocuments.
However, this will limit the flexibility of your database, because if for example an author name needs to be changed, you need to change all the Book documents in your database to update the author's name. With population, you only need to change the Author document.
I am trying to create a database for my Giveaway Bot. It consist of 2 collections, Main (holding settings) and Giveaway which is nested in under the Main collection. I can create my giveaway's without problems. However I want to add some data later on using findOneAndUpdate.
Running the code below I always get this error: Updating the path 'giveaways.duration' would create a conflict at 'giveaways'. Can anyone help solving this issue ?
schema.js
const giveawaySchema = new mongoose.Schema({
_id: String,
destination: String,
duration: String,
winners: String,
price: String,
})
const mainSchema = new mongoose.Schema({
_id: String,
log_channel_id: String,
admin_roles: [],
giveaways: [giveawaySchema],
const Main = mongoose.model("mainSchema", mainSchema);
const Giveaway = mongoose.model("giveawaySchema", giveawaySchema);
module.exports = { Main, Giveaway }
});
Part of my code used for updating:
const mongoose = require("mongoose")
const {Main, Giveaway} = require("../models/schema.js")
const newestGiveaway = await Main.findOneAndUpdate(
{
_id: guildId,
'giveaways._id': giveaway_id,
},
{
"$set":{
"giveaways.duration": "3d",
"giveaways.winners": "20",
"giveaways.price": "Price to Win",
},
},
{
upsert: true,
}
Thank you for your help :)
A small side question. I have fetched the Main document (the parent) before already can I make my search cheaper/ more efficent by only searching through this instead of running the findOneandUpdate method on the whole database ?
Edit 1:
I found that it is neccesary to use the $ operator and have updated my code. However I still get the same error:
{
$set:{
"giveaways.$.duration": "3d",
"giveaways.$.winners": "20,
"giveaways.$.price": "Price to Win",
},
},
Edit 2:
Just to clarify, the creation and nesting of the giveawaySchemas works but I am not able to update the nested document by using the code above.
My child component is already created by using the code below. I now want to update this child (newGiveaway with _id of 1)
const currentGuild = await Main.findOne({_id: guildId})
const newGiveaway = await Giveaway.create({
_id: 1,
destination: 12345678,
});
currentGuild.giveaways.push(newGiveaway)
You can change your schema declaration to use a ref to the giveawaySchema:
const giveawaySchema = new mongoose.Schema({
_id: String,
destination: String,
duration: String,
winners: String,
price: String,
})
const mainSchema = new mongoose.Schema({
_id: String,
log_channel_id: String,
admin_roles: [],
giveaways: [{
type: mongoose.Schema.Types.ObjectId,
ref: 'giveawaySchema'
}],
const Main = mongoose.model("mainSchema", mainSchema);
const Giveaway = mongoose.model("giveawaySchema", giveawaySchema);
module.exports = { Main, Giveaway }
Then, you will just need to update your giveaways directy:
const mongoose = require('mongoose');
const { Giveaway } = require('../models/schema.js');
const newestGiveaway = await Main.findByIdAndUpdate(
giveaway_id,
{
duration: '3d',
winners: '20',
price: 'Price to Win',
},
{
new: true,
}
);
In the mainSchema you define giveaways field as an array of giveawaySchema object. So, you have to treat it as an array, not an object. If you want to treat it as an object, you will have to update mainSchema by removing square bracket at giveawaysSchema.
Relevant Question for how to pushing item into mongo array
When i fetch new alerts, i want to check if the ID of the new alert was already recorded. The issue is that that ID is nested inside an array. There's the alertsDetails array, which contains objects and those objects have an _ID filed which is what i want to check. I am not sure how to achieve that. I got the code below but then i have to iterate over the result to check the exists value. Im sure there must be a better way.
const mongoose = require('mongoose');
const { Schema } = mongoose;
const G2AlertsSchema = new Schema(
{
status: { type: String, required: true },
openDate: { type: Date, required: true },
alertType: { type: Array, required: true },
severity: { type: Array, required: true },
locationName: { type: Array, required: true },
history: { type: Array, required: true },
alertDetails: { type: Array, required: false },
assignedTo: { type: Schema.Types.ObjectId, ref: 'user' },
},
{
timestamps: true,
},
);
const G2Alerts = mongoose.model('G2Alert', G2AlertsSchema);
module.exports = G2Alerts;
This is the code i found on mongodb's website. I just want to see if the ID exists only. Basically when i fetch the new alerts i get an array and i iterate over it, i want to check each item's ID against what's inside the Database. If it's there, skip and go to the next. If it's new, then create a new alert and save it.
const exists = await G2Alerts.aggregate([
{
$project: {
exists: {
$in: ['5f0b4f508bda3805754ab343', '$alertDetails._id'],
},
},
},
]);
EDIT: Another thing. I am getting a eslint warning saying i should use array iteration instead of a for loop. The issue is, i need to use await when looking up the Alert ID. If i use, reduce or filter, i can't use await. If i use async inside the reduce or filter function, then it will return promises in or just an empty array.
This below works, based on the answer provided by Tom Slabbaert
const newAlertsData = [];
for (let item of alertData.data.items) {
const exists = await G2Alerts.find({ 'alertDetails._id': `${item._id}` });
if (exists.length === 0) {
newAlertsData.push(item);
}
}
if (newAlertsData.length !== 0) {......
But this does not
const filteredAlerts = alertData.data.items.reduce((filtered, item) => {
const exists = await G2Alerts.find({ 'alertDetails._id': `${item._id}` });
if (exists.length === 0) {
filtered.push(item);
}
return filtered;
}, []);
You're not far off, here is an example using the correct syntax:
const exists = await G2Alerts.findOne({"alertDetails._id": '5f0b4f508bda3805754ab343'}});
if (!exists) {
... do something
}
This can also be achieve using aggregate with a $match stage instead of a $project stage or even better countDocuments which just returns the count instead of the entire object if you do not require it.
One more thing I'd like to add is that make sure alertDetails._id is string type as you're using string in you're $in. otherwise you'll need to cast them to ObjectId type in mongoose like so:
new mongoose.Types.ObjectId('5f0b4f508bda3805754ab343')
And for Mongo:
import {ObjectId} from "mongodb"
...
new ObjectId('5f0b4f508bda3805754ab343')
EDIT
Try something like this?
let ids = alertData.data.items.map(item => item._id.toString());
let existing = await G2Alerts.distinct("alertsDetails._id", {"alertsDetails._id": {$in: ids}});
const filteredAlerts = alertData.data.items.reduce((filtered, item) => {
if (!existing.includes(item._id.toString())) {
return [item].concat(filtered)
}
return filtered;
}, []);
This way you only need to call the db once and not multiple times.
Final code based on the provided answer.
const ids = alertData.data.items.map(item => item._id);
const existing = await G2Alerts.find({ 'alertDetails._id': { $in: ids } }).distinct(
'alertDetails._id',
(err, alerts) => {
if (err) {
res.send(err);
}
return alerts;
},
);
const filteredAlerts = alertData.data.items.reduce((filtered, item) => {
if (!existing.includes(item._id.toString()) && item.openDate > dateLimit) {
return [item].concat(filtered);
}
return filtered;
}, []);
According to this mongodb article it is possible to auto increment a field and I would like the use the counters collection way.
The problem with that example is that I don't have thousands of people typing the data in the database using the mongo console. Instead I am trying to use mongoose.
So my schema looks something like this:
var entitySchema = mongoose.Schema({
testvalue:{type:String,default:function getNextSequence() {
console.log('what is this:',mongoose);//this is mongoose
var ret = db.counters.findAndModify({
query: { _id:'entityId' },
update: { $inc: { seq: 1 } },
new: true
}
);
return ret.seq;
}
}
});
I have created the counters collection in the same database and added a page with the _id of 'entityId'. From here I am not sure how to use mongoose to update that page and get the incrementing number.
There is no schema for counters and I would like it to stay that way because this is not really an entity used by the application. It should only be used in the schema(s) to auto increment fields.
Here is an example how you can implement auto-increment field in Mongoose:
var CounterSchema = Schema({
_id: {type: String, required: true},
seq: { type: Number, default: 0 }
});
var counter = mongoose.model('counter', CounterSchema);
var entitySchema = mongoose.Schema({
testvalue: {type: String}
});
entitySchema.pre('save', function(next) {
var doc = this;
counter.findByIdAndUpdate({_id: 'entityId'}, {$inc: { seq: 1} }, function(error, counter) {
if(error)
return next(error);
doc.testvalue = counter.seq;
next();
});
});
You can use mongoose-auto-increment package as follows:
var mongoose = require('mongoose');
var autoIncrement = require('mongoose-auto-increment');
/* connect to your database here */
/* define your CounterSchema here */
autoIncrement.initialize(mongoose.connection);
CounterSchema.plugin(autoIncrement.plugin, 'Counter');
var Counter = mongoose.model('Counter', CounterSchema);
You only need to initialize the autoIncrement once.
The most voted answer doesn't work. This is the fix:
var CounterSchema = new mongoose.Schema({
_id: {type: String, required: true},
seq: { type: Number, default: 0 }
});
var counter = mongoose.model('counter', CounterSchema);
var entitySchema = mongoose.Schema({
sort: {type: String}
});
entitySchema.pre('save', function(next) {
var doc = this;
counter.findByIdAndUpdateAsync({_id: 'entityId'}, {$inc: { seq: 1} }, {new: true, upsert: true}).then(function(count) {
console.log("...count: "+JSON.stringify(count));
doc.sort = count.seq;
next();
})
.catch(function(error) {
console.error("counter error-> : "+error);
throw error;
});
});
The options parameters gives you the result of the update and it creates a new document if it doesn't exist.
You can check here the official doc.
And if you need a sorted index check this doc
So combining multiple answers, this is what I ended up using:
counterModel.js
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
const counterSchema = new Schema(
{
_id: {type: String, required: true},
seq: { type: Number, default: 0 }
}
);
counterSchema.index({ _id: 1, seq: 1 }, { unique: true })
const counterModel = mongoose.model('counter', counterSchema);
const autoIncrementModelID = function (modelName, doc, next) {
counterModel.findByIdAndUpdate( // ** Method call begins **
modelName, // The ID to find for in counters model
{ $inc: { seq: 1 } }, // The update
{ new: true, upsert: true }, // The options
function(error, counter) { // The callback
if(error) return next(error);
doc.id = counter.seq;
next();
}
); // ** Method call ends **
}
module.exports = autoIncrementModelID;
myModel.js
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
const autoIncrementModelID = require('./counterModel');
const myModel = new Schema({
id: { type: Number, unique: true, min: 1 },
createdAt: { type: Date, default: Date.now },
updatedAt: { type: Date },
someOtherField: { type: String }
});
myModel.pre('save', function (next) {
if (!this.isNew) {
next();
return;
}
autoIncrementModelID('activities', this, next);
});
module.exports = mongoose.model('myModel', myModel);
Attention!
As hammerbot and dan-dascalescu pointed out this does not work if you remove documents.
If you insert 3 documents with id 1, 2 and 3 - you remove 2 and insert another a new one it'll get 3 as id which is already used!
In case you don't ever remove documents, here you go:
I know this has already a lot of answers, but I would share my solution which is IMO short and easy understandable:
// Use pre middleware
entitySchema.pre('save', function (next) {
// Only increment when the document is new
if (this.isNew) {
entityModel.count().then(res => {
this._id = res; // Increment count
next();
});
} else {
next();
}
});
Make sure that entitySchema._id has type:Number.
Mongoose version: 5.0.1.
This problem is sufficiently complicated and there are enough pitfalls that it's best to rely on a tested mongoose plugin.
Out of the plethora of "autoincrement" plugins at http://plugins.mongoosejs.io/, the best maintained and documented (and not a fork) is mongoose sequence.
I've combined all the (subjectively and objectively) good parts of the answers, and came up with this code:
const counterSchema = new mongoose.Schema({
_id: {
type: String,
required: true,
},
seq: {
type: Number,
default: 0,
},
});
// Add a static "increment" method to the Model
// It will recieve the collection name for which to increment and return the counter value
counterSchema.static('increment', async function(counterName) {
const count = await this.findByIdAndUpdate(
counterName,
{$inc: {seq: 1}},
// new: return the new value
// upsert: create document if it doesn't exist
{new: true, upsert: true}
);
return count.seq;
});
const CounterModel = mongoose.model('Counter', counterSchema);
entitySchema.pre('save', async function() {
// Don't increment if this is NOT a newly created document
if(!this.isNew) return;
const testvalue = await CounterModel.increment('entity');
this.testvalue = testvalue;
});
One of the benefits of this approach is that all the counter related logic is separate. You can store it in a separate file and use it for multiple models importing the CounterModel.
If you are going to increment the _id field, you should add its definition in your schema:
const entitySchema = new mongoose.Schema({
_id: {
type: Number,
alias: 'id',
required: true,
},
<...>
});
test.pre("save",function(next){
if(this.isNew){
this.constructor.find({}).then((result) => {
console.log(result)
this.id = result.length + 1;
next();
});
}
})
I didn't wan to use any plugin (an extra dependencie, initializing the mongodb connection apart from the one I use in the server.js, etc...) so I did an extra module, I can use it at any schema and even, I'm considering when you remove a document from the DB.
module.exports = async function(model, data, next) {
// Only applies to new documents, so updating with model.save() method won't update id
// We search for the biggest id into the documents (will search in the model, not whole db
// We limit the search to one result, in descendant order.
if(data.isNew) {
let total = await model.find().sort({id: -1}).limit(1);
data.id = total.length === 0 ? 1 : Number(total[0].id) + 1;
next();
};
};
And how to use it:
const autoincremental = require('../modules/auto-incremental');
Work.pre('save', function(next) {
autoincremental(model, this, next);
// Arguments:
// model: The model const here below
// this: The schema, the body of the document you wan to save
// next: next fn to continue
});
const model = mongoose.model('Work', Work);
module.exports = model;
Hope it helps you.
(If this Is wrong, please, tell me. I've been having no issues with this, but, not an expert)
Here is a proposal.
Create a separate collection to holds the max value for a model collection
const autoIncrementSchema = new Schema({
name: String,
seq: { type: Number, default: 0 }
});
const AutoIncrement = mongoose.model('AutoIncrement', autoIncrementSchema);
Now for each needed schema, add a pre-save hook.
For example, let the collection name is Test
schema.pre('save', function preSave(next) {
const doc = this;
if (doc.isNew) {
const nextSeq = AutoIncrement.findOneAndUpdate(
{ name: 'Test' },
{ $inc: { seq: 1 } },
{ new: true, upsert: true }
);
nextSeq
.then(nextValue => doc[autoIncrementableField] = nextValue)
.then(next);
}
else next();
}
As findOneAndUpdate is an atomic operation, no two updates will return same seq value. Thus each of your insertion will get an incremental seq regardless of number of concurrent insertions. Also this can be extended to more complex auto incremental logic and the auto increment sequence is not limited to Number type
This is not a tested code. Test before you use until I make a plugin for mongoose.
Update I found that this plugin implemented related approach.
The answers seem to increment the sequence even if the document already has an _id field (sort, whatever). This would be the case if you 'save' to update an existing document. No?
If I'm right, you'd want to call next() if this._id !== 0
The mongoose docs aren't super clear about this. If it is doing an update type query internally, then pre('save' may not be called.
CLARIFICATION
It appears the 'save' pre method is indeed called on updates.
I don't think you want to increment your sequence needlessly. It costs you a query and wastes the sequence number.
I had an issue using Mongoose Document when assigning value to Schema's field through put(). The count returns an Object itself and I have to access it's property.
I played at #Tigran's answer and here's my output:
// My goal is to auto increment the internalId field
export interface EntityDocument extends mongoose.Document {
internalId: number
}
entitySchema.pre<EntityDocument>('save', async function() {
if(!this.isNew) return;
const count = await counter.findByIdAndUpdate(
{_id: 'entityId'},
{$inc: {seq: 1}},
{new: true, upsert: true}
);
// Since count is returning an array
// I used get() to access its child
this.internalId = Number(count.get('seq'))
});
Version: mongoose#5.11.10
None of above answer works when you have unique fields in your schema
because unique check at db level and increment happen before db level validation, so you may skip lots of numbers in auto increments like above solutions
only in post save can find if data already saved on db or return error
schmea.post('save', function(error, doc, next) {
if (error.name === 'MongoError' && error.code === 11000) {
next(new Error('email must be unique'));
} else {
next(error);
}
});
https://stackoverflow.com/a/41479297/10038067
that is why none of above answers are not like atomic operations auto increment in sql like dbs
I use together #cluny85 and #edtech.
But I don't complete finish this issues.
counterModel.findByIdAndUpdate({_id: 'aid'}, {$inc: { seq: 1} }, function(error,counter){
But in function "pre('save...) then response of update counter finish after save document.
So I don't update counter to document.
Please check again all answer.Thank you.
Sorry. I can't add comment. Because I am newbie.
var CounterSchema = Schema({
_id: { type: String, required: true },
seq: { type: Number, default: 0 }
});
var counter = mongoose.model('counter', CounterSchema);
var entitySchema = mongoose.Schema({
testvalue: { type: String }
});
entitySchema.pre('save', function(next) {
if (this.isNew) {
var doc = this;
counter.findByIdAndUpdate({ _id: 'entityId' }, { $inc: { seq: 1 } }, { new: true, upsert: true })
.then(function(count) {
doc.testvalue = count.seq;
next();
})
.catch(function(error) {
throw error;
});
} else {
next();
}
});
I am looking for group by queries through Sequelize and cannot seem to find any documentation.
SELECT column, count(column)
FROM table
GROUP BY column
issue: https://github.com/sequelize/sequelize/issues/348
User.findAll({
group: ['field']
})
i use sequelize#2.0.0-dev9
I think you looking for something like this:
Table.findAll({
attributes: ['column1',
sequelize.fn('count', sequelize.col('column2'))],
group: ["Table.column1"]
}).success(function (result) { });
Update: Newer versions of Sequelize uses .then instead of .success.
Table.findAll({
attributes: ['column1',
sequelize.fn('count', sequelize.col('column2'))],
group: ["Table.column1"]
}).then(function (result) { });
Group by with count sequelize ORM
'field' - custom user input, you can rename this string, it's your field(column) in database
'count' - reserved sequelize keyword string need for get count in sequelize
'cnt' - custom user input, you can rename this string, it's your output count
Sequelize version 3.25.0
User.findAll({
attributes: ['field', [sequelize.fn('count', sequelize.col('field')), 'cnt']],
group: ['field'],
})
Try this -
Table.count(
{
attributes: ['column'],
group: 'column',
}
Example: how to add an convenient alias to the grouped function column.
I did this as a separate response mostly because it wouldn't format well in a comment... otherwise I would have just added it to Sampat's answer.
function getSumsBySomeId() {
const criteria = {
attributes: ['some_id', [sequelize.fn('sum', sequelize.col('some_count')), 'some_count_sum']],
group: ['some_id'],
raw: true
};
return Table.getAll(criteria);
}
YIELDS:
{ some_id: 42, some_count_sum: 100 },
{ some_id: 43, some_count_sum: 150 }
...
etc.
Your code should look something like these using ES6 standard.
Table.findAll({ attributes: ['column1', sequelize.fn('count', sequelize.col('column2'))], group: ["Table.column1"] }).then( (result) => { })
**Try this*
Table.findAll({
group: ['column']
})
You need to use row.get('count') to get the count, row.count won't work
The API mentioned at in this answer is correct, but there were a few tricks I was missing in order to actually get the count results out.
As mentioned at: How do I select a column using an alias you need to use .get() for attribute aliased columns for some reason.
And another thing: you need to use parseInt to get an integer out of PostgreSQL count: Postgres sequelize raw query to get count returns string value due to bigint shenanigans.
Minimal runnable example also demonstrating ORDER BY, WHERE and HAVING:
main.js
#!/usr/bin/env node
// https://cirosantilli.com/sequelize-example
const assert = require('assert')
const { DataTypes, Op } = require('sequelize')
const common = require('./common')
const sequelize = common.sequelize(__filename, process.argv[2])
;(async () => {
const UserLikesPost = sequelize.define('UserLikesPost', {
userId: {
type: DataTypes.INTEGER,
},
postId: {
type: DataTypes.INTEGER,
},
}, {})
await UserLikesPost.sync({force: true})
await UserLikesPost.create({userId: 1, postId: 1})
await UserLikesPost.create({userId: 1, postId: 2})
await UserLikesPost.create({userId: 1, postId: 3})
await UserLikesPost.create({userId: 2, postId: 1})
await UserLikesPost.create({userId: 2, postId: 2})
await UserLikesPost.create({userId: 3, postId: 1})
await UserLikesPost.create({userId: 4, postId: 1})
// Count likes on all posts but:
// - don't consider likes userId 4
// - only return posts that have at least 2 likes
// Order posts by those with most likes first.
const postLikeCounts = await UserLikesPost.findAll({
attributes: [
'postId',
[sequelize.fn('COUNT', '*'), 'count'],
],
group: ['postId'],
where: { userId: { [Op.ne]: 4 }},
order: [[sequelize.col('count'), 'DESC']],
having: sequelize.where(sequelize.fn('COUNT', '*'), Op.gte, 2)
})
assert.strictEqual(postLikeCounts[0].postId, 1)
assert.strictEqual(parseInt(postLikeCounts[0].get('count'), 10), 3)
assert.strictEqual(postLikeCounts[1].postId, 2)
assert.strictEqual(parseInt(postLikeCounts[1].get('count'), 10), 2)
assert.strictEqual(postLikeCounts.length, 2)
await sequelize.close()
})()
common.js
const path = require('path');
const { Sequelize } = require('sequelize');
function sequelize(filename, dialect, opts) {
if (dialect === undefined) {
dialect = 'l'
}
if (dialect === 'l') {
return new Sequelize(Object.assign({
dialect: 'sqlite',
storage: path.parse(filename).name + '.sqlite'
}, opts));
} else if (dialect === 'p') {
return new Sequelize('tmp', undefined, undefined, Object.assign({
dialect: 'postgres',
host: '/var/run/postgresql',
}, opts));
} else {
throw new Error('Unknown dialect')
}
}
exports.sequelize = sequelize
package.json:
{
"name": "tmp",
"private": true,
"version": "1.0.0",
"dependencies": {
"pg": "8.5.1",
"pg-hstore": "2.3.3",
"sequelize": "6.5.1",
"sqlite3": "5.0.2"
}
}
and PostgreSQL 13.4 on Ubuntu 21.10. GitHub upstream.
Generated PostgreSQL query:
SELECT
"postId",
COUNT('*') AS "count"
FROM
"UserLikesPosts" AS "UserLikesPost"
WHERE
"UserLikesPost"."userId" != 4
GROUP BY
"postId"
HAVING
COUNT('*') >= 2
ORDER BY
"count" DESC;
JOIN + GROUP BY + aggregate
See: Sequelize query with count in inner join
To create a query like
SELECT key, COUNT(ref) FROM MyModel GROUP BY key
You can do this as follows
const results = await MyModel.findAll({
attributes: ['key', [Sequelize.fn('COUNT', Sequelize.col('ref')), 'count']],
group: ['key']
});
Optionally you can cast the result to something like (MyModel & { count: number })[]
Finally, to extract the count for each row, you'll need to use the getDataValue function. e.g.
results.map(r => r.getDataValue('count'))