I am looking for group by queries through Sequelize and cannot seem to find any documentation.
SELECT column, count(column)
FROM table
GROUP BY column
issue: https://github.com/sequelize/sequelize/issues/348
User.findAll({
group: ['field']
})
i use sequelize#2.0.0-dev9
I think you looking for something like this:
Table.findAll({
attributes: ['column1',
sequelize.fn('count', sequelize.col('column2'))],
group: ["Table.column1"]
}).success(function (result) { });
Update: Newer versions of Sequelize uses .then instead of .success.
Table.findAll({
attributes: ['column1',
sequelize.fn('count', sequelize.col('column2'))],
group: ["Table.column1"]
}).then(function (result) { });
Group by with count sequelize ORM
'field' - custom user input, you can rename this string, it's your field(column) in database
'count' - reserved sequelize keyword string need for get count in sequelize
'cnt' - custom user input, you can rename this string, it's your output count
Sequelize version 3.25.0
User.findAll({
attributes: ['field', [sequelize.fn('count', sequelize.col('field')), 'cnt']],
group: ['field'],
})
Try this -
Table.count(
{
attributes: ['column'],
group: 'column',
}
Example: how to add an convenient alias to the grouped function column.
I did this as a separate response mostly because it wouldn't format well in a comment... otherwise I would have just added it to Sampat's answer.
function getSumsBySomeId() {
const criteria = {
attributes: ['some_id', [sequelize.fn('sum', sequelize.col('some_count')), 'some_count_sum']],
group: ['some_id'],
raw: true
};
return Table.getAll(criteria);
}
YIELDS:
{ some_id: 42, some_count_sum: 100 },
{ some_id: 43, some_count_sum: 150 }
...
etc.
Your code should look something like these using ES6 standard.
Table.findAll({ attributes: ['column1', sequelize.fn('count', sequelize.col('column2'))], group: ["Table.column1"] }).then( (result) => { })
**Try this*
Table.findAll({
group: ['column']
})
You need to use row.get('count') to get the count, row.count won't work
The API mentioned at in this answer is correct, but there were a few tricks I was missing in order to actually get the count results out.
As mentioned at: How do I select a column using an alias you need to use .get() for attribute aliased columns for some reason.
And another thing: you need to use parseInt to get an integer out of PostgreSQL count: Postgres sequelize raw query to get count returns string value due to bigint shenanigans.
Minimal runnable example also demonstrating ORDER BY, WHERE and HAVING:
main.js
#!/usr/bin/env node
// https://cirosantilli.com/sequelize-example
const assert = require('assert')
const { DataTypes, Op } = require('sequelize')
const common = require('./common')
const sequelize = common.sequelize(__filename, process.argv[2])
;(async () => {
const UserLikesPost = sequelize.define('UserLikesPost', {
userId: {
type: DataTypes.INTEGER,
},
postId: {
type: DataTypes.INTEGER,
},
}, {})
await UserLikesPost.sync({force: true})
await UserLikesPost.create({userId: 1, postId: 1})
await UserLikesPost.create({userId: 1, postId: 2})
await UserLikesPost.create({userId: 1, postId: 3})
await UserLikesPost.create({userId: 2, postId: 1})
await UserLikesPost.create({userId: 2, postId: 2})
await UserLikesPost.create({userId: 3, postId: 1})
await UserLikesPost.create({userId: 4, postId: 1})
// Count likes on all posts but:
// - don't consider likes userId 4
// - only return posts that have at least 2 likes
// Order posts by those with most likes first.
const postLikeCounts = await UserLikesPost.findAll({
attributes: [
'postId',
[sequelize.fn('COUNT', '*'), 'count'],
],
group: ['postId'],
where: { userId: { [Op.ne]: 4 }},
order: [[sequelize.col('count'), 'DESC']],
having: sequelize.where(sequelize.fn('COUNT', '*'), Op.gte, 2)
})
assert.strictEqual(postLikeCounts[0].postId, 1)
assert.strictEqual(parseInt(postLikeCounts[0].get('count'), 10), 3)
assert.strictEqual(postLikeCounts[1].postId, 2)
assert.strictEqual(parseInt(postLikeCounts[1].get('count'), 10), 2)
assert.strictEqual(postLikeCounts.length, 2)
await sequelize.close()
})()
common.js
const path = require('path');
const { Sequelize } = require('sequelize');
function sequelize(filename, dialect, opts) {
if (dialect === undefined) {
dialect = 'l'
}
if (dialect === 'l') {
return new Sequelize(Object.assign({
dialect: 'sqlite',
storage: path.parse(filename).name + '.sqlite'
}, opts));
} else if (dialect === 'p') {
return new Sequelize('tmp', undefined, undefined, Object.assign({
dialect: 'postgres',
host: '/var/run/postgresql',
}, opts));
} else {
throw new Error('Unknown dialect')
}
}
exports.sequelize = sequelize
package.json:
{
"name": "tmp",
"private": true,
"version": "1.0.0",
"dependencies": {
"pg": "8.5.1",
"pg-hstore": "2.3.3",
"sequelize": "6.5.1",
"sqlite3": "5.0.2"
}
}
and PostgreSQL 13.4 on Ubuntu 21.10. GitHub upstream.
Generated PostgreSQL query:
SELECT
"postId",
COUNT('*') AS "count"
FROM
"UserLikesPosts" AS "UserLikesPost"
WHERE
"UserLikesPost"."userId" != 4
GROUP BY
"postId"
HAVING
COUNT('*') >= 2
ORDER BY
"count" DESC;
JOIN + GROUP BY + aggregate
See: Sequelize query with count in inner join
To create a query like
SELECT key, COUNT(ref) FROM MyModel GROUP BY key
You can do this as follows
const results = await MyModel.findAll({
attributes: ['key', [Sequelize.fn('COUNT', Sequelize.col('ref')), 'count']],
group: ['key']
});
Optionally you can cast the result to something like (MyModel & { count: number })[]
Finally, to extract the count for each row, you'll need to use the getDataValue function. e.g.
results.map(r => r.getDataValue('count'))
Related
I need to make a select from MySQL with the selection values in an array. I have data like this:
const CompaniesRelation = [{CompanyId: ""},{CompanyId: ""},{CompanyId: ""},{CompanyId: ""},{CompanyId: ""}];
With Companies Relation I need to select/find, I need to get all the information for each CompanyId's:
const Companies: Array<Company> = await getRepository(Company).find({ where:{ CompanyId: CompaniesRelation[0].CompanyId, IsActive: true} });
I'm using TypeOrm in Angular. I need to select the information for each object into CompaniesRelation. I need to use fewest selects to DB, in other words, foreach is not the way.
For the final result I need to have an array with all the information of each company in Companies Relation, like this:
[{CompanyId: "", Name: "", fk:""},{CompanyId: "", Name: "", fk:""},{CompanyId: "", Name: "", fk:""}]
How you use WHERE x IN explained in find options for find*, and adding WHERE expression for QueryBuilder.
First get your Company Ids into an array of integers:
const companyIds = [1, 2, 3];
or
const companySel = [{ CompanyId: 1 }, { CompanyId: 2 }, { CompanyId: 3 }];
const companyIds = companySel.map(a => a.CompanyId);
Then you can use the In operator with find
import {In} from "typeorm";
const companyList = await getRepository(Company)
.find({ where: { CompanyId: In (companyIds ) } });
Or you can use the TypeOrm QueryBuilder with "IN" (note the :... syntax)
const companyList = await getRepository(Company)
.createQueryBuilder()
.where("CompanyId IN (:...ids )", { ids: companyIds ) })
.getMany();
Hey I was wondering how do I use findById for a schema inside an array? For example, I have the following Schema:
const GameSchema = new mongoose.Schema({
users: [
{
user: { type: mongoose.Schema.ObjectId, ref: 'User' },
role: {
type: String,
required: true,
enum: ['user', 'moderator', 'creator'],
default: 'user',
},
},
]
}]
I want to find the user with a mongoose function like findById, such as the following:
const user = await game.users.findById({ user: req.user.id })
It doesn't seem to work since users is not a mongodb model. I know I can find the user by using find() like the following:
const user = await game.users.find(
(gameUser) => gameUser.user == req.user.id
)
The only problem is that the type of gameUser and req.user.id is not the same and I can't use '==='. Is there some way to go through the array and use the mongoose function findById?
As docs explains, findById method:
Finds a single document by its _id field
So you have to use findOne() instead of findById().
Also, to return only one field from the entire array you can use projection into find.
Check this example. This query find an object by its id (i.e. user field) and return only the object, not the whole array.
db.collection.find({
"users": { "$elemMatch": { "user": 1 } }
},
{
"users.$": 1
})
Using mongoose you can do:
yourModel.findOne(({
"users": { "$elemMatch": { "user": 1 } }
},
{
"users.$": 1
})).then(result => {
console.log(result)
}).catch(e => {
// error
})
I am trying to write a query with that would roughly do:
SELECT Challenge.id, Challenge.name, count(AcceptedChallenge.userId) AS attempts, count(a.met) AS met, count(b.active) AS active, count(c.id) AS WHOLE
FROM Challange
LEFT JOIN AcceptedChallenge ON Challenge.id = AcceptedChallenge.challengeId,
LEFT JOIN AcceptedChallenge AS a ON Challenge.id = a.challengeId
LEFT JOIN AcceptedChallenge AS b ON Challenge.id = b.challengeId
LEFT JOIN AcceptedChallenge AS c ON Challenge.id = c.challengeId
WHERE a.met = true
AND b.userId = id and b.active = true
AND c.userId = id;
Tried multiple versions, including the below:
const Sequelize = require('sequelize');
const ChallengeController = async ({ user: { id } }) => {
const challenges = await Challenge
.findAll({
attributes: {
include: [[Sequelize.fn('count', Sequelize.col('AcceptedChallenges.userId')), 'attempts'],
[Sequelize.fn('count', Sequelize.col('a.met')), 'met']]
},
include: [{
model: AcceptedChallenge, attributes: [],
required: false,
}, {
model: AcceptedChallenge, attributes: [],
as: 'a',
where: { userId: id, met: true },
required: false,
}],
group: ['Challenge.id']
})
.catch((e) => {
console.log("error:", e);
throw new HTTP404Error('Challenges not found');
});
return challenges;
};
It is not recognizing my associations. Please advise. The version here results in: SequelizeEagerLoadingError: AcceptedChallenge is associated to Challenge using an alias. You've included an alias (a), but it does not match the alias(es) defined in your association (AcceptedChallenges).
When including the AcceptedChallenge model just once, it calculates attempts just fine. I am perplexed as to how I could do the include/JOIN multiple times to get the result, which I need from a single SQL request.
This works for me when the association is repeated, once for each alias needed in your query (example below, but obviously your association may be different).
ChallengeController.hasMany(AcceptedChallenge, {as: 'a', foreignKey: 'challengeId'});
ChallengeController.hasMany(AcceptedChallenge, {as: 'b', foreignKey: 'challengeId'});
ChallengeController.hasMany(AcceptedChallenge, {as: 'c', foreignKey: 'challengeId'});
Are you doing something similar?
This is my model:
import mongoose from 'mongoose';
const UserData = new mongoose.Schema({
additionalData: Array,
name: String,
});
mongoose.model('UserData', UserDataSchema);
mongoose.set('useFindAndModify', false);
export default mongoose.model('UserData');
How can I update additionalData property of UserData model so for example another array is appended.
Let's say additional array is
[ 1, 2, 3]
You can use $addToSet
await UserData.update({name:"something"},{ $addToSet: { additionalData: [1,2,3] })
or
await UserData.updateOne({name:"something"},{ $addToSet: { additionalData: [1,2,3] })
If you have no condition do this ,this will add to all documents
await UserData.update({},{ $addToSet: { additionalData: [1,2,3] })
If you want to remove multiple values from the array,use $pull
await UserData.update({},{ $pull: { additionalData: { $in: [ 1, 2,3 ] }}})
If you wish to remove single value
await UserData.update({},{ $pull: { additionalData:1 }})
If this is not what you mean, let me know in the comments, I'll edit the answer.
const newArray = [ 1, 2, 3];
const nameToUpdate = 'foo';
mongoose.model('UserData').updateMany({
name: nameToUpdate
}, {$set: {
additionalData: newArray
}});
I want to insert 1500000 documents in MongoDB. First, I query a database and get a list of 15000 instructors from there and for each instructor I want to insert 100 courses by each of them.
I run two loops: first it loops through all instructors and secondly, in each iteration it will insert 100 docs for that id as in the code below:
const instructors = await Instructor.find();
//const insrtuctor contains 15000 instructor
instructors.forEach((insructor) => {
for(let i=0; i<=10; i++) {
const course = new Course({
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: insructor._id,
prise: Math.floor(Math.random()*11),
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
});
course.save().then(result => {
console.log(result._id);
Instructor.findByIdAndUpdate(insructor._id, { $push: { courses: course._id } })
.then(insructor => {
console.log(`Instructor Id : ${insructor._id} add Course : ${i} `);
}).catch(err => next(err));
console.log(`Instructor id: ${ insructor._id } add Course: ${i}`)
}).catch(err => console.log(err));
}
});
Here is my package.json file where I put something I found on the internet:
{
"scripts": {
"start": "nodemon app.js",
"fix-memory-limit": "cross-env LIMIT=2048 increase-memory-limit"
},
"devDependencies": {
"cross-env": "^5.2.0",
"faker": "^4.1.0",
"increase-memory-limit": "^1.0.6",
}
}
This is my course model definition
const mongoose = require('mongoose');
const Course = mongoose.model('courses', new mongoose.Schema({
title: {
type: String,
required: true,
minlength: 3
},
author: {
type: mongoose.Schema.Types.ObjectId,
ref: 'instructor'
},
description: {
type: String,
required: true,
minlength: 5
},
ratings: [{
user: {
type: mongoose.Schema.Types.ObjectId,
ref: 'users',
required: true,
unique: true
},
rating: {
type: Number,
required: true,
min: 0,
max: 5
},
description: {
type: String,
required: true,
minlength: 5
}
}],
tags: [String],
rating: {
type: Number,
min: 0,
default: 0
},
ratedBy: {
type: Number,
min: 0,
default: 0
},
prise: {
type: Number,
required: function() { this.isPublished },
min: 0
},
isPublished: {
type: Boolean,
default: false
}
}));
module.exports = Course;
For big amount of data You've to use cursors.
Idea is to process document asap as You get one from db.
Like You're asking db to give instructors and db sends back with small batches and You operate with that batch and process them until reach the end of all batches.
Otherwise await Instructor.find() will load all data to memory and populate that instances with mongoose methods that You don't need.
Even await Instructor.find().lean() will not give memory benefit.
Cursor is mongodb's feature when You do find on collection.
With mongoose it's accessible using: Instructor.collection.find({})
Watch this video.
Below I've written solution for batch processing data using cursor.
Add this somewhere inside the module:
const createCourseForInstructor = (instructor) => {
const data = {
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: instructor._id,
prise: Math.floor(Math.random()*11), // typo: "prise", must be: "price"
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
};
return Course.create(data);
}
const assignCourseToInstructor = (course, instructor) => {
const where = {_id: instructor._id};
const operation = {$push: {courses: course._id}};
return Instructor.collection.updateOne(where, operation, {upsert: false});
}
const processInstructor = async (instructor) => {
let courseIds = [];
for(let i = 0; i < 100; i++) {
try {
const course = await createCourseForInstructor(instructor)
await assignCourseToInstructor(course, instructor);
courseIds.push(course._id);
}
catch (error) {
console.error(error.message);
}
}
console.log(
'Created ', courseIds.length, 'courses for',
'Instructor:', instructor._id,
'Course ids:', courseIds
);
};
and in Your asynchronous block replace Your loop with:
const cursor = await Instructor.collection.find({}).batchSize(1000);
while(await cursor.hasNext()) {
const instructor = await cursor.next();
await processInstructor(instructor);
}
P.S. I'm using native collection.find and collection.updateOne for performance to avoid mongoose use extra heap for mongoose methods and fields on model instances.
BONUS:
Even if with this cursor solution Your code will get out of memory issue again, run Your code like in this example (define size in megabytes according server's ram):
nodemon --expose-gc --max_old_space_size=10240 app.js
The reason is that you are not awaiting the promises returned by save, and immediately continue with the next iterations of the for and forEach loops. This means you are launching a huge amount of (pending) save operations, which will indeed grow the memory usage by the mongodb library.
It would be better to wait for a save (and the chained findByIdAndUpdate) to resolve before continuing with the next iterations.
Since you are apparently in an async function context, you can use await for this, provided that you replace the forEach loop with a for loop (so that you remain in the same function context):
async function yourFunction() {
const instructors = await Instructor.find();
for (let instructor of instructors) { // Use `for` loop to allow for more `await`
for (let i=0; i<10; i++) { // You want 10 times, right?
const course = new Course({
title: faker.lorem.sentence(),
description: faker.lorem.paragraph(),
author: instructor._id,
prise: Math.floor(Math.random()*11),
isPublished: 'true',
tags: ["java", "Nodejs", "javascript"]
});
const result = await course.save();
console.log(result._id);
instructor = await Instructor.findByIdAndUpdate(instructor._id, { $push: { courses: course._id } });
console.log(`Instructor Id : ${instructor._id} add Course : ${i}`);
}
}
}
Now all the save operations are serialised: the next only starts when the previous has completed.
Note that I have not included the error handling you had: this should best be done with a catch call chained to the call of this async function.