We Keep Coding

How to speed up Sequelize findAndCountAll?

I have pagination made with Sequelize
router.js
router.post('/getall', async (req, res) => {
try {
const { q, page, limit, order_by, order_direction } = req.query;
const { candidate, position, filters } = req.body
let include = [
{
model: SortedLvl,
where: {
lvl: filters.jobLvl.name,
months: { [Op.gte]: filters.jobMinExp.value },
},
},
{
model: SortedLastJob,
where: { jobposition: filters.jobType.name }
},
{
model: SortedSkills,
}
]
let search = {};
let order = [];
let filterCandidate = {}
if (candidate) {
if (candidate != undefined) {
t = candidate.split(/[ ,]+/)
let arr = new Array()
// console.log('t', t);
t.map((el, index) => {
console.log('el', el);
if (typeof el == 'number') {
arr.push({ first_name: { [Op.iLike]: `%` + `${el}` + `%` } }, { last_name: { [Op.iLike]: `%` + `${el}` + `%` } });
} else {
arr.push({ first_name: { [Op.iLike]: `%` + `%${el}%` + `%` } }, { last_name: { [Op.iLike]: `%` + `%${el}%` + `%` } });
}
});
filterCandidate = {
[Op.or]: arr
};
}
}
let filterPosition = {}
if (position) {
if (position != undefined) {
filterPosition = { position: { [Op.iLike]: `%${position}%` } }
}
}
if (filterCandidate.length > 0 || filterPosition.length > 0) {
search = { where: { ...(filterCandidate || []), ...(filterPosition || []) } }
}
console.log('search', search);
console.log('candidate', filterCandidate);
console.log('position', filterPosition);
if (order_by && order_direction) {
order.push([order_by, order_direction]);
}
const transform = (records) => {
return records.map(record => {
return {
id: record.id,
name: record.name,
date: moment(record.createdAt).format('D-M-Y H:mm A')
}
});
}
const products = await paginate(Candidate, page, limit, search, order, include);
return res.json({
success: true,
// message: '',
data: products
})
} catch (error) {
console.log('Failed to fetch products', error);
return res.status(500).send({
success: false,
message: 'Failed to fetch products'
})
}
});
paginate.js
const paginate = async (model, pageSize, pageLimit, search = {}, order = [], include, transform, attributes, settings) => {
try {
const limit = parseInt(pageLimit, 10) || 10;
const page = parseInt(pageSize, 10) || 1;
// create an options object
let options = {
offset: getOffset(page, limit),
limit: limit,
distinct: true,
include: include,
};
// check if the search object is empty
if (Object.keys(search).length) {
options = { ...options, ...search };
}
if (attributes && attributes.length) {
options['attributes'] = attributes;
}
if (order && order.length) {
options['order'] = order;
}
let data = await model.findAndCountAll(options);
if (transform && typeof transform === 'function') {
data = transform(data.rows);
}
return {
previousPage: getPreviousPage(page),
currentPage: page,
nextPage: getNextPage(page, limit, count),
total: count,
limit: limit,
data: data.rows
}
} catch (error) {
console.log(error);
}
}
const getOffset = (page, limit) => {
return (page * limit) - limit;
}
const getNextPage = (page, limit, total) => {
if ((total / limit) > page) {
return page + 1;
}
return null
}
const getPreviousPage = (page) => {
if (page <= 1) {
return null
}
return page - 1;
}
module.exports = paginate;
First problem - on queries with included models sometimes it response with wrong count
and also it is so slow
Please, help me with solving this issue
Thank you
I tried use separate: true, required: true and etc - I just get empty arrays on included

In your sequelize request there are lots of json query. Mysql cant create directly index your json fields. You can create a referance your query property like below and after that you can add a index for reference column.
And change your referance column as created reference. Also you can create mysql index another requests columns.
ALTER TABLE [table_name] ADD COLUMN email VARCHAR(255)
GENERATED ALWAYS as (properties->>"$.[jsonObjectName].[jsonProperty]");
ALTER TABLE [table_name] ADD INDEX email ([jsonProperty]) USING BTREE;

How to chain multiple updateMany() requests to MongoDB (and wait for each response to come back before moving on)

I am trying to make 3 different updateMany requests inside a get request, each with a different query. They all work except that my third updateMany request only achieves the desired behaviour after the page reloads/refreshes two times.
Here is my code:
app.get('/', (req, res) => {
let todaysDate = new Date().toString().split(' ').slice(0, 4).join(' ')
let todaysDateMs = new Date(todaysDate + ', 00:00:00').getTime()
habitsCollection.updateMany({}, {
$set: {
todaysDate,
todaysDateMs
}
}).then(res => {
habitsCollection.updateMany({ lastClicked: { $ne: todaysDate } }, {
$set: {
clicked: 'false'
}
}).then(res => {
habitsCollection.updateMany({ $expr: { $gte: [{ $subtract: ["$todaysDateMs", "$lastClickedMs"] }, 172800000] } }, {
$set: {
streak: 0
},
})
})
})
habitsCollection.find({}).toArray()
.then(results => {
console.log(results)
let filtered = results.filter(result => result.clicked === 'false')
habitsLeft = filtered.length
res.render('index.ejs', { habits: results, dayVar: 'days', habitsLeft })
})
})
I am expecting that on each page load, if a document has a lastClickedMs key/value subtract todaysDateMs key/value and is greater than or equal to 172800000, that the streak value will be reset to 0. This does happen but only after the page has loaded twice.

I think the problem is that you are not waiting for the promises to resolve before you call habitsCollection.find({}).toArray(). You can use Promise.all to handle multiple promises.
app.get('/', (req, res) => {
let todaysDate = new Date().toString().split(' ').slice(0, 4).join(' ');
let todaysDateMs = new Date(todaysDate + ', 00:00:00').getTime();
Promise.all([
habitsCollection.updateMany({}, {
$set: {
todaysDate,
todaysDateMs
}
}),
habitsCollection.updateMany({ lastClicked: { $ne: todaysDate } }, {
$set: {
clicked: 'false'
}
}),
habitsCollection.updateMany({ $expr: { $gte: [{ $subtract: ["$todaysDateMs", "$lastClickedMs"] }, 172800000] } }, {
$set: {
streak: 0
},
})
]).then(() => {
habitsCollection.find({}).toArray()
.then(results => {
console.log(results);
let filtered = results.filter(result => result.clicked === 'false');
habitsLeft = filtered.length;
res.render('index.ejs', { habits: results, dayVar: 'days', habitsLeft });
});
});
});

I want to know why the log array doesn't return the specified user with the given id?

app.get("/api/users/:_id/logs", (req, res) => {
const id = req.params._id;
const { from, to, limit } = req.query;
** Here I tried to search for the matched user and it works successfully: **
User.findById({ _id: id }, (err, user) => {
if (!user || err) {
res.send("Unknown User Id !!");
} else {
**Then I tried to filter the log array with date **
// const username = user.username;
let responObject = {};
if (from) {
responObject["$gte"] = new Date(from).toDateString();
}
if (to) {
responObject["$lte"] = new Date(to).toDateString();
}
let filter = {
_id: id,
};
if (from || to) {
filter.date = responObject;
}
let nonNullLimit = limit ?? 500;
**try to build the array log and return it to the user but it always be empty and never return the exercises for the user **
Exercise.find(filter)
.limit(+nonNullLimit)
.exec((err, data) => {
if (err || !data) {
res.json([]);
} else {
const count = data.length;
const rowLog = data;
const { username, _id } = user;
const log = rowLog.map((item) => ({
description: item.description,
duration: item.duration,
date: new Date(item.date).toDateString(),
}));
console.log(log)
if (from && to) {
res.json({
username,
from: new Date(from).toDateString(),
to: new Date(to).toDateString(),
count,
_id,
log,
});
} else {
res.json({
username,
count,
_id,
log,
});
}
}
});
}
});
});
this is the result when I try to log all the exercises for the user
{"username":"ahmed","count":0,"_id":"62a9aab2743ddfc9df5165f2","log":[]}

Node.js how to use mongoose to do filter and pagination

I get all data successfully in my node.js code using Profile.find(),but when I want to add filter in api, it get 500 error.
Profile.js
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
//Create Schema
const ProfileSchema = new Schema({
type: {
type: String,
},
description: {
type: String,
},
cash:{
type: String,
required: true,
}
})
module.exports = Profile = mongoose.model("profile",ProfileSchema);
api.js
const router = express.Router();
const Profile = require('../../models/Profile');
router.get("/",passport.authenticate('jwt',{session:false}),(req, res) => {
Profile.find().then(profiles => {
res.json(profiles)
})//works
})
router.get("/",passport.authenticate('jwt',{session:false}),(req, res) => {
var condition = {'cash' : '5000'};
Profile.find(condition,function(err,res){//error for this
res.json(res)
})
})
I am new to mongoose and do not sure how to do the filter and pagination for my code.
Appreciate for any kind help.
Update
Thanks to #Amila Senadheera, I finally find the solution to do filter and pagination using below code:
var condition = { cash: '5000' };
Profile.find(condition).skip(1).limit(10).then(profiles => {
res.json(profiles)
})
To use aggregation, it works like below:
router.post(
"/",
passport.authenticate("jwt", { session: false }),
async (req, res) => {
// page index
let pageIndex = 1;
// profiles per page
let pageSize = 10;
if (req.body.pageIndex !== null || req.body.pageIndex !== "undefined") {
pageIndex = req.body.pageIndex;
}
if (req.body.pageSize !== null || req.body.pageSize !== "undefined") {
pageSize = req.body.pageSize;
}
let condition = { description: "1" };
try {
const getAllProfile = await Profile.aggregate([{
$facet: {
data:[
{ $match: condition },
{ $skip: (pageIndex - 1) * pageSize },
{ $limit: pageSize },
],
totalCount:[
{ $match: condition },
{ $count: 'totalCount' }
]
}
}]) ;
res.json(getAllProfile);
} catch (error) {
res.status(404).json(error);
}
}
);

The find function can only accept query and projection as arguments. You are not allowed to give a callback as an argument. then should be chained to the Promise returned by find.
Try like this
var condition = {'cash' : '5000'};
Profile.find(condition).then(profiles => {
res.json(profiles)
});
Profiles with pagination.
you can use the MongoDB aggregation framework for that (which will make it possible to get additional information like the total entries and count of the entries on the current page (the last page might have a lesser number than the page limit)). You need to send start and limit as the request body.
router.get(
"/",
passport.authenticate("jwt", { session: false }),
async (req, res) => {
// page index
let start = 0;
// profiles per page
let limit = 10;
if (req.body.start !== null || req.body.start !== "undefined") {
start = req.body.start;
}
if (req.body.limit !== null || req.body.limit !== "undefined") {
limit = req.body.limit;
}
let condition = { cash: "5000" };
try {
const aggregatedData = await Profile.aggregate([
{
$facet: {
data: [
{ $match: condition },
{
$project: {
type: 1,
description: 1,
cash: 1
}
},
{ $skip: start * limit },
{ $limit: limit }
],
metaData: [
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
]
}
}
]);
return res.status(200).json({
data: aggregatedData[0].data,
metaData: {
total: aggregatedData[0].metaData[0].count,
start: start
}
});
} catch (error) {
return res.status(500).json({
error: error
});
}
}
);

How to send complex object over IPC in electron?

I'm using ipcRenderer.send() to send an array of objects back to ipcMain. Here's my code:
const loadData = async () => {
let promises = [];
['stocks', 'crypto', 'vehicles', 'property'].forEach(item => {
promises.push(getTableData(item))
})
let data = {}
await Promise.allSettled(promises).then(results => {
for (let i in results) {
var result = results[i]
if (result.status === 'fulfilled') {
console.log(result.value.type)
// result.value.data will be an array of objects
console.log(result.value.data)
data[result.value.type] = result.value.data
} else {
console.error(result)
}
}
}).finally(_ => {
console.log(data)
ipcRenderer.send('asynchronous-message', data)
})
}
When result.value.data is printed via console.log, it shows the correct data (comes from an SQL query):
{ stocks: [{id: 1, ticker: "BRK.B", action: "ADD", price: 173.97, shares: 6}, ...], ...}
However, when it gets printed in ipcMain.on('asynchronous-message', ...), it prints empty arrays for the values:
{ stocks: [], crypto: [], vehicles: [], property: [] }
How would I send an IPC message with a complex object? Is it not being serialized correctly?
For reference, here is my ipcMain.on('asynchronous-message', ...) code:
ipcMain.on('asynchronous-message', async (event, data) => {
console.log(data)
})
In addition, here is getTableData():
const getTableData = (table) => {
let toReturn = {
type: `${table}`,
data: []
}
return new Promise((resolve, reject) => {
try {
db.run(`PRAGMA table_info('${table}');`, err => {
if (err) {
reject(err)
} else {
db.each(`SELECT * FROM ${table}`, (err, row) => {
if (err) {
reject(err)
} else {
toReturn.data.push(row)
}
})
resolve(toReturn)
}
})
} catch (e) {
reject(e)
}
})
}
Where each row is an object that looks like:
{id: 1, ticker: "AAPL", action: "ADD", price: 100.0,
shares: 10, datetime: "2020-05-14 23:24:50", platform: ""}

With #Estradiaz's help, I realized that the promise was being resolved before the SQL query had been processed. To fix this, I switched from using db.each() to db.all(). Here is my updated getTableData():
const getTableData = (table) => {
let toReturn = {
type: `${table}`,
data: []
}
return new Promise((resolve, reject) => {
try {
DB.run(`PRAGMA table_info('${table}');`, err => {
if (err) {
console.error(err)
reject(err)
} else {
DB.all(`SELECT * FROM ${table}`, (err, rows) => {
if (err) {
console.error(err)
reject(err)
} else {
toReturn.data = rows
resolve(toReturn)
}
})
}
})
} catch (e) {
console.error(e)
reject(e)
}
})
}