How to speed up Sequelize findAndCountAll? - javascript

I have pagination made with Sequelize
router.js
router.post('/getall', async (req, res) => {
try {
const { q, page, limit, order_by, order_direction } = req.query;
const { candidate, position, filters } = req.body
let include = [
{
model: SortedLvl,
where: {
lvl: filters.jobLvl.name,
months: { [Op.gte]: filters.jobMinExp.value },
},
},
{
model: SortedLastJob,
where: { jobposition: filters.jobType.name }
},
{
model: SortedSkills,
}
]
let search = {};
let order = [];
let filterCandidate = {}
if (candidate) {
if (candidate != undefined) {
t = candidate.split(/[ ,]+/)
let arr = new Array()
// console.log('t', t);
t.map((el, index) => {
console.log('el', el);
if (typeof el == 'number') {
arr.push({ first_name: { [Op.iLike]: `%` + `${el}` + `%` } }, { last_name: { [Op.iLike]: `%` + `${el}` + `%` } });
} else {
arr.push({ first_name: { [Op.iLike]: `%` + `%${el}%` + `%` } }, { last_name: { [Op.iLike]: `%` + `%${el}%` + `%` } });
}
});
filterCandidate = {
[Op.or]: arr
};
}
}
let filterPosition = {}
if (position) {
if (position != undefined) {
filterPosition = { position: { [Op.iLike]: `%${position}%` } }
}
}
if (filterCandidate.length > 0 || filterPosition.length > 0) {
search = { where: { ...(filterCandidate || []), ...(filterPosition || []) } }
}
console.log('search', search);
console.log('candidate', filterCandidate);
console.log('position', filterPosition);
if (order_by && order_direction) {
order.push([order_by, order_direction]);
}
const transform = (records) => {
return records.map(record => {
return {
id: record.id,
name: record.name,
date: moment(record.createdAt).format('D-M-Y H:mm A')
}
});
}
const products = await paginate(Candidate, page, limit, search, order, include);
return res.json({
success: true,
// message: '',
data: products
})
} catch (error) {
console.log('Failed to fetch products', error);
return res.status(500).send({
success: false,
message: 'Failed to fetch products'
})
}
});
paginate.js
const paginate = async (model, pageSize, pageLimit, search = {}, order = [], include, transform, attributes, settings) => {
try {
const limit = parseInt(pageLimit, 10) || 10;
const page = parseInt(pageSize, 10) || 1;
// create an options object
let options = {
offset: getOffset(page, limit),
limit: limit,
distinct: true,
include: include,
};
// check if the search object is empty
if (Object.keys(search).length) {
options = { ...options, ...search };
}
if (attributes && attributes.length) {
options['attributes'] = attributes;
}
if (order && order.length) {
options['order'] = order;
}
let data = await model.findAndCountAll(options);
if (transform && typeof transform === 'function') {
data = transform(data.rows);
}
return {
previousPage: getPreviousPage(page),
currentPage: page,
nextPage: getNextPage(page, limit, count),
total: count,
limit: limit,
data: data.rows
}
} catch (error) {
console.log(error);
}
}
const getOffset = (page, limit) => {
return (page * limit) - limit;
}
const getNextPage = (page, limit, total) => {
if ((total / limit) > page) {
return page + 1;
}
return null
}
const getPreviousPage = (page) => {
if (page <= 1) {
return null
}
return page - 1;
}
module.exports = paginate;
First problem - on queries with included models sometimes it response with wrong count
and also it is so slow
Please, help me with solving this issue
Thank you
I tried use separate: true, required: true and etc - I just get empty arrays on included

In your sequelize request there are lots of json query. Mysql cant create directly index your json fields. You can create a referance your query property like below and after that you can add a index for reference column.
And change your referance column as created reference. Also you can create mysql index another requests columns.
ALTER TABLE [table_name] ADD COLUMN email VARCHAR(255)
GENERATED ALWAYS as (properties->>"$.[jsonObjectName].[jsonProperty]");
ALTER TABLE [table_name] ADD INDEX email ([jsonProperty]) USING BTREE;

Related

Rxdb infinitely pulling in replicateRxCollection

I'm working with rxdb and I have pull and push handlers for the backend I have used supabase
I have setup the code for replication as follows:
replication.ts
import { RxDatabase } from "rxdb";
import { RxReplicationPullStreamItem } from "rxdb/dist/types/types";
import { replicateRxCollection } from "rxdb/plugins/replication";
import { Subject } from "rxjs";
import { supabaseClient, SUPABASE_URL } from "src/config/supabase";
import { DbTables } from "src/constants/db";
import {
blockPullHandler,
blockPushHandler,
} from "./repilicationhandlers/block";
import { CheckpointType, RxBlockDocument, RxBlocksCollections } from "./types";
export async function startReplication(
database: RxDatabase<RxBlocksCollections>
) {
const pullStream$ = new Subject<
RxReplicationPullStreamItem<RxBlockDocument, CheckpointType>
>();
supabaseClient
.from(DbTables.Block)
.on("*", (payload) => {
console.log("Change received!", payload);
const doc = payload.new;
pullStream$.next({
checkpoint: {
id: doc.id,
updated: doc.updated,
},
documents: [doc] as any,
});
})
.subscribe((status: string) => {
console.log("STATUS changed");
console.dir(status);
if (status === "SUBSCRIBED") {
pullStream$.next("RESYNC");
}
});
const replicationState = await replicateRxCollection({
collection: database.blocks,
replicationIdentifier: "supabase-replication-to-" + SUPABASE_URL,
deletedField: "archived",
pull: {
handler: blockPullHandler as any,
stream$: pullStream$.asObservable(),
batchSize: 10,
},
push: {
batchSize: 1,
handler: blockPushHandler as any,
},
});
replicationState.error$.subscribe((err) => {
console.error("## replicationState.error$:");
console.log(err);
});
return replicationState;
}
blockPullHandler:
export const blockPullHandler = async (
lastCheckpoint: any,
batchSize: number
) => {
const minTimestamp = lastCheckpoint ? lastCheckpoint.updated : 0;
console.log("Pulling data", batchSize, lastCheckpoint);
const { data, error } = await supabaseClient
.from(DbTables.Block)
.select()
.gt("updated", minTimestamp)
.order("updated", { ascending: true })
.limit(batchSize);
if (error) {
console.log(error);
throw error;
}
const docs: Array<Block> = data;
return {
documents: docs,
hasMoreDocuments: false,
checkpoint:
docs.length === 0
? lastCheckpoint
: {
id: lastOfArray(docs).id,
updated: lastOfArray(docs).updated,
},
};
};
blockPushHandler:
export const blockPushHandler = async (
rows: RxReplicationWriteToMasterRow<RxBlockDocumentType>[]
) => {
if (rows.length !== 1) {
throw new Error("# pushHandler(): too many push documents");
}
const row = rows[0];
const oldDoc: any = row.assumedMasterState;
const doc: Block = row.newDocumentState;
console.log(row, oldDoc, doc);
// insert
if (!row.assumedMasterState) {
const { error } = await supabaseClient.from(DbTables.Block).insert([doc]);
console.log("Error 1", error);
if (error) {
// we have an insert conflict
const conflictDocRes: any = await supabaseClient
.from(DbTables.Block)
.select()
.eq("id", doc.id)
.limit(1);
return [conflictDocRes.data[0]];
} else {
return [];
}
}
// update
console.log("pushHandler(): is update");
const { data, error } = await supabaseClient
.from(DbTables.Block)
.update(doc)
.match({
id: doc.id,
replicationRevision: oldDoc.replicationRevision,
});
console.log("Error 2", error);
if (error) {
console.log("pushHandler(): error:");
console.log(error);
console.log(data);
throw error;
}
console.log("update response:");
console.log(data);
if (data.length === 0) {
// we have an updated conflict
const conflictDocRes: any = await supabaseClient
.from(DbTables.Block)
.select()
.eq("id", doc.id)
.limit(1);
return [conflictDocRes.data[0]];
}
return [];
};
But the issue is when I start the application and the pull handler is called correctly but it doesn't stop calling the pull handler and it sends continuous request one after another even after it has fetched the documents even when I set hasMoreDocuments to false It keeps sending requests and running the replicator. Is there something wrong with my configuration?
database.ts:
export const createDatabase = async () => {
const database = await createRxDatabase({
name: "sundaedb",
storage: getRxStorageDexie(),
});
await database.addCollections({
blocks: {
schema: blockSchema as any,
conflictHandler: conflictHandler as any,
},
documents: {
schema: documentSchema as any,
conflictHandler: conflictHandler as any,
},
});
database.blocks.preInsert((docData) => {
docData.replicationRevision = createRevision(
database.hashFunction,
docData as any
);
return docData;
}, false);
database.blocks.preRemove((docData) => {
console.log(" PRE REMOVE !!");
console.log(JSON.stringify(docData, null, 4));
const oldRevHeight = parseRevision(docData.replicationRevision).height;
docData.replicationRevision =
oldRevHeight + 1 + "-" + database.hashFunction(JSON.stringify(docData));
console.log(JSON.stringify(docData, null, 4));
return docData;
}, false);
database.blocks.preSave((docData) => {
const oldRevHeight = parseRevision(docData.replicationRevision).height;
docData.replicationRevision =
oldRevHeight + 1 + "-" + database.hashFunction(JSON.stringify(docData));
return docData;
}, false);
return database;
};

I want to know why the log array doesn't return the specified user with the given id?

app.get("/api/users/:_id/logs", (req, res) => {
const id = req.params._id;
const { from, to, limit } = req.query;
** Here I tried to search for the matched user and it works successfully: **
User.findById({ _id: id }, (err, user) => {
if (!user || err) {
res.send("Unknown User Id !!");
} else {
**Then I tried to filter the log array with date **
// const username = user.username;
let responObject = {};
if (from) {
responObject["$gte"] = new Date(from).toDateString();
}
if (to) {
responObject["$lte"] = new Date(to).toDateString();
}
let filter = {
_id: id,
};
if (from || to) {
filter.date = responObject;
}
let nonNullLimit = limit ?? 500;
**try to build the array log and return it to the user but it always be empty and never return the exercises for the user **
Exercise.find(filter)
.limit(+nonNullLimit)
.exec((err, data) => {
if (err || !data) {
res.json([]);
} else {
const count = data.length;
const rowLog = data;
const { username, _id } = user;
const log = rowLog.map((item) => ({
description: item.description,
duration: item.duration,
date: new Date(item.date).toDateString(),
}));
console.log(log)
if (from && to) {
res.json({
username,
from: new Date(from).toDateString(),
to: new Date(to).toDateString(),
count,
_id,
log,
});
} else {
res.json({
username,
count,
_id,
log,
});
}
}
});
}
});
});
this is the result when I try to log all the exercises for the user
{"username":"ahmed","count":0,"_id":"62a9aab2743ddfc9df5165f2","log":[]}

Node.js how to use mongoose to do filter and pagination

I get all data successfully in my node.js code using Profile.find(),but when I want to add filter in api, it get 500 error.
Profile.js
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
//Create Schema
const ProfileSchema = new Schema({
type: {
type: String,
},
description: {
type: String,
},
cash:{
type: String,
required: true,
}
})
module.exports = Profile = mongoose.model("profile",ProfileSchema);
api.js
const router = express.Router();
const Profile = require('../../models/Profile');
router.get("/",passport.authenticate('jwt',{session:false}),(req, res) => {
Profile.find().then(profiles => {
res.json(profiles)
})//works
})
router.get("/",passport.authenticate('jwt',{session:false}),(req, res) => {
var condition = {'cash' : '5000'};
Profile.find(condition,function(err,res){//error for this
res.json(res)
})
})
I am new to mongoose and do not sure how to do the filter and pagination for my code.
Appreciate for any kind help.
Update
Thanks to #Amila Senadheera, I finally find the solution to do filter and pagination using below code:
var condition = { cash: '5000' };
Profile.find(condition).skip(1).limit(10).then(profiles => {
res.json(profiles)
})
To use aggregation, it works like below:
router.post(
"/",
passport.authenticate("jwt", { session: false }),
async (req, res) => {
// page index
let pageIndex = 1;
// profiles per page
let pageSize = 10;
if (req.body.pageIndex !== null || req.body.pageIndex !== "undefined") {
pageIndex = req.body.pageIndex;
}
if (req.body.pageSize !== null || req.body.pageSize !== "undefined") {
pageSize = req.body.pageSize;
}
let condition = { description: "1" };
try {
const getAllProfile = await Profile.aggregate([{
$facet: {
data:[
{ $match: condition },
{ $skip: (pageIndex - 1) * pageSize },
{ $limit: pageSize },
],
totalCount:[
{ $match: condition },
{ $count: 'totalCount' }
]
}
}]) ;
res.json(getAllProfile);
} catch (error) {
res.status(404).json(error);
}
}
);
The find function can only accept query and projection as arguments. You are not allowed to give a callback as an argument. then should be chained to the Promise returned by find.
Try like this
var condition = {'cash' : '5000'};
Profile.find(condition).then(profiles => {
res.json(profiles)
});
Profiles with pagination.
you can use the MongoDB aggregation framework for that (which will make it possible to get additional information like the total entries and count of the entries on the current page (the last page might have a lesser number than the page limit)). You need to send start and limit as the request body.
router.get(
"/",
passport.authenticate("jwt", { session: false }),
async (req, res) => {
// page index
let start = 0;
// profiles per page
let limit = 10;
if (req.body.start !== null || req.body.start !== "undefined") {
start = req.body.start;
}
if (req.body.limit !== null || req.body.limit !== "undefined") {
limit = req.body.limit;
}
let condition = { cash: "5000" };
try {
const aggregatedData = await Profile.aggregate([
{
$facet: {
data: [
{ $match: condition },
{
$project: {
type: 1,
description: 1,
cash: 1
}
},
{ $skip: start * limit },
{ $limit: limit }
],
metaData: [
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
]
}
}
]);
return res.status(200).json({
data: aggregatedData[0].data,
metaData: {
total: aggregatedData[0].metaData[0].count,
start: start
}
});
} catch (error) {
return res.status(500).json({
error: error
});
}
}
);

Why doesn't array.map change anything?

I'm trying to format the data i got from YouTube Data API v3 but i'm unable to change anything of it.
const videoIds = youtubeResponse.items.map(item => item.id);
VideoRepo.getById(videoIds, (err, videos) => {
/*
videos is an array of objects that contain youtube videos from YT API and MongoDB(mongoose)
*/
console.log(videos.map((v) => {
v.time = moment(v.time).fromNow();
v.duration = moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss');
return v;
}));
});
VideoRepo class:
static getById(id, callback) {
if (Array.isArray(id)) {
// Multiple ids were specified
async.waterfall([
(next) => {
// Get existing videos' data
Video.find({ _id: { $in: id } }).select('-__v').sort({ createdAt: 1 }).exec((err, data) => {
if (err) return next(err);
next(null, data);
});
},
(existingData, next) => {
if (existingData.length === 0) {
// All videos are new, skip to the next step
return next(null, [], id);
}
// Remove existing data from ID array
const obj = existingData.map(el => el._id);
next(null, existingData, id.filter(el => !obj.includes(el)));
},
(existingData, newIDs, next) => {
if (newIDs.length === 0) {
return next(null, existingData);
}
// Get new videos' data from YT API
youtube.videos.list({ id: newIDs.join(','), part: 'snippet,contentDetails,statistics' }, (err, videoResp) => {
if (err) return next(err);
// Final data
const data = id;
// New videos' data
const newData = videoResp.data.items.map(item => this.fixVideoData(item));
// Add new videos to the DB
Video.insertMany(newData, (err) => {
if (err) return next(err);
// Merge new data with existing data
const merged = existingData.concat(newData);
// Fix order
for (let i = 0; i < merged.length; i += 1) {
const d = merged[i];
data[data.indexOf(d._id)] = d;
}
// Success!
next(null, data);
});
});
},
], (err, data) => callback(err, data));
}
}
static fixVideoData(videoData) {
const data = {
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
slug: slugify(videoData.snippet.title, { lower: true }),
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
const possibleThumbs = ['maxres', 'standard', 'high', 'medium', 'default'];
for (let j = 0; j < possibleThumbs.length; j += 1) {
if (Object.prototype.hasOwnProperty.call(videoData.snippet.thumbnails, possibleThumbs[j])) {
data.thumbnail = videoData.snippet.thumbnails[possibleThumbs[j]].url;
break;
}
}
if (videoData.snippet.thumbnails.medium) {
data.preThumbnail = videoData.snippet.thumbnails.medium.url;
} else if (videoData.snippet.thumbnails.high) {
data.preThumbnail = videoData.snippet.thumbnails.high.url;
} else {
data.preThumbnail = data.thumbnail;
}
return data;
}
This is what videos array contains:
// videoData: https://developers.google.com/youtube/v3/docs/videos#resource
{
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
Expected results:
[...{ [..other keys] duration: "00:05:43", time: "3 days ago" }]
Actual output (nothing is changed, exactly the same array as videos):
[...{ [..other keys] duration: 343, time: 2018-12-26T13:37:32.000Z }]
Why is it not working and how can i fix it?
You can return a new object where you override only those 2 specific keys,
video.map(v => ({
...v,
duration: moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss'),
time: moment(v.time).fromNow()
}))
So apparently Model.find() returns mongoose documents instead of javascript objects and i should have used Query.lean().
Video.find({ _id: { $in: id } }).lean().select('-__v').sort({ createdAt: 1 })
.exec()

Ionic 3 - get data in a variable with this (\"9876543210\", \"9876543211\") format

I want to retrieve all user's contact numbers in a variable with this format: \"9876543210\", \"9876543211\", \"9876543211\",...
I have loaded all the contact numbers from database,
Here is my code:
loadContacts()
{
let contacts: any = [];
firebase.database().ref('users').orderByKey().once('value', (items: any) => {
//console.log(items);
items.forEach((item) => {
if(item.val().contact_no != '0')
{
contacts.push({
contactNo: item.val().contact_no
});
}
this.contactList = contacts;
console.log("Contacts: ",this.contactList);
});
},
(error) => {
console.log("Error: ", error);
});
}
loadContacts() retrieves contacts in this format:
I want all the contact numbers in this format: \"9876543210\", \"9876543211\", \"9876543211\",...
and store it in a variable.
Thanks in advance.
You can convert the result to string
if(item.val().contact_no != '0')
{
let editedContact:string = '\"'+item.val().contact_no+'\"';
contacts.push({
contactNo: editedContact
});
}
console.log will skip the \, but you can get it in the template
<pre>{{contactList|json}}</pre>
loadContacts()
{
let contacts: any = [];
firebase.database().ref('users').orderByKey().once('value', (items: any) => {
//console.log(items);
items.forEach((item) => {
if(item.val().contact_no != '0')
{
contacts.push(item.val().contact_no);
}
this.contactList = contacts;
//console.log("Contacts: ",this.contactList);
});
var length = this.contactList.length;
console.log("Length: ",length);
if(length > 0)
{
for(var i = 0; i <= length; i++)
{
if(i > 0)
{
this.contacts += '\\\"' + this.contactList[i] + '\\\",' ;
}
}
var contact = this.contacts;
this.contacts = contact.slice(9,-16);
console.log("Formatted Contacts: ",this.contacts);
}
},
(error) => {
console.log("Error: ", error);
});
}

Categories