We Keep Coding

Why Api response is slow when using Async await with mongodb

When i am using async and await in nodejs its response is coming slow PRODUCT SCHEMA
const mongoose = require('mongoose');
const productSchema = mongoose.Schema(
{
_id: mongoose.Schema.Types.ObjectId,
name: {type: String, require: true},
category_fields: { type:Object },
name_list_fields: [{ type:mongoose.Schema.Types.ObjectId, ref:'list' }],
full_name: { type:String },
actual_price: { type: Number, default: 0 },
selling_price: { type: Number, default: 0 },
product_image_url_1: { type: String, default: "" },
sku: { type: String, default: "" },
brand:{type:mongoose.Schema.Types.ObjectId, ref:'brand' },
linked_offers: [{ type:mongoose.Schema.Types.ObjectId, ref:'offer' }],
hot_deal: { type:Boolean,default:false}, });
CODE
return new Promise(function(resolve, reject){
productschema.find({delete_status: { $ne: 1}})
.sort({_id:-1})
.populate('brand similar_product linked_offers','name actual_price product_image_url_1 actual_price selling_price full_name offer_image_url offer_type offer_amount_percentage description')
.skip( parseInt(req.params.index) )
.limit( parseInt(req.params.limit) )
.then(async productsrows => {
if(productsrows.length == 0){
resolve({
"success":false,
"message":"No Data"
})
}else{
try{
var arrayListOfProduct = [];
for(var i =0;i<productsrows.length;i++){
var item = productsrows[i].toObject()
item.in_stock = await commonFunctionAdmin.fetchProductCountInStock(productsrows[i]._id)
arrayListOfProduct.push(item)
}
resolve({
"success":true,
"message":"Products fetched success",
"count":await fetchTotalProductCount({delete_status: { $ne: 1}}),
"data":arrayListOfProduct
});
}catch(e){
console.log(e)
resolve({
"success":false,
"message":"something went wrong"
})
}
}
})
}) //STOCK COUNT
FUNCTION
return new Promise(function(resolve, reject){
stockSchema.aggregate([
{$unwind: "$product"},
{$unwind: "$product.imei"},
{$match: {"product.imei.sold_status":false,"product.product":ObjectId(product_id)}},
{$group: {
_id: null,
count: { $sum: 1 }
}
}]).then(async rowsTotalRevenue => {
if(rowsTotalRevenue.length > 0){
resolve(rowsTotalRevenue[0].count)
}else{
resolve(0)
}
}).catch(e=>{
console.log(e)
resolve(0)
})
});

Usually when you use await keyword, you save something like 200ms from each request (my experience).
To understand what is happening in your application, you can put a timer in every essential step of your function, measure the time difference from start to finish. This is very simple to do, just check what hours the code started to be run.
Async/await makes asynchronous code look and behave a little more like synchronous code. This is where all its power lies.
I tried to understand the code posted above and saw that you have some iterations in the array, know that all of this might be a bottleneck for your answer.
async function myCoolFunction () {
var initial = new Date().getTime();
var finalTime;
// create a new promise inside of the async function
let promise = new Promise((resolve, reject) => {
setTimeout(() => resolve(true), 1000) // resolve
});
// wait for the promise to resolve
let result = await promise;
finalTime = newDate().getTime();
}
myCoolFunction();
}

Request Promise not returning a value (Node JS)

I have been trying to get this to work for while now, my request promise is not returning the streamer it has found. When i console.log(streamer) inside of the .then part of the request it works. Any ideas?
function getStreamerByName(name){
var streamer;
var options = {
url: "https://api.twitch.tv/helix/users?login=" + name,
method: 'GET',
headers: {
'Client-ID': 'CLIENT_ID',
}
};
requestP(options)
//SPREAD
.spread(function(res, body) {
streamer_data = JSON.parse(body);
})
//THEN
.then(function (body) {
streamer = ({
twitch_id: streamer_data["data"][0]["id"],
name: streamer_data["data"][0]["display_name"],
image: streamer_data["data"][0]["profile_image_url"],
description: streamer_data["data"][0]["description"]
});
Streamer.create(streamer);
return streamer;
})
//CATCH
.catch(function (err) {
console.log(err);
return streamer;
});
}

You need to return the promise chain:
function getStreamerByName(name){
var streamer;
var options = {
url: "https://api.twitch.tv/helix/users?login=" + name,
method: 'GET',
headers: {
'Client-ID': 'CLIENT_ID',
}
};
//A return was added here
return requestP(options)
//SPREAD
.spread(function(res, body) {
streamer_data = JSON.parse(body);
})
//THEN
.then(function (body) {
streamer = ({
twitch_id: streamer_data["data"][0]["id"],
name: streamer_data["data"][0]["display_name"],
image: streamer_data["data"][0]["profile_image_url"],
description: streamer_data["data"][0]["description"]
});
Streamer.create(streamer);
return streamer;
})
//CATCH
.catch(function (err) {
console.log(err);
return streamer;
});
}
When you all this function externally, you'll also need to use a .then() to get the result or use an async function and await.
so
async caller() {
var value = await getStreamerByName('stuff')
}
or
caller() {
getStreamerByName('stuff').then((result) => { //do stuff })
}

I'm not sure if this is what you mean but i just tried this and its not getting the streamer still, its coming up with an error message saying:
TypeError: Cannot read property 'then' of undefined
function getStreamers() {
getStreamerByName(search).then((streamer) => {
console.log(streamer)
})
}
getStreamers();

async function getStreamerByName (name) {
return new Promise(function (resolve, reject) {
Your code
});
}
Var myStreamer = getStreamerByName ('myStreamer');

Callback NodeJS & insert data with mongoose

I know this topic as already asked many times before but I didn't find the right answer to do what I want.
Actually, I try to save two different list of JSON object in MongoDB via Mongoose. To perform both at the same time I use 'async'.
However, when I save it with the command insertMany() I get an error because he calls the callback of async before finishing the insertMany(). Therefore answer[0] is not defined.
What will be the proper way of doing it ?
Here is my code with the async:
const mongoose = require("mongoose");
const async = require("async");
const utils = require("../utils");
const experimentCreate = function(req, res) {
let resData = {};
let experimentList = req.body.experiment;
let datasetList = req.body.datasetList;
async.parallel(
{
dataset: function(callback) {
setTimeout(function() {
answer = utils.createDataset(datasetList);
callback(answer[0], answer[1]);
}, 100);
},
experiment: function(callback) {
setTimeout(function() {
answer = utils.createExp(experimentList);
callback(answer[0], answer[1]);
}, 100);
}
},
function(err, result) {
if (err) {
console.log("Error dataset or metadata creation: " + err);
sendJSONresponse(res, 404, err);
} else {
console.log("Experiment created.");
resData.push(result.dataset);
resData.push(result.experiment);
console.log(resData);
sendJSONresponse(res, 200, resData);
}
}
);
};
Then the two functions called createExp and createDataset are the same in another file. Like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
return [err, null];
} else {
console.log("All dataset created.");
return [null, ds];
}
});
};

There's a few problems with your code. For one, you're not returning anything in your createDataset function. You're returning a value in the callback of insertMany but it doesn't return that value to the caller of createDataset as it's within another scope. To solve this issue, you can wrap your Dataset.insertMany in a promise, and resolve or reject depending on the result of Data.insertMany like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
return new Promise((resolve, reject) => {
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
reject(err);
} else {
console.log("All dataset created.");
resolve(ds);
}
});
});
};
Now your return object is no longer going to be an array so you won't be able to access both the error and the result via answer[0] and answer[1]. You're going to need to chain a then call after you call createDataset and use callback(null, answer) in the then call (as that means createDataset executed successfully) or use callback(err) if createDataset throws an error like below:
dataset: function(callback) {
setTimeout(function() {
utils.createDataset(datasetList).then(answer => {
callback(null, answer);
}).catch(err => callback(err)); // handle error here);
}, 100);
}
Note: You'll most likely need to alter your createExp code to be structurally similar to what I've produced above if it's also utilizing asynchronous functions.

Node.js looping async calls

I'm busy working on an endpoint for a reporting system. Node being async is giving me issues, although I'd rather not force it to be synchronous.
We're using MongoDB and Mongoose. I've got to query regex over collection A, then for each document that gets returned, query multiple contained documents to populate a JSON object/array to be returned.
I can use populate for most of the data, except the final looped queries which is where the async kicks in and returns my report early. Is there an elegant way to do this? Or should I be splitting into a different function and calling that multiple times to stick to the functions should do only one thing rule?
Example Code:
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
A.map(function(a)){
report[a.name] = [];
D.aggregate([
{
$match: {
id: B._id
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
], function(err, result) {
C.map(function(c){
report[a.name].push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
});
}
return report;
});
The issue here is with the logic / async. Not with the syntax, hence the semi-pseudo code.
Any help or advice would be greatly appreciated.

You need to familiarize yourself with promises, and with async in general.
Because you are returning an array, that's the value you are going to get.
You have a few options when dealing with Async, but in your case, you want to look at two solutions:
// callbacks
getSetOfIDs((err, ids) => {
let remaining = ids.length;
let things = [];
let failed = false;
ids.forEach(id => {
getThingByID(id, (err, thing) => {
if (failed) { return; }
if (err) {
failed = true;
handleFailure(err);
} else {
remaining -= 1;
things.push(thing);
if (!remaining) {
handleSuccess(things);
}
}
});
});
});
Note, I'm not returning things, I'm passing it into a callback.
You can use higher-order functions to clean this sort of thing up.
// cleaned up callbacks
function handleNodeCallback (succeed, fail) {
return function (err, data) {
if (err) {
fail(err);
} else {
succeed(data);
}
};
}
function handleAggregateCallback (succeed, fail, count) {
let items = [];
let failed = false;
const ifNotFailed = cb => data => {
if (!failed) { cb(data); }
};
const handleSuccess = ifNotFailed((item) => {
items.push(item);
if (items.length === count) { succeed(items); }
});
const handleFailure = ifNotFailed((err) => {
failed = true;
fail(err);
});
return handleNodeCallback(handleSuccess, handleFailure);
}
A little helper code later, and we're ready to go:
// refactored callback app code (note that it's much less scary)
getSetOfIDs((err, ids) => {
const succeed = (things) => app.display(things);
const fail = err => app.apologize(err);
if (err) { return fail(err); }
let onThingResponse = handleAggregateCallback(succeed, fail, ids.length);
ids.forEach(id => getThingByID(id, onThingResponse));
});
Note that aside from higher-order functions, I'm never returning anything, I'm always passing continuations (things to do next, with a value).
The other method is Promises
// Promises
getSetOfIDs()
.then(ids => Promise.all(ids.map(getThingByID)))
.then(things => app.display(things))
.catch(err => app.apologize(err));
To really get what's going on here, learn Promises, the Promise.all static method, and array.map().
Both of these sets of code theoretically do the exact same thing, except that in this last case getSetOfIDs and getThingByID don't take callbacks, they return promises instead.

usually in async calls, after return statement any operations are cancelled.
maybe you can return report object only when all is done and well.
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
A.map(function(a)){
report[a.name] = D.aggregate([
{
$match: {
id: B._id
}
},
{
$group: {
_id: null,
count: { $sum: 1 }
}
}
], function(err, result) {
if(err){
return [];
}
var fields = []
C.map(function(c){
fields.push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
return fields;
});
}
return report;
});

Just use promises:
A.find({ name: regex }).populate({ path: 'B', populate: { path: 'B.C', model: 'C' } }).exec(function(err, A) {
var report = [];
return Promise.all([
A.map(function(a)){
return new Promise(function(resolve, reject) {
report[a.name] = [];
D.aggregate([{ $match: { id: B._id }},{$group: {_id: null,count: { $sum: 1 }}}],
function(err, result) {
if(err) {
reject(err)
} else {
C.map(function(c){
report[a.name].push({
'field1': c.field1,
'field2': c.field2,
'field3': c.field3,
'count': result.count
});
});
resolve(report)
}
});
}
})])
})
.then(function(report){
console.log(report)
})
.catch(function(err){
console.log(err)
})

How wait several requests are resolved to send an response in javascript?

I need to do the the same request to external api with differents parameter (node.js and express.js)
If all answers to my requests are correct, then I send the angular client code 200, however, if any erroneous send him the error.
I try something like that but don't work:
for(var i = 0; i<array.length; i++){
github.repos.createHook({
user: user,
repo: array[i],
name: "web",
config: {
url: "http://9ec2067d.ngrok.io/api/v1/callback",
content_type: "json"
},
events: ["*"],
active:true,
headers: {
"X-GitHub-OTP": "two-factor-code"
}
}, function(err, res) {
if (err) {
console.log(err);
errGithub = true;
response.status(404).json(err);
}
else {
if(i==array.length && !errGithub){
response.json({message:"OK"});
}
}
Any idea?
Many thanks

try this out:
// Include the async package
// Make sure you add "async" to your package.json
async = require("async");
// 1st para in async.each() is the array of items
async.each(array,
// 2nd param is the function that each item is passed to
function(item, callback){
// Call an asynchronous function,
github.repos.createHook({
user: user,
repo: item,
name: "web",
config: {
url: "http://9ec2067d.ngrok.io/api/v1/callback",
content_type: "json"
},
events: ["*"],
active:true,
headers: {
"X-GitHub-OTP": "two-factor-code"
}
}, function(err, res) {
if (err) {
console.log(err);
errGithub = true;
}
callback(); //required
});
},
// 3rd param is the function to call when everything's done
function(err){
if(err){
console.log('Error:' + err);
}
// All tasks are done now
if(errGithub ===true){
response.status(404).json(err);
}else{
response.json({message:"OK"});
}
}
);

You can use Promises, it is a clean and easy way to do it.
First create a queue with promises:
var queue = array.map((item) => {
return new Promise((resolve, reject) => {
github.repos.createHook({
user: user,
repo: item,
name: "web",
config: {
url: "http://9ec2067d.ngrok.io/api/v1/callback",
content_type: "json"
},
events: ["*"],
active:true,
headers: {
"X-GitHub-OTP": "two-factor-code"
}
}, (err, res) => {
if (err) {
reject(err)
console.log(err)
} else {
resolve(res);
}
})
})
})
Then execute it:
Promise.all(queue).then((values) => {
response.json({message:"OK"})
},(err) => {
response.status(404).json(err)
});
This example just explains how you could use promises, the code has not been tested.
check also the documentation on: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all