We Keep Coding

Node.JS wait for each parallel mysql query to finish

I need assistance with the following: I have 4 MySQL queries which I wish to run in parallel but it is essential that all queries are completed before continuing in the main function; The four queries get different parts of the data (they are structured differently and access different tables) that is then used in the main function. Namely, I used async.parallel (even tried adding the await keyword) but still ended up continuing before the final callback is called in async.parallel. I even considered cluster and worker threads, but each query function is too different from one another.
The functions in async.parallel are MySQL connection queries in the form of:
connection.query(q, function(err, res) {
if(err) {
callback(err, [])
} else {
// do something
callback(null, res)
}
})
Each function is shaped differently and this is only a reduced form for the sake of brevity. Example of the function:
async main_function() {
await async.parallel({
"1": function(callback) { conn.query(q1, function(...) { console.log("1"); callback(null, 1) }) },
"2": function(callback) { conn.query(q2, function(...) { console.log("2"); callback(null, 2) }) },
"3": function(callback) { conn.query(q3, function(...) { console.log("3"); callback(null, 3) }) },
"4:" function(callback) { conn.query(q4, function(...) { console.log("4"); callback(null, 4) }) }
}, function(err, results) { /*final callback; do something*/ console.log("Finished"); })
console.log("Continuing");
//continue
}
Expected output:
3
2
4
1
Finished
Continuing
Actual output:
Continuing
3
2
4
1
Finished
If there's an entire other method (even without async.parallel), I'm open to it. I will likely have multiple distinct parallel MySQL queries later in the function and in other parts of the program, so having a parametrized form such as with async.parallel is extremely useful.
Javascript being single-threaded means that solutions like 'a while loop that checks whether the queries are finished' do not work. This main_function will be called very often so forking will cause issues.
EDIT: The Jakub's solution is rather simple as in my program works as the following:
async main_function() {
await Promise.all([
new Promise((resolve, reject) => {
connection.query(q1 , function (error, result) {
if (error != null) {
console.log("mysql query error", error);
return reject(error);
} else {
console.log("1");
resolve(result);
}
})
}),
new Promise((resolve, reject) => {
connection.query(q1 , function (error, result) {
if (error != null) {
console.log("mysql query error", error);
return reject(error);
} else {
console.log("2");
resolve(result);
}
})
}), ... // Two more times
]).then((values) => {
data[0] = values[0];
data[1] = values[1];
data[2] = values[2];
data[3] = values[3];
}).catch(function (err) { console.log(err); });
console.log("Continuing"); //continue
}
This is very crude right now, and if anybody wishes to suggest cleaner/better solutions for future programmers, go ahead.

These query functions are callback based (judging from your code), you can promisify them:
import { promisify } from 'utils';
async main_function() {
const res = await Promise.all([
promisify(conn.query)(q1),
promisify(conn.query)(q2),
// all your other queries
]);
console.log("Continuing");
//continue
}

How to insert http post data before the callback?

I built a API gateway,Function compute and Tablestore(Nosql) using Alibaba cloud to receive http post data and insert it into the a Nosql database.
While testing my code, the insert function did not invoke before the callback response. I know I should use Async/Await, but just don't fully understand how to use it.
I've tried .promise() the insert function did invoke but the callback response of the http did not invoke.
How can I put the data into my database and also gets the callback of function handler?
Some of my code:
var TableStore=require('tablestore');
async function handler(event, context, callback) {
var putReq = {
tableName: 'wifi',
condition: new TableStore.Condition(TableStore.RowExistenceExpectation.IGNORE, null),
primaryKey: [{ 'date':'123' }],
attributeColumns: [
{'col1': JSON.stringify(data)}, // string column
{'col2': 100}, // integer column
],
};
await client.putRow(putReq,function(err, data) {
if (err) return callback(err);
console.log('putRow suceess: %j', data);
callback(null, data);
});
const response = Object.assign(baseresponse, {
body: {
status: "ok",
body:event.body
}
});
callback(null, response);
}
module.exports.handler = handler;

How to run request sequentially in a loop?

This is a pseudo code of what I am trying to achieve. First I need to get a list of URLs from the request body then pass those URLs to request function (using request module) which will get the data from each url and then save those data to MongoDB. After all the requests are finished including saving data to the server only then it should send a response.
app.post('/', (req, resp) => {
const { urls } = req.body;
urls.forEach((url, i) => {
request(url, function (err, resp, body) {
if (err) {
console.log('Error: ', err)
} else {
// function to save data to MongoDB server
saveUrlData(body);
console.log(`Data saved for URL number - ${i+1}`)
}
})
});
// Should be called after all data saved from for loop
resp.send('All data saved')
})
I have tried this code and of course the resp.send() function will run without caring if the request has completed. Using this code I get a result on the console like this:
Data saved for URL number - 3
Data saved for URL number - 1
Data saved for URL number - 5
Data saved for URL number - 2
Data saved for URL number - 4
I could write them in nested form but the variable urlscan have any number of urls and that's why it needs to be in the loop at least from my understanding. I want the requests to run sequentially i.e. it should resolve 1st url and then second and so on and when all urls are done only then it should respond. Please help!

app.post('/', async (req, resp) => {
const {
urls
} = req.body;
for (const url of urls) {
try {
const result = await doRequest(url)
console.log(result)
} catch (error) {
// do error processing here
console.log('Error: ', err)
}
}
})
function doRequest(url) {
return new Promise((resolve, reject) => {
request(url, function(err, resp, body) {
err ? reject(err) ? resolve(body)
})
})
}
using async await

You should look at JavaScript Promises
Otherwise, you can do a recursive request like so:
app.post('/', (req, resp) => {
const { urls } = req.body;
sendRequest(urls, 0);
})
function sendRequest(urlArr, i){
request(urlArr[i], function (err, resp, body) {
if (err) {
console.log('Error: ', err)
}
else {
saveUrlData(body);
console.log(`Data saved for URL number - ${i+1}`)
}
i++;
if(i == urlArr.length) resp.send('All data saved') //finish
else sendRequest(urlArr, i); //send another request
})
}
All I had to do is create a separate function I can call over and over again, passing the url array and a base index 0 as arguments. Each success callback increments the index variable which I pass in the same function again. Rinse and repeat until my index hits the length of the url array, I'll stop the recursive loop from there.

You want to wait till all api response you get and stored in db, so you should do async-await and promisify all the response.
You can use Request-Promise module instead of request. So you will get promise on every requested api call instead of callback.
And use promise.all for pushing up all request(module) call inside array.
Using async-await you code execution will wait till all api call get response and stored in db.
const rp = require('request-promise');
app.post('/', async (req, res) => {
try{
const { urls } = req.body;
// completed all will have all the api resonse.
const completedAll = await sendRequest(urls);
// now we have all api response that needs to be saved
// completedAll is array
const saved = await saveAllData(completedAll);
// Should be called after all data saved from for loop
res.status(200).send('All data saved')
}
catch(err) {
res.status(500).send({msg: Internal_server_error})
}
})
function sendRequest(urlArr, i){
const apiCalls = [];
for(let i=0;i < urlArr.length; i++){
apiCalls.push(rp(urlArr[i]));
}
// promise.all will give all api response in order as we pushed api call
return Promise.all(apiCalls);
}
You can refer these links:
https://www.npmjs.com/package/request-promise
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all

Looking at the intention(a crawler) you can use Promise.all because the urls are not dependant upon each other.
app.post('/', (req, resp) => {
const { urls } = req.body;
const promises = urls.map((url, i) => {
return new Promise((resolve, rej)=>{
request(url, function (err, resp, body) {
if (err) {
rej(err);
} else {
resolve(body);
}
})
})
.then((body)=>{
//this should definitely be a promise as you are saving data to mongo
return saveUrlData(body);
})
});
// Should be called after all data saved from for loop
Promise.all(promises).then(()=>resp.send('All data saved'));
})
Note: Need to do error handling as well.

there are multiple ways to solve this.
you can use async/await
Promises
you can also use the async library
app.post('/', (req, res, next) => {
const { urls } = req.body;
async.each(urls, get_n_save, err => {
if (err) return next(err);
res.send('All data saved');
});
function get_n_save (url, callback) {
request(url, (err, resp, body) => {
if (err) {
return callback(err);
}
saveUrlData(body);
callback();
});
}
});

async waterfall not following order when mysql query

I am trying to do a waterfall async but i don't get the expected output that i want.
Basically my waterfall works as expected if i use an array instead of the query
so i guess i am doing something wrong on the callback of the query but i don't know what.
Code when it works with what i expect using array:
function range(start, end) {
var foo = [];
for (var i = start; i <= end; i++) {
foo.push(i);
}
return foo;
}
users = range(1,2)
obj = [1,2];
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
results = {sku_config:'A',img:'http//blabla',sku_config:'B',img:'http//bloblo'}
callback(null, results);
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
request
loop
request
Finish
loop
request
loop
request
Finish
User For Loop Completed
But when i try to query the data with mysql here comes the problem
code:
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, callback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
});
callback(null);
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
}
);
}, function(err){
console.log("User For Loop Completed");
});
output:
loop
loop
Finish
loop
loop
Finish
User For Loop Completed
request
request
request
request
All the request gets executed at the end :(
If you have idea on what i could fix.
Thanks

The first problem you have is that your callbacks have the exact same name, this could cause major problems. The callbacks you are meaning to call can not be differentiated, which could cause your program to execute pieces of code that shouldn't be executed until later.
The second problem is that the callback is placed outside of the request.get function. The nature of node js means that it does not wait until the request.get function returns and instead just calls the callback straight away. By placing the callback inside of the request.get function it is forced to wait until the request function returns and then the callback is called. A revised version of your code is below.
async.forEachLimit(users, 1, function(user, userCallback){
async.waterfall(
[
function(callback) { // query the data to get the category and specific number of rows
connection.query(query_sku,
['Fashion',1,2],
function(err, results, fields) {
if (err)
throw err;
callback(null, results);
});
},
function(obj,callback) {
async.eachSeries(obj, function (sku, seriesCallback) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
seriesCallback(null);
});
}, function(responsetoendofloop){
callback(null);
});
},
],
function (err) {
console.log('Finish');
userCallback(null);
});
}, function(err){
console.log("User For Loop Completed");
});

Your callback(null); inside async.eachSeries are after request.
To fix just put inside request like this.
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
callback(null);
});
Plus to be clear what you actually calling rename callback functions. For example callback inside eachSeries call next
function(obj,callback) {
async.eachSeries(obj, function (sku, next) {
var url = sku.img;
var sku = sku.sku_config;
console.log("loop");
request.get(url, {encoding: null} , function(error, response, body) {
console.log('request');
next(null);
});
}, function(responsetoendofloop){
callback(null);
});
}
Hope this helps.

Understanding Node.JS async.parallel

I need to request data from two web servers. The tasks are independent; therefore, I am using aync.parallel. Now I am only writing 'abc', 'xyz', and 'Done' to the body of my web page.
Since tasks are performed at the same time, can I run into a strange output? E.g.,
xab
cyz
The code.
var async = require('async');
function onRequest(req, res) {
res.writeHead(200, {
"Content-Type" : "text/plain"
});
async.parallel([ function(callback) {
res.write('a');
res.write('b');
res.write('c\n');
callback();
}, function(callback) {
res.write('x');
res.write('y');
res.write('z\n');
callback();
} ], function done(err, results) {
if (err) {
throw err;
}
res.end("\nDone!");
});
}
var server = require('http').createServer(onRequest);
server.listen(9000);

If you want to be absolutely certain in the order in which the results are printed, you should pass your data (abc\n and xyz\n) through the callbacks (first parameter is the error) and handle/write them in the final async.parallel callback's results argument.
async.parallel({
one: function(callback) {
callback(null, 'abc\n');
},
two: function(callback) {
callback(null, 'xyz\n');
}
}, function(err, results) {
// results now equals to: results.one: 'abc\n', results.two: 'xyz\n'
});