We Keep Coding

Node js async convert to sync

I need to push data to array synchronously. First API request get image key base one that need to get image data within loop.
var deasync = require('deasync');
router.get('/a', function(req, res) {
var username="user";
var passw ="pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
//this is first api request
client.get(global.apiUrl+"V1/ProductItem", args,
function (data, response) {
//this is second api request
data.forEach(function(img) {client.get(global.apiUrl+"V1/ImagePreview/"+img.AvatarKey,args,
function (data2, response){
img['data']=data2.Image;
deasync(pushData(img));
});
});
});
function pushData(img){
op.push(img);//array push
}
res.render('test1', { "out":JSON.stringify(op) });
});

As much as I think deasync is a poor choice of solving your particular issue, the key to using it, is to "deasync" asynchronous functions. As Array.push is synchronous, deasync'ing Array.push makes no sense
having read the documentation for deasync, it's fairly simple to use
var deasync = require('deasync');
// create a sync client.get
function syncClientGet(client, url, args) {
var inflight = true;
var ret;
client.get(url, args, function(data, response) {
// as your original code ignores response, ignore it here as well
ret = data;
inflight = false;
});
deasync.loopWhile(() => inflight);
return ret;
}
router.get('/a', function(req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
let data = syncClientGet(client, global.apiUrl + "V1/ProductItem", args);
data.forEach(function(img) {
let data2 = syncClientGet(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args);
img['data'] = data2.Image;
op.push(img);
});
res.render('test1', {
"out": JSON.stringify(op)
});
});
However, embracing asynchronicity, the code you posted could be easily written as
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var op = [];
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
client.get(global.apiUrl + "V1/ProductItem", args, function (data, response) {
data.forEach(function (img) {
client.get(global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args, function (data2, response) {
img['data'] = data2.Image;
op.push(img);
if (img.length == data.length) {
res.render('test1', {
"out": JSON.stringify(op)
});
}
});
});
});
});
or, using Promises
router.get('/a', function (req, res) {
var username = "user";
var passw = "pass";
var args = {
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
// create a Promisified client get
var clientGetPromise = function clientGetPromise(client, url, args) {
return new Promise(function (resolve, reject) {
return client.get(url, args, function (data, response) {
return resolve(data);
});
});
};
clientGetPromise(client, global.apiUrl + "V1/ProductItem", args).then(function (data) {
return Promise.all(data.map(function (img) {
return clientGetPromise(client, global.apiUrl + "V1/ImagePreview/" + img.AvatarKey, args).then(function (data2) {
img['data'] = data2.Image;
return img;
});
}));
}).then(function (op) { // op is an Array of img because that's how Promise.all rolls
return res.render('test1', { "out": JSON.stringify(op) });
});
});

Trouble with Node JS promises

I cannot find a solution to why this function returns before my message array is updated with the necessary values.
var calculateDistance = function (message, cLongitude, cLatitude, cSessionID) {
return new Promise(function (resolve, reject) {
distance.key = options.apiKey;
distance.units('metric');
var origins = [];
origins.push(cLatitude + ',' + cLongitude);
message.forEach(function (obj) {
obj.sessionId = cSessionID;
var destinations = [];
destinations.push(obj.geoLocation.latitude + ',' + obj.geoLocation.longitude);
distance.matrix(origins, destinations, function (err, distances) {
if (err) {
return console.log(err);
}
if (!distances) {
return console.log('no distances');
}
if (distances.status == 'OK') {
for (var i = 0; i < origins.length; i++) {
for (var j = 0; j < destinations.length; j++) {
var origin = distances.origin_addresses[i];
var destination = distances.destination_addresses[j];
if (distances.rows[0].elements[j].status == 'OK') {
var distance = distances.rows[i].elements[j].distance.text;
console.log('Distance from ' + origin + ' to ' + destination + ' is ' + distance);
obj.distance = distance;
} else {
console.log(destination + ' is not reachable by land from ' + origin);
obj.distance = 'N/A';
}
}
}
}
});
});
return resolve(message);
});
}
Could someone point out to me what i am doing wrong here.
Regards
Jimmy

var async = require('async');
var calculateDistance = function (message, cLongitude, cLatitude, cSessionID) {
return new Promise(function (resolve, reject) {
distance.key = options.apiKey;
distance.units('metric');
var origins = [];
origins.push(cLatitude + ',' + cLongitude);
async.each(message, function(obj, callback) {
obj.sessionId = cSessionID;
var destinations = [];
destinations.push(obj.geoLocation.latitude + ',' + obj.geoLocation.longitude);
distance.matrix(origins, destinations, function (err, distances) {
if (err) {
callback(err);
}
if (!distances) {
callback('no distances');
}
if (distances.status == 'OK') {
for (var i = 0; i < origins.length; i++) {
for (var j = 0; j < destinations.length; j++) {
var origin = distances.origin_addresses[i];
var destination = distances.destination_addresses[j];
if (distances.rows[0].elements[j].status == 'OK') {
var distance = distances.rows[i].elements[j].distance.text;
console.log('Distance from ' + origin + ' to ' + destination + ' is ' + distance);
obj.distance = distance;
} else {
console.log(destination + ' is not reachable by land from ' + origin);
obj.distance = 'N/A';
}
}
}
callback(null);
}
});
},function(err){
if(err){
return reject(err);
}else{
return resolve(message);
}
});
});
};

This is happening because your distance.matrix(origins, destinations, callback )is asynchronous . In above code distance.matrix method is getting pushed to event loop and continues it's execution and before that method callbacks gets executed resolve(message) is returned .

You need to read up on promises. It looks to me as if you are thinking of promises as magical way to set up a callback. "Magic" tends to mean "something I don't need to understand." In this case that's not true.
That executor function of yours (that is the function which begins with 'function(resolve,reject)') should set up one asynchronous request. If, as is normal, the request has a callback you put the 'resolve' and 'reject' in the callback. The result will be a promise object which has methods 'then' and 'catch' where your post-request processing goes.
Since you want to fill up a matrix with the results of a lot of async requests, you will need to read about 'Promise.all' so you can react when all of them have resolved.

Callback/Promises Implementation in Express.js

I'm writing an Express.js application using x-ray as a scraper to get some info.
I want to create a model for each website I'm scraping (different website = different data/procedures to scrape).
Here's the code of the module:
module.exports.getData = function(title){
var Xray = require('x-ray');
var x = Xray();
var url = "http://www.mywebsite.it/online/search?text="+title;
var scraped = '';
x(url, '.listing-products.listing-rows.clearfix h2.title',
[{
title: 'a',
link: 'a#href'
}]
)(function (err, obj) {
for (var i = obj.length - 1; i >= 0; i--) {
scraped += obj[i]['title'] + " "; //we get just the title of the links for now
};
sendScraped(scraped);
});
}
problem is that the controller's function getData does not return anything because it is
called and the execution goes forward without waiting the completation of the x() scraping function.
I'm trying to implement a callback function sendScraped(scraped) to get my controller to wait for the completation, but
i don't know how to call for it from the model.
This is what i tried in the controller:
var mywebsite = require('../models/mywebsite')
exports.searchTitle = function(req, res) {
mywebsite.getData(req.params.title);
};
global.sendScraped = function endScraping(data) {
return res.send(data);
}

I'd suggest you change your getData() method to take a callback. It can then call that callback with the scraped data:
module.exports.getData = function (title, callback) {
var Xray = require('x-ray');
var x = Xray();
var url = "http://www.mywebsite.it/online/search?text=" + title;
var scraped = '';
x(url, '.listing-products.listing-rows.clearfix h2.title', [{
title: 'a',
link: 'a#href'
}])(function (err, obj) {
if (err) return callback(err);
for (var i = obj.length - 1; i >= 0; i--) {
scraped += obj[i]['title'] + " "; //we get just the title of the links for now
};
callback(null, scraped);
});
}
You can then use that in your request handler like this:
var mywebsite = require('../models/mywebsite')
exports.searchTitle = function(req, res) {
mywebsite.getData(req.params.title, function(err, scrapeData) {
if (err) {
// do something with error
} else {
res.send(scrapeData);
}
});
};
And, here's a version using promises:
var Xray = require('x-ray');
function xp(url, selector, args) {
return new Promise(function(resolve, reject) {
var x = Xray();
x(url, selector, args)(function(err, obj) {
if (err) {
reject(err);
} else {
resolve(obj);
}
});
});
}
module.exports.getData = function (title) {
var url = "http://www.mywebsite.it/online/search?text=" + title;
var selector = '.listing-products.listing-rows.clearfix h2.title';
return xp(url, selector, [{title: 'a', link: 'a#href'}]).then(function(obj) {
var scraped = "";
for (var i = obj.length - 1; i >= 0; i--) {
scraped += obj[i]['title'] + " "; //we get just the title of the links for now
}
return scraped;
});
}
You can then use that in your request handler like this:
var mywebsite = require('../models/mywebsite');
exports.searchTitle = function(req, res) {
mywebsite.getData(req.params.title).then(function(scrapeData) {
res.send(scrapeData);
}, function(err) {
// handle error here
});
};

Node.js Function Flow

When I get a request, I want it to generate a 4-character code, then check if it already exists in the database. If it does, then generate a new code. If not, add it and move on. This is what I have so far:
var code = "";
var codeFree = false;
while (! codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code=" + code, function(err, result) {
if (! err) {
if (result.rows.length > 0) {
codeFree = false;
} else {
codeFree = true;
}
} else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
This does not do nearly what I want it to do. How can I handle something like this?

You are doing an async task.
When you have an asyncronous task inside your procedure, you need to have a callback function which is going to be called with the desired value as its argument.
When you found the free code, you call the function and passing the code as its argument, otherwise, you call the getFreeCode function again and passing the same callback to it. Although you might consider cases when an error happens. If your the db call fails, your callback would never get called. It is better to use a throw/catch mechanism or passing another argument for error to your callback.
You can achieve what you need to do by doing it this way:
function getFreeCode(callback) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
if(!err) {
if(result.rows.length > 0) {
getFreeCode(callback);
} else {
callback(code);
}
}else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
// in your main:
getFreeCode(function (code) {
console.log(' this code was free: ' + code)
})

I recommend you look into two alternatives to help deal with asynchronous code.
node generator functions using the 'yield' keyword
promises
Using generators requires running a recent version of node with the --harmony flag. The reason I recommend generators is because you can write code that flows the way you expect.
var x = yield asyncFunction();
console.log('x = ' + x);
The previous code will get the value of x before logging x.
Without yielding the console.log would write out x before the async function was finished getting the value for x.
Your code could look like this with generators:
var client = {
execute: function (query) {
var timesRan = 0;
var result = [];
return function () {
return setTimeout(function () {
result = ++timesRan < 4 ? ['length_will_be_1'] : [];
return result;
},1);
};
}
};
function* checkCode () {
var code;
var codeFree = false;
while(!codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
try {
var result = yield client.execute("select * from codes where code="+code);
codeFree = result.rows.length > 0 ? false : true;
}catch(e) {
console.log('DB ERR: %s', err);
} finally {
console.log(codeFree);
}
console.log('here');
}
}
checkCode().next();
You would leave off the client object. I only added that to make a working example that fakes an async call.
If you have to use an older version of node or do not like the yield syntax then promises could be a worthy option.
There are many promise libraries. The reason I recommend promises is that you can write code that flows the way you expect:
asyncGetX()
.then(function (x) {
console.log('x: ' + x);
});
The previous code will get the value of x before logging x.
It also lets you chain async functions and runs them in order:
asyncFunction1()
.then(function (result) {
return asyncFunction2(result)
})
.then(function (x) { /* <-- x is the return value from asyncFunction2 which used the result value of asyncFunction1 */
console.log('x: ' + x);
});
Your code could look like this with the 'q' promise library:
var Q = require('q');
var client = {
timesRan: 0,
execute: function (query, callback) {
var self = this;
var result = {};
setTimeout(function () {
console.log('self.timesRan: ' + self.timesRan);
result.rows = ++self.timesRan < 4 ? ['length = 1'] : [];
callback(null, result);
},1);
}
};
function checkCode () {
var deferred = Q.defer();
var codeFree = false;
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log('rand: %s', rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
console.log('err: '+err+', result: ' + JSON.stringify(result));
console.log('result.rows.length: ' + result.rows.length);
if(!err) {
if(result.rows.length > 0) {
codeFree = false;
console.log('result.rows: %s, codeFree: %s', result.rows, codeFree);
checkCode();
} else {
codeFree = true;
console.log('line 36: codeFree: ' + codeFree);
deferred.resolve(code);
}
}else {
console.log('DB ERR: %s', err);
deferred.reject(err);
}
console.log(codeFree);
});
console.log('waiting for promise');
return deferred.promise;
}
checkCode()
.then(function (code) {
console.log('success with code: ' + code);
})
.fail(function(err) {
console.log('failure, err: ' + err);
});
Also omit the client object here. I only added that to make a working example that fakes an async call.
Promises and generators definitely take some time to get used to. It's worth it because they make the code a lot easier to follow in the end than code written with nested callbacks.

Callbacks in node.js

I've read and heard that I shouldn't return values in functions because it is a blocking operation and that it will potentially refuse any requests until the operation has finished.
So here's a small function I've coded and I'd like to know if I am handling it correctly. I'm saying this because I just started using node and I want to code in the correct way, also because it feels weird to have a testing condition inside the function and another one to test the callback.
function isWithinSplit(path, target, separator, callBack)
{
var response = "";
var readStream = fs.createReadStream(path);
readStream.on('data', function (data) {
response += data;
});
//Data complete, process it
readStream.on('end', function (close)
{
var array = response.split(separator);
for (var idx=0 ; idx < array.length; idx++)
{
if(array[idx] != "" && array[idx] == target)
callBack("true");
else
callBack("false");
}
});
}
Call:
fileHelper.isWithinSplit(__dirname + ROOM_LIST_PATH, "hello", "|", function(data){
if(data == "true")
console.log("hurray!");
});
I just want to know if this is how people do and if it's efficient.

You forgot
error handling
using ===
caching array length
naming anonymous functions
function isWithinSplit(path, target, separator, callBack) {
var response = "";
var readStream = fs.createReadStream(path);
readStream.on('data', function _aggregateData(data) {
response += data;
});
//Data complete, process it
readStream.on('end', function _findTarget(close) {
var array = response.split(separator);
for (var idx = 0, len = array.length; idx < len; idx++) {
if (array[idx] === target) {
return callBack(null, true);
}
}
callback(null, false);
});
readStream.on('error', callBack);
}
fileHelper.isWithinSplit(__dirname + ROOM_LIST_PATH, "hello", "|", printIfSuccess);
function printIfSuccess(err, data) {
if (data === true) {
console.log("hurray!");
}
}
You can also improve it using Array.prototype.any
readStream.on('end', function(close) {
callback(null, response.split(seperator).any(function _doesItMatchTarget(val) {
return val === target;
}));
});