We Keep Coding

Node.js and Express: wait for asynchronous operation before responding to HTTP request

Say there is a HTTP GET callback defined as:
router.get('/latestpost', function(req, res, next) {
var data = new FbData();
get_latest_post (data);
get_post_image (data);
res.json(data);
};
Both get_ functions use the fb package to generate a HTTP request and execute a callback when finished. How can the above GET callback be modified in order to wait for the responses from Facebook and only then send a response to the client?
At the time being I solved the problem by executing the get_ functions in series and passing them the res (response) argument, with the last function sending the response:
router.get('/latestpost', function(req, res, next) {
var data = new FbData();
get_latest_post (res, data);
};
function get_latest_post (res, data) {
FB.api(_url, function (res_fb) {
if(!res_fb || res_fb.error) {
console.log(!res_fb ? 'error occurred' : res_fb.error);
return;
}
// Do stuff with data
get_post_image (res, data);
});
}
function get_post_image (res, data) {
FB.api(_url, function (res_fb) {
if(!res_fb || res_fb.error) {
console.log(!res_fb ? 'error occurred' : res_fb.error);
return;
}
// Do stuff with data
/* At the end send the post data to the client */
res.json(data);
});
}
I have found a similar question, but I'm wrapping my head around it, since I can't find a proper way to apply the solution to my problem. I have tried using the patterns described in this manual, but I can't get it to execute using promises, or async/await. Can someone please point me in the right direction?

Your API can easily be modified to return a promise:
function get_post_image (res, data) {
return new Promise((resolve, reject) => {
FB.api(_url, function (res_fb) {
if(!res_fb || res_fb.error) {
reject(res_fb && res_fb.error);
} else resolve(res_fb/*?*/);
});
}
Now that you have a promise, you can await it:
router.get('/latestpost', async function(req, res, next) {
const data = new FbData();
const image = await get_post_image (data);
res.json(data);
});

Node.js / Express.js can't get body response from API call via Request middleware

I'm trying to learn Node.js via Express.js framework. Currently I need to make an API call to get some data usefull for my app.
The API call is made with Request middleware, but when I'm out of the request my variable become undefined ... Let me show you :
var request = require('request');
var apiKey = "FOOFOO-FOOFOO-FOO-FOO-FOOFOO-FOOFOO";
var characters = [];
var gw2data;
var i = 0;
module.exports.account = function() {
request('https://api.guildwars2.com/v2/characters/?access_token=' + apiKey, function (error, response, body) {
gw2data = JSON.parse(body);
console.log('out request ' + gw2data); // {name1, name2 ...}
for (i; i < gw2data.length; i++) {
getCharacInfo(gw2data[i], i);
}
});
console.log('out request ' + characters); // undefined
return characters;
};
function getCharacInfo (name, position) {
request('https://api.guildwars2.com/v2/characters/' + name + '/?access_token=' + apiKey, function (error, response, body) {
if (!error && response.statusCode == 200) {
characters[position] = JSON.parse(body);
}
});
}
I don't understand why the gw2data variable become undefined when I go out of the request ... Someone can explain me ?
EDIT : I come to you because my problem has changed, I now need to make an API call loop in my first API call, same async problem I guess.
The previous code has evoluate with previous answers :
module.exports.account = function(cb) {
var request = require('request');
var apiKey = "FOOFOO-FOOFOO-FOO-FOO-FOOFOO-FOOFOO";
var characters = [];
var i = 0;
request('https://api.guildwars2.com/v2/characters/?access_token=' + apiKey, function(err, res, body) {
var numCompletedCalls = 1;
for (i; i < JSON.parse(body).length; i++) {
if (numCompletedCalls == JSON.parse(body).length) {
try {
console.log(characters); // {} empty array
return cb(null, characters);
} catch (e) {
return cb(e);
}
}else {
getCharacInfo(JSON.parse(body)[i]);
}
numCompletedCalls++;
}
});
};
function getCharacInfo (name) {
request('https://api.guildwars2.com/v2/characters/' + name + '/?access_token=' + apiKey, function (err, res, body) {
if (!err) {
console.log(characters); // {data ...}
characters.push(JSON.parse(body));
}
});
}
And the call of this function :
var express = require('express');
var router = express.Router();
router.get('/', function(req, res, next) {
var charactersInfos = require('../models/account');
charactersInfos.account(function(err, gw2data) {
if (err) {
console.error('request failed:', err.message);
} else {
res.render('index', {
persos : gw2data
});
}
});
});
The problem is, when I return the characters array, it's always an empty array, but when I check in my function getCharacInfo, the array contains the data ...

The reason gw2data is undefined in the second console.log is because your logging it too early. request is an asynchronous operation therefore it will return immediately...however, that doesn't mean the callback will.
So basically what your doing is logging gw2data before it's actually been set. When dealing with asynchronous operations the best approach is to generally make your own method asynchronous as well which can be accomplished in a couple of ways - the simplest being having your function accept a callback which expects the data in an asynchronous way e.g.
module.exports.account = function(cb) {
request(..., function(err, res, body) {
// return an error if `request` fails
if (err) return cb(err);
try {
return cb(null, JSON.parse(body));
} catch (e) {
// return an error if `JSON.parse` fails
return cb(e);
}
});
}
...
var myModule = require('mymodule');
myModule.account(function(err, g2wdata) {
if (err) {
console.error('request failed', err.message);
} else {
console.log('out request', g2wdata);
}
});
Based on your syntax I'm assuming you aren't working with ES6, worth looking at this (particularly if your just starting to learn). With built-in promises & also async-await support coming relatively soon this sort of stuff becomes relatively straightforward (at least at the calling end) e.g.
export default class MyModule {
account() {
// return a promise to the caller
return new Promise((resolve, reject) => {
// invoke request the same way but this time resolve/reject the promise
request(..., function(err, res, body) {
if (err) return reject(err);
try {
return resolve(JSON.parse(body));
} catch (e) {
return reject(e);
}
});
});
}
}
...
import myModule from 'mymodule';
// handle using promises
myModule.account()
.then(gw2data => console.log('out request', gw2data))
.catch(e => console.error('request failed', e));
// handle using ES7 async/await
// NOTE - self-executing function wrapper required for async support if using at top level,
// if using inside another function you can just mark that function as async instead
(async () => {
try {
const gw2data = await myModule.account();
console.log('out request', gw2data);
} catch (e) {
console.log('request failed', e);
}
})();
Finally, should you do decide to go down the Promise route, there are a couple of libraries out there that can polyfill Promise support into existing libraries. One example I can think of is Bluebird, which from experience I know works with request. Combine that with ES6 you get a pretty neat developer experience.

The request API is asynchronous, as is pretty much all I/O in Node.js. Check out the Promise API which is IMO the simplest way to write async code. There are a few Promise adapters for request, or you could use something like superagent with promise support.

how to write unit test to "request" node module with sinon.stub?

I have this function in my code :
let request = require("request");
let getDrillDownData = function (userId, query, callback) {
query.id = userId;
let urlQuery = buildUrlFromQuery(query);
request.get({
url: urlQuery,
json: true
}, function (error, response, data) {
if (!error && response.statusCode === 200) {
return callback(null, calculateExtraData(data));
} else if (error) {
return callback(error, null);
}
});
};
and I wish to write some unit test which verify that when the function is called with correct parameters, it is running OK,
and if there is an error, it did return the error
I wrote this unit test code :
describe.only('Server Service Unit Test', function(){
var sinon = require('sinon'),
rewire = require('rewire');
var reportService;
var reportData = require('./reportData.json');
beforeEach(function(){
reportService = rewire('../../services/reports.server.service');
});
describe('report methods', function(){
var reportData;
var query = { id: "test"};
var userId = 'testuser';
var getDrillDownData;
var request;
beforeEach(function(){
getDrillDownData = reportService.__get__('getDrillDownData');
});
it ('should get drill down data by userId and query', function(done){
var getCallback = sinon.stub();
request = {
get: sinon.stub().withArgs({
url: query,
json: true
}, getCallback.withArgs("error", {statusCode: 200}, reportData))
};
reportService.__set__('request', request);
getDrillDownData(userId, query, function(err, returnData){
(err === null).should.eql(true);
//(getCallback.withArgs(undefined, {statusCode: 200}, reportData).calledOnce).equal(true);
done();
});
});
});
But I keep getting this error:
Error: timeout of 2000ms exceeded. Ensure the done() callback is being called in this test.
Can someone help?
Thanks

I would stub request.get() directly:
describe('report methods', function() {
// Make `request.get()` a Sinon stub.
beforeEach(function() {
sinon.stub(request, 'get');
});
// Restore the original function.
afterEach(function() {
request.get.restore();
});
it ('should get drill down data by userId and query', function(done) {
// See text.
request.get.yields(null, { statusCode : 200 }, { foo : 'bar' });
// Call your function.
getDrillDownData('userId', {}, function(err, data) {
...your test cases here...
done();
});
});
});
Using request.get.yields() (which calls the first function argument that Sinon can find in the argument list; in this case, it's the (error, response, data) callback that gets passed to request.get() in your function) you can tell Sinon which arguments to use to call the callback.
That way, you can check if the callback to request.get() handles all arguments properly.
You can use .withArgs() too (request.get.withArgs(...).yields(...)), although you have to be sure that you're using it correctly; otherwise, if the exact arguments don't match, Sinon will call the original request.get() instead of using the stubbed version.
Instead, I prefer using stub.calledWith() to check for the correct arguments after the call has been made. That integrates much better with Mocha as well, as you can use explicit assertions.

Node script needs to return success/failure to Angular service that called it

I'm trying to make a request to a node function that will write to a file, but I'm not sure the best way to get the success or failure back to angular. The setup looks like this:
//controller (in an ng-click handler)
writeFileRoute.writeFile(file)
.then(function(response){
console.log('success', response);
}, function(error){
console.log('error', error);
});
//service
app.service('WriteFileService', function($http) {
this.writeFile = function(data) {
return $http.post('/writeFile', data)
.then(function(response) {
return {
'success': response
};
}, function(response) {
return {
'error': response
};
});
};
})
//server.js
app.post('/writeFile', function(req, res){
components.writefile(req.body, function(err){
//do something with error here?
});
//by this point I have become increasingly confused by what is going on
//node write script
var writefile = function(data, callback){
//data = JSON.stringify(data).message;
fs.writeFile('./testFile.txt', data.message, function(err){
callback(err);
});
};
module.exports = exports = writefile;
So the file is actually writing. I just don't know how to combine these separately working components into something that can notify angular of success or failure. I thought about using q, however I don't know if I should use angular $q or node Q. I would really like to use Q and/or $q, but I don't know which is the right solution or where to plug them in.
Edit (3 Feb)
This is the code that I am currently working with:
Angular:
angular.module('test', [])
.controller('ctrl', function($scope, WriteService){
$scope.testMessage = "test"
$scope.writeTheFile = function(){
WriteService.write()
.then(function(err){
if (err){
console.log(err);
}else{
console.log('no error');
}
});
}
})
.service('WriteService', function($http){
this.write = function(){
$http.get('./write/', function(response){
return {'success': response};
}, function(error){
return {'error': error};
});
};
});
Node write module
var writes = {
nfcall: function(data){
data = data.repeat(5);
return Q.nfcall(fs.writeFile, dest, data);
}
}
Node Server
app.get('/write/', function(req, res, next){
var data = 'some text string ' + +new Date() + '\n';
writeModule.nfcall(data)
.then(function(response){
return {'response': response};
})
.catch(function(err){
return {'index.js error': err};
});
});

Promises are a very good alternative to callbacks, and Q is an excellent Promise library to use on your node.js backend:
var Q = require('q');
Your server will look like something like this:
app.post('/writeFile', function(req, res, next) {
components.writefile(req.body)
.then(function() {
// Success! 'Writefile' worked fine, just send some 200 OK response.
res.send('OK');
})
.catch(function(err) {
// If 'Writefile' fails, this will be called!
// Just pass the error to the Error Handling middleware
next(err);
});
});
var writefile = function(data){
return Q.nfcall(fs.writefile, './testFile.txt', data.message);
};
module.exports = exports = writefile;
Note that nfcall() method is used for calling Node.js-style functions and getting back a promise.
Your components.writefile now returns a Promise, and what you do in app.post could be two things (have a look at the Promise Core Methods):
Catching any errors if your promise is rejected (catch method is called)
Istead, if the promise was resolved, then method is called.
I don't see anything wrong on your fronted, in fact, it already uses Promises: note that $http returns a Promise (have a look at it's Documentation).

In app.post, you must return something back to the client through res. Use res.send(data or error)
See: http://expressjs.com/en/api.html
This also mean you do not need a promise from the server to the client. The promise for writeFileRoute will be resolved with the res return.

HTTP request in separate function[node.js/expressjs]

By using async module I have something like this and it works perfectly.
But when i try to restructure the code or make it reusable, it finish executing before it finished the HTTP requests. Nodejs do lots of things in asynchronous way so finding a solution to this is bit hard to me.
What I have up to now.
var async = require('async'),
http = require('http');
exports.unitedStates = function(req, res) {
var texas = {
//GET method data here / ex: host, path, headers....
};
var washington = {
//GET method data here / ex: host, path, headers....
};
async.parallel({
getSource: function(callback) {
http.request(texas, function(respond) {
//Http request
}).end();
},
getScreen: function(callback) {
http.request(washington, function(respond) {
//Http request
}).end();
}
},
function(err, results) {
//Return the results
/* REPLY TO THE REQUEST */
res.send( /* data here */ );
});
}
is there a perfect way to make this piece of code to reusable?
Example
exports.unitedStates = function(req, res) {
var tokyo = japan();
//send the result to front end
res.send(tokyo);
}
function japan(){
//async calls comes here and return the value...
return result;
}

Instead of returning value from function, pass a callback.
exports.unitedStates = function (req, res) {
// pass callback here
japan(function (value) {
res.send(value);
});
}
function japan(cb) {
//async call here
cb(result);
}