We Keep Coding

Using results of a callback in node in a separate function

I'm probably missing the point somewhere here so I'm looking for advice.
I have a nodejs server which is listening for client connections and, based on the data received, makes calls to an API.
The very first call to that API gets an ID which needs to be used on subsequent calls to group them together.
Where I'm struggling is that the call to the API is necessarily asynchronous and in the callback I'm assigning the ID to a variable. While that async call is being processed by the API server, more data is coming in from the client and needs more API calls made BUT I can't fire them until I know the results from the first call as the second calls depend on it.
What's the proper way to handle this? I feel like I should be using Q to promise the results of the first API call to the second, but I'm not sure how it should be structured. Or should I just be queueing up the API calls until the first completes? How would I do that?
Example problem code :
var server = net.createServer();
//set up the callback handler
server.on('connection', handleConnection);
handleConnection(conn) {
//do some stuff...
firstAPICall();
conn.on('data', handleData);
}
handleData(data) {
//do some stuff...
otherAPIcall();
}
firstAPICall() {
client.get("http://myAPI/getID", function (data, response) {
conn.myID = data[0].myID;
}
}
}
otherAPICall() {
//How do I make sure I actually have a value
//in conn.myID from the first function???
client.post("http://myAPI/storeData", { data: {myID:conn.myID, data:someData} }, function (data, response) {
//do some stuff...
}
}
}

Yes, you should be using promises for this. Make a promise for the id that is asynchronously resolved from the first call, and then use it in the subsequent calls:
handleConnection(conn) {
//do some stuff...
var idPromise = firstAPICall();
conn.on('data', function handleData(data) {
//do some stuff...
otherAPIcall(idPromise).then(function(result) {
…
});
});
}
firstAPICall() {
return Q.Promise(function(resolve, reject) {
client.get("http://myAPI/getID", function (data, response) {
resolve(data[0].myID);
});
});
}
otherAPICall(idPromise) {
return idPromise.then(function(myID) {
return new Promise(function(resolve, reject) {
client.post("http://myAPI/storeData", {
data: {myID:myID, data:someData}
}, function (data, response) {
//do some stuff...
resolve(…);
});
});
});
}
Probably you should factor out creating a promise for the result of a client.get call in an extra function. Also make sure to handle errors correctly there and call reject with them. If client would use the node callback conventions, Q even has some nice helper functions for that.

Try using promises. Then use 'then' to call the otherAPICall()

I think you can assume they will be sending data immediately after connecting. So you can simplify and just check in otherAPICall if you have an ID, if not, you can just use a callback. Promises or the async/await keywords might make things sort of nicer down the line but aren't required for this.
var server = net.createServer();
//set up the callback handler
server.on('connection', handleConnection);
handleConnection(conn) {
conn.on('data', handleData(connm, data));
}
handleData(conn, data) {
//do some stuff...
otherAPIcall(conn);
}
checkID(conn, cb) {
if (!conn.myID) {
client.get("http://myAPI/getID", function (data, response) {
conn.myID = data[0].myID;
cb();
});
} else {
cb();
}
}
otherAPICall(conn) {
checkID(conn, function() {
client.post("http://myAPI/storeData", { data: {myID:conn.myID, data:someData} }, function (data, response) {
//do some stuff...
});
});
}

promises can chain values and are always resolved after the callback occurs with the returned value,
function async(value) {
var deferred = $q.defer();
var asyncCalculation = value / 2;
deferred.resolve(asyncCalculation);
return deferred.promise;
}
var promise = async(8)
.then(function(x) {
return x+1;
})
.then(function(x) {
return x*2;
})
.then(function(x) {
return x-1;
});
promise.then(function(x) {
console.log(x);
});
This value passes through all the success callbacks and so the value 9 is logged ((8 / 2 + 1) * 2 - 1).

How do I Promisify an Async operation, using the Async library?

GrabRedisDataByArray = Promise.promisify(function(data, callback) {
var temp_results = [];
async.each(data, function(result, done) {
redis.hgetall('username:' + result, function(err, results) {
temp_results.push(results);
done();
})
}, function(err) {
callback(temp_results)
});
});
Except, it's returning
Unhandled rejection (<[{"server":"9300","user_id":"31","char...>, no stack trace )
--That object data is just some user information stored in the username: key.
And my method for data retrieval is:
GrabRedisDataByArray(data).then(function(data){
console.log(data)
});
I think my problem is with GrabRedisDataByArray and how it doesn't return anything in the main function scope, but only returns something through its second parameter? If that makes sense? I'm just kind of lost here and trying to understand how I can return that async operation to the main function instead of only returning it through the second parameter callback (Which is I think my promises isn't working correctly)
Edit: This is using the Bluebird library

Well, promisify expects node-style callbacks, so when you're calling callback(temp_results) you are passing an error parameter. It would have to be callback(null, temp_results).
That said, you should not use the async library at all when you're working with promises anyway. Just embrace promises. Or if you insist on using async, at least do it properly:
var grabRedisDataByArray = Promise.promisify(function(data, callback) {
async.map(data, function(result, done) {
redis.hgetall('username:' + result, done);
}, callback);
});
But when promisifying, you should always promisify on the lowest level - in your case redis. Then you don't have to mess around with async callbacks but can simply use Bluebird's Promise.map:
Promise.promisifyAll(redis);
function grabRedisDataByArray(data) {
return Promise.map(data, function(result) {
return redis.asyncHgetall('username:' + result);
});
}

Callback runs first? - learnyounode juggling

Can someone help me understand why my solution does not work? It seems like the callback function is running before the juggle function is finished.
My code works fine if I remove the comments. It's just that I don't understand why the log function does not get called after the juggle function is finished. That is how callbacks are supposed to work right?
Thanks in advance :)
var http = require('http')
links = process.argv.slice(2)
var contents = []
//var cbacks = 0
function juggle(callback) {
links.forEach(function(link, i, links) {
http.get(link, function(response) {
response.setEncoding("utf8")
var str = ""
response.on("data", function(data) {
str = str.concat(data)
})
response.on("end", function(){
contents[i] = str
//cbacks++
//if(cbacks === 3) {
// callback()
//}
})
})
})
callback()
}
function log() {
contents.forEach(function(content, i, contents) {
console.log(contents[i])
})
}
juggle(log)

http.get is asynchronous. forEach is executed against your links which calls http.get, which registers a connection to be processed. It doesn't actually complete the connection/request.
if you need to execute the callback when all the forEach functions complete you could use a library like async to accomplish it.
async supports a forEach method. Using async, the first param to forEach would take an additional callback function which should be called to denote item processing has been finished. You could place that callback in the response.on('end') callback. When all of those callbacks have been called, or when an error has occurred async.forEach will execute the onComplete callback you provide to it as the 3rd parameter, accomplishing your goal.

So you have global scope which is what I would use to actually juggle the requests.
When each link gets registered to an EventEmitter, you can store it inside of a map.
var link_map = {};
var request_counter = 0;
links.forEach( function (link, index) {
link_map[link] = '';
...
then in your requests you can append data from a specific request
response.on('data', function (chunk) {
link_map[link] += chunk.toString();
...
and finally at each end, check if all requests have finished
response.on('end', function () {
request_counter += 1;
if ( links.length === request_counter ) {
// do your logging stuff here and you have all
// the data that you need inside link_map
...
the link variable inside of the anonymous function declared in the forEach is stored for that closure. So each time that the 'data' event is emitted, the link variable is going to refer to the request for a specific link that was registered to the callback. That is why I chose to use a map data structure and map specific data to each link which we are using as a key.
EventEmitters and callbacks can get kind of harry if you are unfamiliar with them. Keep practicing though and it will eventually become easier.
And using an array as you did is not incorrect or anything, I just prefer to use Objects with key => value pairs when I can.
RUN THIS CODE on your machine to see it in action.
const http = require('http');
var links = [
'http://www.google.com',
'http://www.example.com',
'http://www.yahoo.com'
];
var link_map = {};
var request_counter = 0;
links.forEach( function (link, index) {
link_map[link] = '';
http.get(link, function(response) {
response.on('data', function (chunk) {
link_map[link] += chunk.toString();
});
response.on('end', function () {
request_counter += 1;
if ( links.length === request_counter ) {
links.forEach( function(link) {
require('fs').writeFileSync(link.split('//')[1],link_map[link]);
});
}
});
});
});
You can see an output of files from links in the parent directory.

You didn't wait for http response and call callback function immediately. At this point of code contents array is empty.

Node.JS How to set a variable outside the current scope

I have some code that I cant get my head around, I am trying to return an array of object using a callback, I have a function that is returning the values and then pushing them into an array but I cant access this outside of the function, I am doing something stupid here but can't tell what ( I am very new to Node.JS )
for (var index in res.response.result) {
var marketArray = [];
(function () {
var market = res.response.result[index];
createOrUpdateMarket(market, eventObj , function (err, marketObj) {
marketArray.push(marketObj)
console.log('The Array is %s',marketArray.length) //Returns The Array is 1.2.3..etc
});
console.log('The Array is %s',marketArray.length) // Returns The Array is 0
})();
}

You have multiple issues going on here. A core issue is to gain an understanding of how asynchronous responses work and which code executes when. But, in addition to that you also have to learn how to manage multiple async responses in a loop and how to know when all the responses are done and how to get the results in order and what tools can best be used in node.js to do that.
Your core issue is a matter of timing. The createOrUpdateMarket() function is probably asynchronous. That means that it starts its operation when the function is called, then calls its callback sometime in the future. Meanwhile the rest of your code continues to run. Thus, you are trying to access the array BEFORE the callback has been called.
Because you cannot know exactly when that callback will be called, the only place you can reliably use the callback data is inside the callback or in something that is called from within the callback.
You can read more about the details of the async/callback issue here: Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference
To know when a whole series of these createOrUpdateMarket() operations are all done, you will have to code especially to know when all of them are done and you cannot rely on a simple for loop. The modern way to do that is to use promises which offer tools for helping you manage the timing of one or more asynchronous operations.
In addition, if you want to accumulate results from your for loop in marketArray, you have to declare and initialize that before your for loop, not inside your for loop. Here are several solutions:
Manually Coded Solution
var len = res.response.result.length;
var marketArray = new Array(len), cntr = 0;
for (var index = 0, index < len; index++) {
(function(i) {
createOrUpdateMarket(res.response.result[i], eventObj , function (err, marketObj) {
++cntr;
if (err) {
// need error handling here
}
marketArray[i] = marketObj;
// if last response has just finished
if (cntr === len) {
// here the marketArray is fully populated and all responses are done
// put your code to process the marketArray here
}
});
})(index);
}
Standard Promises Built Into Node.js
// make a version of createOrUpdateMarket that returns a promise
function createOrUpdateMarketAsync(a, b) {
return new Promise(function(resolve, reject) {
createOrUpdateMarket(a, b, function(err, marketObj) {
if (err) {
reject(err);
return;
}
resolve(marketObj);
});
});
}
var promises = [];
for (var i = 0; i < res.response.result.length; i++) {
promises.push(createorUpdateMarketAsync(res.response.result[i], eventObj));
}
Promise.all(promises).then(function(marketArray) {
// all results done here, results in marketArray
}, function(err) {
// an error occurred
});
Enhanced Promises with the Bluebird Promise library
The bluebird promise library offers Promise.map() which will iterate over your array of data and produce an array of asynchronously obtained results.
// make a version of createOrUpdateMarket that returns a promise
var Promise = require('bluebird');
var createOrUpdateMarketAsync = Promise.promisify(createOrUpdateMarket);
// iterate the res.response.result array and run an operation on each item
Promise.map(res.response.result, function(item) {
return createOrUpdateMarketAsync(item, eventObj);
}).then(function(marketArray) {
// all results done here, results in marketArray
}, function(err) {
// an error occurred
});
Async Library
You can also use the async library to help manage multiple async operations. In this case, you can use async.map() which will create an array of results.
var async = require('async');
async.map(res.response.result, function(item, done) {
createOrUpdateMarker(item, eventObj, function(err, marketObj) {
if (err) {
done(err);
} else {
done(marketObj);
}
});
}, function(err, results) {
if (err) {
// an error occurred
} else {
// results array contains all the async results
}
});

Call async function while pushing the same function on array

I have this code:
var queue = [];
var allParserd = [];
_.each(webs, function (web) {
queue.push(function () {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
allParserd.push(parsed);
});
});
});
Promise.all(queue).then(function (data) {
console.log(allParserd);
});
Basically I need to fetch all my webs and be sure to give the result after that every parsing is done. the function parseWebsite return the correct data, but in this way is not called and allParsed return just as an empty array. I'm sure that I miss some things, I've started to use the promises just from some days.
If you need some more information just tell me.
P.s.
I want that all the functions to start at the same time; I don't want to wait for each one response for going forward.

Tagged with Bluebird so let's use it:
First, let's convert your callback API to promises:
Promise.promisifyAll(WebsitesUtils);
Now, let's use .map to map every item in webs to it being parsed parseWebsite:
Promise.map(webs, function(item){
return WebsitesUtils.parseWebsiteAsync(item); // note the suffix
}).then(function(results){
// all the results are here.
}).catch(function(err){
// handle any errors
});
As you can see - this is trivial to do with Bluebird.

Promise.all doesn't take a queue of functions to execute. It expects an array of promises which represent the results of the many concurrently running (still pending) requests.
The first step is to have a function that actually returns a promise, instead of only executing a callback. We can use
function parseWebsite(web) {
return new Promise(function(fulfill, reject) {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
if (err)
reject(err);
else
fulfill(parsed);
});
});
}
or simply use promisification that does this generically:
var parseWebsite = Promise.promisify(WebsitesUtils.parseWebsite, WebsitesUtils);
Now we can go to construct our array of promises by calling that function for each site:
var promises = [];
_.each(webs, function (web) {
promises.push(parseWebsite(web));
});
or just
var promises = _.map(webs, parseWebsite);
so that in the end we can use Promise.all, and get back our allParsed array (which even is in the same order as webs was!):
Promise.all(promises).then(function(allParsed) {
console.log(allParsed);
});
Bluebird even provides a shortcut function so you don't need promises:
Promise.map(webs, parseWebsite).then(function(allParsed) {
console.log(allParsed);
});

Here's how might do it with async:
var async = require('async');
var webs = ...
async.map(webs, function(web, callback) {
WebsitesUtils.parseWebsite(web, callback);
}, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});
and if parseWebsite() isn't a prototype method dependent on WebsitesUtils then you could simplify it further:
async.map(webs, WebsitesUtils.parseWebsite, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});