I'm trying to iterate threw a list of item and do some actions on them by calling an API like this example :
for (i = 0; i < arr.length; i++) {
if (arr[i].id == 42) {
api.requestAction(arr[i].id, function(error, response){ });
}
}
Problem is the loop obviously ended before all the requests are done and the program exits. What should I do to manage it ? I saw the "Promise" method but don't really know how I can use it in this case or maybe there's an other solution.
Thank you by advance !
With node-fetch (a promisify http api) you can together with async/await halt the for loop until it's done but this requires node v6+ with --harmony-async-await flag added
const fetch = require('node-fetch')
async function foo() {
for (let item of arr) {
if (item.id == 42) {
let res = await fetch(url)
let body = await res.text()
console.log(body)
}
}
console.log('done (after request)')
}
now every time you add the async keyword in front of a function it will always return a promise that resolve/rejects when everything is done
foo().then(done, fail)
alternetive you can just wrap you api fn in a promise if you don't want to install node-fetch
await new Promise((rs, rj) => {
api.requestAction(arr[i].id, function(error, response){
error ? rj(error) : rs(response)
})
})
Install bluebird
npm install bluebird --save
Code
//require npm
var Promise = require("bluebird");
//code
//"promisify" converts traditional callback function into a Promise based function
var _requestAction = Promise.promisify(api.requestAction);
//loop over array
Promise.map(arr, function (value) {
if (value.id == 42) {
//async request
return _requestAction(value.id).then(function (_result) {
//success
console.log(_result);
}).catch(function (e) {
//error
console.error(e);
});
}
});
You could use async.js. It's an asyncronous control flow library which provides control flows for things like sequential loops, looping in parralel, and many other common flow control mechanism, check it out.
See code below, the code assumes that you're variable 'arr' is defined somewhere in scope.
npm install async
var async = require("async");
//Loop through each item, waiting for your
//asyncronous function to finish before continuing
//to move onto the next item in the array
//NOTE: This does not loop sequentially, if you want that function with asyncjs then user eachSeries
async.each(arr,
//Item is the current item being iterated over,
//callback is the callback you call to finish the current iteration, it accepts an error and result parameter callback(error, result);
function (item, callback) {
api.requestAction(item.id, function(error, response){
//Check for any errors...
if (error) return callback(error);
callback(null);
});
},
function (err, result) {
//You've now finished the loop
if (err) {
//Do something, you passed an error object to
//in one of the loop's iterations
}
//No errors, move on with your code..
});
Use Bluebird Promises:
var Promise = require('bluebird');
Promise.map(arrayOfIds, function(item){
return api.requestAction(item);
})
.then(function(response){
// all the requests are resolved here
})
if u want sequential execution of the ids then use Promise.mapSeries (is slow as it waits for task to finish)
Related
I am trying to build an API through which I can get whois detail in the JSON output like below
For this, I installed the whois package from npm (https://www.npmjs.com/package/whois[whois Link]2). I tried to convert the string to object and print it in JSON format but I am not getting any output on the web but it console i can get the data easily. Can you guys please fix my error.
function whoisFunction() {
var whois = require('whois')
whois.lookup(url,async function(err, data) {
try {
a = await data.split('\n')
}
catch (e) {
console.log(e)
return e
}
c=[]
for(i = 0; i < a.length; i++){
c.push(a[i])
}
await console.log(typeof (c))
console.log(c)
return a
})
}
// res.json({'Headers':+whoisFunction()})
res.status(200).json(whoisFunction())
There are async and awaits sprinkled seemingly randomly all over your function.
You should realize that the only thing that is asynchronous here is whois.lookup().
console.log is not asynchronous. Array.prototype.split is not asynchronous. The callback (err, data) => {...} is not asynchronous.
If you want to use the callback pattern, then you need to use res.send() inside of the callback
(err, data) => {
res.send(data)
}
But we got fed up with callback-patterns because of how messy it is to nest them. So instead we went over to using promises. If you have callbacks but want use promises, then you wrap the callback in a promise. You do it once, and you do it as tight to the offending callback as you can:
function promisifiedLookup(url){
return new Promise( (resolve, reject) => {
whois.lookup(url, function(err, data) {
if(err) reject(err)
else resolve(data)
})
})
}
So, to use async/await we need that:
the calling function is declared async
the called function is returning a promise (or else there is nothing to wait for)
async function whoisFunction() {
let data = await promisifiedLookup(url) // _NOW_ we can use await
data = data.split('\n')
// ...
return data; // Because this funtion is declared async, it will automatically return promise.
}
If your express-handler is defined as async, then you now can use await here as well:
res.status(200).json(await whoisFunction())
I have a tricky situation that needs to collect keys that belongs to certain types (types in a given array), then filter the collected keys and pass to a deletion function.
The collection process calls shell codes and process the results in a callback within a loop. I will need to wait until the whole loop of callback finishes then pass to the deletion function.
I am using shelljs in the node codes, basically look like the below:
var del_arr = [];
for (var i in types) {
shell.exec(somecode with
var [i], {
silent: true
},
function(code, stdout, stderr) {
somecode-processing/filtering stdout and pushes the results to del_arr;
});
//loop through array types[] and need to wait for all shell codes' callbacks to finish;
}
//then pass del_arr to deletion function
I wasn't able to build a async function in this format b/s of the shelljs callback. I also don't know how to use promise in this situation.
Can you tell me how to achieve this non-blocking process?
Thanks
Turn child_process.exec into a promise:
function execWrapper(command, options) {
return new Promise((resolve, reject) => {
shell.exec(command, options, (error, out, err) => {
if (error) return reject(error);
resolve({out: out, err: err});
})
})
}
Then you can iterate over types and map each one to a promise:
const promises = types.map(type => execWrapper(type, {slient: true}));
Now wait for each promise to resolve, or for one to reject:
Promise.all(promises).then((del_arr) => {
// del_arr is now a array of objects with the stdout and stderr of each type.
//
})
A good implementation of this case :
async function myAsyncFunction() {
const promises = types.map((type) => myAsyncRequest(type));
let del_arr = Promise.all(promises);
}
A good article that explains this :
https://medium.freecodecamp.org/avoiding-the-async-await-hell-c77a0fb71c4c
Try to convert shell.exec to Promise like
function shellPromise(command,option) {
return Promise((resolv,reject)=>{
shell.exec(command,option,(code,stdout,stderr)=>
resolv({code:code,stdout:stdout,stderr:stderr})
);
};
};
Then you can use something like
for (var i in types){
var result=await shellPromise(somecode with var[i], {silent:true});
// somecode-processing/filtering stdout and pushes the results to del_arr;
}
You can also use async package in npm. It provides a function eachSeries that might come handy in your situation, without useing promises and dealing with callbacks only.
async.eachSeries(hugeArray, function iteratee(item, callback) {
if (inCache(item)) {
callback(null, cache[item]); // if many items are cached, you'll overflow
} else {
doSomeIO(item, callback);
}
}, function done() {
//...
});
For more details on how to use this function: https://caolan.github.io/async/
I have some code that I cant get my head around, I am trying to return an array of object using a callback, I have a function that is returning the values and then pushing them into an array but I cant access this outside of the function, I am doing something stupid here but can't tell what ( I am very new to Node.JS )
for (var index in res.response.result) {
var marketArray = [];
(function () {
var market = res.response.result[index];
createOrUpdateMarket(market, eventObj , function (err, marketObj) {
marketArray.push(marketObj)
console.log('The Array is %s',marketArray.length) //Returns The Array is 1.2.3..etc
});
console.log('The Array is %s',marketArray.length) // Returns The Array is 0
})();
}
You have multiple issues going on here. A core issue is to gain an understanding of how asynchronous responses work and which code executes when. But, in addition to that you also have to learn how to manage multiple async responses in a loop and how to know when all the responses are done and how to get the results in order and what tools can best be used in node.js to do that.
Your core issue is a matter of timing. The createOrUpdateMarket() function is probably asynchronous. That means that it starts its operation when the function is called, then calls its callback sometime in the future. Meanwhile the rest of your code continues to run. Thus, you are trying to access the array BEFORE the callback has been called.
Because you cannot know exactly when that callback will be called, the only place you can reliably use the callback data is inside the callback or in something that is called from within the callback.
You can read more about the details of the async/callback issue here: Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference
To know when a whole series of these createOrUpdateMarket() operations are all done, you will have to code especially to know when all of them are done and you cannot rely on a simple for loop. The modern way to do that is to use promises which offer tools for helping you manage the timing of one or more asynchronous operations.
In addition, if you want to accumulate results from your for loop in marketArray, you have to declare and initialize that before your for loop, not inside your for loop. Here are several solutions:
Manually Coded Solution
var len = res.response.result.length;
var marketArray = new Array(len), cntr = 0;
for (var index = 0, index < len; index++) {
(function(i) {
createOrUpdateMarket(res.response.result[i], eventObj , function (err, marketObj) {
++cntr;
if (err) {
// need error handling here
}
marketArray[i] = marketObj;
// if last response has just finished
if (cntr === len) {
// here the marketArray is fully populated and all responses are done
// put your code to process the marketArray here
}
});
})(index);
}
Standard Promises Built Into Node.js
// make a version of createOrUpdateMarket that returns a promise
function createOrUpdateMarketAsync(a, b) {
return new Promise(function(resolve, reject) {
createOrUpdateMarket(a, b, function(err, marketObj) {
if (err) {
reject(err);
return;
}
resolve(marketObj);
});
});
}
var promises = [];
for (var i = 0; i < res.response.result.length; i++) {
promises.push(createorUpdateMarketAsync(res.response.result[i], eventObj));
}
Promise.all(promises).then(function(marketArray) {
// all results done here, results in marketArray
}, function(err) {
// an error occurred
});
Enhanced Promises with the Bluebird Promise library
The bluebird promise library offers Promise.map() which will iterate over your array of data and produce an array of asynchronously obtained results.
// make a version of createOrUpdateMarket that returns a promise
var Promise = require('bluebird');
var createOrUpdateMarketAsync = Promise.promisify(createOrUpdateMarket);
// iterate the res.response.result array and run an operation on each item
Promise.map(res.response.result, function(item) {
return createOrUpdateMarketAsync(item, eventObj);
}).then(function(marketArray) {
// all results done here, results in marketArray
}, function(err) {
// an error occurred
});
Async Library
You can also use the async library to help manage multiple async operations. In this case, you can use async.map() which will create an array of results.
var async = require('async');
async.map(res.response.result, function(item, done) {
createOrUpdateMarker(item, eventObj, function(err, marketObj) {
if (err) {
done(err);
} else {
done(marketObj);
}
});
}, function(err, results) {
if (err) {
// an error occurred
} else {
// results array contains all the async results
}
});
I'm using Meteor (1.0.3) in general, but for one particular case I'm using a raw server side route to render a file -- so I'm outside a Meteor method.
I'm using node fs.writeFile/fs.readFile and exec commands to call out to Linux command-line utilities too.
My only point in brining this up is that the node calls are async of course. And so I'm opted to use the node Q library in order to manage async callbacks.
This all worked until I added a line to call out to the MongoDB database.
A call like so:
var record_name = Mongo_Collection_Name.findOne({_personId: userId}, {fields: {'_id': 0}});
Produces the following error:
[Error: Can't wait without a fiber]
The error only occurs when I wrap the function in a Promise.
For example, something like this will throw:
getRecordExample = function () {
var deferred = Q.defer();
var record_name = Mongo_Collection_Name.findOne({_personId: userId}, {fields: {'_id': 0}});
// do something
// if no error
deferred.resolve(record_name);
return deferred.promise;
}
If I use the Meteor Fibers library I don't get the error:
getRecordExample = function () {
var deferred = Q.defer();
Fiber = Npm.require('fibers');
var record_name
Fiber(function () {
record_name = Mongo_Collection_Name.findOne({_personId: userId});
}).run()
// do something
// if no error
deferred.resolve(record_name);
return deferred.promise;
}
but, the record_name variable is undefined outside the fiber, so I don't have a way to pass the variable outside of the Fiber scope as far as I can tell.
A More Precise Example
This is a little long, so you have to scroll down to see it all. I'm basically building a workflow here so there are processes and subprocesses.
// both/routes.js
Router.route('/get-route', function(req, res) {
// get the userId then start the workflow below
// using Promises here because these were firing concurrently
Q(userId)
.then(process_1)
.then(process_2)
.done();
}, { name: 'server-side-ir-route', where: 'server' }
// server.js
process_1 = function (userId) {
sub_process_1(userId);
sub_process_2(userId);
return userId;
}
process_2 = function (userId) {
sub_process_3(userId);
sub_process_4(userId);
return userId;
}
sub_process_1 = function (userId) {
var result = get_record_1(userId);
// do stuff with result
// using Q library to call out to async fs.writeFile, return Promise
fs_writeFile_promise(result)
.catch(function (error) {
console.log('error in sub_process_1_write', error);
})
.done(function () {
console.log('done with sub_process_1');
}
return userId;
}.future() // <-- if no future() here, the exception is thrown.
sub_process_2 = function (userId) {
var result = get_record_2(userId);
// do stuff with result
// using Q library to call out to async fs.writeFile, return Promise
fs_writeFile_promise(result)
.catch(function (error) {
console.log('error in sub_process_1_write', error);
})
.done(function () {
console.log('done with sub_process_1');
}
return userId;
}.future()
// async because of I/O operation (I think)
get_record_1 = function (userId) {
var record_1 = Mongo_Collection_Name.findOne({'userId': userId});
// do stuff
return record_1;
}
get_record_2 = function (userId) {
var record_2 = Mongo_Collection_Name.findOne({'userId': userId});
// do stuff
return record_2;
}
// async operation using Q library to return a Promise
fs_writeFile_promise = function (obj) {
var deferred = Q.defer();
fs.writeFile(obj.file, obj.datas, function (err, result) {
if (err) deferred.reject(err);
else deferred.resolve('write data completed');
});
return deferred.promise;
}
For now, lets assume that the process_2 function is exactly like process_1
Also, we should assume I have console.log('step_start') and console.log('step_end') in each function. This is what it would look like on the command line:
start processes
end processes
start processes 1
end processes 1
start processes 2
start sub processes 1
getting record 1
start sub processes 2
getting record 2
returning record 1
end sub processes 1
called writeData in sub process 1
returning record 2
called writeData in sub process 2
end processes 2
ending sub processes 1
The reason I had to place a Fiber (future) on the sub_process_1() function was because when I placed the function process_1() in the Q chain at the top I got the Error: Can't wait without a fiber].
If I remove the process_1() in the Q chain at the top and remove the .future() from sub_process_1() no exception is thrown.
Questions
Why does calling out to a Mongo collection within a Promise cause a
fiber error within a Meteor application?
Does calling a async function within a sync function in general cause the sync function to become a async function?
How do I solve this problem?
The most common way to solve this is wrap your asynchronous callbacks that use Meteor functions in Meteor.bindEnvironment().
If you are using the Meteor core WebApp package to handle your server side route, the code would be like this (also in meteorpad):
WebApp.connectHandlers.use(
'/test',
Meteor.bindEnvironment(function(req, res, next) {
var someSyncData = Players.findOne();
res.write(JSON.stringify(someSyncData));
res.end();
})
);
Working with fibers or promises yourself is unnecessary unless you are trying to get multiple async events to run concurrently.
To deal with file reading or other functions that are not already synchronous, Meteor also provides Meteor.wrapAsync() to make them synchronous.
There are also packages and a help page that give you other high level alternatives.
I have this code:
var queue = [];
var allParserd = [];
_.each(webs, function (web) {
queue.push(function () {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
allParserd.push(parsed);
});
});
});
Promise.all(queue).then(function (data) {
console.log(allParserd);
});
Basically I need to fetch all my webs and be sure to give the result after that every parsing is done. the function parseWebsite return the correct data, but in this way is not called and allParsed return just as an empty array. I'm sure that I miss some things, I've started to use the promises just from some days.
If you need some more information just tell me.
P.s.
I want that all the functions to start at the same time; I don't want to wait for each one response for going forward.
Tagged with Bluebird so let's use it:
First, let's convert your callback API to promises:
Promise.promisifyAll(WebsitesUtils);
Now, let's use .map to map every item in webs to it being parsed parseWebsite:
Promise.map(webs, function(item){
return WebsitesUtils.parseWebsiteAsync(item); // note the suffix
}).then(function(results){
// all the results are here.
}).catch(function(err){
// handle any errors
});
As you can see - this is trivial to do with Bluebird.
Promise.all doesn't take a queue of functions to execute. It expects an array of promises which represent the results of the many concurrently running (still pending) requests.
The first step is to have a function that actually returns a promise, instead of only executing a callback. We can use
function parseWebsite(web) {
return new Promise(function(fulfill, reject) {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
if (err)
reject(err);
else
fulfill(parsed);
});
});
}
or simply use promisification that does this generically:
var parseWebsite = Promise.promisify(WebsitesUtils.parseWebsite, WebsitesUtils);
Now we can go to construct our array of promises by calling that function for each site:
var promises = [];
_.each(webs, function (web) {
promises.push(parseWebsite(web));
});
or just
var promises = _.map(webs, parseWebsite);
so that in the end we can use Promise.all, and get back our allParsed array (which even is in the same order as webs was!):
Promise.all(promises).then(function(allParsed) {
console.log(allParsed);
});
Bluebird even provides a shortcut function so you don't need promises:
Promise.map(webs, parseWebsite).then(function(allParsed) {
console.log(allParsed);
});
Here's how might do it with async:
var async = require('async');
var webs = ...
async.map(webs, function(web, callback) {
WebsitesUtils.parseWebsite(web, callback);
}, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});
and if parseWebsite() isn't a prototype method dependent on WebsitesUtils then you could simplify it further:
async.map(webs, WebsitesUtils.parseWebsite, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});