I have found two handy modules called run-parallel and run-series to run arrays functions and return arrays of results.
Looking at the little number of contributors and stars in the Github projects, I wonder if there is a canonical way of doing these tasks instead of installing these modules?
Maybe there is a native way in Node or in ES6 to do this that I am missing?
Examples
First see some examples - scroll below for an explanation.
Callbacks:
Example with async and functions that take Node-style callbacks:
async.parallel([
(cb) => {
setTimeout(() => {
cb(null, 'one');
}, 200);
},
(cb) => {
setTimeout(() => {
cb(null, 'two');
}, 100);
},
],
(err, results) => {
if (err) {
// there was an error:
console.log('Error:', err);
return;
}
// we have ['one', 'two'] in results:
console.log('Results:', JSON.stringify(results));
});
Promises:
Example using functions that return promises - with Bluebird's delay() function:
const { delay } = require('bluebird');
Promise.all([
delay(200, 'one'),
delay(100, 'two'),
]).then((results) => {
// we have ['one', 'two'] in results:
console.log('Results:', JSON.stringify(results));
}).catch((err) => {
// there was an error:
console.log('Error:', err);
});
ES2017 async/await:
Using async/await:
const { delay } = require('bluebird');
try {
const results = await Promise.all([
delay(200, 'one'),
delay(100, 'two'),
]);
// we have ['one', 'two'] in results:
console.log('Results:', JSON.stringify(results));
} catch (err) {
// there was an error:
console.log('Error:', err);
}
I'm using JSON.stringify() to make it explicit what is the format of the data in results.
Note that even though the first value comes last, the original order is still preserved.
The last example must be run inside of a function declared with an async keyword, or wrapped in (async () => { ... })() like this:
(async () => {
try {
const results = await Promise.all([
delay(200, 'one'),
delay(100, 'two'),
]);
// we have ['one', 'two'] in results:
console.log('Results:', JSON.stringify(results));
} catch (err) {
// there was an error:
console.log('Error:', err);
}
})();
Generators and coroutines:
With no support for async/await you can use some generator-based coroutines, like those that comes from Bluebird:
const { delay, coroutine } = require('bluebird');
coroutine(function* () {
try {
const results = yield Promise.all([
delay(200, 'one'),
delay(100, 'two'),
]);
// we have ['one', 'two'] in results:
console.log('Results:', JSON.stringify(results));
} catch (err) {
// there was an error:
console.log('Error:', err);
}
})();
Explanation
There are many ways to do that and it all depends on the kind of functions that you want to run.
If you want to run a traditional Node-style functions that take error-first callbacks as their last arguments then the most popular way to run those in parallel or in series is the async module on npm:
https://www.npmjs.com/package/async
There is no built-in support for things like that in ES6 because those error-first callbacks are really a Node thing, not very popular in JavaScript outside of Node.
The ES6/ES7/ES8 goes in the direction of functions that return promises (instead of functions that take callbacks) and there is a new async/await syntax to make them look kind-of-like synchronous, with try/catch working for error handling.
So, the most popular way in Node to combine functions taking callbacks is async module:
https://www.npmjs.com/package/async
To work with promises a popular modules is Bluebird:
https://www.npmjs.com/package/bluebird
For more advanced tasks there is Task.js:
https://github.com/mozilla/task.js
See those answers for more info:
try/catch blocks with async/await
node.js ~ constructing chained sequence of Promise resolves
How to run Generator Functions in Parallel?
node.js ~ constructing chained sequence of Promise resolves
Using async/await + Bluebird to promisifyAll
jQuery: Return data after ajax call success
Remember that functions can just be passed around like any other object in JavaScript, so it's actually pretty straightforward to do things like this.
For example, to run things in series you could just use map to generate an array of results by executing each of your functions - like this...
let functions = [
function() { return 'abc'; },
function() { return 123; },
function() { return { key: 'value' }; }
];
let results = functions.map(fn => fn());
Obviously, what you get from using these libraries is nice - hopefully well tested - handling for when things go wrong etc. but it's perfectly possible to do it without them.
You can use Array.prototype.map() to loop through all functions in your array sequentially, using an anonymous function that executes each function in the array as a parameter.
Using Function.prototype.apply() instead of directly calling the function allows you to pass an array to each function with its parameters.
Example
var functions = [
function(x){return 3 * x},
function(x){return 4 * x},
function(x){return 8 * x},
function(x){return 10 * x},
function(){return Array.prototype.map.call(arguments, function(name){
return "Hello, " + name
})}
];
var results = functions.map(function(value, index) {
return value.apply(value, this[index]);
}, [[5], [1], [8], [8], ["Dolly", "major Tom"]]);
document.body.innerHTML = '<pre>' + JSON.stringify(results, null, '\t') + '</pre>';
See also this Fiddle.
The structure depends on what you want to implement.
:D
functions = {};
async function getArticle(id = 0) {
// const result = await articles.get(id); get result of a db.
const result = { id: 10, name: "apple" }
if (result) return result;
return false;
}
async function validatePermission(id = 0) {
// const result = await users.get(id); get result of a db.
const result = { id: 25 }
if (result) return result;
return false;
}
functions.getArticle = async params => {
const { userId, articleId } = params;
const [validUser, article] = await Promise.all([
validatePermission(userId),
getArticle(articleId),
]);
if (validUser === false || article === false) {
return { status: 400, message: 'Error', data: [] }
}
return { status: 200, message: 'Success', data: article };
}
const params = { userId: 25, articleId: 10 };
functions.getArticle(params)
.then(result => {
console.log(result);
});
Related
I'm trying to understand why the following two code blocks yield different results.
Code Block one works as expected and returns an array of the providers looked up from the database. On the other hand, Code Block two returns an array of functions. I feel like I'm missing something simple here in my understanding of Promise.all() and async / await.
The differences in the code blocks are:
Block 1: Array of promise functions is created and then wrapped in async functions using the map operator.
Block 2: Array of promises functions are created as async functions. Therefore, the map operator isn't called.
In case you aren't familiar with the Sequelize library, the findOne() method that gets called returns a promise.
Also worth mentioning, I know that I could use a single find query with and "name in" where clause to get the same results without creating an array of promises for multiple select queries. I'm doing this simply as a learning exercise on async / await and Promise.all().
CODE BLOCK 1: Using map() inside Promise.all()
private async createProfilePromises(profiles){
let profileProviderFindPromises = [];
//Build the Profile Providers Promises Array.
profiles.forEach(profile => {
profileProviderFindPromises.push(
() => {
return BaseRoute.db.models.ProfileProvider.findOne({
where: {
name: {[BaseRoute.Op.eq]: profile.profileProvider}
}
})}
);
});
//Map and Execute the Promises
let providers = await Promise.all(profileProviderFindPromises.map(async (myPromise) =>{
try{
return await myPromise();
}catch(err){
return err.toString();
}
}));
//Log the Results
console.log(providers);
}
CODE BLOCK 2: Adding the async function without the use of map()
private async createProfilePromises(profiles){
let profileProviderFindPromises = [];
//Build the Profile Providers Promises Array.
profiles.forEach(profile => {
profileProviderFindPromises.push(
async () => {
try{
return await BaseRoute.db.models.ProfileProvider.findOne({
where: {
name: {[BaseRoute.Op.eq]: profile.profileProvider}
}
});
}catch(e){
return e.toString();
}
}
);
});
//Execute the Promises
let providers = await Promise.all(profileProviderFindPromises);
//Log the Results
console.log(providers);
}
Your code basically boils down to:
const array = [1, 2, 3];
function fn() { return 1; }
array.map(fn); // [1, 1, 1]
array.push(fn);
console.log(array); // [1, 2, 3, fn]
You push a function (whether that is async or not does not matter), instead you want to push the result of calling the function:
array.push(fn());
or in your case:
array.push((async () => { /*...*/ })());
How I'd write your code:
return Promise.all(profiles.map(async profile => {
try{
return await BaseRoute.db.models.ProfileProvider.findOne({
where: {
name: { [BaseRoute.Op.eq]: profile.profileProvider }
}
});
} catch(e) {
// seriously: does that make sense? :
return e.toString();
}
}));
Description:
I have a Mocha Test within my Node-App that should test whether a DB-Export of Mongo-DB-JSON-Documents is done correctly.
In my test I besides other tests also test if the download-directory is not empty.
Expected result:
The test should await the downloads and only then check whether the directory is empty.
Actual Result:
The test returns always green.
My Question:
I understood that we have callbacks but promises are better.
I understood that async await is some syntactic sugar to promises.
And I understood that there is even RxJS (which I do not use here)
Somehow I have to deal with the callback from mogodb-backup.
See https://www.npmjs.com/package/mongodb-backup
I do not understand what I am doing wrong so that the tests always turn green (running in parallel to the download)
mocha-test.js
describe('Database.downloadDocumentsOfType_KEYS()', function () {
it('should result in data/exportFromCosmos/KEYS/admin/ag-data/ not being empty', function () {
const config = {
documents: ['DEFAULT', 'KEYS'],
exportpathDEFAULT: 'data/exportFromCosmos/DEFAULT/',
exportpathKEYS: 'data/exportFromCosmos/KEYS/',
uploadpath: 'data/uploadToAzureBlob/',
crosscheckFile: 'data/crosscheckFile.txt'
}
async function f() {
await Database.downloadDocumentsOfType_KEYS().then(expect(dir(config.exportpathKEYS + 'admin/ag-data/')).to.not.be.empty)
}
f()
})
})
Databasemodule-to-be-tested.js
const mongodbbackup = require('mongodb-backup')
const Database = {
downloadDocumentsOfType_DEFAULT: () => {
new Promise((resolve) => mongodbbackup({
uri: process.env.DB_CONNECTION_STRING_READ,
root: 'data/exportFromCosmos/DEFAULT',
parser: 'json',
callback: function(err) {
if (err) {
reject()
} else {
resolve()
}
}
)}
}
async function f() {
await Database.downloadDocumentsOfType_KEYS().then(e)
}
f()
This fires off the asynchronous function immediately and
it('...', function (){})
finishes immediately.
So you need to use
describe('...',async function(){
it('...',async function(){
const f = async function(){
await Database.downloadDocumentsOfType_KEYS();
expect(dir(config.exportpathKEYS + 'admin/ag-data/')).to.not.be.empty);
};
await f();
});
});
Also,
new Promise((resolve) => mongodbbackup({...
should be
new Promise((resolve,reject) => mongodbbackup({
Otherwise reject is undefined
I have a foreach loop, where I call an async function. How can I make sure that all the async functions called the specified callback function, and after that, run something?
Keep a counter.
Example:
const table = [1, 2, 3];
const counter = 0;
const done = () => {
console.log('foreach is done');
}
table.forEach((el) => {
doSomeAsync((err, result) => {
counter++;
if (counter === 3) {
done();
}
});
});
As the other answer says, you can use the async package which is really good. But for the sake of it I recommend using Promises and use the Vanila Promise.all(). Example:
const table = [1, 2, 3];
Promise.all(table.map((el) => {
return new Promise((resolve, reject) => {
doSomeAsync((err, result) => {
return err ? reject(err) : resolve(result);
});
});
}))
.then((result) => {
// when all calls are resolved
})
.catch((error) => {
// if one call encounters an error
});
You can use Async library for this. It has various useful utility functions.
There is a Queue function in it which can be used to execute a set of tasks and you get a callback when all tasks are executed where you can do whatever you want. You can also control the concurrency of your queue(how many tasks are executed at a time in parallel).
Here is a sample code-
// create a queue object with concurrency 2
var q = async.queue(function(task, callback) {
console.log('hello ' + task.name);
callback();
}, 2);
// The callback function which is called after all tasks are processed
q.drain = function() {
console.log('all tasks have been processed');
};
// add some tasks to the queue
q.push({name: 'foo'}, function(err) {
console.log('finished processing foo');
});
q.push({name: 'bar'}, function (err) {
console.log('finished processing bar');
});
I want to check if an async function throws using assert.throws from the native assert module.
I tried with
const test = async () => await aPromise();
assert.throws(test); // AssertionError: Missing expected exception..
It (obviously?) doesn't work because the function exits before the Promise is resolved.
Yet I found this question where the same thing is attained using callbacks.
Any suggestion?
(I'm transpiling to Node.js native generators using Babel.)
node 10 and newer
Since Node.js v10.0, there is assert.rejects which does just that.
Older versions of node
async functions never throw - they return promises that might be rejected.
You cannot use assert.throws with them. You need to write your own asynchronous assertion:
async function assertThrowsAsynchronously(test, error) {
try {
await test();
} catch(e) {
if (!error || e instanceof error)
return "everything is fine";
}
throw new AssertionError("Missing rejection" + (error ? " with "+error.name : ""));
}
and use it like
return assertThrowsAsynchronously(aPromise);
in an asynchronous test case.
Based on Bergi answer I've suggest more universal solution that utilizes original assert.throws for error messages:
import assert from 'assert';
async function assertThrowsAsync(fn, regExp) {
let f = () => {};
try {
await fn();
} catch(e) {
f = () => {throw e};
} finally {
assert.throws(f, regExp);
}
}
Usage:
it('should throw', async function () {
await assertThrowsAsync(async () => await asyncTask(), /Error/);
});
The answers given work, but I came across this issue today and came up with another solution, that I think is a little simpler.
// Code being tested
async function thisFunctionThrows() {
throw new Error('Bad response')
}
// In your test.
try {
await thisFunctionThrows()
assert.equal(1 == 0) // Never gets run. But if it does you know it didn't throw.
} catch (e) {
assert(e.message.includes('Bad response'))
}
Since the question is still getting attention, I'd like to sum up the two best solutions, especially to highlight the new standard method.
Node v10+
There's a dedicated method in the assert library, assert.rejects.
For older versions of Node
A fill from vitalets answer:
import assert from 'assert';
async function assertThrowsAsync(fn, regExp) {
let f = () => {};
try {
await fn();
} catch(e) {
f = () => {throw e};
} finally {
assert.throws(f, regExp);
}
}
You are going to want to use, assert.rejects() which is new in Node.js version 10.
At the high level, instead of assert.throws, we want something like assert.rejects, hopefully you can take this and run with it:
const assertRejects = (fn, options) => {
return Promise.resolve(fn()).catch(e => {
return {
exception: e,
result: 'OK'
}
})
.then(v => {
if (!(v && v.result === 'OK')) {
return Promise.reject('Missing exception.');
}
if (!options) {
return;
}
if (options.message) {
// check options
}
console.log('here we check options');
});
};
it('should save with error', async () => {
// should be an error because of duplication of unique document (see indexes in the model)
return await assertRejects(async () => {
patientSubscriber = await PatientSubscriber.create({
isSubscribed: true,
patient: patient._id,
subscriber: user._id
});
}, {
message: /PatientSubscriber validation failed/
});
});
I'm using babel to transpile my node.js#0.10.x code and I'm stuck with promises.
I need allSettled-type functionality that I could use in q and bluebird or angular.$q for example.
On babel's core-js Promise, there is no allSettled method.
Currently I'm using q.allSettled as a workaround:
import { allSettled } from 'q';
Is there something like that in babel polyfill? Alternatively, which is a good algorithm for me to try to implement?
2019 Answer
There was a proposal to add this function to the ECMAScript standard, and it has been accepted! Check out the Promise.allSettled docs for details.
Original Answer
If you take a look at the implementation of q.allSettled you'll see it's actually quite simple to implement. Here's how you might implement it using ES6 Promises:
function allSettled(promises) {
let wrappedPromises = promises.map(p => Promise.resolve(p)
.then(
val => ({ status: 'fulfilled', value: val }),
err => ({ status: 'rejected', reason: err })));
return Promise.all(wrappedPromises);
}
2020 answer:
What the other answers are trying to do is to implement Promise.allSettled themselves. This was already done by the core-js project.
What you need to do is to make babel polyfill Promise.allSettled for you via core-js. The way you configure it to do so is through #babel/preset-env like so:
presets: [
['#babel/preset-env', {
useBuiltIns: 'usage',
corejs: {version: 3, proposals: true},
}],
],
In your build artifact this will add a call to require("core-js/modules/esnext.promise.all-settled") which monkeypatches the .allSettled function to the promises API.
const allSettled = promises =>
Promise.all(promises.map(promise => promise
.then(value => ({ state: 'fulfilled', value }))
.catch(reason => ({ state: 'rejected', reason }))
));
Or if you insist on polyfilling it:
if (Promise && !Promise.allSettled) {
Promise.allSettled = function (promises) {
return Promise.all(promises.map(function (promise) {
return promise.then(function (value) {
return { state: 'fulfilled', value: value };
}).catch(function (reason) {
return { state: 'rejected', reason: reason };
});
}));
};
}
Taken from here
Alternatively, which is a good algorithm for me to try to implement?
create a new promise with an executor function
use a counter/result array in the scope of the executor
register a then() callback with each parent promise saving the results in the array
resolve/reject promise from step 1 when counter indicates that all parent promises are done
Here's my attempt at something similar, I have Newsletter service and in my case I wanted my allSettled promise to resolve with an array of all the results (rejections and resolutions), IN ORDER, once all the email_promises are settled (all the emails had gone out):
Newsletter.prototype.allSettled = function(email_promises) {
var allSettledPromise = new Promise(function(resolve, reject) {
// Keep Count
var counter = email_promises.length;
// Keep Individual Results in Order
var settlements = [];
settlements[counter - 1] = undefined;
function checkResolve() {
counter--;
if (counter == 0) {
resolve(settlements);
}
}
function recordResolution(index, data) {
settlements[index] = {
success: true,
data: data
};
checkResolve();
}
function recordRejection(index, error) {
settlements[index] = {
success: false,
error: error
};
checkResolve();
}
// Attach to all promises in array
email_promises.forEach(function(email_promise, index) {
email_promise.then(recordResolution.bind(null, index))
.catch(recordRejection.bind(null, index));
});
});
return allSettledPromise;
}
my implementation will be below
Promise.prototype.myAllSettled = function (arr = []) {
return new Promise(function processIterable(resolve, reject) {
let result = [];
arr.forEach((item) => {
item
.then((value) => {
result.push({ status: "fulfilled", value: value });
if (arr.length === result.length) resolve(result);
})
.catch((err) => {
result.push({ status: "rejected", reason: `${err}` });
if (arr.length === result.length) resolve(result);
});
});
});
};
Here's another take at the same functionality: spex.batch
The source code would be too much to re-post here, so here's just an example from the batch processing of how to use it:
var spex = require('spex')(Promise);
// function that returns a promise;
function getWord() {
return Promise.resolve("World");
}
// function that returns a value;
function getExcl() {
return '!';
}
// function that returns another function;
function nested() {
return getExcl;
}
var values = [
123,
"Hello",
getWord,
Promise.resolve(nested)
];
spex.batch(values)
.then(function (data) {
console.log("DATA:", data);
}, function (reason) {
console.log("REASON:", reason);
});
This outputs:
DATA: [ 123, 'Hello', 'World', '!' ]
Now let's make it fail by changing getWord to this:
function getWord() {
return Promise.reject("World");
}
Now the output is:
REASON: [ { success: true, result: 123 },
{ success: true, result: 'Hello' },
{ success: false, result: 'World' },
{ success: true, result: '!' } ]
i.e. the entire array is settled, reporting index-bound results.
And if instead of reporting the entire reason we call getErrors():
console.log("REASON:", reason.getErrors());
then the output will be:
REASON: [ 'World' ]
This is just to simplify quick access to the list of errors that occurred.