I'm currently using two approaches to parallelize the execution of async functions in JavaScript Node.js. I do not intend to find which of the Promise or callback approach is best, but I'm likely searching for a better approach to parallelize functions execution in Node.js without using child_process.fork(), so that it can be used in the same process.
Assumed that my functions are Promise, hence a typical function will be like
var myFunc = () => new Promise((resolve, reject) => {
setTimeout(() => resolve("some task"), 500)
})
myFunc().then(_ => console.log(_)).catch(err => console.err(err))
One approach is using Promise.all in this way:
var promiseAllP = function(items, block) {
var promises = [];
items.forEach(function(item, index) {
promises.push(function(item, i) {
return new Promise(function(resolve, reject) {
return block.apply(this, [item, index, resolve, reject]);
});
}(item, index))
});
return Promise.all(promises);
} //promiseAll
promiseAllP([...Array(3).keys()], (item, index, resolve, reject) => {
setTimeout(() => resolve("task "+index), 500 * index)
})
.then(_ => console.log(_)).catch(err => console.err(err))
the other approach would be using async execution with a callback like
var tasks = [
(cb) => setTimeout(() => cb(null, "task 1"), 500 * 1),
(cb) => setTimeout(() => cb(null, "task 2"), 500 * 2),
(cb) => setTimeout(() => cb(null, "task 3"), 500 * 3),
];
function parallel(tasks, cb) {
var results, pending, keys
var isSync = true
if (Array.isArray(tasks)) {
results = []
pending = tasks.length
} else {
keys = Object.keys(tasks)
results = {}
pending = keys.length
}
function done(err) {
function end() {
if (cb) cb(err, results)
cb = null
}
if (isSync) process.nextTick(end)
else end()
}
function each(i, err, result) {
results[i] = result
if (--pending === 0 || err) {
done(err)
}
}
if (!pending) {
// empty
done(null)
} else if (keys) {
// object
keys.forEach(function(key) {
tasks[key](function(err, result) {
each(key, err, result)
})
})
} else {
// array
tasks.forEach(function(task, i) {
task(function(err, result) {
each(i, err, result)
})
})
}
isSync = false
}
parallel(tasks, (err, res) => {
if (err) console.error(error)
else console.log(res)
});
The last one can be easily adapted to support Promise instead of callback. The problem is that I didn't find any improvement in executing my tasks through the first approach or the second one, so which is the best solution among them or is out of there a better approach to execute parallel tasks without using Node.js child_process.fork()?
The tasks to run have different execution times, they do not share memory, and I can run a callback or Promise.resolve to notify the caller that the task ended gracefully or Promise.reject if some error occurred.
Related
I have an exercise where I have to build some async functions (not using await/async, Promises are allowed).
funcs need to be a list of those x functions I make.
Each of the function has to receive callback as an argument.
The callback is made with (err,res) where err could be either null or the statement of error (I don't know how to determine any error there).
Function together need to return an Array with the results of those functions, for which the pred is fulfilled.
Before I go to together function, I have to ask you, how to get into the value of Promise if I don't want to console.log it, but I want for example to either return it in callback or just put it into a const. If I make res.then(...) I am only able to console.log that result, but am not able to put it into let result (because of async probably the result is always returned as 0).
My code:
function fun1(value,cb) {
const promise1 = new Promise((resolve, reject) => {
setTimeout(() => resolve( value * 100), 500)
})
return cb(null,promise1)
}
function fun2(value,cb) {
const promise2 = new Promise((resolve, reject) => {
setTimeout(() => resolve( value * 100), 500)
})
return cb(null, promise2)
}
function fun3(value,cb) {
const promise3 = new Promise((resolve, reject) => {
setTimeout(() => resolve( value * 100), 2000)
})
return cb(null, promise3)
}
function callback(err, res) {
let result = 0;
res.then(x => {
if(x>200) {
console.log(x)
result = x
}
else {
console.log(err)
result = err
}
})
}
console.log(fun1(1,callback))
//const together = (funcs, pred) => (n) => { ... };
I need to write a JavaScript module that exports a function createJobQueue, which creates an asynchronous job queue.
This job queue should have 3 methods:
addJob
cancelJob
processAllJob
addJob adds a job to the end of the queue. It should return a promise that resolves with the value returned by job, whenever job ends up getting executed. (But addJob itself should not trigger execution of any jobs.) If job throws an error, then the promise returned by addJob should be rejected.
cancelJob removes a job from the queue. This should reject the promise returned by addJob. If no matching job is found, it does nothing.
When processAllJobs is called, the queue should process jobs (by invoking them) one-at-a-time in FIFO order until there are none left, then resolve with the number of jobs successfully processed (i.e., that did not reject or throw an error).
If any job cannot be processed (because job rejects or throws an error when invoked) the promise returned by addJob should be rejected. However, this should not stop processAllJobs from processing the rest of the queue.
What I tried so far
addJob is working good but I can't make cancelJob to reject job added by addJob and I can't make proccessAllJobs work one-at-a-time.
Here's the code:
function createJobQueue() {
return new class jobQueue{
constructor() {
this.jobs = [];
}
addJob(job) {
return new Promise((resolve, reject) => {
if (job()) {
this.jobs.push(job);
//console.log(this.jobs)
resolve(job());
}
throw new Error;
reject();
})
}
cancelJob(job) {
return new Promise(resolve => {
const index = this.jobs.findIndex(j => j === job);
if (index === -1) {
resolve();
} else {
this.jobs = this.jobs.filter((job, i) => i !== index);
}
})
}
processAllJobs() {
return new Promise(resolve => {
let count = 0;
this.jobs.forEach((f, index) => {
if(index === this.jobs.length-1) {
f()
.then(() => {
++count
})
.catch((e) => {
resolve(count);
})
}
//console.log("count", count);
f()
.then(() => {
count++
})
.catch()
})
resolve(count);
});
}
getJobs() {
console.log(this.jobs, "Next only jobs")
console.log(this.jobs.forEach(f => f().then(console.log)));
}
}
}
const queue = createJobQueue();
queue.addJob(function() {
return Promise.resolve("One")
});
queue.addJob(function() {
return Promise.resolve("Two")
});
queue.addJob(function() {
return Promise.resolve("Three")
});
//queue.getJobs();
console.log(queue.processAllJobs());
module.exports = { createJobQueue };
Your implementation of addJobs goes against the requirements: it does execute the job, while it shouldn't. Realise that the promise constructor callback is executed immediately (synchronously).
Secondly, the trouble you have in sequencing the job execution can be more easily avoided by using async/await.
I would also suggest creating the job queue as a Map keyed by job: that way you can easily find an entry by a given job.
Finally, as you need to be able to resolve/reject a job's promise at some later time, when processAllJobs is called, you will need a reference to the appropriate resolve/reject functions. You could store those in the job queue: one pair per job.
Here is how it could look:
class JobQueue{
constructor() {
this.jobs = new Map; // so we can key by job.
}
addJob(job) {
return new Promise((resolve, reject) =>
this.jobs.set(job, {resolve, reject})
);
}
cancelJob(job) {
let deferred = this.jobs.get(job);
if (this.jobs.delete(job)) deferred.reject("job cancelled");
}
async processAllJobs() {
let count = 0;
for (let [job, {resolve, reject}] of this.jobs) {
try {
resolve(await job());
count++;
} catch(e) {
reject(e);
}
}
return count;
}
}
function createJobQueue() {
return new JobQueue;
}
// Demo
const queue = createJobQueue();
// Instead of Promise.resolve, use a bit more flexible
// promise-generation function:
const delay = (ms, value, err) => new Promise((resolve, reject) =>
setTimeout(() => {
if (err) reject(err); else resolve(value);
console.log(err || value);
}, ms)
);
queue.addJob(() => delay(500, "One"));
queue.addJob(() => delay(500, "Two"));
// add an example of a rejecting promise:
queue.addJob(() => delay(500, null, "Failing job"));
queue.addJob(() => delay(500, "Three"));
queue.processAllJobs().then(count =>
console.log("count:", count)
);
I'm trying to merge the data from two JSON files (customer list) and then, create as many task as customers there are to be executed later via async.js
In order to have all the tasks array full, I'm controlling the end of the forEach with a promise:
var parsePromise = new Promise(function (resolve, reject) {
mongoDB.MongoClient.connect(dbURL, (error, db) => {
originalData.forEach(function (element, index) {
var restoredCustomer = Object.assign(element, backupData[index]);
tasksArray.push((function (db, element) {
return function () {
db.collection('customers').insert(element, function (error) {
if (error) {
console.error("Error: ", error);
}
})
}
})(db, restoredCustomer));
forEachCounter--;
if (forEachCounter === 0) {
resolve(tasksArray);
}
});
});
});
Then, when the promise ends, I execute the async.js parallel method:
parsePromise.then(function (tasksArray) {
async.parallel(tasksArray, (err, results) => {
console.log("Operation completed successfully: ", results);
})
})
The weird thing is that the code is working fine, and the inserts are being done on MongoDB, however, the console.log("Operation completed successfully: ", results); placed in the parallel callback is never shown.
Try this
var parsePromise = new Promise(function (resolve, reject) {
mongoDB.MongoClient.connect(dbURL, (error, db) => {
originalData.forEach(function (element, index) {
var restoredCustomer = Object.assign(element, backupData[index]);
tasksArray.push(function(cb){
db.collection('customers').insert(restoredCustomer, function (error) {
if (error) return cb(err);
cb();
})
});
forEachCounter--;
if (forEachCounter === 0) {
resolve(tasksArray);
}
});
});
});
According to async js documentation
https://caolan.github.io/async/global.html
tasks arguments in parallel function in async.js must be AsyncFunction.
AsyncFunction requires to have the final parameter which is a callback. The structure of the callback follows the standard of function(err, args).
Also you can try simplifying your code to:
new Promise((resolve, reject) => {
mongoDB.MongoClient.connect(dbURL, (error, db) => {
if (error) return reject(error);
resolve(db);
});
}).then(db => {
var tasksArray = [];
for (let i = 0; i != originalData.length; ++i){
var restoredCustomer = Object.assign(originalData[i], backupData[index]);
tasksArray.push(function(cb){
db.collection('customers').insert(restoredCustomer, function (error) {
if (error) return cb(err);
cb();
})
});
}
return new Promise((resolve, reject) => {
async.parallel(tasksArray, (err, results) => {
if (err) return reject(err);
resolve(results)
})
})
})
I have a list of promises.
var p1 = {
run: function () {
return new Promise(function (resolve, reject) {
setTimeout(function () {
resolve("Promise 1 done");
}, 2000);
})
}
};
var p2 = {
run: function () {
return new Promise(function (resolve, reject) {
setTimeout(function () {
reject("Promise 2 reject");
}, 1000);
})
}
};
var p3 = {
run: function () {
return new Promise(function (resolve, reject) {
setTimeout(function () {
resolve("Promise 3 done");
}, 1500);
})
}
};
I want to execute [p1,p2,p3] in sequence. I writed a function Process.sequence to act like Promise.all() (resolve when all promises resolve, and reject right after a promise rejects)
Process = {
sequence: function(promises){
window.promises = promises;
return new Promise(function (resolve, reject) {
promises.reduce(function (sequence, promise) {
return sequence.then(function () {
return promise.run();
}).then(function (result) {
console.log(result);
if (promises.indexOf(promise) == promises.length - 1) {
resolve("All Done");
}
}).catch(function (reason) {
reject(reason);
});
}, Promise.resolve());
});
}
};
But when i run Process.sequence...
Process.sequence([p1,p2,p3]).then(function(result){
console.log(result);
}, function(reason){
console.log(reason);
});
... the p3 still executed even p2 had rejected before.
Here is the result i expect:
Promise 1 done
Promise 2 reject
But this is the real result:
Promise 1 done
Promise 2 reject
Promise 3 done
What wrong with my function Process.sequence?
UPDATE
Thank #JaromandaX for your support. The function Process.sequence should be like this.
Process = {
sequence: function(promises) {
return promises.reduce(function(sequence, promise) {
return sequence.then(function() {
return promise.run();
}).then(function(result) {
console.log(result);
});
}, Promise.resolve()).then(function() {
return "All Done";
});
}
};
As you want the results to contain all of the fulfilled values, and the promises only to be created ("run") when all previous ones were fulfilled, you should make some changes:
Make your loop asynchronous, as you can only know whether to continue with the next promise or not when the previous one has resolved.
Stop looping when a promise rejects
Concatenate the results in an array as you progress
Furthermore, I would not call a variable "promise" when it is not a promise object... that will only bring confusion. Call it task or something. The promise here is what the task.run() method returns.
Here is how I would suggest to do it:
// The p1, p2, p3 objects have been written more concisely using a helper function:
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
const p1 = { run: _ => wait(2000).then(_ => "Promise 1 fulfilled") };
const p2 = { run: _ => wait(1000).then(_ => { throw "Promise 2 rejected" }) };
const p3 = { run: _ => wait(1500).then(_ => "Promise 3 fulfilled") };
const Process = {
sequence: function (tasks) {
return (function loop(results) {
return results.length >= tasks.length
// All promises were fulfilled: return the results via a promise
? Promise.resolve(results)
// Create the next promise
: tasks[results.length].run()
// When it fulfills, collect the result, and chain the next promise
.then(value => loop(results.concat(value)))
// When it rejects, return a rejected promise with
// the partial results and the reason of rejection
.catch(reason => { throw results.concat(reason) });
})([]); // Start with an empty results array
}
};
console.log('Wait for the results to come in...');
Process.sequence([p1, p2, p3]).then(function(result){
console.log('Fulfilled: ', result);
}).catch(function(reason){
console.log('Rejected: ', reason);
});
As browsers have started to support async/await you could also use this more procedural looking code:
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
const p1 = { run: _ => wait(2000).then(_ => "Promise 1 fulfilled") };
const p2 = { run: _ => wait(1000).then(_ => { throw "Promise 2 rejected" }) };
const p3 = { run: _ => wait(1500).then(_ => "Promise 3 fulfilled") };
const Process = {
sequence: async function (tasks) {
const results = [];
for (let task of tasks) {
try {
results.push(await task.run());
} catch(err) {
throw results.concat(err);
}
}
return results;
}
};
console.log('Wait for the results to come in...');
Process.sequence([p1, p2, p3]).then(function(result){
console.log('Fulfilled: ', result);
}).catch(function(reason){
console.log('Rejected: ', reason);
});
I have a situation where I think the only choice for me is to nest some Promises within each other. I have a Promise that needs to be performed and a method that does something until that Promise is complete. Something like this:
let promise = new Promise((resolve, reject) => {
// Do some stuff
});
doSomethingUntilPromiseisDone(promise);
However, within my Promise, I need to execute another method that returns another Promise:
let promise = new Promise((resolve, reject) => {
fetchValue(url)
.then((value) => {
// Do something here
}).catch((err) => {
console.error(err);
});
});
doSomethingUntilPromiseisDone(promise);
But now, in the fetchValue method's then statement, I have another method I need to execute that, guess what, returns another Promise:
let promise = new Promise((resolve, reject) => {
fetchValue(url)
.then((value) => {
saveToCache(value)
.then((success) => {
console.log('success!!');
resolve('success');
});
}).catch((err) => {
console.error(err);
});
});
doSomethingUntilPromiseisDone(promise);
So in the end, I have a Promise, within a Promise, within a Promise. Is there someway I can structure this better so that it is more straightforward? It seems like nesting them within each other is counter to Promise's intended chaining approach.
Use .then()
let doStuff = (resolve, reject) => {/* resolve() or reject() */};
let promise = new Promise(doStuff);
doSomethingUntilPromiseisDone(
promise
.then(value => fetchValue(url))
.then(value => value.blob())
.then(saveToCache)
)
.then(success => console.log("success!!"))
.catch(err => console.error(err))
you can use generator to flatten your nested promises (Bluebird.couroutine or Generators)
//Bluebird.couroutine
const generator = Promise.coroutine(function*() {
try {
const value = yield fetchValue(url);
const success = yield saveToCache(value);
console.log('success:', success);
} catch(e) {
console.error(err);
}
}));
generator();
Each function will call the next one with the result of the method before.
var promises = [1,2,3].map((guid)=>{
return (param)=> {
console.log("param", param);
var id = guid;
return new Promise(resolve => {
// resolve in a random amount of time
setTimeout(function () {
resolve(id);
}, (Math.random() * 1.5 | 0) * 1000);
});
}
}).reduce(function (acc, curr, index) {
return acc.then(function (res) {
return curr(res[index-1]).then(function (result) {
console.log("result", result);
res.push(result);
return res;
});
});
}, Promise.resolve([]));
promises.then(console.log);