We Keep Coding

How to break Async series on SIGTERM?

Assume I have the following scenario -
async.series(
[
function (cbi) {
students.getAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
students.deleteAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.getAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.deleteAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
};
]
);
And I want a graceful cleanup when SIGTERM is sent. That is a clean up of all the students or all teachers whichever is in progress when the signal was sent should complete and the next one should not begin.
function (cbi) {
students.getAll('student', function (err, response) {
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
cbi(err, response);
});
}
I was thinking that I should set a global variable to keep track of the SIGTERM signal.
process.on('SIGTERM', function onSigterm () {
GLOBAL_VAR_SIGTERM = true;
}
Is there any better way to break an async series to break a SIGTERM signal?

As #adamrights points out in his answer, the main problem in your code is you didn't call cbi(err, response) with a truthy err 1st param, which is essential to stop async.series continuing to next task in the queue.
Now your code should work, but you have a repeating pattern in your code that goes:
function (cbi) {
students.getAll('student', function (err, response) {
// these 3 lines appear in every callback function
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
// end of repeat pattern
});
}
Your callback passed to every async task always do the same 3-liner thing. We know the DRY rule, it's always a good idea to extract the repeating pattern into another function to reuse it as much as possible.
So instead of repeatedly declaring anonymous functions, you should declare a factory function.
function callbackFactory(cbi) {
return function(err, response) {
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
}
}
// use arrow function to write more concise code
async.series(
[
cbi => students.getAll('student', callbackFactory(cbi)),
cbi => students.deleteAll('student', callbackFactory(cbi)),
cbi => teachers.getAll('teacher', callbackFactory(cbi)),
cbi => teachers.deleteAll('teacher', callbackFactory(cbi)),
]
);
Advance Topic: use decorator to handle cross cutting concern
Let's explore this topic a bit more. Obviously, abort early on receiving SIGTERM is a cross cutting concern that should be separated from business logic. Suppose your business logic varies from task to task:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length) // <- extra work
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length) // <- different extra work
cbi(err, response);
})
]
);
Because the callback is varying, it can be difficult to be extracted into a factory function like before. From this perspective, we'd better inject the abort-early behavior to each task, keep it easy to write normal business logic.
This is where decorator pattern comes handy. But global variable isn't the best tool to implement it, we'll use event listener.
The basic interface of the decorator looks like:
// `task` will be things like `cbi => students.getAll('student', ... )`
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
...
task(originalCbi)
}
}
Below is our implementation checklist:
we're going to call originalCbi if we receive SIGTERM
but when we don't receive SIGTERM, the originalCbi is still callable inside callback of any async task like normal
if originalCbi is ever called once, we should unsubscribe from SIGTERM to prevent memory leak
The implementation:
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
// subscribe to `SIGTERM`
var listener = () => originalCbi(new Error("SIGTERM: Aborting remaining tasks"));
process.once('SIGTERM', listener);
var wrappedCbi = (err, response) => {
// unsubscribe if `cbi` is called once
process.off('SIGTERM', listener);
return originalCbi(err, response);
};
// pass `cbi` through to `task`
task(wrappedCbi);
}
}
// Usage:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length)
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length)
cbi(err, response);
})
].map(decorateTaskAbortEarly) // <--- nice API
);

If you want to respond to the SIGTERM event from within the async.series() then you are correct, the easiest way would be to track with a global variable.
But you need to set the first parameter (error-first callback) in the cbi(err, response) function to true in order to break the series.
so:
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
should be more like:
if (err) logger.error(err);
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
// You could just do err = true
// But best practice is to use an Error instance.
Then because cbi(err, response) will be called with err value equal totrue the remaining tasks will not be run.

I liked the other answers. This is another way to achieve the same. I am using my own example:
var async = require('async');
var ifAsync = require('if-async')
var GLOBAL_VAR_SIGTERM = false;
async.series({
one: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('one');
callback(null, 1);
}, 1000);
}),
two: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('two');
callback(null, 2);
}, 1000);
}),
three: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('three');
callback(null, 3);
}, 1000);
}),
four: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('four');
callback(null, 4);
}, 1000);
}),
}, function (err, results) {
if (err) {
//Handle the error in some way. Here we simply throw it
//Other options: pass it on to an outer callback, log it etc.
throw err;
}
console.log('Results are ' + JSON.stringify(results));
});
process.on('SIGTERM', function onSigterm () {
console.log('SIGTERM caught');
GLOBAL_VAR_SIGTERM = true;
});
function notsigterm(callback) {
if (!GLOBAL_VAR_SIGTERM) return callback(null, true)
else return callback(null, false)
}
I am using a package called ifAsync which allows you to use a predicate notsigterm to decide if a callback should be invoked or not. If notsigterm returns true then the callback will be invoked else it will be skipped. This is similar answer to others but somehow I find this cleaner. Let me know if you have questions.

Cannot access db result in Nodejs, it always returns null

var robject=[];
async.waterfall([
function (callback) {
for(var i in serial){
Router.find({},{r_serial_no:serial[i]},function (err,routerData) {
robject = robject.concat(routerData);
});
}
console.log('Robject= '+robject); //THIS RETURNS NULL
callback(null, robject);
},
function (blogs, callback) {
res.render('index', {dispatched_data:dispatched_data });
callback(null, 'Ended..' );
}
], function (err, result) {
console.log(result);
});
this is my waterfall model, here i need to access the robject from schema.find method to outside of that method. but it always return null..
how to access that??

You have the syntax error:
for(var i in serial){
Router.find({},{r_serial_no: i},function (err,routerData) {
robject = robject.concat(routerData);
});
}
the "for" loop defines "i" as next item in the array each iteration

The problem I see here is in for...in loop. Your callback will be fired even if your process i.e. Router.find is not completed. You can try below code, It might help.
Unlike your serial object please create a array called serials.
var robject=[];
async.waterfall([
function (callback) {
async.each(serials,
function(serial, localCb){
Router.find({},{r_serial_no:serial},function (err,routerData) {
robject = robject.concat(routerData);
localCb()
});
},
function(err){
console.log('Robject= '+robject);
callback(null, robject);
}
);
},
function (blogs, callback) {
res.render('index', {dispatched_data:dispatched_data });
callback(null, 'Ended..' );
}
], function (err, result) {
console.log(result);
});

Binding a callback function when using Async Waterfall

I was trying for a while to figure out a good solution for abstracting a function from an async waterfall (async library) function (to keep my code DRY), but I kept getting an error saying that cb was not defined. Also, when passing just this to bind, the function that async was defined in was the scope, and when passing cb as well.
Eventually, I found a solution which works (adding cb as a function to the this object), but it looks a bit messy and there is probably a better way of doing it. Any suggestions?:
// registerController.js
const async = require('async');
const registerService = require('../services/register');
// Api endpoint code
...
// is there a better way than (this.cb = cb)
let callbackHandler = (err, res) => {
if (err) this.cb(err);
this.cb(null, res);
};
// asynchronously perform registration
async.waterfall([
(cb) => {
registerService.createAccount(username, email, callbackHandler.bind(this.cb = cb));
},
(res, cb) => {
registerService.doSomethingElse(domain, callbackHandler.bind(this.cb = cb);
},
....
// registerService.js
module.exports = {
createAccount: (username, email, callback) => {
httpService.request({
host: myHost,
method: 'POST',
path: '/api/signup',
body: {
username,
email
}
}).then((res) => {
return callback(null, res);
}).catch((err) => {
return callback(err);
});
},
...
}
Note: code refactored into Services file for unit testing purposes and lean controllers (taking an MVC approach)

You're not supposed to need any refactoring, async is already abstract. Just drop the callbackHandler and pass the cb directly into your function:
async.waterfall([
(cb) => {
myService.createAccount(fullName, email, cb);
},
(res, cb) => {
myService.doSomethingElse(domain, cb);
},
…
], …);
However, you get much simpler code if you just abandon async.js here and embrace promises:
// registerController.js
const registerService = require('../services/register');
// Api endpoint code
…
registerService.createAccount(username, email)
.then(res =>
registerService.doSomethingElse(domain) // supposed to return a promise as well
)
…
// registerService.js
exports.createAccount = (username, email) => { // no callback
return httpService.request({
// ^^^^^^ returns the promise
host: myHost,
method: 'POST',
path: '/api/signup',
body: {
username,
email
}
});
};
…

Node JS Sync Work flow with Async request

Currently try to learn Node JS and getting my head around Async/Sync workflow.
Try to the follow:
Step 1:
- Get data 1 with function 1
- Get data 2 with function 2
- Get data 3 with function 3
Step2:
- Work out logic with data 1,2,3
Step 3
- Do final call
I been looking at Q and Async packages but still havent really find an example.
Can some one show me how they will go about this issue in Node JS?
Thanks

Not entirely clear on your implementation, but depending on how specific your ordering needs to be you could try something like this:
var data1 = null;
var data2 = null;
var data3 = null;
async.series([
function(httpDoneCallback){
async.parallel([
function(data1Callback){
$http(...).then(function(response){
// some logic here
data1 = response;
data1Callback();
})
},
function(data2Callback){
$http(...).then(function(response){
// some logic here
data2 = response;
data2Callback();
})
},
function(data3Callback){
$http(...).then(function(response){
// some logic here
data3 = response;
data3Callback();
})
}
], function(){
//all requests dome, move onto logic
httpDoneCallback();
})
},
function(logicDoneCallback){
// do some logic, maybe more asynchronous calls with the newly acquired data
logicDoneCallback();
}
], function(){
console.log('all done');
})

Do you want function 1, 2, and 3 to trigger at the same time? If so then this should help:
var async = require('async');
async.parallel([
function(cb1) {
cb1(null, "one")
},
function(cb2){
cb2(null, "two")
},
function(cb3){
cb3(null, "three")
}
], function(err, results) {
console.log(results); // Logs ["one", "two", "three"]
finalCall();
});
To explain, every function in the array submitted as the first param to the parallel method will also receive a callback function. Activating the callback function signifies that you're done fetching your data or doing whatever you need to do in said function. All three functions will trigger at the same time, and once all three callbacks are called, the final function is called. The callback accepts two parameters: "error", and "result." If everything's successful, pass "null" as the error parameter. The results will be given to the final function as an array containing each of the results for your individual functions.

You can setup a chain of Promises to do things sequentially:
var funcA = () => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from A')
}, 1000)
});
}
var funcB = (dataFromA) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(dataFromA + ' data from B')
}, 2000)
})
}
var funcC = (dataFromB) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(dataFromB + ' data from C')
}, 500)
})
}
// Doing the functions on after another
funcA().then(funcB).then(funcC).then((data) => {
console.log(data);
})
Or if you want to do them all at the same time you can use Promise.all():
var promises = [];
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from A')
}, 1000)
}));
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from B')
}, 1000)
}));
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from C')
}, 1000)
}));
// Execute the array of promises at the same time, and wait for them all to complete
Promise.all(promises).then((data) => {
console.log(data);
})

Probably the best thing to do is use Promises like #Tyler here states. However, for conceptual understanding it is best to first understand the node callback pattern.
Because some tasks take time, we give the task a function and say 'When you are done, put the data you retrieved into this function'. These functions that we give to other functions are called callbacks. They must be constructed to accept the data, and also an error in case there is a problem while fetching the data. In Node the error is the first callback parameter and the data is the second.
fs.readFile('/file/to/read.txt', function callback(error, data) {
if (error) console.log(error);
else console.log(data);
});
In this example, once node reads the file, it will feed that data into the callback function. In the callback we must account for the case that there was a problem and handle the error.
In your question you want to do multiple async tasks and use their results. Therefore you must take this pattern and nest several of them. So, continuing this example, if there is no error you will begin another async task.
fs.readFile('/file/to/read.txt', function callback(error, data) {
if (error) console.log(error);
else {
someOtherAsyncThing(function callback(error, data2) {
if (error) console.log(error);
else {
console.log(data + data2)
}
});
}
});

Async series's callback doesn't fire

My problem here is that when I use this code I always get the callback in ensureAppIsValid and the one in the Async series seems to be never fired
var ReviewProcess = function (args) {
'use strict';
assert(args.application, 'Need an application to review');
this.app = args.application;
};
ReviewProcess.prototype.ensureAppIsValid = function (callback) {
'use strict';
if (this.app.isValid()) {
callback(null, this.app);
} else {
callback(this.app.validationMessage(), null);
}
};
ReviewProcess.prototype.processApplication = function (callback) {
'use strict';
async.series([
this.ensureAppIsValid(callback)
], function (err, callback) {
if (err) {
return callback(null, {
success: false,
message: err
});
}
callback(null, {
success: true,
message: 'Welcome to Mars'
});
});
};

It looks like you're using the word 'callback' too many times and the code isn't doing what you're expecting it to. You are passing the top level callback into the ensureAppIsValid() function, so once that function executes it doesn't go to async's callback. It also looks like you don't need the extra callback in async's follow up.
How about this:
ReviewProcess.prototype.processApplication = function (callback) {
'use strict';
async.series([
this.ensureAppIsValid(cb)
], function (err) {
if (err) {
return callback(null, {
success: false,
message: err
});
}
callback(null, {
success: true,
message: 'Welcome to Mars'
});
});
};

Async series requires a lists of functions as tasks to be ran.
You pass
this.ensureAppIsValid(callback)
But this is the call of the function, not the function itself.
Try this:
Async.series([
this.ensureAppIsValid.bind.apply(this.ensureAppIsValid, [null, [callback]])
], ... )

You shouldn't pass argument callback to this.ensureAppIsValid(). Instead, use here local callback parameter. For example, named cb.
Try:
var ReviewProcess = function (args) {
'use strict';
assert(args.application, 'Need an application to review');
this.app = args.application;
};
ReviewProcess.prototype.ensureAppIsValid = function (callback) {
'use strict';
if (this.app.isValid()) {
callback(null, this.app);
} else {
callback(this.app.validationMessage(), null);
}
};
ReviewProcess.prototype.processApplication = function (callback) {
'use strict';
async.series([
this.ensureAppIsValid(cb)
], function (err, callback) {
callback(null, {
success: !err,
message: err? err : 'Welcome to Mars'
});
}
});
};
I have also slightly changed your eventual callback in async.series. Now it's more compact.