Assume I have the following scenario -
async.series(
[
function (cbi) {
students.getAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
students.deleteAll('student', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.getAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
},
function (cbi) {
teachers.deleteAll('teacher', function (err, response) {
if (err) {
logger.error(err);
}
cbi(err, response);
});
};
]
);
And I want a graceful cleanup when SIGTERM is sent. That is a clean up of all the students or all teachers whichever is in progress when the signal was sent should complete and the next one should not begin.
function (cbi) {
students.getAll('student', function (err, response) {
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
cbi(err, response);
});
}
I was thinking that I should set a global variable to keep track of the SIGTERM signal.
process.on('SIGTERM', function onSigterm () {
GLOBAL_VAR_SIGTERM = true;
}
Is there any better way to break an async series to break a SIGTERM signal?
As #adamrights points out in his answer, the main problem in your code is you didn't call cbi(err, response) with a truthy err 1st param, which is essential to stop async.series continuing to next task in the queue.
Now your code should work, but you have a repeating pattern in your code that goes:
function (cbi) {
students.getAll('student', function (err, response) {
// these 3 lines appear in every callback function
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
// end of repeat pattern
});
}
Your callback passed to every async task always do the same 3-liner thing. We know the DRY rule, it's always a good idea to extract the repeating pattern into another function to reuse it as much as possible.
So instead of repeatedly declaring anonymous functions, you should declare a factory function.
function callbackFactory(cbi) {
return function(err, response) {
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
if (err) logger.error(err);
cbi(err, response);
}
}
// use arrow function to write more concise code
async.series(
[
cbi => students.getAll('student', callbackFactory(cbi)),
cbi => students.deleteAll('student', callbackFactory(cbi)),
cbi => teachers.getAll('teacher', callbackFactory(cbi)),
cbi => teachers.deleteAll('teacher', callbackFactory(cbi)),
]
);
Advance Topic: use decorator to handle cross cutting concern
Let's explore this topic a bit more. Obviously, abort early on receiving SIGTERM is a cross cutting concern that should be separated from business logic. Suppose your business logic varies from task to task:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length) // <- extra work
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length) // <- different extra work
cbi(err, response);
})
]
);
Because the callback is varying, it can be difficult to be extracted into a factory function like before. From this perspective, we'd better inject the abort-early behavior to each task, keep it easy to write normal business logic.
This is where decorator pattern comes handy. But global variable isn't the best tool to implement it, we'll use event listener.
The basic interface of the decorator looks like:
// `task` will be things like `cbi => students.getAll('student', ... )`
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
...
task(originalCbi)
}
}
Below is our implementation checklist:
we're going to call originalCbi if we receive SIGTERM
but when we don't receive SIGTERM, the originalCbi is still callable inside callback of any async task like normal
if originalCbi is ever called once, we should unsubscribe from SIGTERM to prevent memory leak
The implementation:
function decorateTaskAbortEarly(task) {
return (originalCbi) => {
// subscribe to `SIGTERM`
var listener = () => originalCbi(new Error("SIGTERM: Aborting remaining tasks"));
process.once('SIGTERM', listener);
var wrappedCbi = (err, response) => {
// unsubscribe if `cbi` is called once
process.off('SIGTERM', listener);
return originalCbi(err, response);
};
// pass `cbi` through to `task`
task(wrappedCbi);
}
}
// Usage:
async.series(
[
cbi => students.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateStudentCount(response.data.length)
cbi(err, response);
}),
cbi => teachers.getAll('student', (err, response) => {
if (err) {
logger.error(err);
return cbi(err);
}
updateTeacherCount(response.data.length)
cbi(err, response);
})
].map(decorateTaskAbortEarly) // <--- nice API
);
If you want to respond to the SIGTERM event from within the async.series() then you are correct, the easiest way would be to track with a global variable.
But you need to set the first parameter (error-first callback) in the cbi(err, response) function to true in order to break the series.
so:
if (err || GLOBAL_VAR_SIGTERM === true) {
logger.error(err);
}
should be more like:
if (err) logger.error(err);
if (GLOBAL_VAR_SIGTERM) err = new Error("SIGTERM: Aborting remaining tasks");
// You could just do err = true
// But best practice is to use an Error instance.
Then because cbi(err, response) will be called with err value equal totrue the remaining tasks will not be run.
I liked the other answers. This is another way to achieve the same. I am using my own example:
var async = require('async');
var ifAsync = require('if-async')
var GLOBAL_VAR_SIGTERM = false;
async.series({
one: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('one');
callback(null, 1);
}, 1000);
}),
two: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('two');
callback(null, 2);
}, 1000);
}),
three: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('three');
callback(null, 3);
}, 1000);
}),
four: ifAsync(notsigterm).then(function (callback) {
setTimeout(function () {
console.log('four');
callback(null, 4);
}, 1000);
}),
}, function (err, results) {
if (err) {
//Handle the error in some way. Here we simply throw it
//Other options: pass it on to an outer callback, log it etc.
throw err;
}
console.log('Results are ' + JSON.stringify(results));
});
process.on('SIGTERM', function onSigterm () {
console.log('SIGTERM caught');
GLOBAL_VAR_SIGTERM = true;
});
function notsigterm(callback) {
if (!GLOBAL_VAR_SIGTERM) return callback(null, true)
else return callback(null, false)
}
I am using a package called ifAsync which allows you to use a predicate notsigterm to decide if a callback should be invoked or not. If notsigterm returns true then the callback will be invoked else it will be skipped. This is similar answer to others but somehow I find this cleaner. Let me know if you have questions.
Related
I am trying to figure out how to make the second Request run after the first request returned a response.I tried to search on google, but I'm kinda new to Postman and I'm not sure what should I look for.
I tried to do something like:
pm.sendRequest(req1, function(err, res) {
pm.sendRequest(req2, function(err, done)
)});
But It didnt work
while(documentLength>0)
{
pm.sendRequest(listDocumentsRequest, function(err, res){
pm.environment.set('dossierId',res.json().documentsList[index].subDossierId)
pm.environment.set('documentId',res.json().documentsList[index].documentId)
});
pm.sendRequest(getDocumentRequest);
index++;
documentLength--;
}
So I'm trying to make the first Request(listDocumentsRequest) then wait till I got an answer, then run the second request (getDocumentRequest) ,wait till I got an answer then move to the next iteration.
Do you guys have any idea?
Best Regards
Edited after Chilly answer
while(documentLength>0)
{
const interval = setTimeout(() => {}, Number.MAX_SAFE_INTEGER);
function resolvedPromise() {
return new Promise((resolve, reject) => {
pm.sendRequest(listDocumentsRequest, (err, res) => {
if (err) {
console.log(err);
reject();
} else {
pm.environment.set('dossierId',res.json().documentsList[index].subDossierId)
pm.environment.set('documentId',res.json().documentsList[index].documentId)
resolve();
}
});
});
}
resolvedPromise()
.then(pm.request(getDocumentRequest))
.then(() => clearTimeout(interval))
.catch(err => {
console.log(err);
clearTimeout(interval);
});
index++;
documentLength--;
}
// Example with a plain string URL
pm.sendRequest('https://postman-echo.com/get', (error, response) => {
if (error) {
console.log(error);
} else {
// Example with a plain string URL
pm.sendRequest('https://postman-echo.com/get/1', (error, response) => {
if (error) {
console.log(error);
} else {
console.log(response);
}
});
}
});
You can simply chain the requests what is the error you are facing? in your case it looks like
while (documentLength > 0) {
pm.sendRequest(listDocumentsRequest, function (err, res) {
pm.environment.set('dossierId', res.json().documentsList[index].subDossierId)
pm.environment.set('documentId', res.json().documentsList[index].documentId)
pm.sendRequest(getDocumentRequest, function (err, res) {});
index++;
documentLength--;
});
}
I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone
You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}
To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.
I want to store all the data fetched from the database in arr_obj, then use this variable int async.forEachLimit function. For this reason i used async.series function, everything works fine except sleep(1000), code sleeps as soon as second function of async.series is called and then gives all the result together.Being a beginner in NodeJs i don't have much idea about all this.
var sleep = require('system-sleep');
//
//
var arr_obj = [];
async.series([
function (callback) {
Service.listAllUser(req.body, function (err, data) {
if(err) return callback(err);
arr_obj = data.toJSON();
callback();
});
},
function (callback1) {
console.log(arr_obj);
async.forEachLimit(arr_obj, 1, function (item, callback) {
Quality_Service.qualityService(item, function (err, data) {
if (err) return next(err);
console.log(data);
});
sleep(1000);
callback();
});
callback1();
}
], function (err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return next(err);
res.send("okay");
});
Try to use callback inside the foreach, with setTimeout
async.forEachLimit(arr_obj, function (item, callback) {
Quality_Service.qualityService(item, function (err, data) {
if (err) return next(err);
setTimeout(callback, 1000, err);
});
});
Is it a bad code design if I put multiple calls to the same service in a single function? Each call to the service returns a value. Are there other better ways of implementing this?
For example:
$scope.getAdditionalInfo = () => {
ExampleService
.methodName($scope.parameter)
.then(function (res) {
//do things here
}, function (err) {
alert(err.message);
console.log(err);
});
ExampleService
.methodName2($scope.parameter)
.then(function (res) {
//do things here
}, function (err) {
alert(err.message);
console.log(err);
});
ExampleService
.methodName3($scope.parameter)
.then(function (res) {
//do things here
}, function (err) {
alert(err.message);
console.log(err);
});
}
How can I change my logic to retry if the err.retryable = true in the following code:
async.each(queues, function (queue, callback) {
sqs.getQueueUrl({'QueueName': queue.queue}, function (err, qurl) {
if (err) {
if (err.retryable) {
// How to retry sqs.getQueueUrl({'QueueName': queue.queue}...?
} else {
console.error(err, err.stack);
callback(err);
}
}else{
//Do lots of things here
}
})
}, function (err) {
//...
})
In addition to the advice by dfsq to name your callback and use it an asynchronously recursive manner, see also async.retry from the async module by Caolan McMahon. Example:
async.retry(3, apiMethod, function(err, result) {
// do something with the result
});
More complex example:
async.auto(
{
users: api.getUsers.bind(api),
payments: async.retry(3, api.getPayments.bind(api))
}, function(err, results) {
// do something with the results
}
);
More details in the docs.
UPDATE
A better solution for your use case:
I wrote a utility function that you can use to make your original method support any number of retries (with err.retryable support).
You can use it this way:
var retryingFunction = withRetries(sqs, sqs.getQueueUrl);
(Note that you need to provide both sqs and sqs.getQueueUrl)
And now you can use the retryingFunction just like you would use sqs.getQueueUrl but with a number of retries as the first arguments. The retries will only be done when err.retryable is true.
So now, instead of:
sqs.getQueueUrl({'QueueName': queue.queue}, function (err, qurl) {
// ...
});
you can use:
retryingFunction(3, {'QueueName': queue.queue}, function (err, qurl) {
// ...
});
where 3 is the number of retries.
And this is the function that I wrote to make the above possible:
function withRetries(obj, method) {
if (!method) {
method = obj;
obj = null;
}
if (typeof method != "function") {
throw "Bad arguments to function withRetries";
}
var retFunc = function() {
var args = Array.prototype.slice.call(arguments);
var retries = args.shift();
var callback = args.pop();
if (typeof retries != "number" || typeof callback != "function") {
throw "Bad arguments to function returned by withRetries";
}
var retryCallback = function (err, result) {
if (err && err.retryable && retries > 0) {
retries--;
method.apply(obj, args);
} else {
callback(err, result);
}
};
args.push(retryCallback);
method.apply(obj, args);
};
return retFunc;
}
See this LIVE DEMO to play with it and see how it works.
It works fine in the demo, I hope it will also work for your code.
You can give queue callback a name and provide it in retry request again. Try this:
async.each(queues, function (queue, callback) {
sqs.getQueueUrl({'QueueName': queue.queue}, function queueCallback(err, qurl) {
if (err) {
if (err.retryable) {
sqs.getQueueUrl({'QueueName': queue.queue}, queueCallback);
} else {
console.error(err, err.stack);
callback(err);
}
} else {
//Do lots of things here
}
});
}, function (err) {
//...
});