How can I change my logic to retry if the err.retryable = true in the following code:
async.each(queues, function (queue, callback) {
sqs.getQueueUrl({'QueueName': queue.queue}, function (err, qurl) {
if (err) {
if (err.retryable) {
// How to retry sqs.getQueueUrl({'QueueName': queue.queue}...?
} else {
console.error(err, err.stack);
callback(err);
}
}else{
//Do lots of things here
}
})
}, function (err) {
//...
})
In addition to the advice by dfsq to name your callback and use it an asynchronously recursive manner, see also async.retry from the async module by Caolan McMahon. Example:
async.retry(3, apiMethod, function(err, result) {
// do something with the result
});
More complex example:
async.auto(
{
users: api.getUsers.bind(api),
payments: async.retry(3, api.getPayments.bind(api))
}, function(err, results) {
// do something with the results
}
);
More details in the docs.
UPDATE
A better solution for your use case:
I wrote a utility function that you can use to make your original method support any number of retries (with err.retryable support).
You can use it this way:
var retryingFunction = withRetries(sqs, sqs.getQueueUrl);
(Note that you need to provide both sqs and sqs.getQueueUrl)
And now you can use the retryingFunction just like you would use sqs.getQueueUrl but with a number of retries as the first arguments. The retries will only be done when err.retryable is true.
So now, instead of:
sqs.getQueueUrl({'QueueName': queue.queue}, function (err, qurl) {
// ...
});
you can use:
retryingFunction(3, {'QueueName': queue.queue}, function (err, qurl) {
// ...
});
where 3 is the number of retries.
And this is the function that I wrote to make the above possible:
function withRetries(obj, method) {
if (!method) {
method = obj;
obj = null;
}
if (typeof method != "function") {
throw "Bad arguments to function withRetries";
}
var retFunc = function() {
var args = Array.prototype.slice.call(arguments);
var retries = args.shift();
var callback = args.pop();
if (typeof retries != "number" || typeof callback != "function") {
throw "Bad arguments to function returned by withRetries";
}
var retryCallback = function (err, result) {
if (err && err.retryable && retries > 0) {
retries--;
method.apply(obj, args);
} else {
callback(err, result);
}
};
args.push(retryCallback);
method.apply(obj, args);
};
return retFunc;
}
See this LIVE DEMO to play with it and see how it works.
It works fine in the demo, I hope it will also work for your code.
You can give queue callback a name and provide it in retry request again. Try this:
async.each(queues, function (queue, callback) {
sqs.getQueueUrl({'QueueName': queue.queue}, function queueCallback(err, qurl) {
if (err) {
if (err.retryable) {
sqs.getQueueUrl({'QueueName': queue.queue}, queueCallback);
} else {
console.error(err, err.stack);
callback(err);
}
} else {
//Do lots of things here
}
});
}, function (err) {
//...
});
Related
Trying to write unit tests for reading a json file with the readFile function however I get the error: Timeout - Async callback was not invoked within the 5000ms timeout specified by jest.setTimeout. I must be doing something wrong when mocking the json file.
The function:
function jsonReader(filePath, cb) {
fs.readFile(filePath, (err, fileData) => {
if (err) {
return cb && cb(err);
}
try {
const object = JSON.parse(fileData);
return object;
} catch (err) {
return cb && cb(err);
}
});
}
module.exports = jsonReader;
Then the testfile:
const jsonReader = require('.././ReadJson');
jest.mock('fs', () => {
const MOCK_FILE_INFO = { 'test.json': JSON.stringify({ name: 'myname' }) };
return {
readFile: (fpath, opts) => {
if (fpath in MOCK_FILE_INFO) {
return MOCK_FILE_INFO[fpath];
}
}
};
});
test('Test file', (done) => {
function callback(data) {
expect(data.name).toBe('myname');
done();
}
jsonReader('test.json', callback);
});
I tried to change the timeout but if I put it higher the execution also takes longer and it's still giving the same error.
You're trying to use your functions synchronously?
jest.mock('fs', () => {
const MOCK_FILE_INFO = { 'test.json': JSON.stringify({ name: 'myname' }) };
return {
readFile: (fpath, callback) => {
if (fpath in MOCK_FILE_INFO) {
callback(null, MOCK_FILE_INFO[fpath]);
}
}
};
});
function jsonReader(filePath, cb) {
fs.readFile(filePath, (err, fileData) => {
if (err) {
return cb && cb(err);
}
try {
const object = JSON.parse(fileData);
cb(object);
} catch (err) {
return cb && cb(err);
}
});
}
module.exports = jsonReader;
I would like create find method in my User object. This function should returns user. But for this example it returns only the text.
But I don't know how return value for waterfall.
When I run
console.log(User.find("575578a9f95d6de1354327ef"));
I got 'undefined' in my output, but I except 'function find shoud return this value', What I should to do if I want get 'function find shoud return this value' text on my output
User = {
collectionName: 'users',
find: function(id){
async.waterfall(
[
function(callback) {
MongoClient.connect('mongodb://127.0.0.1:27017/lingogo', function(err,db) {
if(err) { throw err}
callback(null, db, id);
});
},
function(db,id, callback) {
var collection = db.collection(User.collectionName);
collection.find({'_id': ObjectID(id)}).toArray(function (err, result) {
if (err) { throw err };
if (result[0] && result[0]._id != '') {
return callback(null,result[0]);
}
return callback(null,null);
})
},
],
function (err, user) {
return 'function find shoud return this value';
}
);
}
}
console.log(User.find("575578a9f95d6de1354327ef"));
Function find must have a callback too, that you call in a callback of waterfall. You cannot return a value synchronously from an asynchronous function.
find: function (id, callback) {
async.waterfall(..., function (...) {
callback(null, return_value);
});
}
That should be called like
User.find("575578a9f95d6de1354327ef", function (err, return_value) {
console.log(return_value);
});
I am having issues with callback functions - I have a main function which has two callback functions inside. This is the main function
socket.on('play next video', function(data) {
removeVideo(cue[0], getCueFromDb(function() {
io.sockets.emit('next video');
}));
});
My removeVideo function looks like this:
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
});
if (callback)
callback();
else
return
}
and the getCueFromDb function looks like this
function getCueFromDb(callback) {
Video.find({}).exec(function(err, videos) {
if (err) {
console.log(err)
}
if (videos.length) {
cue.length = 0 // empty array
videos.forEach(function(video) {
cue.push(video.id) // push all the videos from db into cue array
});
io.sockets.emit('send cue', {cue: cue});
}
else {
console.log("No more videos in database!")
}
if (callback)
callback();
else
return
});
}
However the functions aren't getting called in the correct order - am I doing something wrong?
Your callback needs to be inside your find.remove()
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
if (callback)
callback();
else
return
});
You have to change removeVideo so callback we be called only after delete.
That's the code:
function removeVideo(id, callback) {
Video.find({'id' : id}).remove(function(err, data) {
if (err)
console.log(err)
console.log("Removed video", id)
if (callback)
callback();
else
return
}
});
So callback will be called only after vide is really removed.
I am having a weird problem with calling a callback inside another callback from mongoose.
Setup : MEAN Stack.
myFunc = function (cb) {
var projection = {
'_id': 0,
'var1': 1,
'var2': 1
}
var order = {
'var1': 1
}
User.find({})
.select(projection).sort(order)
.exec(function(err, docs){
if(err){
console.log(err);
cb(err,docs);
} else {
console.log(docs);
cb(err,docs);
}
});
};
going to the lines where cb(err,docs) will result in
"ReferenceError: cb is not defined"
the weird part is
I have functions with even deeper nested callbacks that can invoke the "cb" normaly.
myFunc = function(cb){
model1.count({var1:'test'}, function (err, count) {
if(count) {
model2.findOne({dat1:'hoho'}, function (err, doc){
if (err) {
console.error(err);
cb(err,doc);
} else {
cb(err,doc);
}
});
} else {
cb({message: "No items found"}, null);
}
})
}
The code above will be invoked like so...
function init(something){
myfunc(function(err, doc) {
if (err){
console.log(err.message);
} else {
//do something about doc
}
});
}
ugh, it seems that the calling function did not properly follow the rules.
it called
myFunc(json, function(err,doc){
//do something
})
wrong param count...
Suppose this simple piece of code:
/*jslint node: true */
"use strict";
function MyClass(db) {
var self = this;
this._initError = new Error("MyClass not initialized");
db.loadDataAsyncronously(function(err, data) {
if (err) {
self._initError =err;
} else {
self._initError = null;
self.data = data;
}
});
}
MyClass.prototype.getA = function(cb) {
if (this._initError) {
return cb(this._initError);
}
return cb(null, this.data.a);
};
MyClass.prototype.getB = function(cb) {
if (this._initError) {
return cb(this._initError);
}
return cb(null, this.data.b);
};
var db = {
loadDataAsyncronously: function(cb) {
// Emulate the load process from disk.
// Data will be available later
setTimeout(function() {
cb(null, {
a:"Hello",
b:"World"
});
},1000);
}
};
var obj = new MyClass(db);
obj.getA(function (err, a) {
if (err) {
console.log(err);
} else {
console.log("a: " + a);
}
});
obj.getB(function (err, b) {
if (err) {
console.log(err);
} else {
console.log("a: " + b);
}
});
This gives an error because obj is not initialized when getA and getB methods are called. I would like that any method called before the object is initialized, be delayed automatically until the class finish its initialization.
One way to solve it is this way:
/*jslint node: true */
"use strict";
function MyClass(db) {
var self = this;
self._pendingAfterInitCalls = [];
db.loadDataAsyncronously(function(err, data) {
if (!err) {
self.data = data;
}
self._finishInitialization(err);
});
}
MyClass.prototype.getA = function(cb) {
this._waitUntiliInitialized(function(err) {
if (err) {
return cb(err);
}
return cb(null, this.data.a);
});
};
MyClass.prototype.getB = function(cb) {
this._waitUntiliInitialized(function(err) {
if (err) {
return cb(err);
}
return cb(null, this.data.b);
});
};
MyClass.prototype._finishInitialization = function(err) {
this._initialized=true;
if (err) {
this._initError = err;
}
this._pendingAfterInitCalls.forEach(function(call) {
call(err);
});
delete this._pendingAfterInitCalls;
};
MyClass.prototype._waitUntiliInitialized = function(cb) {
var bindedCall = cb.bind(this);
if (this._initialized) {
return bindedCall(this._initError);
}
this._pendingAfterInitCalls.push(bindedCall);
};
var db = {
loadDataAsyncronously: function(cb) {
// Emulate the load process from disk.
// Data will be available later
setTimeout(function() {
cb(null, {
a:"Hello",
b:"World"
});
},1000);
}
};
var obj = new MyClass(db);
obj.getA(function (err, a) {
if (err) {
console.log(err);
} else {
console.log("a: " + a);
}
});
obj.getB(function (err, b) {
if (err) {
console.log(err);
} else {
console.log("a: " + b);
}
});
But it seems to me a lot of overhead to be written for each class following this pattern.
Is there a more elegant way to handle this functionality?
Does exist any library to simplify this functionality?
Preparing this question, came to my head what probable would be a good answer. The idea is to use the concept of a factory function. The code above would be rewritten this way.
/*jslint node: true */
"use strict";
function createMyClass(db, cb) {
var obj = new MyClass();
obj._init(db, function(err) {
if (err) return cb(err);
cb(null, obj);
});
}
function MyClass() {
}
MyClass.prototype._init = function(db, cb) {
var self=this;
db.loadDataAsyncronously(function(err, data) {
if (err) return cb(err);
self.data = data;
cb();
});
};
MyClass.prototype.getA = function(cb) {
if (this._initError) return cb(this._initError);
cb(null, this.data.a);
};
MyClass.prototype.getB = function(cb) {
if (this._initError) return cb(this._initError);
cb(null, this.data.b);
};
var db = {
loadDataAsyncronously: function(cb) {
// Emulate the load process from disk.
// Data will be available later
setTimeout(function() {
cb(null, {
a:"Hello",
b:"World"
});
},1000);
}
};
var obj;
createMyClass(db,function(err, aObj) {
if (err) {
console.log(err);
return;
}
obj=aObj;
obj.getA(function (err, a) {
if (err) {
console.log(err);
} else {
console.log("a: " + a);
}
});
obj.getB(function (err, b) {
if (err) {
console.log(err);
} else {
console.log("a: " + b);
}
});
});
I share this Q/A because I thing it can be interesting to some body else. If you thing there exist better solutions, libraries to handle this situations, or any other idea, I will appreciate it.
The usual strategy here is to NOT use any async operation from a constructor. If an object needs an async operation in order to initialize it, then you use one of two options:
The async portion of the initialization is done in an .init(cb) method that must be called by the creator of the object.
You use a factory function that takes a callback that is called when the async portion of the operation has completed (like your proposed answer).
If you create a lot of these, perhaps the factory function makes sense because it saves you a little repeated code. If you don't create a lot of them, I prefer the first option because I think it makes it a lot clearer in the code exactly what is happen (you create an object and then you initialize it asynchronously and then the code continues only when the async operation has completed).
For the first option, it could look like this:
function MyClass(...) {
// initialize instance variables here
}
MyClass.prototype.init = function(db, callback) {
var self = this;
db.loadDataAsyncronously(function(err, data) {
if (err) {
self._initError = err;
} else {
self._initError = null;
self.data = data;
}
callback(err);
});
}
// usage
var obj = new MyClass(...);
obj.init(db, function(err) {
if (!err) {
// continue with rest of the code that uses this object
// in here
} else {
// deal with initialization error here
}
});
Personally, I would probably use a design using promises so the calling code could look like this:
var obj = new MyClass(...);
obj.init(db).then(function() {
// object successfully initialized, use it here
}, function(err) {
// deal with initialization error here
});
You can use a Promise internally in MyClass and have all functions wait for the promise to complete before proceeding.
http://api.jquery.com/promise/
You could inject the precondition into the functions:
function injectInitWait(fn) {
return function() {
var args = Array.prototype.slice.call(arguments, 0);
this._waitUntiliInitialized(function(err) {
if (err) return args[args.length - 1](err);
fn.apply(this, args);
});
}
}
injectInitWait(MyClass.prototype.getA);
injectInitWait(MyClass.prototype.getB);
That being said, your factory approach would be superior.