We Keep Coding

learnyounode JUGGLING ASYNC order is unstable

I'm trying learnyounode(https://github.com/workshopper/learnyounode)
Exercise 9 ##JUGGLING ASYNC (try 3 get request and show response in the same order)
I wrote below code but result order is not stable. What's is wrong?
var http = require('http');
var bl = require('bl');
var urls = [];
for (var i = 2; i < process.argv.length; i++ ) {
urls.push(process.argv[i]);
};
var results = [];
for (var i = 0; i < urls.length; i++) {
http.get(urls[i], function(response) {
response.pipe(bl(function(err, data) {
if (err) {
console.error(err)
};
// if this code results[i] = data.toString() // console.log shows “”
results.splice(i, 0 , data.toString());
if ( results.length === urls.length) {
for (var i = 0; i < results.length ; i++) {
console.log(results[i]);
}
};
}));
});
}
official answer is below.
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++) {
console.log(results[i])
}
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err) {
return console.error(err)
}
results[index] = data.toString()
count++
if (count === 3) {
printResults()
}
}))
})
}
for (var i = 0; i < 3; i++) {
httpGet(i)
}

Using Async With waterfall and Recursion in nodejs

I've created a script to migrate data from Dynamo to a Mysql DB.
First I was not using Async, but I started getting bottlenecks on the sql side, so I decided to "throttle" the dymano part using the async lib.
The problem: I have a recursion in the middle of the path, as long as dynamo has data I have to continue the process (ultra simple ETL), but I don't know how to perform the recursion inside the waterfall.
My code :
function main() {
async.waterfall([getMaxTimestamp, scanDynamoDB, printout, saveToMySQL], function(err, result) {
if(err) console.log(err)
console.log(result)
});
}
function getMaxTimestamp(callback) {
console.time("max query");
connection.query("SELECT MAX(created_at) as start_date from Tracking;", function(err, data) {
console.timeEnd("max query");
callback(err, data);
})
}
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
callback(err, data);
// if (!err) {
// if (data != undefined && data.Count > 0) {
// printout(data.Items) // Print out the subset of results.
// if (data.LastEvaluatedKey) { // Result is incomplete; there is more to come.
// query.ExclusiveStartKey = data.LastEvaluatedKey;
// scanDynamoDB(query);
// }
// } else {
// console.log('No fresh data found on Dynamo')
// } else console.dir(err);
});
};
function assembleSql() {
insertSql = "insert into Tracking (";
for (var i = 0; i < headers.length; i++) {
insertSql += headers[i];
if (i < headers.length - 1)
insertSql += ",";
}
insertSql += ") values ?;"
previousInsertSql = insertSql;
}
function saveToMySQL(items, callback) {
assembleSql();
//connection.connect();
console.time("insert sql")
connection.query(insertSql, [items], function(err, result) {
console.timeEnd("insert sql")
if (err){
callback(err, null)
return;
}
totalInserts += result.affectedRows;
callback(err, totalInserts)
//connection.end();
})
}
function printout(items, callback) {
var headersMap = {};
var values;
var header;
var value;
var out = [];
if (headers.length == 0) {
if (items.length > 0) {
for (var i = 0; i < items.length; i++) {
for (var key in items[i]) {
headersMap[key] = true;
}
}
}
for (var key in headersMap) {
headers.push(key);
}
}
for (index in items) {
values = [];
for (i = 0; i < headers.length; i++) {
value = "";
header = headers[i];
// Loop through the header rows, adding values if they exist
if (items[index].hasOwnProperty(header)) {
if (items[index][header].N) {
value = items[index][header].N;
} else if (items[index][header].S) {
value = items[index][header].S;
} else if (items[index][header].SS) {
value = items[index][header].SS.toString();
} else if (items[index][header].NS) {
value = items[index][header].NS.toString();
} else if (items[index][header].B) {
value = items[index][header].B.toString('base64');
} else if (items[index][header].M) {
value = JSON.stringify(items[index][header].M);
} else if (items[index][header].L) {
value = JSON.stringify(items[index][header].L);
} else if (items[index][header].BOOL !== undefined) {
value = items[index][header].BOOL.toString();
}
}
values.push(value)
}
out.push(values)
}
callback(null, out);
}
main();
The commented part is where the recursion happens, but I don't know where to place this inside my flow !
Any help would be appreciated !

Just don't call callback function inside scanDynamoDB while fetching data. You can implement additional function and call it recursive while errors is not appears, like below
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
var result = []; // for accumulate data of each query
function readNext(err, data) {
if (err)
return callback(err);
if (!data || !data.Count)
return callback(null, result);
// add data to result
dynamoDB.scan(query, readNext);
}
dynamoDB.scan(query, readNext);
};

Actually I was able to figure it out by myself.
async.whilst(function() { return canInsert}, function (callback){
scanDynamoDB(query, callback)
}, function(err, res) {}
function scanDynamoDB(data, callback) {
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
if (!err) {
if (data != undefined && data.Count > 0) {
canInsert = data.LastEvaluatedKey;
if (data.LastEvaluatedKey) // Result is incomplete; there is more to come.
query.ExclusiveStartKey = data.LastEvaluatedKey;
}
} else console.dir(err);
});
};
I could have done it just with a while(canInsert). Anyway, I avoided recursion and memory usage is way way lower.

How to properly execute an async method within a for-loop?

I am trying to execute an asynchronous method within a for-loop construct and then display the result. I believe the problem is that the for-loop increments before the cryto.randomBytes method calls the callback. How would I properly execute this for-loop ?
var crypto = require('crypto');
var nimble = require('nimble');
var codes = [];
nimble.series([
function(callback){
for(var i = 0; i < 100;i++){
crypto.randomBytes(64, function(ex, buf) {
if (ex) throw ex;
codes[i] = buf.toString('hex');
});
}
callback();
},
function(callback){
for(var i = 0; i < codes.length;i++){
console.log("Ticket " + i + ":" + codes[i]);
}
callback();
}]);

Yes, you are right that the loop completes before the callbacks are called. You can use an anonymous function to create a scope where each iteration gets its own copy of the variable.
Also, you would call the callback after the last value has been added to the result, not after the loop:
function(callback){
var cnt = 0;
for(var i = 0; i < 100;i++){
(function(i){
crypto.randomBytes(64, function(ex, buf) {
if (ex) throw ex;
codes[i] = buf.toString('hex');
if (++cnt == 100) {
callback();
}
});
})(i);
}
}

Instead of:
function(callback){
for(var i = 0; i < 100;i++){
crypto.randomBytes(64, function(ex, buf) {
if (ex) throw ex;
codes[i] = buf.toString('hex');
});
}
callback();
},
You might try something like:
function(callback){
for(var i = 0, len = 100; i < len; i++){
crypto.randomBytes(64, function(ex, buf) {
if (ex) throw ex;
codes.push(buf.toString('hex'));
if (codes.length === len)
callback();
});
}
},

Using an IIFE with recursivity, should work :
var crypto = require('crypto');
var nimble = require('nimble');
var codes = [];
nimble.series([
function (callback) {
// Using an IIFE
(function recursive(index) {
if (index < 100) { // i < 100
crypto.randomBytes(64, function (ex, buf) {
if (ex) throw ex;
codes[index] = buf.toString('hex');
recursive(index + 1);
});
} else callback();
})(0); // i = 0
},
function (callback) {
for (var i = 0; i < codes.length; i++) {
console.log("Ticket " + i + ":" + codes[i]);
}
callback();
}]);

Local array not being pushed to

I have the following code. The array prices does not seem to be pushed to the prices array despite successfully retrieving the ustock.unitprice.
getLatestMarketPrices: function(username, callback) {
var prices = [];
db.find('portfolio', {user: username}, function(err, stocks) {
for(var i = 0; i < stocks.length; i++) {
module.exports.getQuote(stocks[i].stock, function(err, ustock) {
console.log(ustock.unitprice); // Retrieves 1.092
prices.push(ustock.unitprice); // Should push to prices array?
});
}
console.log(prices); // Prices is still [] despite earlier push.
callback(null, prices);
});
},
Is this a scoping issue? I'm not really sure why prices is not pushed to.
Thanks very much.

If you know jquery, you could try deferred object
getLatestMarketPrices: function(username, callback) {
var prices = [];
var defer = $.Deferred();
//Attach a handler to be called when the deferred object is resolved
defer.done(function(){
console.log(prices);
callback(null, prices);
});
db.find('portfolio', {user: username}, function(err, stocks) {
for(var i = 0; i < stocks.length; i++) {
module.exports.getQuote(stocks[i].stock, function(err, ustock) {
console.log(ustock.unitprice); // Retrieves 1.092
prices.push(ustock.unitprice); // Should push to prices array?
//resolve when we retrieve all
if (prices.length == stocks.length){
defer.resolve();
}
});
}
});
},
Update: or don't need deferred object at all:
getLatestMarketPrices: function(username, callback) {
var prices = [];
db.find('portfolio', {user: username}, function(err, stocks) {
for(var i = 0; i < stocks.length; i++) {
module.exports.getQuote(stocks[i].stock, function(err, ustock) {
console.log(ustock.unitprice); // Retrieves 1.092
prices.push(ustock.unitprice); // Should push to prices array?
//callback only when we receive all
if (prices.length == stocks.length){
console.log(prices);
callback(null, prices);
}
});
}
});
},

Node.js - The function does not return data in select from table

I want to retrieve the unique id from the database and store them into the object (userslist).
Why can not I retrieve data into an object?
Object userslist is empty {}
I have this function in Node.js:
getOnlineUsers: function()
{
var userslist = {};
client.query(
'SELECT DISTINCT Aid FROM online',
function selectCb(err, results, fields) {
if (err) {
throw err;
}
if (results.length > 0) {
for (var i = 0; i < results.length; i++) {
var reader = results[i];
userslist[reader['Aid']] = {
fbid: reader['Aid']
}
}
}
}
);
return userslist;
}

Here's one example of what was discussed in the comments above.
Pass a callback to getOnlineUsers...
my_obj.getOnlineUsers(function(u_list) {
/* do something with the user list */
});
And have getOnlineUsers receive the callback argument, and invoke it...
// receive the callback----v
getOnlineUsers: function(cb_func) {
client.query(
'SELECT DISTINCT Aid FROM online',
function selectCb(err, results, fields) {
if (err) {
throw err;
}
var userslist = {}
if (results.length > 0) {
for (var i = 0; i < results.length; i++) {
var reader = results[i];
userslist[reader['Aid']] = {
fbid: reader['Aid']
}
}
}
cb_func(userslist); // <-- invoke the callback, passing the data
});
}
This is one approach, but basically illustrates that the continuation of the flow of code needs to take place in the callback passed to client.query.