We Keep Coding

http requests inside loop of variable size, how to do something only after last request has finished (JavaScript)

I'm using request to make HTTP requests inside a loop of variable size. (I will be reading words from a file, right now I just have an array). I'm adding onto an object (response) with each loop, and want to return the object only after the last request has been completed. I've been playing around with promises but I'm not sure how to chain them since it is a variable number of requests.
const request = require('request');
// eventually this array will be populated by reading a file
var words = ["hello", "there"];
var response = {};
// Loop thruogh input string
for (var i = 0; i < words.length; i += 1) {
// get next word
var curWord = words[i];
// if the current word is not already in the response
if (!(curWord in response)) {
// request info from dictionary
var options = {
url: 'https://api.dictionaryapi.dev/api/v2/entries/en/' + curWord,
json: true
}
request(options, (err, res, body) => {
if (err) { return console.log(err); }
// find part of speech
var partOfSpeech;
try {
partOfSpeech = body[0].meanings[0].partOfSpeech
} catch (err) {
partOfSpeech = "undefined";
}
// add to response
response[curWord] = partOfSpeech;
});
}
}
// do this part only after last request has been completed
console.log(response);

Basically you need to count the responses knowing that when counter reaches zero you are done with them all.
EDIT: make sure you are referring the same curWord parameter in the response to match the request. This is done using the IIFE there.
I have setup an example for you
const request = function fake_request(options, callback) {
setTimeout(function() {
callback(null, options.url.slice(options.url.lastIndexOf('/')))
}, Math.random() * 1000)
};
var words = ["hello", "there", "angela"];
var count = words.length;
var response = {}
for (var i = 0; i < words.length; i += 1) {
var curWord = words[i];
if (!(curWord in response)) {
var options = {
url: 'https://api.dictionaryapi.dev/api/v2/entries/en/' + curWord,
json: true
};
(function(curWord) {
request(options, (err, res, body) => {
count--
console.log(count)
partOfSpeech = res;
response[curWord] = partOfSpeech;
if (count == 0) {
// do this part only after last request has been completed
console.log(response);
}
});
})(curWord);
}
}

Using Async With waterfall and Recursion in nodejs

I've created a script to migrate data from Dynamo to a Mysql DB.
First I was not using Async, but I started getting bottlenecks on the sql side, so I decided to "throttle" the dymano part using the async lib.
The problem: I have a recursion in the middle of the path, as long as dynamo has data I have to continue the process (ultra simple ETL), but I don't know how to perform the recursion inside the waterfall.
My code :
function main() {
async.waterfall([getMaxTimestamp, scanDynamoDB, printout, saveToMySQL], function(err, result) {
if(err) console.log(err)
console.log(result)
});
}
function getMaxTimestamp(callback) {
console.time("max query");
connection.query("SELECT MAX(created_at) as start_date from Tracking;", function(err, data) {
console.timeEnd("max query");
callback(err, data);
})
}
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
callback(err, data);
// if (!err) {
// if (data != undefined && data.Count > 0) {
// printout(data.Items) // Print out the subset of results.
// if (data.LastEvaluatedKey) { // Result is incomplete; there is more to come.
// query.ExclusiveStartKey = data.LastEvaluatedKey;
// scanDynamoDB(query);
// }
// } else {
// console.log('No fresh data found on Dynamo')
// } else console.dir(err);
});
};
function assembleSql() {
insertSql = "insert into Tracking (";
for (var i = 0; i < headers.length; i++) {
insertSql += headers[i];
if (i < headers.length - 1)
insertSql += ",";
}
insertSql += ") values ?;"
previousInsertSql = insertSql;
}
function saveToMySQL(items, callback) {
assembleSql();
//connection.connect();
console.time("insert sql")
connection.query(insertSql, [items], function(err, result) {
console.timeEnd("insert sql")
if (err){
callback(err, null)
return;
}
totalInserts += result.affectedRows;
callback(err, totalInserts)
//connection.end();
})
}
function printout(items, callback) {
var headersMap = {};
var values;
var header;
var value;
var out = [];
if (headers.length == 0) {
if (items.length > 0) {
for (var i = 0; i < items.length; i++) {
for (var key in items[i]) {
headersMap[key] = true;
}
}
}
for (var key in headersMap) {
headers.push(key);
}
}
for (index in items) {
values = [];
for (i = 0; i < headers.length; i++) {
value = "";
header = headers[i];
// Loop through the header rows, adding values if they exist
if (items[index].hasOwnProperty(header)) {
if (items[index][header].N) {
value = items[index][header].N;
} else if (items[index][header].S) {
value = items[index][header].S;
} else if (items[index][header].SS) {
value = items[index][header].SS.toString();
} else if (items[index][header].NS) {
value = items[index][header].NS.toString();
} else if (items[index][header].B) {
value = items[index][header].B.toString('base64');
} else if (items[index][header].M) {
value = JSON.stringify(items[index][header].M);
} else if (items[index][header].L) {
value = JSON.stringify(items[index][header].L);
} else if (items[index][header].BOOL !== undefined) {
value = items[index][header].BOOL.toString();
}
}
values.push(value)
}
out.push(values)
}
callback(null, out);
}
main();
The commented part is where the recursion happens, but I don't know where to place this inside my flow !
Any help would be appreciated !

Just don't call callback function inside scanDynamoDB while fetching data. You can implement additional function and call it recursive while errors is not appears, like below
function scanDynamoDB(data, callback) {
if (data[0].start_date != null && data[0].start_date)
query.ExpressionAttributeValues[':v_ca'].N = data[0].start_date;
console.time("dynamo read");
var result = []; // for accumulate data of each query
function readNext(err, data) {
if (err)
return callback(err);
if (!data || !data.Count)
return callback(null, result);
// add data to result
dynamoDB.scan(query, readNext);
}
dynamoDB.scan(query, readNext);
};

Actually I was able to figure it out by myself.
async.whilst(function() { return canInsert}, function (callback){
scanDynamoDB(query, callback)
}, function(err, res) {}
function scanDynamoDB(data, callback) {
console.time("dynamo read");
dynamoDB.scan(query, function(err, data) {
console.timeEnd("dynamo read");
if (!err) {
if (data != undefined && data.Count > 0) {
canInsert = data.LastEvaluatedKey;
if (data.LastEvaluatedKey) // Result is incomplete; there is more to come.
query.ExclusiveStartKey = data.LastEvaluatedKey;
}
} else console.dir(err);
});
};
I could have done it just with a while(canInsert). Anyway, I avoided recursion and memory usage is way way lower.

Node.js Function Flow

When I get a request, I want it to generate a 4-character code, then check if it already exists in the database. If it does, then generate a new code. If not, add it and move on. This is what I have so far:
var code = "";
var codeFree = false;
while (! codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code=" + code, function(err, result) {
if (! err) {
if (result.rows.length > 0) {
codeFree = false;
} else {
codeFree = true;
}
} else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
This does not do nearly what I want it to do. How can I handle something like this?

You are doing an async task.
When you have an asyncronous task inside your procedure, you need to have a callback function which is going to be called with the desired value as its argument.
When you found the free code, you call the function and passing the code as its argument, otherwise, you call the getFreeCode function again and passing the same callback to it. Although you might consider cases when an error happens. If your the db call fails, your callback would never get called. It is better to use a throw/catch mechanism or passing another argument for error to your callback.
You can achieve what you need to do by doing it this way:
function getFreeCode(callback) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
if(!err) {
if(result.rows.length > 0) {
getFreeCode(callback);
} else {
callback(code);
}
}else {
console.log('DB ERR: %s', err);
}
console.log(codeFree);
});
console.log('here');
}
// in your main:
getFreeCode(function (code) {
console.log(' this code was free: ' + code)
})

I recommend you look into two alternatives to help deal with asynchronous code.
node generator functions using the 'yield' keyword
promises
Using generators requires running a recent version of node with the --harmony flag. The reason I recommend generators is because you can write code that flows the way you expect.
var x = yield asyncFunction();
console.log('x = ' + x);
The previous code will get the value of x before logging x.
Without yielding the console.log would write out x before the async function was finished getting the value for x.
Your code could look like this with generators:
var client = {
execute: function (query) {
var timesRan = 0;
var result = [];
return function () {
return setTimeout(function () {
result = ++timesRan < 4 ? ['length_will_be_1'] : [];
return result;
},1);
};
}
};
function* checkCode () {
var code;
var codeFree = false;
while(!codeFree) {
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log(rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
try {
var result = yield client.execute("select * from codes where code="+code);
codeFree = result.rows.length > 0 ? false : true;
}catch(e) {
console.log('DB ERR: %s', err);
} finally {
console.log(codeFree);
}
console.log('here');
}
}
checkCode().next();
You would leave off the client object. I only added that to make a working example that fakes an async call.
If you have to use an older version of node or do not like the yield syntax then promises could be a worthy option.
There are many promise libraries. The reason I recommend promises is that you can write code that flows the way you expect:
asyncGetX()
.then(function (x) {
console.log('x: ' + x);
});
The previous code will get the value of x before logging x.
It also lets you chain async functions and runs them in order:
asyncFunction1()
.then(function (result) {
return asyncFunction2(result)
})
.then(function (x) { /* <-- x is the return value from asyncFunction2 which used the result value of asyncFunction1 */
console.log('x: ' + x);
});
Your code could look like this with the 'q' promise library:
var Q = require('q');
var client = {
timesRan: 0,
execute: function (query, callback) {
var self = this;
var result = {};
setTimeout(function () {
console.log('self.timesRan: ' + self.timesRan);
result.rows = ++self.timesRan < 4 ? ['length = 1'] : [];
callback(null, result);
},1);
}
};
function checkCode () {
var deferred = Q.defer();
var codeFree = false;
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
var code = "";
for (var i = 0; i < 4; i++) {
var rand = Math.floor(Math.random() * chars.length);
console.log('rand: %s', rand);
code += chars.charAt(rand);
}
console.log("Code: %s generated.", code);
client.execute("select * from codes where code="+code, function(err, result) {
console.log('err: '+err+', result: ' + JSON.stringify(result));
console.log('result.rows.length: ' + result.rows.length);
if(!err) {
if(result.rows.length > 0) {
codeFree = false;
console.log('result.rows: %s, codeFree: %s', result.rows, codeFree);
checkCode();
} else {
codeFree = true;
console.log('line 36: codeFree: ' + codeFree);
deferred.resolve(code);
}
}else {
console.log('DB ERR: %s', err);
deferred.reject(err);
}
console.log(codeFree);
});
console.log('waiting for promise');
return deferred.promise;
}
checkCode()
.then(function (code) {
console.log('success with code: ' + code);
})
.fail(function(err) {
console.log('failure, err: ' + err);
});
Also omit the client object here. I only added that to make a working example that fakes an async call.
Promises and generators definitely take some time to get used to. It's worth it because they make the code a lot easier to follow in the end than code written with nested callbacks.

Angularjs promise not resolving in time

I am trying to retrieve data for use in a service that can be used throughout my app. The problem is that I can't get the data to be resolved in time for the routine that uses it. Here is the routine that uses it:
function getTranslation(lookup, language) {
var i = 0;
if (vm.translations == null) {
//vm.translations = getAllTranslations();
dataService.getTranslations()
.then(function (data) {
vm.translations = data;
});
}
var len = vm.translations.length;
for (var i=0; i < len; i++) {
if (vm.translations[i].labelName == lookup) {
if (language == "English") {
return vm.translations[i].english;
} else {
if (language == "Spanish") {
return vm.translations[i].espanol;
}
}
}
}
return null;
}
Here is the calling method within that service:
function getAllTranslations() {
return vm.translations = dataService.getTranslations()
.then(function (data) {
vm.translations = data;
return vm.translations;
});
}
And here is the method in the dataService:
function getTranslations() {
return $http.get('/api/labeltext')
.then (getTranslationComplete)
.catch(getTranslationFailed);
function getTranslationComplete(response) {
var deferred = $q.defer();
return response.data;
}
function getTranslationFailed(error) {
alert("XHR failed for frequent pawner report: " + error.responseText);
}
}
I am still learning angularjs and want to be able to populate the data in the service and then call it from other controllers. However, when I get to my for loop the array is empty and only gets populated after its completed.

That is because the promise will not be resolved before your for loop fires. By placing the loop within the .then(), you will have access to the response and your loop values will be defined. This is not DRY since there would be code duplication if you add an else to the function and would have to add in the same loop code. For that, I would refactor the loop into an external function and just call it from within the proper areas of getTranslation().
function getTranslation(lookup, language) {
var i = 0;
if (vm.translations == null) {
//vm.translations = getAllTranslations();
dataService.getTranslations()
.then(function (data) {
vm.translations = data;
var len = vm.translations.length;
for (var i=0; i < len; i++) {
if (vm.translations[i].labelName == lookup) {
if (language == "English") {
return vm.translations[i].english;
} else {
if (language == "Spanish") {
return vm.translations[i].espanol;
}
}
}
}
});
}
return null;
}

Since your data originates from a promise ($http), all of your subsequent code that needs to access that data has to be within a then function.
angular.controller('myController', function(dataService) {
var translationsPromise;
/**
* Caches translations from /api/labeltext and performs a lookup
* #param lookup
* #param language
*/
function getTranslation(lookup, language) {
if (translationsPromise == null) {
translationsPromise = dataService.getTranslations()
}
translationsPromise.then(function(data) {
vm.translations = data;
var len = vm.translations.length;
for (var i = 0; i < len; i++) {
if (vm.translations[i].labelName == lookup) {
if (language == "English") {
return vm.translations[i].english;
} else {
if (language == "Spanish") {
return vm.translations[i].espanol;
}
}
}
}
});
}
getTranslation('Some Label', 'English').then(function(translation) {
// The translation that was found is accessible in this block
});
});

When data is asynchronously derived, it's always better to cache a promise rather than the data. Thus, should another request for the same data be made before the original promise is resolved, then that promise can be retrieved from cache, and returned or otherwise exploited. The same is not true of cached async data, which is not guaranteed to have arrived when another request is made.
function getTranslation(lookup, language) {
if (!vm.translationsPromise) {
vm.translationsPromise = dataService.getTranslations();
}
return vm.translationsPromise.then(function (data) {
var lang = {'English':'english', 'Spanish':'espanol'};
return data.reduce(function(str, item) {
return (str !== '') ? str : (item.labelName == lookup) ? item[lang[language]] : str;
}, '');
});
}

How to handle each response in successCallBack of promises in case of dynamic passage of promises to .when()

Here are the cloud functions namely 'batchReq1' and batchPromises.
In any case, if I know the exact number of promises pushed (Consider the size of results to be '2' in function batchPromises(results)) and executed through when(), I can handle the success response by passing that number of result parameters (In the below example request1, request2) in successCallBack of .then().
If I have to process the number of promises pushed to .when() dynamically, then, how can we handle this in SuccessCallBack? Unlike earlier scenario, we can't expect fixed number of results in the then method (batchPromises(results).then(function (result1, result2) {....)
batchReq1
Parse.Cloud.define("batchReq1", function (request, response) {
var results = request.params.imageArray;
batchPromises(results).then(function (result1, result2) {
console.log("Final Result:: Inside Success");
console.log("Final Result:: Inside Success result 1::::"+result1);
console.log("Final Result:: Inside Success result 2::::"+result2);
response.success();
}
// batchPromises(results).then(function (arraySuccess) {
//
// console.log("Final Result:: Inside Success");
// console.log("Final Result:: Inside Success:: Length:: "+arraySuccess.length);
// //Fetch all responses from promises and display
// var _ = require('underscore.js');
// _.each(arraySuccess, function (result) {
//
// console.log("Final Result:: " + result)
//
// });
//
//
// response.success();
//
// }
, function (error) {
console.log("Final Result:: Inside Error");
response.error(error);
});
});
batchPromises
function batchPromises(results) {
var promise = Parse.Promise.as();
var promises = [];
var increment = 0;
var isFromParallelExecution = false;
var _ = require('underscore.js');
_.each(results, function (result) {
var tempPromise = Parse.Promise.as("Promise Resolved ");
promises.push(tempPromise);
}
promise = promise.then(function () {
return Parse.Promise.when(promises);
});
}
increment++;
});
return promise;
}

this is how i handle this...
Parse.Cloud.define("xxx", function(request, response)
{
var channels = ["channel1", "channel2", "channel3", "channel4", "channel5"];
var queries = new Array();
for (var i = 0; i < channels.length; i++)
{
var query = new Parse.Query(channels[i]);
queries.push(query.find());
}
Parse.Promise.when(queries).then(function()
{
var msg = "";
for (var j = 0; j < arguments.length; j++)
{
for (var k = 0; k < arguments[j].length; k++)
{
var object = arguments[j][k];
msg = msg + " " + object.get('somefield');
}
}
response.success(msg);
});
});

Either you just use the arguments object to loop over the results, or you build your arraySuccess without when - it doesn't make much sense here anyway as you batch the requests (executing them sequentially), instead of executing them in parallel:
function batchPromises(tasks) {
var _ = require('underscore.js');
_.reduce(tasks, function (promise, task) {
return promise.then(function(resultArr) {
var tempPromise = Parse.Promise.as("Promise Resolved for "+taks);
return tempPromise.then(function(taskResult) {
resultArr.push(taskResult);
return resultArr;
});
});
}, Parse.Promise.as([]));
}
If you actually wanted to execute them in parallel, use a simple
function batchPromises(tasks) {
var _ = require('underscore.js');
return Parse.Promise.when(_.map(tasks, function (task) {
return Parse.Promise.as("Promise Resolved for "+taks);
}).then(function() {
return [].slice.call(arguments);
});
}