loop 100+ getJSON calls and call a another function when completely done - javascript

I need to read a grid and take that data and call a $getJSON url. The grid could have over 100 lines of data. The getJSON returns a list of comma separated values that I add to an array. Once the loop is finished I take the array and process it for the duplicates. I need to use the duplicates in another process. I know that I can't determine the order of the data that is coming back but I need to know that all of the calls have been make.
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
if (i == rowscount - 1) {
// call the related function
}
});
}
I can't figure out how to be sure that the process has finished. The call to the related function has been done early at times.

Well if we keep track of how many have completed we can fire off the code when the last one is done.
let complete = 0;
for (let i = 0; i < rowscount; i++){
$.getJSON(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" +
terms,
function (data) {
var pmids = data.esearchresult.idlist;
var pmidlist = pmids.join();
pmid_List.push(pmidlist);
complete += 1;
if (complete == rowscount) {
// call the related function
}
});
}

I'd use fetch and Promise.all
const link = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=";
Promise.all(Array.from({
length: 3
}, () => fetch(link + 'foo').then(e => e.json()))).then(e => {
//called when all requests are done
console.log(e);
})

Try this
function getJson(url, i) {
return $.getJSON(url, function (data) {
//var pmids = data.esearchresult.idlist;
//var pmidlist = pmids.join();
//pmid_List.push(pmidlist);
console.log('completed', i)
return data;
});
}
function run() {
let promises = []
for (let i = 0; i < rowscount; i++) {
const terms = 'foot';
const url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=500&term=" + terms;
promises.push(getJson(url, i));
}
return promises;
}
Promise.all(run()).then(() => console.log('All are completed'));
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>

Related

node.js async; counting callbacks

How does counting instances of a callback function work in node.js?
I was working on the 9th exercise of learnyounode (below the official solution).
As I understand it, the httpGet function is called three times, running through process.argv[2], [3] and [4]. But how could count ever === 3? Don't the individual functions just get to one? How does one call of httpGet know of the other ones?
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++)
console.log(results[i])
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err)
return console.error(err)
results[index] = data.toString()
count++
if (count == 3)
printResults()
}))
})
}
for (var i = 0; i < 3; i++)
httpGet(i)
But how could count ever === 3?
count is defined outside of httpGet and thus its value is independent of the those function calls. count++ is the same as count = count + 1, i.e. every call to httpGet increases the value of count by 1. The third time the function is called, count's value will be 3.
We can easily replicate this:
var count = 0;
function httpGet() {
count++;
console.log('count: ', count);
if (count === 3) {
console.log('count is 3');
}
}
httpGet();
httpGet();
httpGet();
First, prefer not using var.
var is defined in the global scope so it’s value updated between calls
Read more about var here

Values of a promise in JS within for loop

I am lost in the promised land and could really use some guidance. I have exhausted searching numerous SO questions (2-3 hours of reading solutions + docs) related to this seemingly common issue and feel I just am not getting it.
Overview
Below I have code that takes in an Object type (resources), grabs a few values from this Object and then calculates distance and duration from the GoogleMaps Distance Matrix. The results of the function googleRequest() are a promise containing two values (distance and duration).
I would like to get these two values back within the for loop, execute pushToRows(), and then return an array called final_rows.
Problem
final_rows shows UNDEFINED for the duration and distance keys within each row. I speculate this is occurring because I am attempting to access the values in dist_dur inappropriately. I would appreciate any help on resolving this issue. Thanks.
Code
final_rows = []
function getDistTime(resources){
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
var dist_time_data = googleRequest(origin1, destinationA).then((values) => {
return values
})
pushToRows(resources.data[i], dist_time_data)
}
// console.log(final_rows)
}
function pushToRows(resources, dist_dur){
resources["DISTANCE_MI"] = dist_dur[0];
resources["ACTUAL_DUR_HR"] = dist_dur[1];
resources["FINANCE_DUR_HR"] = (dist_dur[0] / 45.0).toFixed(2)
final_rows.push(resources)
}
So, what you would need to do is just store promises in an array in the for loop and then wait for these promises to resolve using Promise.all but this would parallelize your requests to google distance api.
function getDistTime(resources){
const promiseArr = [];
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
promiseArr.push(googleRequest(origin1, destinationA));
}
// Not sure how would you use the data pushed in rows but since you are not waiting for promises to be resolved, data would be updated later on
return Promise.all(promiseArr)
.then((resultsArr) => {
resultsArr.forEach((result, i) => pushToRows(resources.data[i], result));
})
}
function pushToRows(resources, dist_dur){
resources["DISTANCE_MI"] = dist_dur[0];
resources["ACTUAL_DUR_HR"] = dist_dur[1];
resources["FINANCE_DUR_HR"] = (dist_dur[0] / 45.0).toFixed(2)
final_rows.push(resources)
}
I would recommend to use async-await which are syntactic sugar to promises but make your code easy to understand and remove the complications that come with promise chaining.
If you move your pushToRows() inside where you return values, you will have access to that data.
googleRequest(origin1, destinationA).then((values) => {
pushToRows(resources.data[i], values);
});
Until that promise resolves, dist_time_data would be undefined
You could also convert to Promise.all() which takes an array of promises and resolves when all of the promises are complete:
function getDistTime(resources){
const promises = [];
for (var i = 0; i < resources.data.length; i++) {
var origin1 = $("#citystate").val();
var destinationA = resources.data[i]['DEMOBILIZATION CITY'] + ',' + resources.data[i]['DEMOBILIZATION STATE'];
promises.push(googleRequest(origin1, destinationA));
}
return Promise.all(promises).then((results) => {
return results.map((result, i) => {
return {
...resources.data[i],
DISTANCE_MI: result[0],
ACTUAL_DUR_HR: result[1],
FINANCE_DUR_HR: (result[0] / 45.0).toFixed(2)
};
});
});
}
getDistTime(resources).then(result => {
//result is now "final_rows"
});

learnyounode exercise 9 - output is showing undefined, problems understanding async or the logic

I've been struggling with the JUGGLING ASYNC task on learnyounode. As far as I can tell, I'm doing things almost right and it looks like the issue sits somewhere with the async compounded by myself trying to avoid bl or other modules. I did the previous task without another module and would like to continue this trend.
const http = require('http');
const url = process.argv[2];
let content = [];
let count = 0;
const arguments = process.argv;
let urlArray = arguments.filter(function pullUrls(element, index, array) {
return index >= 2;
});
function printResults() {
for (var i = 0; i < 3; i++) {
console.log(content[i]);
}
}
function httpGet(index) {
http.get(urlArray[index], function(response){
response.on('data', function(data){
newData = data.toString();
content[index] = content[index] + newData;
})
response.on('end', function(){
count++;
if (count === 3){
printResults();
}
})
})
}
for (var i = 0; i < 3; i++) {
httpGet(i);
}
Common output is very close to what is expected, but includes an undefined string at the beginning and I can't figure out why.
undefinedLets get some ute when we're going parma. Grab us a slab how it'll be bull bar. He's got a massive bushie where stands out like a pot.
undefinedShe'll be right mongrel heaps as cross as a hit the turps. Stands out like a booze also you little ripper flick. As stands out like ironman when lets throw a bikkie.
undefinedHe hasn't got a bounce with gutful of struth. Stands out like a aerial pingpong piece of piss built like a battler.
As you can see, it's finding the array 'undefined' first, which is true, but then appending onto it.
My best guess is that it's the content[index] = content[index] + newData; line somehow holding on to the undefined nature of let content = [] before content[i] is figured out. Now I've written it out and thought it through, this might be a simple JS problem I'm just overlooking or don't-know-that-I-don't-know.
Any help would be good.
Your getting right output, content[index] is undefined initially, you could do a undefined check, before concatenating newData. Here is complete code with changes.
const http = require('http');
const url = process.argv[2];
let content = [];
let count = 0;
const arguments = process.argv;
let urlArray = arguments.filter(function pullUrls(element, index, array) {
return index >= 2;
});
function printResults() {
for (var i = 0; i < 3; i++) {
console.log(content[i]);
}
}
function httpGet(index) {
http.get(urlArray[index], function(response){
response.on('data', function(data){
let newData = data.toString();
//check if undefined
if(typeof content[index] !== 'undefined'){
content[index] = content[index] + newData;
}else{
//add newData if undefined
content[index] = newData;
}
})
response.on('end', function(){
count++;
if (count === 3){
printResults();
}
})
})
}
for (var i = 0; i < 3; i++) {
httpGet(i);
}

Multiple queries in a loop Parse Cloud Code

I'm having a hard time trying to understand promises, I'm sure I need to use them for this but I don't know how and other answers don't help me at all.
I'd like to loop over an array, query all the results for each value of the array, then after calculating the average value for these results, add the average in an array. After every iterations, this array is sent as a response.
Here is my code which could help understand me here:
Parse.Cloud.define('getScorePeopleArray', function(request, response) {
var peopleArray = request.params.peoplearray;
var query = new Parse.Query("Scores");
var resultat;
var index, len;
var resultarray = [];
var people;
for (index = 0, len = peopleArray.length; index < len; ++index) {
people = peopleArray[index];
query.equalTo("People",people);
query.find({
success: function(results) {
var sum = 0;
for (var i = 0; i < results.length; ++i) {
sum += results[i].get("Score");
}
resultat = (sum / results.length)*5;
if(!resultat){
resultarray.push("null");
}else{
resultarray.push(resultat);
}
},
error: function() {
response.error("score lookup failed");
}
}).then();
}
response.success(resultarray);
});
Of course response.success is not called when every queries are done, but as soon as possible (since queries are asynchronous if I'm right).
I know I have to change it with promises, but I don't understand at all how this works.
Thanks a lot in advance !
var _ = require('underscore');
Parse.Cloud.define('getScorePeopleArray', function(request, response) {
var peopleArray = request.params.peoplearray; // what is this an array of?
var resultArray = [];
return Parse.Promise.as().then(function() { // this just gets the ball rolling
var promise = Parse.Promise.as(); // define a promise
_.each(peopleArray, function(people) { // use underscore, its better :)
promise = promise.then(function() { // each time this loops the promise gets reassigned to the function below
var query = new Parse.Query("Scores");
query.equalTo("People", people); // is this the right query syntax?
return query.find().then(function(results) { // the code will wait (run async) before looping again knowing that this query (all parse queries) returns a promise. If there wasn't something returning a promise, it wouldn't wait.
var sum = 0;
for (var i = 0; i < results.length; i++) {
sum += results[i].get("Score");
}
var resultat = (sum / results.length) * 5;
if (!resultat){
resultArray.push("null");
} else {
resultArray.push(resultat);
}
return Parse.Promise.as(); // the code will wait again for the above to complete because there is another promise returning here (this is just a default promise, but you could also run something like return object.save() which would also return a promise)
}, function (error) {
response.error("score lookup failed with error.code: " + error.code + " error.message: " + error.message);
});
}); // edit: missing these guys
});
return promise; // this will not be triggered until the whole loop above runs and all promises above are resolved
}).then(function() {
response.success(resultArray); // edit: changed to a capital A
}, function (error) {
response.error("script failed with error.code: " + error.code + " error.message: " + error.message);
});
});

Chrome JavaScript developer console: Is it possible to call console.log() without a newline?

I'd like to use console.log() to log messages without appending a new line after each call to console.log(). Is this possible?
No, it's not possible. You'll have to keep a string and concatenate if you want it all in one line, or put your output elsewhere (say, another window).
In NodeJS you can use process.stdout.write and you can add '\n' if you want.
console.log(msg) is equivalent to process.stdout.write(msg + '\n').
Yes, it's possible (check out the demo below) -- by implementing your own virtual console on top of the native browser console, then syncing it to the real one.
This is much easier than it sounds:
maintain a display buffer (e.g. an array of strings representing one line each)
call console.clear() before writing to erase any previous contents
call console.log() (or warn, error, etc) to fill the console with the contents from your display buffer
Actually, I've been doing this for some time now. A short, rudimentary implementation of the idea would be something along the following lines, but still capable of animating the console contents:
// =================================================
// Rudimentary implementation of a virtual console.
// =================================================
var virtualConsole = {
lines: [],
currentLine: 0,
log: function (msg, appendToCurrentLine) {
if (!appendToCurrentLine) virtualConsole.currentLine++;
if (appendToCurrentLine && virtualConsole.lines[virtualConsole.currentLine]) {
virtualConsole.lines[virtualConsole.currentLine] += msg;
} else {
virtualConsole.lines[virtualConsole.currentLine] = msg;
}
console.clear();
virtualConsole.lines.forEach(function (line) {
console.log(line);
});
},
clear: function () {
console.clear();
virtualConsole.currentLine = 0;
}
}
// =================================================
// Little demo to demonstrate how it looks.
// =================================================
// Write an initial console entry.
virtualConsole.log("Loading");
// Append to last line a few times.
var loadIndicatorInterval = setInterval(function () {
virtualConsole.log(".", true); // <- Append.
}, 500);
// Write a new line.
setTimeout(function () {
clearInterval(loadIndicatorInterval);
virtualConsole.log("Finished."); // <- New line.
}, 8000);
It sure has its drawbacks when mixing with direct console interaction, and can definitely look ugly -- but it certainly has its valid uses, which you couldn't achieve without it.
You can put as many things in arguments as you'd like:
console.log('hi','these','words','will','be','separated','by','spaces',window,document)
You'll get all that output on one line with the object references inline and you can then drop down their inspectors from there.
The short answer is no.
But
If your use-case involves attempting to log perpetually changing data while avoiding console-bloat, then one way to achieve this (in certain browsers) would be to use console.clear() before each output.
function writeSingleLine (msg) {
console.clear();
console.log(msg);
}
writeSingleLine('this');
setTimeout( function () { writeSingleLine('is'); }, 1000);
setTimeout( function () { writeSingleLine('a'); }, 2000);
setTimeout( function () { writeSingleLine('hack'); }, 3000);
Note that this would probably break any other logging functionality that was taking place within your application.
Disclaimer: I would class this as a hack.
collect your output in an array and then use join function with a preferred separator
function echo(name, num){
var ar= [];
for(var i =0;i<num;i++){
ar.push(name);
}
console.log(ar.join(', '));
}
echo("apple",3)
check also Array.prototype.join() for mode details
var elements = ['Fire', 'Wind', 'Rain'];
console.log(elements.join());
// expected output: Fire,Wind,Rain
console.log(elements.join(''));
// expected output: FireWindRain
console.log(elements.join('-'));
// expected output: Fire-Wind-Rain
If your only purpose to stop printing on many lines, One way is to group the values if you don't want them to fill your complete console
P.S.:- See you browser console for output
let arr = new Array(10).fill(0)
console.groupCollapsed('index')
arr.forEach((val,index) => {
console.log(index)
})
console.groupEnd()
console.group
console.groupCollapsed
Something about #shennan idea:
function init(poolSize) {
var pool = [];
console._log = console.log;
console.log = function log() {
pool.push(arguments);
while (pool.length > poolSize) pool.shift();
draw();
}
console.toLast = function toLast() {
while (pool.length > poolSize) pool.shift();
var last = pool.pop() || [];
for (var a = 0; a < arguments.length; a++) {
last[last.length++] = arguments[a];
}
pool.push(last);
draw();
}
function draw() {
console.clear();
for(var i = 0; i < pool.length; i++)
console._log.apply(console, pool[i]);
}
}
function restore() {
console.log = console._log;
delete console._log;
delete console.toLast;
}
init(3);
console.log(1);
console.log(2);
console.log(3);
console.log(4); // 1 will disappeared here
console.toLast(5); // 5 will go to row with 4
restore();
A simple solution using buffered output. Works with deno and should work with node.js. (built for porting pascal console programs to javascript)
const write = (function(){
let buffer = '';
return function (text='\n') {
buffer += text;
let chunks = buffer.split('\n');
buffer = chunks.pop();
for (let chunk of chunks)
{console.log(chunk);}
}
})();
function writeln(text) { write(text + '\n'); }
To flush the buffer, you should call write() at the end of program.
If you mix this with console.log calls, you may get garbage output.
if you want for example console log array elements without a newline you can do like this
const arr = [1,2,3,4,5];
Array.prototype.log = (sep='') => {
let res = '';
for(let j=0; j<this.lengthl j++){
res += this[j];
res += sep;
}
console.log(res);
}
// console loging
arr.log(sep=' '); // result is: 1 2 3 4 5
Useful for debugging or learning what long chained maps are actually doing.
let myConsole = (function(){
let the_log_buffer=[[]], the_count=0, the_single_line=false;
const THE_CONSOLE=console, LINE_DIVIDER=' ~ ', ONE_LINE='ONE_LINE',
PARAMETER_SEPARATOR= ', ', NEW_LINE = Symbol();
const start = (line_type='NOT_ONE_LINE') => {
the_log_buffer=[[]];
the_count=0;
the_single_line = line_type == ONE_LINE;
console = myConsole;
}
const stop = () => {
isNewline();
console = THE_CONSOLE;
};
const isNewline = a_param => {
if (the_single_line && a_param==NEW_LINE) return;
const buffer_parts = the_log_buffer.map(one_set=> one_set.join(PARAMETER_SEPARATOR))
const buffer_line = buffer_parts.join(LINE_DIVIDER);
if (the_single_line) {
THE_CONSOLE.clear();
}
THE_CONSOLE.log( buffer_line );
the_log_buffer = [[]];
the_count=0;
}
const anObject = an_object => {
if (an_object instanceof Error){
const error_props = [...Object.getOwnPropertyNames(an_object)];
error_props.map( error_key => an_object['_' + error_key] = an_object[error_key] );
}
the_log_buffer[the_count].push(JSON.stringify(an_object));
}
const aScalar = a_scalar => {
if (typeof a_scalar === 'string' && !isNaN(a_scalar)) {
the_log_buffer[the_count].push("'" + a_scalar + "'");
} else {
the_log_buffer[the_count].push(a_scalar);
}
}
const notNewline = a_param => typeof a_param === 'object' ? anObject(a_param):aScalar(a_param);
const checkNewline = a_param => a_param == NEW_LINE ? isNewline(a_param) : notNewline(a_param);
const log = (...parameters_list) => {
the_log_buffer[the_count]=[];
parameters_list.map( checkNewline );
if (the_single_line){
isNewline(undefined);
}else{
const last_log = parameters_list.pop();
if (last_log !== NEW_LINE){
the_count++;
}
}
}
return Object.assign({}, console, {start, stop, log, ONE_LINE, NEW_LINE});
})();
function showConcatLog(){
myConsole.stop();
myConsole.start();
console.log('a');
console.log('bb');
console.dir({i:'not', j:'affected', k:'but not in step'})
console.log('ccc');
console.log([1,2,3,4,5,'6'], {x:8, y:'9'});
console.log("dddd", 1, '2', 3, myConsole.NEW_LINE);
console.log("z", myConsole.NEW_LINE, 8, '7');
console.log(new Error("error test"));
myConsole.stop();
}
myConsole.start(myConsole.ONE_LINE);
var stop_callback = 5;
function myCallback(){
console.log(stop_callback, 'Date.now()', myConsole.NEW_LINE, Date.now());
stop_callback--;
if (stop_callback>0){
window.setTimeout(myCallback, 1000);
}else{
showConcatLog();
}
}
window.setTimeout(myCallback, 1000);
You can use a spread operator to display output in the single line. The new feature of javascript ES6. see below example
for(let i = 1; i<=10; i++){
let arrData = [];
for(let j = 1; j<= 10; j++){
arrData.push(j+"X"+i+"="+(j*i));
}
console.log(...arrData);
}
That will print 1 to 10 table in single line.
// Source code for printing 2d array
window.onload = function () {
var A = [[1, 2], [3, 4]];
Print(A);
}
function Print(A) {
var rows = A.length;
var cols = A[0].length;
var line = "";
for (var r = 0; r < rows; r++) {
line = "";
for (var c = 0; c < cols; c++) {
line += A[r][c] + " ";
}
console.log(line);
}
}

Categories