New to Node.js here. I'm looking for the correct way to make N asynchronous API calls from within another function, and combining their results to use further downstream. In my case, N would be reasonably small and blocking for their execution not too bad.
In synchronous execution the implementation for combine() below should work.
If I only needed the results from one API call it would be straightforward to implement the following logic in a callback function supplied to callAPI(). Where I stumble is when I need all the results combined before before executing foo(total, [...args]).
I looked into async.whilst but wasn't able to get that to work. I'm skeptical if that actually is the correct fit to my needs. I've also looked into Promises which seems to be the correct lead but it would be nice to get reassurances before crawling into that cavernous rabbit hole. Be it that Promises is the correct way, which module is the standard to use in Node.js projects?
var http = require('http');
function callAPI(id) {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
return body;
}).on('error', (err) => {
console.error(err);
});
});
}
function combine(inputs) {
var total = 0;
for (i=0; i < inputs.length; i++) {
total += callAPI(inputs[i]['id']);
};
console.log(total);
// call some function, foo(total, [...args])
}
Edit 1:
I attempted to follow samanime's answer below and modify the API call to return a Promise. See:
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
resolve(body);
}).on('error', (err) => {
reject(err);
});
});
});
}
function combine(inputs) {
var combined = [];
for (i=0; i < inputs.length; i++) {
total += callAPI(inputs[i]['id']);
.then(result => {
combined.push(result);
});
};
var total = combined.reduce((a, b) => a + b, 0);
console.log(total);
// call some function, foo(total, [...args])
}
This seems to get me halfway there. If I console.log(combined) inside the then() block I can see the list building up with results from the API calls. However, I still can't access the complete combined at the "end" of the for loop. Can I attach a callback to something to run after the full list has been built? Is there a better way?
Edit 2 (My solution - per Patrick Roberts suggestion)
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/'.concat(id)
}
var req = http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = parseInt(Buffer.concat(body));
resolve(body);
}).on('error', (err) => {
reject(err);
});
});
});
}
function combine(inputs) {
var combined = [];
Promise.all(inputs.map(input => callAPI(input.id)))
.then((combined) => {
var total = combined.reduce((a, b) => a + b, 0);
// foo(total, [...args])
});
};
It sounds like you can just chain together a bunch of promises, passing the data along.
Basically something like:
const combined = [];
asyncOne()
.then(result => { combined.push(result); return asyncTwo())
.then(result => { combined.push(result); return asyncThree())
// and so on
As long as each function returns a promise, you'll be all set.
If you want to run them in parallel, use Promise.all(), which will do the same thing for you:
Promise.all([asyncOne(), asyncTwo(), asyncThree() /* , etc */])
.then(combined => /* combined is an array with the results of each */)
This is by far the preferred pattern for this sort of thing.
Your edit is looking a lot better, but try this:
function callAPI(id) {
return Promise((resolve, reject) => {
var options = {
host: 'example.com',
path: '/q/result/' + id
}
http.get(options, (res) => {
var body = [];
res.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
resolve(body);
}).on('error', reject);
});
});
}
function combine(inputs) {
Promise.all(inputs.map(input => callAPI(input.id))).then((combined) => {
// completed array of bodies
console.log(combined);
// foo(combined.length, [...args]);
}).catch((error) => {
console.log(error);
});
}
I would add a counter that keeps track of remaining API calls. Whenever an API call finishes, decrement and if its 0, you're done.
const numCalls = 10;
let remaining = numCalls;
let data = [];
function getRandomInt(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min)) + min;
}
function ajax() {
// Simulate ajax with a setTimeout for random amount of time.
setTimeout(() => {
// This is the callback when calling http.get
data.push(getRandomInt(0, 10)); // Some data from server
if (--remaining <= 0) {
// Am I the last call? Use data.
console.log(data);
console.log(data.length);
}
}, getRandomInt(1000, 3000));
}
for (let i = 0; i < numCalls; i++) {
ajax();
}
Related
I have a question about some code that I have. I'm going to post code and break it down below in a second, however i'd like to explain it in advance. My code is a function called getPing, its in a node server and its goes to a website and give me back an array of objects. It sorts through those objects, and based on the lowest number (ping) It pushes them into an array. Once everything is finished, it will sort through the array and pick a random object. That object as you will see in the code is called selectedserver, it then takes that object and then it SHOULD resolve it, and send the data back to the client. Note that all of this is happening in the same file.
As you will see in a second, once a certain condition is met there is a return, but right above that there is a resolve() that I can't seem to get working. Here is my code.
First, we'll start with where the promise starts.
var getPing = function (id,index) {
return new Promise(function (resolve, reject) {
var keepAliveAgent = new https.Agent({ keepAlive: true })
options.agent = keepAliveAgent
index = index || 0;
var r = https.request(options, function (res) {
var data = []
res.on('data', function (d) {
data.push(d)
}).on('end', function () {
var buf = Buffer.concat(data)
var encodingheader = res.headers['content-encoding']
if (encodingheader == 'gzip') {
zlib.gunzip(buf, function (err, buffer) {
var o = JSON.parse(buffer.toString())
// o is what is returned
if (o.TotalCollectionSize - 20 <= index) {
console.log(o.TotalCollectionSize - 20, '<=', index)
var selectedserver = games.gameservers[Math.floor(Math.random() * games.gameservers.length)]
console.log(selectedserver)
resolve(selectedserver)
return;
}
if (index < o.TotalCollectionSize) {
index = index + 10;
console.log(index, o.TotalCollectionSize)
o.Collection.sort(function (a, b) {
return a.Ping > b.Ping
})
if (typeof (o.Collection[0]) != "undefined") {
var playerscapacity = o.Collection[0].PlayersCapacity.charAt(0)
if (playerscapacity != o.Collection[0].Capacity) {
games.gameservers.push(o.Collection[0])
}
}
getPing(id, index)
}
})
}
})
})
r.end()
//reject('end of here')
})}
As you can see here:
if (o.TotalCollectionSize - 20 <= index) {
console.log(o.TotalCollectionSize - 20, '<=', index)
var selectedserver = games.gameservers[Math.floor(Math.random() * games.gameservers.length)]
console.log(selectedserver)
resolve(selectedserver)
return;
}
Once the o.Totalcollectionsize - 20 is <= to the index, Its suppose to take the games that it pushed into the games.gameservers array, and its suppose to resolve it. The code works besides the resolve part, I know this because all of the console.log's in that code work.
Now this is my node server, that's supposed to send the resolved data BACK to the client.
var server = io.listen(47999).sockets.on("connection", function (socket) {
var ip = socket.handshake.address;
var sid = socket.id;
console.log("Connection from " + ip + "\n\tID: " + sid);
http.createServer(function (req, res) {
res.setHeader('Content-Type', 'application/json');
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "X-Requested-With")
//res.writeHead(200, { 'Content-Type': 'text/plain' });
var data = []
if (req.method == "POST") {
res.writeHead(200, { 'Content-Type': 'text/plain' });
req.on('data', function (dat) {
data.push(dat)
})
req.on('end', function () {
var gamedata = Buffer.concat(data).toString();
var game = JSON.parse(gamedata)
getPing(game.placeId, 0).then(function (r) {
console.log(r)
res.end(JSON.stringify(r))
}).catch(function (e) {
console.log(e)
})
console.log(game.placeId)
})
}
}).listen(6157)
console.log('server running')})
As you can see, in my node server when you send a post request to it, it will start the promise.
getPing(game.placeId, 0).then(function (r) {
console.log(r)
res.end(JSON.stringify(r))
}).catch(function (e) {
console.log(e)
})
However, it never gets to this point. I'm new to promises so I'm not where I'm going wrong here. I've tried everything (or so i thought). I would like to learn how promises fully work, because clearly I don't understand them enough. I'm just trying to get this to work at this point.
const https = require('https');
const zlib = require("zlib");
function downloadPage(url) {
return new Promise((resolve, reject) => {
https.get(url,(res)=>{
let raw = "";
let gunzip = res.pipe(zlib.createGunzip());
gunzip.on('data',(chunk)=>{
raw += chunk;
})
.on('end',()=>{
resolve(raw);
})
.on('error',(err)=>{
reject(err);
})
})
});
}
async function myBackEndLogic() {
const html = await downloadPage('https://api.stackexchange.com/2.2/search?page=1&pagesize=2&order=desc&sort=relevance&intitle=javascript%2Bfilter&site=stackoverflow')
return html;
}
myBackEndLogic().then((data)=>console.log(data));
Try something like this.
My objective is to read data from two files and compare the data. My input files are result3.json and result4.json.The data in these files are coma separated.
result3.json
[
"temp1.txt",
"temp2.txt",
]
node:
function readFromFile(file) {
var fileNames = [];
//setTimeout(function() {
fs.readFile(file,function(err, data){
if (err) {
return console.log(err);
}
var temp = JSON.parse(data.toString().split(","));
// console.log(temp.length);
for (let index = 0; index < temp.length; index++) {
//console.log(temp[index]);
fileNames.push(temp[index]);
//console.log(fileNames[index]);
}
Done(); // to block the async call
});
//},3000);
//console.log(fileNames.length);
return fileNames;
}
var baseListOfFiles = readFromFile('./output/result3.json'); // Assume this is the base file
var currentListOfFiles = readFromFile('./output/result4.json'); // Assume this is the current file
function Done(){
//console.log('Out baseListOfFiles + ' + baseListOfFiles.length);
for (let index = 0; index < baseListOfFiles.length; index++) {
console.log("[baseListOfFiles] " + baseListOfFiles[index]);
}
//console.log('Out currentListOfFiles+ ' + currentListOfFiles.length);
for (let index = 0; index < currentListOfFiles.length; index++) {
console.log("[currentListOfFiles] " + currentListOfFiles[index]);
}
}
Above is my code. It seems to be async call, so it always return 0 fileNames.
Is there any way to control it?
Here's example code using Promises:
const fs = require('fs');
function readFromFile(file) {
return new Promise((resolve, reject) => {
fs.readFile(file, function (err, data) {
if (err) {
console.log(err);
reject(err);
}
else {
resolve(JSON.parse(data));
}
});
});
}
const promises = [
readFromFile('./output/result3.json'),
readFromFile('./output/result4.json')
];
Promise.all(promises).then(result => {
console.log(result);
baseListOfFiles = result[0];
currentListOfFiles = result[1];
// do more stuff
});
First, an array promises is built; each Promise reads the file, then calls resolve with the result.
This array is passed to Promise.all(), which then calls the callback, passing the array of results in the same order.
You're right, readFile is async. What you're looking for is readFileSync: https://nodejs.org/api/fs.html#fs_fs_readfilesync_path_options
With that can can do:
const data = fs.readFileSync(file);
//do something with data
There are a few ways to 'promisify' readFile if you like, the options are discussed here: Using filesystem in node.js with async / await
The return Promise.all([photoArray]) returns an empty array, seemingly not waiting for the callFB to return its promise that then pushes into the array.
I am not sure what I am doing wrong but am relatively new to Promises with for loops and Ifs.
I am not sure exactly if I am using the correct number of Promises but I seem to not be able to get the 3rd tier Promise.all to wait for the for loop to actually finish (in this scenario, the for loop has to look through many item so this is causing an issue where it is not triggering callFeedback for all the items it should before context.done() gets called.
I have tried using Q.all also for the Promise.all([photoArray]) but have been unable to get that working.
module.exports = function (context, myBlob) {
var res = myBlob
var promiseResolved = checkPhoto(res,context);
var promiseResolved2 = checkVideo(res,context);
Promise.all([promiseResolved, promiseResolved2]).then(function(results){
context.log(results[0], results[1]);
// context.done();
});
});
};
};
function checkPhoto(res, context){
return new Promise((resolve, reject) => {
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
var callFB = callFeedback(context, feedbackId);
Promise.all([callFB]).then(function(results){
photoArray.push(results[0]);
});
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return Promise.all([photoArray]).then(function(results){
context.log("end results: " + results);
resolve(photoArray);
});
} else {
resolve('No photos');
}
})
}
function checkVideo(res, context){
return new Promise((resolve, reject) => {
same as checkPhoto
})
}
function callFeedback(context, feedbackId) {
return new Promise((resolve, reject) => {
var requestUrl = url.parse( URL );
var requestBody = {
"id": feedbackId
};
// send message to httptrigger to message bot
var body = JSON.stringify( requestBody );
const requestOptions = {
standard
};
var request = https.request(requestOptions, function(res) {
var data ="";
res.on('data', function (chunk) {
data += chunk
// context.log('Data: ' + data)
});
res.on('end', function () {
resolve("callFeedback: " + true);
})
}).on('error', function(error) {
});
request.write(body);
request.end();
})
}
The code suffers from promise construction antipattern. If there's already a promise (Promise.all(...)), there is never a need to create a new one.
Wrong behaviour is caused by that Promise.all(...).then(...) promise isn't chained. Errors aren't handled and photoArray.push(results[0]) causes race conditions because it is evaluated later than Promise.all([photoArray])....
In case things should be processed in parallel:
function checkPhoto(res, context){
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
var callFB = callFeedback(context, feedbackId);
// likely no need to wait for callFB result
// and no need for Promise.all
photoArray.push(callFB);
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return Promise.all(photoArray); // not [photoArray]
} else {
return 'No photos';
};
}
callFB promises don't depend on each other and thus can safely be resolved concurrently. This allows to process requests faster.
Promise.all serves a good purpose only if it's used to resolve promises in parallel, while the original code tried to resolve the results (results[0]).
In case things should be processed in series the function benefits from async..await:
async function checkPhoto(res, context){
if (res.photos.length > 0) {
var photoArray = [];
for (var j = 0; j < res.photos.length; j++) {
if (res.photos[j].feedbackId !== null){
var feedbackId = res.photos[j].feedbackId;
const callFBResult = await callFeedback(context, feedbackId);
// no need for Promise.all
photoArray.push(callFBResult);
} else {
photoArray.push("Photo " + j + " has no feedback");
}
}
return photoArray; // no need for Promise.all, the array contains results
} else {
return 'No photos';
};
}
Add try..catch to taste.
For the following function, I have to add a timeout after every GET request in array ajaxUrls. All the XHR GET request are in array ajaxUrls.
function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
return Promise.all(ajaxUrls.map(getSearchResultsForOnePage))
.then(function(responses) {
return responses.map(function(response) {
if (response.meta.total === 0) {
return [];
}
return response.result.searchResults.map(function(searchResult) {
return (searchResult);
});
});
})
.then(function(searchProfiles) {
return [].concat.apply([], searchProfiles);
})
.catch(function(responses) {
console.error('error ', responses);
});
}
function getSearchResultsForOnePage(url) {
return fetch(url, {
credentials: 'include'
})
.then(function(response) {
return response.json();
});
}
I want a certain timeout or delay after every GET request. I am facing difficulty in where exactly to add the timeout.
If you want to make requests in serial, you shouldn't use Promise.all, which initializes everything in parallel - better to use a reduce that awaits the previous iteration's resolution and awaits a promise-timeout. For example:
async function getAllSearchResultProfiles(searchAjaxUrl) {
const ajaxUrls = [];
for (let i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
const responses = await ajaxUrls.reduce(async (lastPromise, url) => {
const accum = await lastPromise;
await new Promise(resolve => setTimeout(resolve, 1000));
const response = await getSearchResultsForOnePage(url);
return [...accum, response];
}, Promise.resolve([]));
// do stuff with responses
const searchProfiles = responses.map(response => (
response.meta.total === 0
? []
: response.result.searchResults
));
return [].concat(...searchProfiles);
}
Note that only asynchronous operations should be passed from one .then to another; synchronous code should not be chained with .then, just use variables and write the code out as normal.
I find a simple for loop in an async function to be the most readable, even if not necessarily the most succinct for things like this. As long as the function is an async function you can also create a nice pause() function that makes the code very easy to understand when you come back later.
I've simplified a bit, but this should give you a good idea:
function pause(time) {
// handy pause function to await
return new Promise(resolve => setTimeout(resolve, time))
}
async function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < 5; i++) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
let responses = []
for (url of ajaxUrls) {
// just loop though and await
console.log("sending request")
let response = await getSearchResultsForOnePage(url)
console.log("recieved: ", response)
responses.push(response)
await pause(1000) // wait one second
}
//responses.map() and other manilpulations etc...
return responses
}
function getSearchResultsForOnePage(url) {
//fake fetch
return Promise.resolve(url)
}
getAllSearchResultProfiles("Test")
.then(console.log)
If you want to add a delay in every request then add a setTimout() in your function which fetches data from api
function getSearchResultsForOnePage(url) {
return new Promise((resolve, reject) => {
fetch(url, {
credentials: 'include'
})
.then(response => reresponse.json())
.then(data => {
let timeout = 1000;
setTimeout(() => resolve(data), timeout);
});
}
I am trying to iterate through the JSON files generated by the protractor tests. I pull all the file names into an array and call a method that opens and parses through the each file, post the results to the database and pass back a passed/failed flag.
I have tried all the examples here
Make angular.forEach wait for promise after going to next object and still get the same results.
The method is actually called, but the results are not posted to the db. I have tested the parser.parseResults on an individual file and it successfully posted to the db, so it has to have something to do with the promise not resolving correctly.
Is it not possible to do something like this in the jasmine/protractor framework? Or do I have something wrong in the code?
I have included the code for my latest attempt.
Thank You
Christine
matches.reduce(function (p, val) {
console.log('val', val);
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
console.log('finalResult = ', finalResult);
}, function (err) {
console.log('error in reduce',err);
});
parser.parseResults code
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
try {
if (err != null) {
console.log('error reading file',err);
reject(err);
}
console.log('obj - ',obj);
var results = [];
var Passed = 0;
var Message = '';
var Stack = '';
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
if (specs.length > 0) {
for (var i = 0; i < specs.length; i++) {
var assert = specs[i];
var tcR = new RegExp(/TC[\d]+/);
var tc = assert.description.match(tcR);
if (!assert.failedExpectations.length) {
Passed = 1;
}
else {
assert.failedExpectations.forEach((expectation) => {
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
})
Passed = 0;
}
if (tc != null) {
utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild,
'P', Message, Stack, 0, moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss'), '')
.then(function (resp) {
resolve(Passed);
}, (err) => {
console.log('Posting to Database failed ', err);
reject(err);
});
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
reject(err);
}
}
}
}
}
catch (err) {
console.log('rejecting opening file');
reject(err);
}
});
})
}
If there is not exactly one suite in the obj, with exactly one spec, then your promise is either resolved not at all or multiple times.
Avoid wrapping too many things in the new Promise constructor - always promisify on the smallest possible level, and use promise chaining afterwards.
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function(obj) {
console.log('obj - ',obj);
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length-1];
Passed = 0;
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.catch(err => {
console.log('Posting to Database failed ', err);
throw err;
}));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results);
});
};