We Keep Coding

http requests inside loop of variable size, how to do something only after last request has finished (JavaScript)

I'm using request to make HTTP requests inside a loop of variable size. (I will be reading words from a file, right now I just have an array). I'm adding onto an object (response) with each loop, and want to return the object only after the last request has been completed. I've been playing around with promises but I'm not sure how to chain them since it is a variable number of requests.
const request = require('request');
// eventually this array will be populated by reading a file
var words = ["hello", "there"];
var response = {};
// Loop thruogh input string
for (var i = 0; i < words.length; i += 1) {
// get next word
var curWord = words[i];
// if the current word is not already in the response
if (!(curWord in response)) {
// request info from dictionary
var options = {
url: 'https://api.dictionaryapi.dev/api/v2/entries/en/' + curWord,
json: true
}
request(options, (err, res, body) => {
if (err) { return console.log(err); }
// find part of speech
var partOfSpeech;
try {
partOfSpeech = body[0].meanings[0].partOfSpeech
} catch (err) {
partOfSpeech = "undefined";
}
// add to response
response[curWord] = partOfSpeech;
});
}
}
// do this part only after last request has been completed
console.log(response);

Basically you need to count the responses knowing that when counter reaches zero you are done with them all.
EDIT: make sure you are referring the same curWord parameter in the response to match the request. This is done using the IIFE there.
I have setup an example for you
const request = function fake_request(options, callback) {
setTimeout(function() {
callback(null, options.url.slice(options.url.lastIndexOf('/')))
}, Math.random() * 1000)
};
var words = ["hello", "there", "angela"];
var count = words.length;
var response = {}
for (var i = 0; i < words.length; i += 1) {
var curWord = words[i];
if (!(curWord in response)) {
var options = {
url: 'https://api.dictionaryapi.dev/api/v2/entries/en/' + curWord,
json: true
};
(function(curWord) {
request(options, (err, res, body) => {
count--
console.log(count)
partOfSpeech = res;
response[curWord] = partOfSpeech;
if (count == 0) {
// do this part only after last request has been completed
console.log(response);
}
});
})(curWord);
}
}

socket.io client automatically disconnecting in long Node.js function

I am using socket.io to communicate the swift client of my app with the server. Essentially, the client joins a socket connection upon opening the app and a job is instantly added to a Redis queue (it's a job that takes anywhere from a few seconds to like 15ish seconds). There's a response from the server to the client of the job id. While this job is processing, SOMETIMES the client will disconnect. There doesn't seem to be a rhyme or reason behind this, as the time of disconnection is totally inconsistent and it's also not like the disconnection is happening at a specific point in the function. I thought maybe I was manually disconnecting from the client side so I set up socket emissions right before each disconnect on the client side (when these emissions were emitted to the server, the server prints something that tells me where the disconnect came from). This showed me that the disconnect is automatic, because the emission is never received by the client before ending the socket connection. This is running on Heroku. Here's my code:
//queue initialization
const queue = new Queue('queue', process.env.REDIS_URL)
//client pings this endpoint to get the job id in the queue
app.post('/process', async function(request, response) {
let job = await queue({request: request.body});
console.log("Logging job as " + job.id)
response.json({ id: job.id });
});
queue.process(10, async (job) => { //10 is the max workers per job
console.log("Started processing")
const client = await pool.connect()
let item = job.data.request
let title = item.title
let subtitle = item.subtitle
let id = item.id
io.to(id).emit("Processing1", ""); //added emissions like these because I thought maybe the socket was timing out, but this didn't help
console.log("Processing1");
try {
await client.query('BEGIN')
let geoData = await //promise of geocoding endpoint api function
let lengthOfGeoData = geoData.context.length
io.to(id).emit("Processing2", "");
console.log("Processing2");
var municipality = ""
var area = ""
var locality = ""
var place = ""
var district = ""
var region = ""
var country = ""
//for loop to go through geoData and set the above values
if (municipality != "") {
console.log("Signing in from " + municipality + ", " + area);
} else {
console.log("Signing in from " + area)
}
await scrape(municipality, area, id);
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
}
try {
await client.query('BEGIN')
const array = await //a function that queries a Postgres db for some rows, makes json objects out of them, and pushes to the 'array' variable
var array2 = []
for (a of array) {
let difference = getDifference(title, subtitle, a.title, a.subtitle) //math function
if (difference <= 10) {
array.push(a)
}
}
io.to(id).emit("Processing9", "");
console.log("Processing9");
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log("ERROR: Failed arrayHelperFunction")
console.log(err)
} finally {
client.release()
console.log("About to emit this ish to " + id) //should emit to socket here ideally to notify that the processing is done and results can be polled
io.to(id).emit("finishedLoading", "")
return array2;
}
});
//when the client polls the queue after it's received the 'done' notifier from the server
app.post('/poll', async function(request, response) {
console.log("Polling")
let id = request.body.id
const results = await queue(id);
for (r of results.returnvalue) {
console.log("Sending " + r.title);
}
response.send(results.returnvalue)
});
//scrape
async function scrape(municipality, area, id) {
const client = await pool.connect();
try {
await client.query('BEGIN')
var location = ""
if (municipality != "") {
location = municipality + ", " + area
} else {
location = area
}
let inDatabase = await client.query('SQL statement AS it_does_exist', [params]);
io.to(id).emit("Processing3", "");
console.log("Processing3");
if (inDatabase.rows[0].it_does_exist == false) {
let query = "book clubs near " + location
var terminationTime = new Date()
terminationTime.setHours(terminationTime.getHours() + 4);
let date = ("0" + terminationTime.getDate()).slice(-2);
let month = ("0" + (terminationTime.getMonth() + 1)).slice(-2);
let year = terminationTime.getFullYear();
let hours = terminationTime.getHours();
let minutes = terminationTime.getMinutes();
let seconds = terminationTime.getSeconds();
let timestamp = year + "-" + month + "-" + date + " " + hours + ":" + minutes + ":" + seconds
try {
await client.query(`SQL statement`, [params]);
} catch(err) {
console.log("FAILURE: scrape() at 1.")
console.log(err)
}
var queryLocation = "New York,New York,United States" //default search origination is here
var queryGLCode = "US"
io.to(id).emit("Processing4", "");
console.log("Processing4");
try {
await fetch('https://serpapi.com/locations.json?q='+municipality+'&limit=10', { method : "GET" })
.then(res => res.json())
.then((json) => {
for (let index = 0; index < 10; index++) {
let locationAPIName = json[index].canonical_name
let locationAPICode = json[index].country_code
let resultLatitude = json[index].gps[1];
let resultLongitude = json[index].gps[0];
}
});
} catch(err) {
console.log("FAILURE: scrape() at 2.")
console.log(err)
}
io.to(id).emit("Processing5", "");
console.log("Processing5");
try {
await Promise.all([
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode}).then(data => async function(){
try {
await client.query('BEGIN');
let results = data.events_results
if (results != null) {
console.log("first HAD results")
for (result of results) {
var fixedAddress = result.address[0]
let address = fixedAddress + ", " + result.address[1]
let title = result.title + address
var description = result.description
let geoData = await geocode(address); //mapbox geocode the address
let latitude = Number(geoData.center[0]);
let longitude = Number(geoData.center[1]);
await client.query(`SQL statement`, [params]);
}
io.to(id).emit("Processing6", "");
console.log("Processing6");
} else {
console.log("first DID NOT have results")
}
console.log("FIRST BLOCK")
await client.query('COMMIT');
} catch(err) {
console.log("Results[0] not found.")
console.log(err)
await client.query('ROLLBACK');
}
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "10"}).then(data => async function(){
// same as the one above, just with an offset
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "20"}).then(data => async function(){
// same as the one above, but with a different offset
}())
])
} catch(err) {
console.log("FAILURE: scrape() at 3.")
console.log(err)
}
} else {
console.log("Location already in the database.")
}
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
} finally {
client.release()
return "Resolved";
}
}
//Client establish socket connection
func establishConnection(_ completion: (() -> Void)? = nil) {
let socketUrlString: String = appState.server
self.manager = SocketManager(socketURL: URL(string: socketUrlString)!, config: [.log(false), .reconnects(true), .extraHeaders(["header": "customheader"])])
self.socket = manager?.defaultSocket
self.socket?.connect()
self.socket?.once(clientEvent: .connect, callback: { (data, emitter) in
if completion != nil{
completion!()
}
})
//other socket functions
}
//Client initial post request
func process() {
let server = "serverstring" + "process"
let title = "title"
let subtitle = "subtitle"
let package = BookPackage(title: title, subtitle: subtitle, id: mySocketID) //this is after the initial connection
print("package is \(package)")
guard let url = URL(string: server) else { return }
var urlRequest = URLRequest(url: url)
urlRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
urlRequest.addValue("application/json", forHTTPHeaderField: "Accept")
urlRequest.httpMethod = "POST"
guard let data = try? JSONEncoder().encode(package) else { return }
urlRequest.httpBody = data
let task = URLSession.shared.dataTask(with: urlRequest) {
(data, response, error) in
if let error = error {
print(error)
return
}
guard let data = data else { return }
guard let dataString = String(data: data, encoding: String.Encoding.utf8) else { return }
let jsonData = Data(dataString.utf8)
var decodedJob: Job? = nil
do {
decodedJob = try JSONDecoder().decode(Job.self, from: jsonData) //Job is just a struct in the same form as the json object sent back from the server
} catch {
print(error.localizedDescription)
}
DispatchQueue.main.async {
self.appState.pendingJob = decodedJob
}
}
// start the task
task.resume()
}
The only consistent part of this bug is the logs right before the user disconnects (side note: 'reason of disconnect' and 'DISCONNECTED USER' fire on the socket.on('disconnect') event:
https://i.stack.imgur.com/7fjuU.png
https://i.stack.imgur.com/z5bmL.png
https://i.stack.imgur.com/aHNt3.png
https://i.stack.imgur.com/64WYI.png

You should be blocking the event loop with await. There is a heartbeat that the client sends every once in a while (which is defined with pingTimeout).
Since no ping is received by the server, it is disconnected.
You should isolate this process. Either find a way to use it with a worker/background process or async, additionally increasing pingTimeout on serverside might help you.

The solution to your problem is to modify the pingTimeout when initiating the server.
From Socket.io:
The server sends a ping, and if the client does not answer with a pong within pingTimeout ms, the server considers that the connection is closed.
Similarly, if the client does not receive a ping from the server
within pingInterval + pingTimeout ms, the client also considers that
the connection is closed.
const io = new Server(httpServer, {
pingTimeout: 30000
});

You can change the transport from the default to:
const io = new Server(httpServer, {
transports: ['polling', 'websocket'],
});
This might resolve the issue, else you can also try canging the upgradeTimeout and pingTimeout

How to make promise resolve before returning?

I have a question about some code that I have. I'm going to post code and break it down below in a second, however i'd like to explain it in advance. My code is a function called getPing, its in a node server and its goes to a website and give me back an array of objects. It sorts through those objects, and based on the lowest number (ping) It pushes them into an array. Once everything is finished, it will sort through the array and pick a random object. That object as you will see in the code is called selectedserver, it then takes that object and then it SHOULD resolve it, and send the data back to the client. Note that all of this is happening in the same file.
As you will see in a second, once a certain condition is met there is a return, but right above that there is a resolve() that I can't seem to get working. Here is my code.
First, we'll start with where the promise starts.
var getPing = function (id,index) {
return new Promise(function (resolve, reject) {
var keepAliveAgent = new https.Agent({ keepAlive: true })
options.agent = keepAliveAgent
index = index || 0;
var r = https.request(options, function (res) {
var data = []
res.on('data', function (d) {
data.push(d)
}).on('end', function () {
var buf = Buffer.concat(data)
var encodingheader = res.headers['content-encoding']
if (encodingheader == 'gzip') {
zlib.gunzip(buf, function (err, buffer) {
var o = JSON.parse(buffer.toString())
// o is what is returned
if (o.TotalCollectionSize - 20 <= index) {
console.log(o.TotalCollectionSize - 20, '<=', index)
var selectedserver = games.gameservers[Math.floor(Math.random() * games.gameservers.length)]
console.log(selectedserver)
resolve(selectedserver)
return;
}
if (index < o.TotalCollectionSize) {
index = index + 10;
console.log(index, o.TotalCollectionSize)
o.Collection.sort(function (a, b) {
return a.Ping > b.Ping
})
if (typeof (o.Collection[0]) != "undefined") {
var playerscapacity = o.Collection[0].PlayersCapacity.charAt(0)
if (playerscapacity != o.Collection[0].Capacity) {
games.gameservers.push(o.Collection[0])
}
}
getPing(id, index)
}
})
}
})
})
r.end()
//reject('end of here')
})}
As you can see here:
if (o.TotalCollectionSize - 20 <= index) {
console.log(o.TotalCollectionSize - 20, '<=', index)
var selectedserver = games.gameservers[Math.floor(Math.random() * games.gameservers.length)]
console.log(selectedserver)
resolve(selectedserver)
return;
}
Once the o.Totalcollectionsize - 20 is <= to the index, Its suppose to take the games that it pushed into the games.gameservers array, and its suppose to resolve it. The code works besides the resolve part, I know this because all of the console.log's in that code work.
Now this is my node server, that's supposed to send the resolved data BACK to the client.
var server = io.listen(47999).sockets.on("connection", function (socket) {
var ip = socket.handshake.address;
var sid = socket.id;
console.log("Connection from " + ip + "\n\tID: " + sid);
http.createServer(function (req, res) {
res.setHeader('Content-Type', 'application/json');
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "X-Requested-With")
//res.writeHead(200, { 'Content-Type': 'text/plain' });
var data = []
if (req.method == "POST") {
res.writeHead(200, { 'Content-Type': 'text/plain' });
req.on('data', function (dat) {
data.push(dat)
})
req.on('end', function () {
var gamedata = Buffer.concat(data).toString();
var game = JSON.parse(gamedata)
getPing(game.placeId, 0).then(function (r) {
console.log(r)
res.end(JSON.stringify(r))
}).catch(function (e) {
console.log(e)
})
console.log(game.placeId)
})
}
}).listen(6157)
console.log('server running')})
As you can see, in my node server when you send a post request to it, it will start the promise.
getPing(game.placeId, 0).then(function (r) {
console.log(r)
res.end(JSON.stringify(r))
}).catch(function (e) {
console.log(e)
})
However, it never gets to this point. I'm new to promises so I'm not where I'm going wrong here. I've tried everything (or so i thought). I would like to learn how promises fully work, because clearly I don't understand them enough. I'm just trying to get this to work at this point.

const https = require('https');
const zlib = require("zlib");
function downloadPage(url) {
return new Promise((resolve, reject) => {
https.get(url,(res)=>{
let raw = "";
let gunzip = res.pipe(zlib.createGunzip());
gunzip.on('data',(chunk)=>{
raw += chunk;
})
.on('end',()=>{
resolve(raw);
})
.on('error',(err)=>{
reject(err);
})
})
});
}
async function myBackEndLogic() {
const html = await downloadPage('https://api.stackexchange.com/2.2/search?page=1&pagesize=2&order=desc&sort=relevance&intitle=javascript%2Bfilter&site=stackoverflow')
return html;
}
myBackEndLogic().then((data)=>console.log(data));
Try something like this.

How to maintain Timeout Session when calling Api Call in Loop Contion

i having Api Call which execute in For Loop some of the value which returns 10 sec itself some may take nearly 60 sec i have to maintain proper Timeout and clear session (i.e if results comes at 15 sec means it should goes to next input values and run the code) but currenly its waiting for 45 sec each single record how to optimize it
here my sample code :
if (selectedrows.length >= 1) {
for (var i = 0; i < selectedrows.length; i++) {
var myVar = setTimeout (function (k) {
var ob = { results: "Appending ..." };
child.update(selectedrows[k][4], selectedrows[k][4], ob);
var fullName = selectedrows[k][1] + ' ' + selectedrows[k][2];
math.ResultCall.async(fullName,function (err, res) {
if (err) throw err;
var returnedValue = JSON.parse(res);
console.log(returnedValue);
if(returnedValue.Result == null || returnedValue.Result.FOUND_Result == null)
{
console.log("None found")
}
else{
var obj = { results: “res” };
child.update(selectedrows[k][4], selectedrows[k][4], obj);
}
}
});
}, i * 45000,i);
}
}

Rephrasing your question, you need to return the data when your api gets resolved.
For this please go through https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/resolve
JavaScript, by default it work asynchronously because of its event loop.
You have promises and resolve to get notified when your api returns a data
Hope I helped :)

There are several approaches to implement the solution
1. Async-Await: in-case the records-processing order is important
for( let i=0; i<selectedrows.length; i++)
{
let ob = { results: "Appending ..." };
child.update(selectedrows[i][4], selectedrows[i][4], ob);
let fullName = selectedrows[i][1] + ' ' + selectedrows[i][2];
await new Promise((resolve,reject)=>
{
math.ResultCall.async(fullName,(err, res) => {
if (err) reject(err);
let returnedValue = JSON.parse(res);
console.log(returnedValue);
if(returnedValue.Result == null || returnedValue.Result.FOUND_Result == null) {
console.log("None found")
} else {
let obj = { results: “res” };
child.update(selectedrows[i][4], selectedrows[i][4], obj);
}
resolve();
});
}
**don't forget this means the wrapping function should be async as well (which returns a promise that can be resolved if necessary)
2.Promise.All: if the order is not important
let promArray = [];
for( let i=0; i<selectedrows.length; i++)
{
let ob = { results: "Appending ..." };
child.update(selectedrows[i][4], selectedrows[i][4], ob);
let fullName = selectedrows[i][1] + ' ' + selectedrows[i][2];
promArray.push( new Promise((resolve,reject)=>
{
math.ResultCall.async(fullName,(err, res) => {
if (err) reject(err);
let returnedValue = JSON.parse(res);
console.log(returnedValue);
if(returnedValue.Result == null || returnedValue.Result.FOUND_Result == null) {
console.log("None found")
} else {
let obj = { results: “res” };
child.update(selectedrows[i][4], selectedrows[i][4], obj);
}
resolve();
});
);
}
Promise.all(promArray);
** this will also return a Promise that can be resolved if necessary.

Concatenate several API request with NodeJS

I'm messing with SteamAPI in order to learn some NodeJS. Right now I'm trying to get games's info after an initial request to get the player's profile, once I have the games IDs stored in an array. The point is that I don't know how to "return" an array AFTER the whole ID array is iterated and all results has come from the server.
function getThumbs(game) {
return rq(
'http://store.steampowered.com/api/appdetails?appids=' + game,
{json: true},
function (error, response, bd) {
if(response.statusCode === 200 && bd[game].data) {
return bd[game].data.screenshots;
}
});
}
function getGamesThumbnails(games) {
var deferred = $q.defer(),
queue = [];
for (var y = 0; y < games.length; y++) {
var game = games[y];
var thumbs = getThumbs(game);
queue.push(thumbs);
}
$q.all(queue).then(
function (data) {
deferred.resolve(data);
},
function (err) {
deferred.reject(err)
}
);
return deferred.promise;
}
app.get('/blog',function(client_req,client_res){
rq('http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?key=' + key + '&steamid=blablabla&format=json', function (error, response, body) {
var data = JSON.parse(body);
var games = data.response.games.map(function (game) {
return game.appid;
});
getGamesThumbnails(games).then(function (data) {
console.log(data)
})
});
});

Basically, you should use a callback, because like you are doing in getThumbsyou are returning the object, while you should return the value bd[game].data.screenshots;
function getThumbs(game, cb) {
return rq(
'http://store.steampowered.com/api/appdetails?appids=' + game,
{json: true},
function (error, response, bd) {
if(response.statusCode === 200 && bd[game].data) {
cb(null, bd[game].data.screenshots);
}
});
}
function getGamesThumbnails(games) {
var deferred = $q.defer(),
queue = [];
for (var y = 0; y < games.length; y++) {
var game = games[y];
getThumbs(game, function(err, value) {
queue.push(value);
});
}
$q.all(queue).then(
function (data) {
deferred.resolve(data);
},
function (err) {
deferred.reject(err)
}
);
return deferred.promise;
}
And plust to return the response to the client you have to use the client_res.send(VALUE)
so the bottom part would become like this:
app.get('/blog',function(client_req,client_res){
rq('http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?key=' + key + '&steamid=blablabla&format=json', function (error, response, body) {
var data = JSON.parse(body);
var games = data.response.games.map(function (game) {
return game.appid;
});
getGamesThumbnails(games).then(function (data) {
client_res.send(data);
console.log(data)
})
});
});

Your getThumbs() function does not return a promise. $q.all only works on an array containing promises, whereas rq uses callbacks.
Try this:
function getThumbs(game) {
var deferred = $q.defer(),
rq(
'http://store.steampowered.com/api/appdetails?appids=' + game,
{json: true},
function (error, response, bd) {
if(response.statusCode === 200 && bd[game].data) {
deferred.resolve(bd[game].data.screenshots);
}
});
return deferred.promise;
}

Thank you both!
I tried Yuri's approach, but $q.all it doesn't seem to resolve the array of promises (nothing happens after the last request from getThumbs())
for (var y = 0; y < games.length; y++) {
var game = games[y];
var thumbs = getThumbs(game);
queue.push(thumbs);
}
$q.all(queue).then(
function (data) {
console.log(data)
deferred.resolve(data);
},
function (err) {
deferred.reject(err)
}
);