We Keep Coding

How to make timed requests to a server to avoid rate-limiting using Javascripts setTimeout?

I am trying to use setTimeout in a for loop so that my HTTP requests get sent once per second to avoid rate-limiting. However, it doesn't seem to be working. Could someone please help?
async function initiateSearchExperimental() {
const json = await getCollections();
for (let i = 0; i < json.result.data.length; i++) {
setTimeout(getData(json, i), 1000 * i)
}
}
function getData(json, i) {
fetch(`https://howrare.is${json.result.data[i].url}/?for_sale=on&sort_by=rank`).then(function(response) {
// The API call was successful!
return response.text();
}).then(function(html) {
// Convert the HTML string into a document object
var parser = new DOMParser();
var doc = parser.parseFromString(html, 'text/html');
var priceArray = getPriceArray(doc.querySelectorAll("div.featured_item"))
console.log(json.result.data[i].url, curateArrayTwo(priceArray, json.result.data[i]))
}).catch(function(err) {
// There was an error
console.warn('Something went wrong.', err);
});
}

No need for await and such
I suggest you DO use setTimeout, but do it in the success of the second fetch
let data;
let cnt = 0;
const getData() {
if (cnt >= data.length) return; // stop
fetch(`https://howrare.is${data[cnt].url}/?for_sale=on&sort_by=rank`)
.then(function(response) {
// The API call was successful!
return response.text();
}).then(function(html) {
// Convert the HTML string into a document object
var parser = new DOMParser();
var doc = parser.parseFromString(html, 'text/html');
var priceArray = getPriceArray(doc.querySelectorAll("div.featured_item"))
console.log(data[cnt].url, curateArrayTwo(priceArray, data[cnt]))
cnt++
setTimeout(getData, 1000)
}).catch(function(err) {
// There was an error
console.warn('Something went wrong.', err);
});
};
fetch(collectionurl)
.then(response => response.json())
.then(json => {
data = json.result.data;
getData()
});

socket.io client automatically disconnecting in long Node.js function

I am using socket.io to communicate the swift client of my app with the server. Essentially, the client joins a socket connection upon opening the app and a job is instantly added to a Redis queue (it's a job that takes anywhere from a few seconds to like 15ish seconds). There's a response from the server to the client of the job id. While this job is processing, SOMETIMES the client will disconnect. There doesn't seem to be a rhyme or reason behind this, as the time of disconnection is totally inconsistent and it's also not like the disconnection is happening at a specific point in the function. I thought maybe I was manually disconnecting from the client side so I set up socket emissions right before each disconnect on the client side (when these emissions were emitted to the server, the server prints something that tells me where the disconnect came from). This showed me that the disconnect is automatic, because the emission is never received by the client before ending the socket connection. This is running on Heroku. Here's my code:
//queue initialization
const queue = new Queue('queue', process.env.REDIS_URL)
//client pings this endpoint to get the job id in the queue
app.post('/process', async function(request, response) {
let job = await queue({request: request.body});
console.log("Logging job as " + job.id)
response.json({ id: job.id });
});
queue.process(10, async (job) => { //10 is the max workers per job
console.log("Started processing")
const client = await pool.connect()
let item = job.data.request
let title = item.title
let subtitle = item.subtitle
let id = item.id
io.to(id).emit("Processing1", ""); //added emissions like these because I thought maybe the socket was timing out, but this didn't help
console.log("Processing1");
try {
await client.query('BEGIN')
let geoData = await //promise of geocoding endpoint api function
let lengthOfGeoData = geoData.context.length
io.to(id).emit("Processing2", "");
console.log("Processing2");
var municipality = ""
var area = ""
var locality = ""
var place = ""
var district = ""
var region = ""
var country = ""
//for loop to go through geoData and set the above values
if (municipality != "") {
console.log("Signing in from " + municipality + ", " + area);
} else {
console.log("Signing in from " + area)
}
await scrape(municipality, area, id);
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
}
try {
await client.query('BEGIN')
const array = await //a function that queries a Postgres db for some rows, makes json objects out of them, and pushes to the 'array' variable
var array2 = []
for (a of array) {
let difference = getDifference(title, subtitle, a.title, a.subtitle) //math function
if (difference <= 10) {
array.push(a)
}
}
io.to(id).emit("Processing9", "");
console.log("Processing9");
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log("ERROR: Failed arrayHelperFunction")
console.log(err)
} finally {
client.release()
console.log("About to emit this ish to " + id) //should emit to socket here ideally to notify that the processing is done and results can be polled
io.to(id).emit("finishedLoading", "")
return array2;
}
});
//when the client polls the queue after it's received the 'done' notifier from the server
app.post('/poll', async function(request, response) {
console.log("Polling")
let id = request.body.id
const results = await queue(id);
for (r of results.returnvalue) {
console.log("Sending " + r.title);
}
response.send(results.returnvalue)
});
//scrape
async function scrape(municipality, area, id) {
const client = await pool.connect();
try {
await client.query('BEGIN')
var location = ""
if (municipality != "") {
location = municipality + ", " + area
} else {
location = area
}
let inDatabase = await client.query('SQL statement AS it_does_exist', [params]);
io.to(id).emit("Processing3", "");
console.log("Processing3");
if (inDatabase.rows[0].it_does_exist == false) {
let query = "book clubs near " + location
var terminationTime = new Date()
terminationTime.setHours(terminationTime.getHours() + 4);
let date = ("0" + terminationTime.getDate()).slice(-2);
let month = ("0" + (terminationTime.getMonth() + 1)).slice(-2);
let year = terminationTime.getFullYear();
let hours = terminationTime.getHours();
let minutes = terminationTime.getMinutes();
let seconds = terminationTime.getSeconds();
let timestamp = year + "-" + month + "-" + date + " " + hours + ":" + minutes + ":" + seconds
try {
await client.query(`SQL statement`, [params]);
} catch(err) {
console.log("FAILURE: scrape() at 1.")
console.log(err)
}
var queryLocation = "New York,New York,United States" //default search origination is here
var queryGLCode = "US"
io.to(id).emit("Processing4", "");
console.log("Processing4");
try {
await fetch('https://serpapi.com/locations.json?q='+municipality+'&limit=10', { method : "GET" })
.then(res => res.json())
.then((json) => {
for (let index = 0; index < 10; index++) {
let locationAPIName = json[index].canonical_name
let locationAPICode = json[index].country_code
let resultLatitude = json[index].gps[1];
let resultLongitude = json[index].gps[0];
}
});
} catch(err) {
console.log("FAILURE: scrape() at 2.")
console.log(err)
}
io.to(id).emit("Processing5", "");
console.log("Processing5");
try {
await Promise.all([
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode}).then(data => async function(){
try {
await client.query('BEGIN');
let results = data.events_results
if (results != null) {
console.log("first HAD results")
for (result of results) {
var fixedAddress = result.address[0]
let address = fixedAddress + ", " + result.address[1]
let title = result.title + address
var description = result.description
let geoData = await geocode(address); //mapbox geocode the address
let latitude = Number(geoData.center[0]);
let longitude = Number(geoData.center[1]);
await client.query(`SQL statement`, [params]);
}
io.to(id).emit("Processing6", "");
console.log("Processing6");
} else {
console.log("first DID NOT have results")
}
console.log("FIRST BLOCK")
await client.query('COMMIT');
} catch(err) {
console.log("Results[0] not found.")
console.log(err)
await client.query('ROLLBACK');
}
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "10"}).then(data => async function(){
// same as the one above, just with an offset
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "20"}).then(data => async function(){
// same as the one above, but with a different offset
}())
])
} catch(err) {
console.log("FAILURE: scrape() at 3.")
console.log(err)
}
} else {
console.log("Location already in the database.")
}
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
} finally {
client.release()
return "Resolved";
}
}
//Client establish socket connection
func establishConnection(_ completion: (() -> Void)? = nil) {
let socketUrlString: String = appState.server
self.manager = SocketManager(socketURL: URL(string: socketUrlString)!, config: [.log(false), .reconnects(true), .extraHeaders(["header": "customheader"])])
self.socket = manager?.defaultSocket
self.socket?.connect()
self.socket?.once(clientEvent: .connect, callback: { (data, emitter) in
if completion != nil{
completion!()
}
})
//other socket functions
}
//Client initial post request
func process() {
let server = "serverstring" + "process"
let title = "title"
let subtitle = "subtitle"
let package = BookPackage(title: title, subtitle: subtitle, id: mySocketID) //this is after the initial connection
print("package is \(package)")
guard let url = URL(string: server) else { return }
var urlRequest = URLRequest(url: url)
urlRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
urlRequest.addValue("application/json", forHTTPHeaderField: "Accept")
urlRequest.httpMethod = "POST"
guard let data = try? JSONEncoder().encode(package) else { return }
urlRequest.httpBody = data
let task = URLSession.shared.dataTask(with: urlRequest) {
(data, response, error) in
if let error = error {
print(error)
return
}
guard let data = data else { return }
guard let dataString = String(data: data, encoding: String.Encoding.utf8) else { return }
let jsonData = Data(dataString.utf8)
var decodedJob: Job? = nil
do {
decodedJob = try JSONDecoder().decode(Job.self, from: jsonData) //Job is just a struct in the same form as the json object sent back from the server
} catch {
print(error.localizedDescription)
}
DispatchQueue.main.async {
self.appState.pendingJob = decodedJob
}
}
// start the task
task.resume()
}
The only consistent part of this bug is the logs right before the user disconnects (side note: 'reason of disconnect' and 'DISCONNECTED USER' fire on the socket.on('disconnect') event:
https://i.stack.imgur.com/7fjuU.png
https://i.stack.imgur.com/z5bmL.png
https://i.stack.imgur.com/aHNt3.png
https://i.stack.imgur.com/64WYI.png

You should be blocking the event loop with await. There is a heartbeat that the client sends every once in a while (which is defined with pingTimeout).
Since no ping is received by the server, it is disconnected.
You should isolate this process. Either find a way to use it with a worker/background process or async, additionally increasing pingTimeout on serverside might help you.

The solution to your problem is to modify the pingTimeout when initiating the server.
From Socket.io:
The server sends a ping, and if the client does not answer with a pong within pingTimeout ms, the server considers that the connection is closed.
Similarly, if the client does not receive a ping from the server
within pingInterval + pingTimeout ms, the client also considers that
the connection is closed.
const io = new Server(httpServer, {
pingTimeout: 30000
});

You can change the transport from the default to:
const io = new Server(httpServer, {
transports: ['polling', 'websocket'],
});
This might resolve the issue, else you can also try canging the upgradeTimeout and pingTimeout

Javascript: Add timeout after every request in Promise.all Map function

For the following function, I have to add a timeout after every GET request in array ajaxUrls. All the XHR GET request are in array ajaxUrls.
function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
return Promise.all(ajaxUrls.map(getSearchResultsForOnePage))
.then(function(responses) {
return responses.map(function(response) {
if (response.meta.total === 0) {
return [];
}
return response.result.searchResults.map(function(searchResult) {
return (searchResult);
});
});
})
.then(function(searchProfiles) {
return [].concat.apply([], searchProfiles);
})
.catch(function(responses) {
console.error('error ', responses);
});
}
function getSearchResultsForOnePage(url) {
return fetch(url, {
credentials: 'include'
})
.then(function(response) {
return response.json();
});
}
I want a certain timeout or delay after every GET request. I am facing difficulty in where exactly to add the timeout.

If you want to make requests in serial, you shouldn't use Promise.all, which initializes everything in parallel - better to use a reduce that awaits the previous iteration's resolution and awaits a promise-timeout. For example:
async function getAllSearchResultProfiles(searchAjaxUrl) {
const ajaxUrls = [];
for (let i = 0; i < numResults; i += resultsPerPage) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
const responses = await ajaxUrls.reduce(async (lastPromise, url) => {
const accum = await lastPromise;
await new Promise(resolve => setTimeout(resolve, 1000));
const response = await getSearchResultsForOnePage(url);
return [...accum, response];
}, Promise.resolve([]));
// do stuff with responses
const searchProfiles = responses.map(response => (
response.meta.total === 0
? []
: response.result.searchResults
));
return [].concat(...searchProfiles);
}
Note that only asynchronous operations should be passed from one .then to another; synchronous code should not be chained with .then, just use variables and write the code out as normal.

I find a simple for loop in an async function to be the most readable, even if not necessarily the most succinct for things like this. As long as the function is an async function you can also create a nice pause() function that makes the code very easy to understand when you come back later.
I've simplified a bit, but this should give you a good idea:
function pause(time) {
// handy pause function to await
return new Promise(resolve => setTimeout(resolve, time))
}
async function getAllSearchResultProfiles(searchAjaxUrl) {
var ajaxUrls = [];
for (var i = 0; i < 5; i++) {
ajaxUrls.push(searchAjaxUrl + "&start=" + i);
}
let responses = []
for (url of ajaxUrls) {
// just loop though and await
console.log("sending request")
let response = await getSearchResultsForOnePage(url)
console.log("recieved: ", response)
responses.push(response)
await pause(1000) // wait one second
}
//responses.map() and other manilpulations etc...
return responses
}
function getSearchResultsForOnePage(url) {
//fake fetch
return Promise.resolve(url)
}
getAllSearchResultProfiles("Test")
.then(console.log)

If you want to add a delay in every request then add a setTimout() in your function which fetches data from api
function getSearchResultsForOnePage(url) {
return new Promise((resolve, reject) => {
fetch(url, {
credentials: 'include'
})
.then(response => reresponse.json())
.then(data => {
let timeout = 1000;
setTimeout(() => resolve(data), timeout);
});
}

Get pdf buffer with puppeteer with session generated url

I've been using puppeteer to try and get pdfs - or its buffer response - from a website which does two requests after clicking on the link for the document (which open in a new tab):
The first request (http://epicdocs.planningni.gov.uk/ViewDocument.pa?uri=4157826&ext=PDF) retrieves the session guid to access the document
The second request (http://epicdocs.planningni.gov.uk/ViewDocument.aspx?guid=4ecd1fe5-43c6-4202-96e3-66b393fb819c) uses that guid to access the document and render the pdf on the browser.
The result of my attempts has been a blank pdf being generated, even if it was created after the page been loaded (checked with Fiddler).
I've tried
Intercepting targetcreated event to get the page
Get the second request url and use page.goto to get the pdf
Wait on a the page response to get the buffer
Set Page.setDownloadBehaviour to allow download instead of rendering it in the browser
Any guidance and help is appreciated.
The code tried is below:
const puppeteer = require("puppeteer");
let browser;
async function getDocument(index, title, page) {
if (index != 19) return "";
console.log("getDocument START");
console.log("#repDocuments__ctl" + index + "_lnkViewDoc\ntitle: " + title);
let docPagePromise = new Promise((resolve, reject) =>
browser.once("targetcreated", async target => {
let targetUrl = await target.url();
if (targetUrl.indexOf("ViewDocument.aspx?") !== -1) {
console.log(targetUrl);
return resolve(target.page());
} else {
console.log("Failed to detect the ViewDocument page");
}
})
);
/* Tried to set the download behaviour to download automatically the pdf but it didn't work */
// await page._client.send("Page.setDownloadBehaviour", {
// behaviour: "allow",
// downloadPath: "./"
// });
await page.click(`#repDocuments__ctl${index}_lnkViewDoc`);
let pdfResults = "";
let pdfPage = await docPagePromise;
/* If I get the target from the page returned from the promise I get the correct ur, however the page url is blank */
// let target = await pdfPage.target();
// let url = await target.url();
// let response = await pdfPage.goto(url);
// console.log(response);
pdfPage.on("console.log", msg => console.log(msg));
/* This is never called */
await pdfPage.on("response", async response => {
console.log("PDF PAGE Response");
let responseBuffer = await response.buffer();
let responseHeaders = response.headers();
console.log("PDF PAGE Response Header: " + responseHeaders);
console.log("PDF PAGE Response Buffer: " + responseBuffer);
return {
responseHeaders,
responseBuffer
};
});
console.log(pdfResults);
let pdfTitle = await pdfPage.title();
console.log("PDFPage URL: " + pdfPage.url());
console.log("PDFPage Title: " + pdfTitle);
let pdfTarget = await pdfPage.target();
console.log("PDFTarget URL: " + (await pdfTarget.url()));
console.log("PDFTarget Type: " + pdfTarget.type());
pdfPage = await pdfTarget.page();
console.log("PDFPage URL: " + pdfPage.url());
await pdfPage.waitFor(3000);
let pdf = await pdfPage.pdf({ path: title + ".pdf" });
console.log(pdf);
return pdf;
}
async function getAdditionalDocumentation(page) {
console.log("getAdditionalDocumentation START");
await page.waitForSelector("#repGroupSummary__ctl1_lnkGroupName");
await page.click("#repGroupSummary__ctl1_lnkGroupName");
await page.waitForSelector("#pnlDocumentList > table > tbody > tr");
await page.waitFor(2000);
const documents = await page.$$eval(
"#pnlDocumentList > table > tbody > tr",
docs =>
docs.map((doc, i) => ({
type: doc.querySelector(".tdl-subgroup > span").innerText,
datePublished: doc.querySelector(
".tdl-date > span[id*='DatePublished']"
).innerText,
dateReceived: doc.querySelector(".tdl-date > span[id*='DateReceived']")
.innerText,
docType: doc.querySelector(".tdl-doctype > span").innerText,
description: doc.querySelector(".tdl-description > span").innerText
// 'docBuffer': window.getDocument(i + 1, doc.querySelector('.tdl-description > span').innerText)
}))
);
for (let i = 0; i < documents.length; i++) {
documents[i].docBuffer = await getDocument(i + 1, documents[i].description, page);
}
await page.click("#btnSummary");
console.log("getAdditionalDocumentation FINISH");
return documents;
}
async function getDocuments(page, browser) {
console.log("getDocuments");
let newPagePromise = new Promise((resolve, reject) =>
browser.once("targetcreated", async target => {
let targetUrl = await target.url();
if (targetUrl.indexOf("ShowCaseFile.aspx?") !== -1) {
console.log(targetUrl);
return resolve(target.page());
} else {
console.log("Failed to detect the ShowCaseFile page");
}
})
);
await page.click("#tab_externalDocuments > span");
await page.waitForSelector("#hp-doc-link");
await page.click("#hp-doc-link");
const newPage = await newPagePromise;
const additionalDocumentation = await getAdditionalDocumentation(newPage);
return {
additionalDocumentation
};
}
async function run() {
try {
browser = await puppeteer.launch();
const page = await browser.newPage();
page.on("console", msg => console.log("PAGE LOG:", ...msg.args));
const planningReference = "LA04/2017/1388/F";
await page.goto(
"http://epicpublic.planningni.gov.uk/publicaccess/search.do?action=simple&searchType=Application"
);
await page.waitForSelector("#simpleSearchString");
await page.type("#simpleSearchString", planningReference);
await page.click("#simpleSearchForm > div.row3 > input.button.primary");
await page.waitForSelector("#simpleDetailsTable");
console.log("getDocuments START");
const documents = await getDocuments(page, browser);
console.log("getDocuments FINISH");
console.log(documents);
console.log(documents.additionalDocumentation.length);
} finally {
browser.close();
}
}
run();

Use exposefunction to write the buffer data to disk with:
page.exposeFunction("writeABString", async (strbuf, targetFile) => {
var str2ab = function _str2ab(str) { // Convert a UTF-8 String to an ArrayBuffer
var buf = new ArrayBuffer(str.length); // 1 byte for each char
var bufView = new Uint8Array(buf);
for (var i=0, strLen=str.length; i < strLen; i++) {
bufView[i] = str.charCodeAt(i);
}
return buf;
}
console.log("In 'writeABString' function...");
return new Promise((resolve, reject) => {
// Convert the ArrayBuffer string back to an ArrayBufffer, which in turn is converted to a Buffer
let buf = Buffer.from(str2ab(strbuf));
// Try saving the file.
fs.writeFile(targetFile, buf, (err, text) => {
if(err) reject(err);
else resolve(targetFile);
});
});
});
With the download link that you have use it in tandem with fetch api to get it as blob and convert it with:
page.evaluate( async () => {
function arrayBufferToString(buffer){ // Convert an ArrayBuffer to an UTF-8 String
var bufView = new Uint8Array(buffer);
var length = bufView.length;
var result = '';
var addition = Math.pow(2,8)-1;
for(var i = 0;i<length;i+=addition){
if(i + addition > length){
addition = length - i;
}
result += String.fromCharCode.apply(null, bufView.subarray(i,i+addition));
}
return result;
}
let geturl = "https://whateverurl.example.com";
return fetch(geturl, {
credentials: 'same-origin', // usefull when we are logged into a website and want to send cookies
responseType: 'arraybuffer', // get response as an ArrayBuffer
})
.then(response => response.arrayBuffer())
.then( arrayBuffer => {
var bufstring = arrayBufferToString(arrayBuffer);
return window.writeABString(bufstring, '/tmp/downloadtest.pdf');
})
.catch(function (error) {
console.log('Request failed: ', error);
});
});
For more info look at this issue on the github puppeteer page. The above solution was also suggested in the issue.
Source

how do i add variables to url requests in javascript, more specifically, expo and react native

i am currently making this request via fetch in expo
const response = await fetch(`https://graph.facebook.com/v2.9/{{{i need to enter the id here}}}/friends?access_token=${token}`);
I have user.id but I am not sure how to call id in the string
here is the entire function, everything works except the {id} in the string
login = async () => {
const ADD_ID = '<APP ID>'
const options = {
permissions: ['public_profile', 'email', 'user_friends'],
}
const {type, token} = await Expo.Facebook.logInWithReadPermissionsAsync(ADD_ID, options)
if (type === 'success') {
const response = await fetch(`https://graph.facebook.com/me?access_token=${token}`)
const user = (await response.json());
const id = user.id;
console.log(user);
console.log(id);
this.authenticate(token)
}
try {
const response = await fetch(`https://graph.facebook.com/v2.9/{id}/friends?access_token=${token}`);
console.log(await response.json())
} catch(error) {
console.error(error);
}
}

Let define your own format String function and move your try catch inside if expression
String.format = function() {
// The string containing the format items (e.g. "{0}")
// will and always has to be the first argument.
var theString = arguments[0];
// start with the second argument (i = 1)
for (var i = 1; i < arguments.length; i++) {
// "gm" = RegEx options for Global search (more than one instance)
// and for Multiline search
var regEx = new RegExp("\\{" + (i - 1) + "\\}", "gm");
theString = theString.replace(regEx, arguments[i]);
}
return theString;
}
console.log(String.format('https://graph.facebook.com/v2.9/{0}/friends?access_token=${1}', id, token))