Node.js: work with slower third party - javascript

I'm working with Node.js since 5 years and from 2 years on big projects with this framework. For two years, I'm confronted to a problem: how to work asynchronously and faster with non-async third party applications who's stacks requests, like MySQL, MongoDB or Apache SolR ?
I'm used to work with promises and to prepared several promises requests, like this:
const promises = []
for (let i = 0; i < 1000; i += 1) {
const promise = mySqlRequest()
promises.push(promise)
}
Promise.all(promises)
.then()
.catch()
This example will work but will send 1000 requests at the same time to MySQL server, who's will stacks these requests and become very slow, will consume very large quantity of RAM.
The best solution is to do only one big request, but in some case it's impossible and I'm forced to make recursive function,
which comes down to be synchronous and slower.
So, what the best way to work fast and asynchronous with Node.js and a stacking third party ?

If sending all requests at once doesn't work and sending them one by one doesn't work either, you'd need something similar to a thread-pool where some arbitrary number of tasks execute simultaneously. This is easily implementable using promises, for example like this:
Promise.pooled = function(arr, num = 5) {
return new Promise(function(resolve, reject) {
var i = -1;
var error = false;
var end = function() {
num--;
if(num === 0) resolve();
}
var next = function() {
if(error) return;
i++;
if(i >= arr.length)
end();
else
arr[i]().then(next).catch(onerr);
}
var onerr = function() {
if(error) return
error = true
reject.call(arguments)
}
for(var j = 0; j < num; j++)
next()
});
}
What this allows you is pass an array of functions as first argument, those functions should take no parameter and return a promise. It will then execute exactly num simultaneously. If one of the promises fail, it will fail its own promise aswell and stop executing (this is changeable easily).
Example:
Promise.after = function(ms) {
return new Promise(function(resolve) {
setTimeout(resolve, ms)
});
}
Promise.pooled = function(arr, num = 5) {
return new Promise(function(resolve, reject) {
var i = -1;
var error = false;
var end = function() {
num--;
if(num === 0) resolve();
}
var next = function() {
if(error) return;
i++;
if(i >= arr.length)
end();
else
arr[i]().then(next).catch(onerr);
}
var onerr = function() {
if(error) return
error = true
reject.call(arguments)
}
for(var j = 0; j < num; j++)
next()
});
}
var test = [
afterH(1000),
afterH(500),
afterH(800),
afterH(600),
afterH(3000),
afterH(300),
afterH(900),
afterH(2000),
afterH(1500),
afterH(900),
afterH(700),
afterH(600),
afterH(700)
];
// helper function, returns a function which when invoked returns a promise
function afterH(ms) {
return function() {
console.log("Starting one job")
return Promise.after(ms);
}
}
Promise.pooled(test, 3).then(function() {console.log("All jobs finished") }).catch(function() {console.log("Job failed")})

Related

Issue with Lodash Throttle or Debounce within the loop

I'm calling a fetch request which I'm trying to limit by using Lodash Throttle or Debounce. I'm looping through some array and calling function instantly which effects a server to respond with 502. I'm trying to slow down the requests with Throttle. The code below should explain my structure. This example does not work and I don't know why?
function doSomething(i) {
console.log('Doing something: ' + i)
}
for (var i = 0; i < 50; i++) {
_.throttle( function() { doSomething(i) }, 15000);
}
The function doSomething() should be called every 15 seconds and additional requests to this function should be stacked.
_.throttle() is not meant to be used this way. Right way to go is to store the result first.
var throttledDoStomething = _.throttle(doSomething, 15000)
for (var i=0; i < 50; i++) {
throttledDoSomething(i)
}
In reply to the comment:
In that case, throttle maybe is not the right choice. You might want to use asynchronous function by using Promise.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
function asyncDoSomething(i) {
return new Promise((resolve, reject) => {
console.log('Doing something: ' + i);
setTimeout(()=>{
resolve(i);
}, 15000)
})
}
async function doSomethingLoop() {
for (var i = 0; i < 50; i++) {
await asyncDoSomething(i);
}
}
doSomethingLoop();
Document suggests that you should first make throttled function.
Putting argument needs Anonymous function(in my case, I used Array function) to be used.
function doSomething(i) {
console.log("Doing something: " + i);
}
const throttledSomething = _.throttle(() => { doSomething(i)}, 5000);
for (var i = 0; i < 50; i++) {
throttledSomething(i);
}

Reading an array of numbers in parallel with Promises

I have an array of numbers and I'd like to read them in parallel, with some workers (Promises), having a maximum of workers working simultaneously allowed.
I want to get the result, an array of numbers, through a Promise, when all workers have finished their work.
The order in which numbers are read is not important.
I've written some code which is close to the solution, but the workers, being promises, start their work immediately after being created, (as you can see from the "Active workers" infos printed which contains more items than maxWorkers). So I cannot control how many are actually running together, which is what I want.
What I actually want is to see a maximum of maxWorkers listed in the "Active workers" info.
A worker can create a max of maxWorkersToCreate workers, and the total workers working can be as high as maxWorkers.
This is just a toy example, the real work I have to do is to read in parallel some web resources. This is why I thought to Promises and this is why there is a setTimeout in the code (to simulate an async operation).
Library suggestions are accepted as well.
Object.defineProperty(Array.prototype, 'flat', {
value: function(depth = 1) {
return this.reduce(function(flat, toFlatten) {
return flat.concat((Array.isArray(toFlatten) && (depth - 1)) ? toFlatten.flat(depth - 1) : toFlatten);
}, []);
}
});
const workerManager = new class WorkerManager {
constructor() {
this.workers = [];
}
showWorkers() {
let w = this.workers.filter(function(n){ return n != undefined });
console.log("Active workers:", w);
}
setWorker(name) {
this.workers[name] = name;
}
removeWorker(name) {
delete this.workers[name];
}
}
const MAX = 10;
var numbers = [];
for (var i = 0; i < MAX; i++) {
numbers.push(i);
}
// last position being read from workers
let maxPositionQueued = 0;
// max number of workers working
const maxWorkers = 3;
// max workers a worker can create
const maxWorkersToCreate = 4;
// current number of workers working
let currentWorkers = 0;
function readAll(positionToRead) {
if (currentWorkers > maxWorkers) {
// Rescheduling worker
return new Promise(function(resolve) {
setTimeout(function() {
resolve(readAll(positionToRead));
}, Math.random() * 10 + 10);
});
}
function readNumber() {
return new Promise(function(resolve) {
workerManager.setWorker(positionToRead);
workerManager.showWorkers();
console.log("Reading pos:", positionToRead);
setTimeout(function() {
let result = numbers[positionToRead];
workerManager.removeWorker(positionToRead);
resolve(result);
}, Math.random() * 500 + 1000);
});
}
currentWorkers++;
return readNumber().then(function(response) {
let workers = [];
while (workers.length < maxWorkersToCreate &&
maxPositionQueued < MAX - 1) {
/*
* here I'd like to queue a worker and not start it
* immediately
*/
workers.push(readAll(++maxPositionQueued));
}
currentWorkers--;
return Promise.all([
response,
workers
].flat()).then(function(responses) {
return responses.flat();
});
}); // then reponse END
}
readAll(0).then(function(response) {
console.log("Numbers:", response);
});

Javascript.Run Multi promises Synchronously

I want to request a website for 40 times.
I want this to be synchronously, like 10 requests 4 times.
This is My code for 1 request - 40 times:
'use strict';
var request = require('request');
var co = require('co');
function callUrl(url) {
return new Promise((resolve, reject) => {
request.get(url, (e, r, b) => {
if (e) reject(e)
else
resolve(`Response from ${url}`);
});
})
}
co(function*() {
for (var i = 1; i < 41; i++) {
let rs = yield callUrl('https://www.google.com/?q=' + i);
// let rs = yield makeUrls(10,i);
console.log(rs);
}
});
I can make an array of promises, but I can't figure it out how to change the value of q to be different.
You don't want to run them synchronously - you want to synchronize them - those are different.
You'd use an array of promises together with Promise#all. When you create a promise the action is already being executed - only yield synchronizes things.
You can make 10 requests at once like so:
co(function*() {
for (var i = 1; i < 41;) {
var promises = [];
for(var lim = i + 10; i < Math.max(lim, 41); i++) {
promises.push(callUrl('https://www.google.com/?q=' + i));
}
let rs = yield Promise.all(promises); // wait for everything
console.log(rs); // an array of 10 results
});
Note that in addition to that, your code is still not very efficient - what happens if 9 out of 10 requests are really fast and one takes a minute? You'll only have one outgoing requests. You can use a library like bluebird which has a more efficient Promise.map method with a concurrency parameter.
this might work w/o using generators
const urls = [/*array of urls*/];
const initialPromise = request[urls[0]];
let promise = initialPromise;
for(let i= 1; i<40;i++){
let thenFunction = response => {
//do something with the response
return request(urls[i])
}
promise = promise.then(thenFunction)
}
the idea behind this is to build the chain of promises so the next one will waif for the previous one to finish

Multiple queries in a loop Parse Cloud Code

I'm having a hard time trying to understand promises, I'm sure I need to use them for this but I don't know how and other answers don't help me at all.
I'd like to loop over an array, query all the results for each value of the array, then after calculating the average value for these results, add the average in an array. After every iterations, this array is sent as a response.
Here is my code which could help understand me here:
Parse.Cloud.define('getScorePeopleArray', function(request, response) {
var peopleArray = request.params.peoplearray;
var query = new Parse.Query("Scores");
var resultat;
var index, len;
var resultarray = [];
var people;
for (index = 0, len = peopleArray.length; index < len; ++index) {
people = peopleArray[index];
query.equalTo("People",people);
query.find({
success: function(results) {
var sum = 0;
for (var i = 0; i < results.length; ++i) {
sum += results[i].get("Score");
}
resultat = (sum / results.length)*5;
if(!resultat){
resultarray.push("null");
}else{
resultarray.push(resultat);
}
},
error: function() {
response.error("score lookup failed");
}
}).then();
}
response.success(resultarray);
});
Of course response.success is not called when every queries are done, but as soon as possible (since queries are asynchronous if I'm right).
I know I have to change it with promises, but I don't understand at all how this works.
Thanks a lot in advance !
var _ = require('underscore');
Parse.Cloud.define('getScorePeopleArray', function(request, response) {
var peopleArray = request.params.peoplearray; // what is this an array of?
var resultArray = [];
return Parse.Promise.as().then(function() { // this just gets the ball rolling
var promise = Parse.Promise.as(); // define a promise
_.each(peopleArray, function(people) { // use underscore, its better :)
promise = promise.then(function() { // each time this loops the promise gets reassigned to the function below
var query = new Parse.Query("Scores");
query.equalTo("People", people); // is this the right query syntax?
return query.find().then(function(results) { // the code will wait (run async) before looping again knowing that this query (all parse queries) returns a promise. If there wasn't something returning a promise, it wouldn't wait.
var sum = 0;
for (var i = 0; i < results.length; i++) {
sum += results[i].get("Score");
}
var resultat = (sum / results.length) * 5;
if (!resultat){
resultArray.push("null");
} else {
resultArray.push(resultat);
}
return Parse.Promise.as(); // the code will wait again for the above to complete because there is another promise returning here (this is just a default promise, but you could also run something like return object.save() which would also return a promise)
}, function (error) {
response.error("score lookup failed with error.code: " + error.code + " error.message: " + error.message);
});
}); // edit: missing these guys
});
return promise; // this will not be triggered until the whole loop above runs and all promises above are resolved
}).then(function() {
response.success(resultArray); // edit: changed to a capital A
}, function (error) {
response.error("script failed with error.code: " + error.code + " error.message: " + error.message);
});
});

throttle requests in Node.js

I have an array. I can loop over it with the foreach method.
data.forEach(function (result, i) {
url = data[i].url;
request(url);
});
The request function is making a http request to the given url.
However making all these requests at the same time leads to all sorts of problems.
So I thought I should slow things down by introducing some-sort of timer.
But I have no idea how will be able to combine a forach loop with setTimeOut/setInterval
Please note am doing this on the server (nodejs) rather on the browser.
Thanks for you help.
As your problem is global, you should adjust your request function to have only 5 request running at a time - using a global, static counter. If your request was before something like
function request(url, callback) {
ajax(url, callback);
}
now use something like
var count = 0;
var waiting = [];
function request(url, callback) {
if (count < 5) {
count++;
ajax(url, function() {
count--;
if (waiting.length)
request.apply(null, waiting.shift());
callback.apply(this, arguments);
});
} else
waiting.push(arguments);
}
data.forEach(function (result, i) {
url = data[i].url;
setTimeout(
function () {
request(url);
},
1000 * (i + 1) // where they will each progressively wait 1 sec more each
);
});
Instead of setTimeout could have them run in sequence. I assume there's a callback parameter to your request() function.
function makeRequest(arr, i) {
if (i < arr.length) {
request(arr[i].url, function() {
i++;
makeRequest(arr, i);
});
}
}
makeRequest(data, 0);
If you need a little more time between requests, then add the setTimeout to the callback.
function makeRequest(arr, i) {
if (i < arr.length) {
request(arr[i].url, function() {
i++;
setTimeout(makeRequest, 1000, arr, i);
});
}
}
makeRequest(data, 0);
you can delay call using setTimeout. following code will insure that each request get called after timerMultiPlier milliseconds from its previous request.
var timerMultiPlier = 1000;
data.forEach(function (result, i) {
setTimeout(function(){
url = data[i].url;
request(url);
}, timerMultiPlier*i );
});
Many of the above solutions, while practical for a few requests, unfourtunatly choke up and brick the page when dealing with tens of thousands of requests. Instead of queuing all of the timers at one, each timer should be qued sequentially one after another. If your goal is to have nice pretty fluffy code with lots of sugar and 'goodie-goodies' then below is the solution for you.
function miliseconds(x) { return x }
function onceEvery( msTime ){
return {
doForEach: function(arr, eachF){
var i = 0, Len = arr.length;
(function rekurse(){
if (i < Len) {
eachF( arr[i], i, arr );
setTimeout(rekurse, msTime);
++i;
}
})();
}
};
}
Nice, pretty, fluffy sugar-coated usage:
onceEvery(
miliseconds( 150 )
).doForEach(
["Lorem", "ipsum", "dolar", "un", "sit", "amet"],
function(value, index, array){
console.log( value, index );
}
)
function miliseconds(x) { return x }
function onceEvery( msTime ){
return {
doForEach: function(arr, eachF){
var i = 0, Len = arr.length;
(function rekurse(){
if (i < Len) {
eachF( arr[i], i, arr );
setTimeout(rekurse, msTime);
++i;
}
})();
}
};
}
You can the offset the execution delay of each item by the index, like this:
data.forEach(function (result, i) {
setTimeout(function() {
url = data[i].url;
request(url);
}, i * 100);
});
This will make each iteration execute about 100 milliseconds after the previous one. You can change 100 to whatever number you like to change the delay.
Some addings to this questions, just for knowledge base.
Created and async version without recursion.
function onceEvery(msTime) {
return {
doForEach: function (eachFunc, paramArray) {
var i = 0, Len = paramArray.length;
(function rekurse() {
if (i < Len) {
eachFunc(paramArray[i], i, paramArray);
setTimeout(rekurse, msTime);
++i;
}
})();
},
doForEachAsync: async function (eachFunc, paramArray, staticParamenters) {
var i = 0, Len = paramArray.length;
while (i < Len) {
await (async function rekurse() {
await eachFunc(paramArray[i], staticParamenters);
setTimeout(() => { }, msTime);
++i;
})();
}
}
};
}
module.exports = {
onceEvery
};
Put this code in a .js and call as simple as:
await throttle.onceEvery(100).doForEachAsync(loadJobFunction, arrayParam, { staticParam1, staticParam2, staticParam3 });
a very simple solution for quick throttling using async / await is to create a promise such as :
const wait = (delay) => new Promise((resolve, _) => {
setTimeout(() => resolve(true), delay)
})
Then await for it when you need to pause (in an async function):
//... loop process
await wait(100)
Note: you need to use a for loop for this to work, a forEach won't wait.

Categories