We Keep Coding

How to execute variable number of async calls(coming dynamically at runtime) serially?

I am making a chrome extension (mv3). Based on user activity, the content.js passes a message to the background.js which then calls an async function to add data in Google Docs using Docs API.
I want each request to execute only after the previous one has finished running. I am using chrome.runtime.sendMessage to send a message from content.js and don't see a way of calling background.js serially from there. So I need a way of executing them one by one in background.js only. The order of these requests is also important (but if the order of the requests gets changed by one/two places, I think that would still be okay from a user perspective).
I tried something and it is working but I am not sure if I am missing some edge cases, because I was unable to find the approach in any other answers -
Semaphore-like queue in javascript?
Run n number of async function before calling another method in nodejs
JavaScript: execute async function one by one
The approach I used is: I use a stack like structure to store requests, use setInterval to check for any pending requests and execute them serially.
content.js:
chrome.runtime.sendMessage({message});
background.js:
let addToDocInterval = "";
let addToDocCalls = [];
async function addToDoc(msg) {
// Await calls to doc API
}
async function addToDocHelper() {
if(addToDocCalls.length === 0)
return;
clearInterval(addToDocInterval)
while(addToDocCalls.length > 0) {
let msg = addToDocCalls.shift();
await addToDoc(msg);
}
addToDocInterval = setInterval(addToDocHelper, 1000);
}
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
addToDocCalls.push(msg);
})
addToDocInterval = setInterval(addToDocHelper, 1000);
Is this approach correct? Or is there any better way to do this?

I'd suggest changing several things.
Don't use timers polling the array. Just initiate processing the array anytime you add a new item to the array.
Keep a flag on whether if you're already processing the array so you don't start duplicate processing.
Use a class to encapsulate this functionality into an object.
Encapsulate the addToDocCalls array and adding to it so your class is managing it and outside code just calls a function to add to it which also triggers the processing. Basically, you're making it so callers don't have to know how the insides work. They just call helper.addMsg(msg) and the class instance does all the work.
Here's an implementation:
async function addToDoc(msg) {
// Await calls to doc API
}
class docHelper {
constructor() {
this.addToDocCalls = [];
this.loopRunning = false;
}
addMsg(msg) {
// add item to the queue and initiate processing of the queue
this.addToDocCalls.push(msg);
this.process();
}
async process() {
// don't run this loop twice if we're already running it
if (this.loopRunning) return;
try {
this.loopRunning = true;
// process all items in the addToDocCalls we have
while(this.addToDocCalls.length > 0) {
let msg = addToDocCalls.shift();
await addToDoc(msg);
}
} finally {
this.loopRunning = false;
}
}
}
const helper = new docHelper();
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
helper.addMsg(msg);
});
So, process() will run until the array is empty. Any interim calls to addMsg while process() is running will add more items to array and will call process() again, but the loopRunning flag will keep it from starting duplicate processing loops. If addMsg() is called while process is not running, it will start the process loop.
P.S. You also need to figure out what sort of error handling you want if addToDoc(msg) rejects. This code protects the this.loopRunning flag if it rejects, but doesn't actually handle a reject error. In code like this that is processing a queue, often times all you can really do is log the error and move on, but you need to decide what is the proper course of action on a rejection.

You don't need to use setTimeout. You do not even need a while loop.
let addToDocInterval = "";
let addToDocCalls = [];
let running = false;
async function addToDoc(msg) {
// Await calls to doc API
}
async function addToDocHelper() {
if(running || addToDocCalls.length === 0)
return;
running = true;
let msg = addToDocCalls.shift();
await addToDoc(msg);
running = false;
addToDocHelper();
}
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
addToDocCalls.push(msg);
addToDocHelper();
});
The code should be self explanatory. There is no magic.

Here is a generic way to run async tasks sequentially (and add more tasks to the queue at any time).
const tasks = [];
let taskInProgress = false;
async function qTask(newTask) {
if (newTask) tasks.push(newTask);
if (tasks.length === 0) return;
if (taskInProgress) return;
const nextTask = tasks.shift();
taskInProgress = true;
try {
await nextTask();
} finally {
taskInProgress = false;
//use setTimeout so call stack can't overflow
setTimeout(qTask, 0);
}
}
//the code below is just used to demonstrate the code above works
async function test() {
console.log(`queuing first task`);
qTask(async () => {
await delay(500); //pretend this task takes 0.5 seconds
console.log('first task started');
throw 'demonstrate error does not ruin task queue';
console.log('first task finished');
});
for (let i = 0; i < 5; i++) {
console.log(`queuing task ${i}`)
qTask(async () => {
await delay(200); //pretend this task takes 0.2 seconds
console.log(`task ${i} ran`);
});
}
await delay(1000); //wait 1 second
console.log(`queuing extra task`);
qTask(async () => {
console.log('extra task ran');
});
await delay(3000); //wait 3 seconds
console.log(`queuing last task`);
qTask(async () => {
console.log('last task ran');
});
}
test();
function delay(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms);
});
}

Await for file to load, then run a function on each 'line' but wait for the return on each line

I have been trying to understand Promises and I'm hitting a brick wall.
==Order I want the code to run==
I need a .txt file to load each line into an array.
WAIT for this to happen.
Run a Function on each entry that returns and array.
WAIT for each index of the array to be processed before doing the next.
==My Functions==
Call this function to start the program.
async function start(){
var data = await getData();
console.log(data);
for (var i = 0; i < data.length; i++){
console.log(await searchGoogle(data[i]));
}
}
'await' for the data from getData
async function getData(){
return new Promise(function(resolve, reject){
fs.readFile('./thingsToGoogle.txt', function(err, data) {
if(err) throw err;
var array = data.toString().split("\n");
resolve(array);
});
});
}
Then call searchGoogle on each index in the array.
async function searchGoogle(toSearch) {
(async() => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://www.google.com/');
await page.type('input[name=q]', toSearch);
try {
console.log('Setting Search' + toSearch);
await page.evaluate(() => {
let elements = document.getElementsByClassName('gNO89b');
for (let element of elements)
element.click();
});
await page.waitForNavigation();
} catch (err) {
console.log(err)
}
try {
console.log("Collecting Data");
const[response] = await Promise.all([
page.waitForNavigation(),
await page.click('.rINcab'),
]);
} catch (err) {
console.log("Error2: " + err)
}
let test = await page.$$('.LC20lb');
// console.log(test);
allresults = [];
for (const t of test) {
const label = await page.evaluate(el => el.innerText, t);
if (label != "") {
allresults.push(label);
}
}
await browser.close();
resolve(allresults);
})();
}
The problem is that this does not work. it does not wait for the file to load.
Picture of Node JS output.
Hopefully the screen shot has uploaded, but you can see it stacking the SearchGoogle function console.logs;
console.log('Setting..')
console.log('Setting..')
console.log('Collecting..')
console.log('Collecting..')
When it should be
console.log('Setting..')
console.log('Collecting..')
console.log('Setting..')
console.log('Collecting..')
This is the 'first' time sort of dealing with promises, i have done a lot of reading up on them and done bits of code to understand them, however when I have tried to apply this knowledge I am struggling. Hope someone can help.
-Peachman-

Queue with concurrent Limit (using p-queue)
You need a queue with concurrency limit. You will read every single line and add them to a queue. We will be using readline and p-queue module for this.
First, create a queue with concurrency of 1.
const {default: PQueue} = require('p-queue');
const queue = new PQueue({concurrency: 1});
Then, create our reader instance.
const fs = require('fs');
const readline = require('readline');
const rl = readline.createInterface({
input: fs.createReadStream('your-input-file.txt')
});
For every line of the file, add an entry to the queue.
rl.on('line', (line) => {
console.log(`Line from file: ${line}`);
queue.add(() => searchGoogle(line));
});
That's it! If you want to process 10 lines at once, just change the concurrency line. It will still read one line at a time, but the queue will limit how many searchGoogle is invoked.
Optional Fixes: Async Await
Your code has the following structure,
async yourFunction(){
(async()=>{
const browser = await puppeteer.launch();
// ... rest of the code
})()
}
While this might run as intended, you will have a hard time debugging because you will be creating an anonymous function every time you run yourFunction.
The following is enough.
async yourFunction(){
const browser = await puppeteer.launch();
// ... rest of the code
}

Here's a way to process them that lets you process N URLs at a time where you can adjust the value of N. My guess is that you want it set to a value of between 5 and 20 in order to keep your CPU busy, but not use too many server resources.
Here's an outline of how it works:
It uses the line-by-line module to read a file line by line and (unlike the built-in readline interface), this module pauses line events when you call .pause() which is important in this implementation.
It maintains a numInFlight counter that tells you how many lines are in the midst of processing.
You set a maxInFlight constant to the maximum number of lines you want to be processed in parallel.
It maintains a resultCntr that helps you keep results in the proper order.
It creates the readline interface and establishes a listener for the line event. This will start the stream flowing with line events.
On each line event, we increment our numInFlight counter. If we have reached the maximum number allowed in flight, we pause the readline stream so it won't produce any more line events. If we haven't reached the max in flight yet, then more line events will flow until we do reach the max.
We pass that line off to your existing searchGoogle() function.
When that line is done processing, we save the result in the appropriate spot in the array, decrement the numInFlight counter and resume the stream (in case it was previously paused).
We check if we're all done (by checking if numInFlight is 0 and if we've reached the end of our file). If we are done, resolve the master promise with the results.
If we're not all done, then there will either be more line events coming or more searchGoogle() functions in flight that will finish, both of which will check again to see if we're done.
Note that the way this is designed to work is that errors on any given URL are just put into the result array (the error object is in the array) and processing continues on the rest of the URLs with an eventual resolved promise. Errors while reading the input file will terminate processing and reject the return promise.
Here's the code:
const fs = require('fs');
const Readline = require('line-by-line');
function searchAll(file) {
return new Promise(function(resolve, reject) {
const rl = new Readline(file);
// set maxInFlight to something between 5 and 20 to optimize performance by
// running multiple requests in flight at the same time without
// overusing memory and other system resources.
const maxInFlight = 1;
let numInFlight = 0;
let resultCntr = 0;
let results = [];
let doneReading = false;
function checkDone(e) {
if (e) {
reject(e);
} else if (doneReading && numInFlight === 0) {
resolve(results);
}
}
rl.on('line', async (url) => {
if (url) {
let resultIndex = resultCntr++;
try {
++numInFlight;
if (numInFlight >= maxInFlight) {
// stop flowing line events when we hit maxInFlight
rl.pause();
}
let result = await searchGoogle(url);
// store results in order
results[resultIndex] = result;
} catch(e) {
// store error object as result
results[resultIndex] = e;
} finally {
--numInFlight;
rl.resume();
checkDone();
}
}
}).on('end', () => {
// all done reading here, may still be some processing in flight
doneReading = true;
checkDone();
}).on('error', (e) => {
doneReading = true;
checkDone(e);
});
});
}
FYI, you can set maxInFlight to a value of 1 and it will read process the URLs one at a time, but the whole point of writing this type of function is so that you can likely get better performance by setting it to a value higher than 1 (I'm guessing 5-20).

How to exit a function if observable result is equals to certain value

I have this piece of code in my angular 6 application:
publish() {
change.subscribe((result: boolean) => {
if(!result) return; // exit publish function
});
// continue
}
I want this publish function to continue executing only if result is true.
How to manage this ?

it is impossible, you should put your publish function code inside your subscription, or there is another way you can do, you can use .toPromise() and async/await, if you will get your data one time and not in a stream
async publish() {
const result = await change.toPromise();
if(result) {
// your publish function code here
});
}

Converting observable to promise is always a bad idea, so better is just to create local variable and assign a value from yourObservable to this local variable.
async publish() {
let doBreak = false;
yourObservable.subscribe((result) => {
if (!result) {
doBreak = true;
}
}
if (doBreak) {
return;
}
}

How to call API every 2 seconds until desired result or timeout in async function

I need to call an api to get a status every 2 seconds if the response is running and first return when response is either complete or failed, or until 30 seconds have passed and the function times out.
This is what I have now which works, but I am sure it can be done much more efficient, but I simply can't figure it out at this point:
const getStatus = async (processId) => {
try {
const response = await fetch(`example.com/api/getStatus/${processId}`);
const status = await response.json();
return await status;
} catch(err) {
// handle error
}
}
Inside another async function using getStatus():
randomFunction = async () => {
let status = null;
let tries = 0;
let stop = false;
while (tries <= 15 && !stop) {
try {
status = await getStatus('some-process-id');
if (status === 'complete') {
stop = true;
// do something outside of loop
}
if (status === 'failed') {
stop = true;
throw Error(status);
}
if (tries === 15) {
stop = true;
throw Error('Request timed out');
}
} catch (err) {
// handle error
}
const delay = time => new Promise(resolve => setTimeout(() => resolve(), time));
if (tries < 15) {
await delay(2000);
}
tries++;
}
}
I would prefer to handle the looping inside getStatus() and in a more readable format, but is it possible?
EDIT:
I tried a solution that looks better and seems to work as I expect, see it here:
https://gist.github.com/AntonBramsen/6cec0faade032dfa3c175b7d291e07bd
Let me know if parts of the solution contains any solutions that are bad practice.

Your question is for javascript. Unfortunately I don't drink coffee, I can only give you the code in C#. But I guess you get the gist and can figure out how to translate this into java
Let's do this as a generic function:
You have a function that is called every TimeSpan, and you want to stop calling this function whenever the function returns true, you want to cancel, whenever some maximum time has passed.
For this maximum time I use a CancellationToken, this allows you to cancel processing for more reasons than timeout. For instance, because the operator wants to close the program.
TapiResult CallApi<TapiResult> <Func<TapiResult> apiCall,
Func<TapiResult, bool> stopCriterion,
CancellationToken cancellationToken)
{
TapiResult apiResult = apiCall;
while (!stopCriterion(apiResult))
{
cancellationToken.ThrowIfCancellationRequested();
Task.Delay(delayTime, cancellationToken).Wait;
apiResult = apiCall;
}
return apiResult;
}
ApiCall is the Api function to call. The return value is a TApiResult. In your case the status is your TApiResult
StopCriterion is a function with input ApiResult and output a boolean that is true when the function must stop. In your case this is when status equals complete or failed
CancellationToken is the Token you can get from a CancellationTokenSource. Whenever you want the procedure to stop processing, just tell the CancellationTokenSource, and the function will stop with a CancellationException
Suppose this is your Api:
Status MyApiCall(int x, string y) {...}
Then the usage is:
Timespan maxProcessTime = TimeSpan.FromSeconds(45);
var cancellationTokenSource = new CancellationTokenSource();
// tell the cancellationTokenSource to stop processing afer maxProcessTime:
cancellationTokenSource.CancelAfter(maxProcessTime);
// Start processing
Status resultAfterProcessing = CallApi<Status>(
() => MyApiCall (3, "Hello World!"), // The Api function to call repeatedly
// it returns a Status
(status) => status == complete // stop criterion: becomes true
|| status == failed, // when status complete or failed
cancellationTokenSource.Token); // get a token from the token source
TODO: add try / catch for CancellationException, and process what should be done if the task cancels
The function will stop as soon as the stopCriterion becomes true, or when the CancellationTokenSource cancels. This will automatically be done after maxTimeOut. However, if you want to stop earlier, for instance because you want to stop the program:
cancellationTokenSource.Cancel();

Using chrome.tabs.executeScript to execute an async function

I have a function I want to execute in the page using chrome.tabs.executeScript, running from a browser action popup. The permissions are set up correctly and it works fine with a synchronous callback:
chrome.tabs.executeScript(
tab.id,
{ code: `(function() {
// Do lots of things
return true;
})()` },
r => console.log(r[0])); // Logs true
The problem is that the function I want to call goes through several callbacks, so I want to use async and await:
chrome.tabs.executeScript(
tab.id,
{ code: `(async function() {
// Do lots of things with await
return true;
})()` },
async r => {
console.log(r); // Logs array with single value [Object]
console.log(await r[0]); // Logs empty Object {}
});
The problem is that the callback result r. It should be an array of script results, so I expect r[0] to be a promise that resolves when the script finishes.
Promise syntax (using .then()) doesn't work either.
If I execute the exact same function in the page it returns a promise as expected and can be awaited.
Any idea what I'm doing wrong and is there any way around it?

The problem is that events and native objects are not directly available between the page and the extension. Essentially you get a serialised copy, something like you will if you do JSON.parse(JSON.stringify(obj)).
This means some native objects (for instance new Error or new Promise) will be emptied (become {}), events are lost and no implementation of promise can work across the boundary.
The solution is to use chrome.runtime.sendMessage to return the message in the script, and chrome.runtime.onMessage.addListener in popup.js to listen for it:
chrome.tabs.executeScript(
tab.id,
{ code: `(async function() {
// Do lots of things with await
let result = true;
chrome.runtime.sendMessage(result, function (response) {
console.log(response); // Logs 'true'
});
})()` },
async emptyPromise => {
// Create a promise that resolves when chrome.runtime.onMessage fires
const message = new Promise(resolve => {
const listener = request => {
chrome.runtime.onMessage.removeListener(listener);
resolve(request);
};
chrome.runtime.onMessage.addListener(listener);
});
const result = await message;
console.log(result); // Logs true
});
I've extended this into a function chrome.tabs.executeAsyncFunction (as part of chrome-extension-async, which 'promisifies' the whole API):
function setupDetails(action, id) {
// Wrap the async function in an await and a runtime.sendMessage with the result
// This should always call runtime.sendMessage, even if an error is thrown
const wrapAsyncSendMessage = action =>
`(async function () {
const result = { asyncFuncID: '${id}' };
try {
result.content = await (${action})();
}
catch(x) {
// Make an explicit copy of the Error properties
result.error = {
message: x.message,
arguments: x.arguments,
type: x.type,
name: x.name,
stack: x.stack
};
}
finally {
// Always call sendMessage, as without it this might loop forever
chrome.runtime.sendMessage(result);
}
})()`;
// Apply this wrapper to the code passed
let execArgs = {};
if (typeof action === 'function' || typeof action === 'string')
// Passed a function or string, wrap it directly
execArgs.code = wrapAsyncSendMessage(action);
else if (action.code) {
// Passed details object https://developer.chrome.com/extensions/tabs#method-executeScript
execArgs = action;
execArgs.code = wrapAsyncSendMessage(action.code);
}
else if (action.file)
throw new Error(`Cannot execute ${action.file}. File based execute scripts are not supported.`);
else
throw new Error(`Cannot execute ${JSON.stringify(action)}, it must be a function, string, or have a code property.`);
return execArgs;
}
function promisifyRuntimeMessage(id) {
// We don't have a reject because the finally in the script wrapper should ensure this always gets called.
return new Promise(resolve => {
const listener = request => {
// Check that the message sent is intended for this listener
if (request && request.asyncFuncID === id) {
// Remove this listener
chrome.runtime.onMessage.removeListener(listener);
resolve(request);
}
// Return false as we don't want to keep this channel open https://developer.chrome.com/extensions/runtime#event-onMessage
return false;
};
chrome.runtime.onMessage.addListener(listener);
});
}
chrome.tabs.executeAsyncFunction = async function (tab, action) {
// Generate a random 4-char key to avoid clashes if called multiple times
const id = Math.floor((1 + Math.random()) * 0x10000).toString(16).substring(1);
const details = setupDetails(action, id);
const message = promisifyRuntimeMessage(id);
// This will return a serialised promise, which will be broken
await chrome.tabs.executeScript(tab, details);
// Wait until we have the result message
const { content, error } = await message;
if (error)
throw new Error(`Error thrown in execution script: ${error.message}.
Stack: ${error.stack}`)
return content;
}
This executeAsyncFunction can then be called like this:
const result = await chrome.tabs.executeAsyncFunction(
tab.id,
// Async function to execute in the page
async function() {
// Do lots of things with await
return true;
});
This wraps the chrome.tabs.executeScript and chrome.runtime.onMessage.addListener, and wraps the script in a try-finally before calling chrome.runtime.sendMessage to resolve the promise.

Passing promises from page to content script doesn't work, the solution is to use chrome.runtime.sendMessage and to send only simple data between two worlds eg.:
function doSomethingOnPage(data) {
fetch(data.url).then(...).then(result => chrome.runtime.sendMessage(result));
}
let data = JSON.stringify(someHash);
chrome.tabs.executeScript(tab.id, { code: `(${doSomethingOnPage})(${data})` }, () => {
new Promise(resolve => {
chrome.runtime.onMessage.addListener(function listener(result) {
chrome.runtime.onMessage.removeListener(listener);
resolve(result);
});
}).then(result => {
// we have received result here.
// note: async/await are possible but not mandatory for this to work
logger.error(result);
}
});

For anyone who is reading this but using the new manifest version 3 (MV3), note that this should now be supported.
chrome.tabs.executeScript has been replaced by chrome.scripting.executeScript, and the docs explicitly state that "If the [injected] script evaluates to a promise, the browser will wait for the promise to settle and return the resulting value."