This code expands on the dynamic.js script. The dynamic.js runs like this: start, then, run, and then the run callback is the dynamic loop. That runs fine. My code doesn't want to loop at the end of the run callback, it wants to loop in the middle, so that I can run more code afterwards. So it's start, then, loop, then, run. It seems that casper is not waiting for my loop to finish before exiting. The loop terminates at different points every time that it runs, and I never see the "after dynamic loop" echo out. Here is some sample output (it ends anywhere between 2 and 8 loops, usually), and then the code:
Output:
start main
start dynamic loop
0
Something.
1
Something.
2
Something.
3
Something.
4
Something.
5
Something.
Code:
var casper = require('casper').create();
var limit = 10;
var i = 0;
// dynamic loop
var loop = function () {
if (i < limit) {
this.echo(i);
this.start("http://www.something.com", function() {
this.echo(this.evaluate(function () { return document.body.innerText; }));
});
i++;
this.run(loop);
} else {
this.echo("dynamic loop done");
this.exit();
}
}
// main flow
casper.start('http://www.something.com', function() {
this.echo("start main");
});
casper.then(function () {
casper.start().then(function() {
this.echo("start dynamic loop");
});
casper.run(loop);
});
casper.then(function () {
this.echo("after dynamic loop");
})
casper.run();
You should use casper.start and casper.run only once in your script. However, you can use them. If you do you run into "undefined" behavior if you nest them in another control flow. I find it is best to use casper.thenOpen and casper.then as a replacement for casper.start and casper.run:
// dynamic loop
var loop = function () {
if (i < limit) {
this.echo(i);
i++;
this.thenOpen("http://www.something.com", function() {
this.echo(this.evaluate(function () { return document.body.innerText; }));
});
this.then(loop);
} else {
this.echo("dynamic loop done");
}
}
// main flow
casper.start('http://www.something.com', function() {
this.echo("start main");
});
casper.then(loop);
casper.then(function () {
this.echo("after dynamic loop");
})
casper.run();
You may have multiple casper instances in one script, but then you would need to synchronize them somehow.
Your script may need a little fix without refactoring. Just remove this.exit(); from loop. You exit prematurely. But I still strongly suggest that you refactor your script.
Related
I am making a chrome extension (mv3). Based on user activity, the content.js passes a message to the background.js which then calls an async function to add data in Google Docs using Docs API.
I want each request to execute only after the previous one has finished running. I am using chrome.runtime.sendMessage to send a message from content.js and don't see a way of calling background.js serially from there. So I need a way of executing them one by one in background.js only. The order of these requests is also important (but if the order of the requests gets changed by one/two places, I think that would still be okay from a user perspective).
I tried something and it is working but I am not sure if I am missing some edge cases, because I was unable to find the approach in any other answers -
Semaphore-like queue in javascript?
Run n number of async function before calling another method in nodejs
JavaScript: execute async function one by one
The approach I used is: I use a stack like structure to store requests, use setInterval to check for any pending requests and execute them serially.
content.js:
chrome.runtime.sendMessage({message});
background.js:
let addToDocInterval = "";
let addToDocCalls = [];
async function addToDoc(msg) {
// Await calls to doc API
}
async function addToDocHelper() {
if(addToDocCalls.length === 0)
return;
clearInterval(addToDocInterval)
while(addToDocCalls.length > 0) {
let msg = addToDocCalls.shift();
await addToDoc(msg);
}
addToDocInterval = setInterval(addToDocHelper, 1000);
}
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
addToDocCalls.push(msg);
})
addToDocInterval = setInterval(addToDocHelper, 1000);
Is this approach correct? Or is there any better way to do this?
I'd suggest changing several things.
Don't use timers polling the array. Just initiate processing the array anytime you add a new item to the array.
Keep a flag on whether if you're already processing the array so you don't start duplicate processing.
Use a class to encapsulate this functionality into an object.
Encapsulate the addToDocCalls array and adding to it so your class is managing it and outside code just calls a function to add to it which also triggers the processing. Basically, you're making it so callers don't have to know how the insides work. They just call helper.addMsg(msg) and the class instance does all the work.
Here's an implementation:
async function addToDoc(msg) {
// Await calls to doc API
}
class docHelper {
constructor() {
this.addToDocCalls = [];
this.loopRunning = false;
}
addMsg(msg) {
// add item to the queue and initiate processing of the queue
this.addToDocCalls.push(msg);
this.process();
}
async process() {
// don't run this loop twice if we're already running it
if (this.loopRunning) return;
try {
this.loopRunning = true;
// process all items in the addToDocCalls we have
while(this.addToDocCalls.length > 0) {
let msg = addToDocCalls.shift();
await addToDoc(msg);
}
} finally {
this.loopRunning = false;
}
}
}
const helper = new docHelper();
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
helper.addMsg(msg);
});
So, process() will run until the array is empty. Any interim calls to addMsg while process() is running will add more items to array and will call process() again, but the loopRunning flag will keep it from starting duplicate processing loops. If addMsg() is called while process is not running, it will start the process loop.
P.S. You also need to figure out what sort of error handling you want if addToDoc(msg) rejects. This code protects the this.loopRunning flag if it rejects, but doesn't actually handle a reject error. In code like this that is processing a queue, often times all you can really do is log the error and move on, but you need to decide what is the proper course of action on a rejection.
You don't need to use setTimeout. You do not even need a while loop.
let addToDocInterval = "";
let addToDocCalls = [];
let running = false;
async function addToDoc(msg) {
// Await calls to doc API
}
async function addToDocHelper() {
if(running || addToDocCalls.length === 0)
return;
running = true;
let msg = addToDocCalls.shift();
await addToDoc(msg);
running = false;
addToDocHelper();
}
chrome.runtime.onMessage.addListener((msg) => {
// Some other logic
addToDocCalls.push(msg);
addToDocHelper();
});
The code should be self explanatory. There is no magic.
Here is a generic way to run async tasks sequentially (and add more tasks to the queue at any time).
const tasks = [];
let taskInProgress = false;
async function qTask(newTask) {
if (newTask) tasks.push(newTask);
if (tasks.length === 0) return;
if (taskInProgress) return;
const nextTask = tasks.shift();
taskInProgress = true;
try {
await nextTask();
} finally {
taskInProgress = false;
//use setTimeout so call stack can't overflow
setTimeout(qTask, 0);
}
}
//the code below is just used to demonstrate the code above works
async function test() {
console.log(`queuing first task`);
qTask(async () => {
await delay(500); //pretend this task takes 0.5 seconds
console.log('first task started');
throw 'demonstrate error does not ruin task queue';
console.log('first task finished');
});
for (let i = 0; i < 5; i++) {
console.log(`queuing task ${i}`)
qTask(async () => {
await delay(200); //pretend this task takes 0.2 seconds
console.log(`task ${i} ran`);
});
}
await delay(1000); //wait 1 second
console.log(`queuing extra task`);
qTask(async () => {
console.log('extra task ran');
});
await delay(3000); //wait 3 seconds
console.log(`queuing last task`);
qTask(async () => {
console.log('last task ran');
});
}
test();
function delay(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms);
});
}
I have an Ajax call which I want run multiple times till it satisfies a particular if condition. The AJAX call gives you the job status-Running, Queued and Complete.
I am unable to get the Job status- Complete. After getting Running status it takes a couple of minutes to get the status Complete. So far I have tried the following JS. I also want to break the loop after the if condition has been met. I am also not sure if I should run the call for 100 times as it might take more than that. Thank you for your help.
My JS:
var pollForJob= gallery.getJob(jobId, function(job){
var jobStat=job.status;
console.log(jobStat);
if(jobStat=="Complete"){
alert("Complete");
} else {
// Attempt it again in one second
setTimeout(pollForJob, 1000);
console.log("still working");
console.log(jobStat);
}
}, function(response){
var error = response.responseJSON && response.responseJSON.message ||
response.statusText;
alert(error);
// Attempt it again in one second
setTimeout(pollForJob, 1000);
});
Like Jeremy Thille said, this is called long polling. A simple way of doing it is to create a function which makes the call out to the server. Then, if it fails, use setTimeout to queue up another request later.
function pollForJob() {
gallery.getJob(jobId, function(job) {
var jobStat = job.status;
if (jobStat == "Complete") {
alert("Complete");
} else {
// Attempt it again in one second
setTimeout(pollForJob, 1000);
}
}, function(response) {
var error = response.responseJSON && response.responseJSON.message || response.statusText;
console.error(error);
// Attempt it again in one second
setTimeout(pollForJob, 1000);
});
}
pollForJob();
So, my code looks something like
for(int n = 0; n < object.length; n++){
/*Other code */
$.get(...,function(data){
//do stuff
});}
Now, the other code executes multiple times like it should. However, when the get command is ran, it is only ran once and that is when n reaches the object.length. This causes all sorts of errors. n is only being incremented in the for loop.
Can you not loop get/post commands? Or if you can, what am doing wrong? Thanks.
The for-loop won't wait for the $.get call to finish so you need to add some async flow control around this.
Check out async.eachSeries. The done callback below is the key to controlling the loop. After the success/fail of each $.get request you call done(); (or if there's an error, you call done(someErr);). This will advance the array iterator and move to the next item in the loop.
var async = require("async");
var list = ["foo", "bar", "qux"];
async.eachSeries(list, function(item, done) {
// perform a GET request for each item in the list
// GET url?somevar=foo
// GET url?somevar=bar
// GET url?somevar=qux
$.get(url, {somevar: item}, function(data) {
// do stuff, then call done
done();
});
}, function(err) {
if (err) {
throw err;
}
console.log("All requests are done");
});
Do you want all the get requests to be done at the same time or one after the other?
You can do all at once with a forEach loop. forEach works better than a normal for loop because it gives you a new instance of elem and n in each iteration.
object.forEach(function(elem, n) {
/*Other code */
$.get(...,function(data){
//do stuff
});
});
But if you want to do the requests one after the other you could do it this way.
(function loop(n) {
if(n >= object.length) return;
/*Other code */
$.get(...,function(data){
//do stuff
loop(n + 1);
});
})(0);
I am having an array which consist of 3 data which I need to insert in pouchdb, so I'm using a for loop to insert the data, but the problem is it is not inserting complete data because the loop finishes before callback.
for(var i=0;i<data.length;i++){
var jsondoc=doc1;//document
console.log(i) //getting console for all data. eg:1,2,3
console.log(data[i])//getting console for all data. eg:hi, hello
jsondoc.messages.push({msg: data[i]}); //updating message, here need to be done somthing
db.put(jsondoc, function(err2, response2){
if (err2) { console.log(JSON.stringify(err2)); }
console.log("this is not repeating") ;
});
}
Since the db insertion runs async, you cannot put the loop on hold until the operation completes. One thing that you can do is to serialise the db inserts with a helper function like this:
function insertItem(data, i, completeCallback) {
// check if we still have items to send
if(i < data.length) {
var jsondoc=doc1;//document
//updating message, here need to be done somthing
jsondoc.messages.push({msg: data[i]});
db.put(jsondoc, function(err2, response2){
if (err2) {
console.log(JSON.stringify(err2));
}
// recursively call to push the next message
insertItem(data, i+1, completeCallback);
});
} else {
// no more items to send, execute the callback
if(typeof completeCallback === "function") {
completeCallback();
}
}
}
You'll have to update your code so that instead of continuing the execution after the call of the function, to pass that code into the callback of the pushMessage function, so if your original code looks like this:
// ... code before the loop
for(var i=0;i<data.length;i++){
// ... original body of the loop
}
// ... code to execute after the loop and that currently causes problems
you'll need to change it like this:
// ... original code that was before the loop
insertItem(data, 0, function() {
// ... original code hat was executed after the loop and that caused problems
// but now it gets executed after all items were inserted in db
}
Another alternative would be to send all inserts in parallel and perform a join() on those operations; you'll still need the callback workaround though. Something along the lines:
function insertItems(data, callback) {
var remainingItems = data.length;
if(remainingItems === 0 && typeof callback === "function") {
callback();
}
for(var i=0;i<data.length;i++){
var jsondoc=doc1;//document
console.log(i) //getting console for all data. eg:1,2,3
console.log(data[i])//getting console for all data. eg:hi, hello
jsondoc.messages.push({msg: data[i]}); //updating message, here need to be done somthing
db.put(jsondoc, function(err2, response2){
if (err2) { console.log(JSON.stringify(err2)); }
remainingItems--;
if(remainingItems === 0 && typeof callback === "function") {
// I know, code redundancy :P
callback();
}
});
}
}
The usage of this second function is the same as for insertItem.
If I understand you correctly, your problem is a scoping issue. jsondoc or data[i], or whichever variable is causing the problem, is changed before your callback can complete.
Take a look at this jsFiddle, which shows how to solve such a scoping problem.
for(var i = 0; i < 3; i++){
(function(){
var j = i;
setTimeout(function(){
callback(j)
}, 500);
})();
}
If you look at your js console when the jsFiddle runs you'll see that the first loop prints 3 times 3, which is the finishing value for i. While the second, where we store the value to a new variable inside a new scope, outputs 1, 2, 3 as expected.
Suppose I have the following code in node.js:
process.on('SIGTERM', function () {
process.exit(0);
});
for (var i = 0; i < 100; i++) {
console.error(i);
}
Will the code always print all the numbers up to 100?
This is easy enough to test:
process.on('SIGTERM', function () {
console.log('*** GOT SIGTERM ***');
process.exit(0);
});
process.kill(process.pid);
for (var i = 0; i < 100; i++) {
console.error(i);
}
The results show that, indeed, the for loop does run to completion, but this is misleading. The for loop blocks I/O. Because Node is single-threaded, process.exit(0) doesn’t get called until the for loop is done.
A more illustrative example:
process.on('SIGTERM', function () {
console.log('*** GOT SIGTERM ***');
process.exit(0);
});
process.kill(process.pid);
setTimeout(function() {
for (var i = 0; i < 100; i++) {
console.error(i);
}
}, 1000);
Here we use setTimeout to wait 1 second before doing the for loop. Node’s event loop will continue to run during that time, so the process.on('SIGTERM') will be called before the for loop begins. And the results are different: the for loop does not run at all.
Once process.exit(0) is called, the app ends and nothing else is executed. The difference between the two examples is when process.exit(0) is called.