stoping async tasks running in parallel - javascript

I'm trying to run through (using foreach) an array of objects and then for each I'd like to call a function that uses request to get a file and then unzips it with zlib, but one at a time, given the nature of node this is currently done asynchronously.
I'd like it to be done something like this...
- foreach - first object
- call function for first object
- when function has completed
- go to the next object in the array
I have tried using the SYNC module to try and solve this but with no luck.
Any ideas on how I can achieve this?
// the function i am trying to run for each in sync
var downloadUnzipFile = function(mID) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
.on('error', function(err) {
console.error(err);
})
.on('finish', function() {
console.log("CSV created: " + fileName(mID));
console.log("Completed merchant: " + mID + " # " + new Date().format('H:i:s').toString());
//console.log("Parsing CSV...");
//csvReader(fileName);
});
} catch (e) {
console.error(e);
}
}
module.exports = function(sMerchants) {
var oMerchants = JSON.parse(JSON.stringify(sMerchants));
sync(function() {
oMerchants.forEach(function eachMerchant(merchant) {
downloadUnzipFile(merchant.merchant_aw_id);
})
})
};

var promiseQueue = (function() {
'use strict';
var promiseQueue = function() {
var queue = [Promise.resolve(true)];
var add = function(cb) {
var args = Array.prototype.slice.call(arguments);
args.shift();
queue.unshift(new Promise(function(resolve) {
queue[0].then(function() {
resolve(cb.apply(null, args));
queue.pop();
});
}));
};
return {
add: add
}
}
return promiseQueue;
}());
usage EXAMPLE:
This is the asynch function that will be called
var theFun = function (time, n) { // use whatever arguments you like that will be called with your function
return new Promise(function(resolve) {
//asynch function goes here INSTEAD of the setTimeout and it's contents, I repeat, INSTEAD of the setTimeout
setTimeout(function() { // this is for demonstrating ONLY
console.log('resolving', n, time); // this is for demonstrating ONLY
resolve(time); // this is for demonstrating ONLY
}, time); // this is for demonstrating ONLY
// remember to resolve("someValueNotImportantAsItIsntUsedAnywhere") on completion of your asynch function
});
}
This is how the items get added to the queue - I did it this way because of MY use case
var pq = promiseQueue();
for(var i = 0; i < 5; i++ ) {
var r = 1000 - i * 150;
console.log('adding ', i, r);
pq.add(theFun, r, i);
}
Hope you find this of some use

First, your function needs to take a callback so it can communicate when it has finished:
var downloadUnzipFile = function(mID, next) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
.on('error', function(err) {
console.error(err);
})
.on('finish', function() {
console.log("CSV created: " + fileName(mID));
console.log("Completed merchant: " + mID + " # " + new Date().format('H:i:s').toString());
//console.log("Parsing CSV...");
//csvReader(fileName);
next();
});
} catch (e) {
console.error(e);
next();
}
}
Then, we need to recursively call each one when the previous has finished:
module.exports = function(sMerchants, next) {
var oMerchants = JSON.parse(JSON.stringify(sMerchants));
var i = 0;
var run = function() {
if(i < oMerchants.length)
downloadUnzipFile(i++, run);
else
next();
};
};
Note that I also added a callback to the exported function, so it can communicate when it is finished. If this is unnecessary, you can drop it.

This may work for you, uses Promise. Need to add resolve and reject callbacks to your downloadUnzipFile-
var exports = (function () {
'use strict';
var pre = document.getElementById('out');
function log(str) {
pre.appendChild(document.createTextNode(str + '\n'));
}
function downloadUnzipFile(id, resolve, reject) {
log('Start: ' + id);
try {
setTimeout(function () {
resolve(id);
}, 3000);
} catch (e) {
reject(e);
}
}
function done(id) {
log('Done: ' + id);
}
function error(e) {
log(e.message);
}
function getPromise(mID) {
return new Promise(function (resolve, reject) {
downloadUnzipFile(mID, resolve, reject);
});
}
return function (sMerchants) {
JSON.parse(sMerchants).reduce(function (next, mID) {
if (!next) {
next = getPromise(mID);
} else {
next = next.then(function (id) {
done(id);
return getPromise(mID);
}, error);
}
return next;
}, null).then(done, error);
};
}());
exports(JSON.stringify([1, 2, 3, 4, 5]));
<script src="https://cdnjs.cloudflare.com/ajax/libs/json2/20150503/json2.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.1.7/es5-shim.min.js"></script>
<script src="https://rawgit.com/jakearchibald/es6-promise/master/dist/es6-promise.min.js"></script>
<pre id="out"></pre>
I added the browser shims to support older browsers that may be viewing this, you shouldn't need them on node.js but you may need to require a Promise shim if you are using an old node.js.

Related

Promise does not resolve node.js

I am trying to resolve some promises by the sendRequest function but it does not work.
For example, if I evoke sendRequest function 4 times, all of the times, I can see the log printed in the console and then going to resolve(data). But only 1 out of 4 times, the program reaches in to sendRequest.then().
Here is the complete code of that file.
change-name.js
var sendRequest = function(fileName){
return new Promise(function (resolve,reject) {
httpRequest(fileName).then(function (data) {
try{
if(.....){
if(.....){
var lastIndex = ....;}
if(.....){
var lastIndex = ....;}
str = fileName.substring(0, lastIndex);
if(.........){
sendRequest(str);}
else{
reject("this is the end");}
}
else{
console.log("end result" + JSON.stringify(data,null,4));
resolve(data);
//===== THIS RESOLVE DOES NOT WORK WHILE THE LOG PRINTS THE DATA =====//
}
}
catch(e){
resolve(data);
}
}).catch(function (err) {
reject(err);
});
});
};
server.js
this files calls the sendRequest function from the change-name.js and the then method is applied here for that function.
fs.readdir(path,(err,files)=>{
if(err){
console.log(err);
return;
}
for(i=0;i<files.length;i++){
sendRequest(files[i]).then(function (data) {
console.log(data + "\n");
}).catch(function(err){
console.log("end Error is " + err + "\n");
});
console.log(files);
}
});
The github link is "https://github.com/abhikulshrestha22/movierator".
Any help would be appreciated. Thanks
The problem is that one of your branches calls sendRequest again rather than resolving or rejecting, but then doesn't make any use of the promise the recursive call returns returns, so the promise you created for the outer call is never resolved. The inner one is (which is why you see the message in the console), but since nothing is using that promise, you don't get the result you expect.
There's also no need for new Promise in your code at all. httpRequest already gives you a promise, and your then handler on it creates a promise (if you return a value). So sendRequest should just return the result of calling then on httpRequest's promise, and within the then callback, either return the value to resolve the new promise with, or throw to reject. In the branch where you're calling sendRequest again, return the promise it returns; the one then creates will then resolve/reject based on that promise:
var sendRequest = function(fileName) {
return httpRequest(fileName).then(function(data) {
if (condition1) {
if (condition2) {
var lastIndex = something;
}
if (condition3) {
var lastIndex = somethingElse;
}
str = fileName.substring(0, lastIndex);
if (condition4) {
return sendRequest(str);
}
else {
throw new Error("this is the end");
}
}
else {
console.log("end result" + JSON.stringify(data, null, 4));
return data;
}
});
};
Here's a live example where httpRequest returns a random number 1-10, and if the number is less than 8, calls sendRequest again; it'll try up to three times before giving up:
function httpRequest() {
return new Promise(function(resolve) {
setTimeout(function() {
resolve(Math.floor(Math.random() * 10) + 1);
}, 500);
});
}
var sendRequest = function(fileName, retries) {
if (typeof retries !== "number") {
retries = 3;
}
return httpRequest(fileName).then(function(data) {
console.log("`then` got: " + data);
if (data > 7) {
console.log("It's > 7, yay! We're done");
return data;
}
console.log("It's <= 7g");
if (--retries === 0) {
console.log("Out of retries, rejecting");
throw new Error("out of retries");
}
console.log("retrying (" + retries + ")");
return sendRequest(fileName, retries);
});
};
document.getElementById("btn").addEventListener("click", function() {
var btn = this;
btn.disabled = true;
console.log("Sending request");
sendRequest("foo")
.then(function(data) {
console.log("Request complete: " + data);
})
.catch(function(err) {
console.log("Request failed: " + err);
// Because we don't throw here, it converts the rejection into a resolution...
})
.then(function() {
// ...which makes this kind of like a "finally"
btn.disabled = false;
});
}, false);
<input type="button" id="btn" value="Start">

Reconnecting an observable

It is possible, when I would unsubscribe an connectable observable and at later time to connect it again.
For example:
var interval = Rx.Observable.interval(1000);
var source = interval
.take(2)
.do(function (x) {
console.log('Side effect');
});
var published = source.publish();
published.subscribe(createObserver('SourceA'));
published.subscribe(createObserver('SourceB'));
// Connect the source
var connection = published.connect();
function createObserver(tag) {
return Rx.Observer.create(
function (x) {
console.log('Next: ' + tag + x);
},
function (err) {
console.log('Error: ' + err);
},
function () {
console.log('Completed');
});
}
Yes it is. This is exactly what operator refCount() does. It subscribes and unsubscribes a ConnectableObservable.
See: https://github.com/ReactiveX/rxjs/blob/master/src/observable/ConnectableObservable.ts

Why isn't my promise getting resolved?

I have two functions - A helper function for downloading files which is as follows
var downloadHelper = function(url, saveDir) {
var deferred = Q.defer();
setTimeout(function() {
deferred.resolve("success");
}, 2000);
return deferred.promise;
}
Now I have a list of files to be downloaded in parallel. I have the logic for that function as follows:
var downloadAll = function() {
var fileDownloadList = []
for(var key in config.files) {
var deferred = Q.defer();
var saveLocation = __base + config.localDir
downloadHelper(
config.files[key],
saveLocation
).then(function() {
deferred.resolve("downloaded: " + fileUrl);
}).catch(function(err) {
deferred.reject(err);
});
fileDownloadList.push(deferred.promise);
}
Q.all(fileDownloadList).done(function() {
console.log("All downloaded");
},function(err) {
console.log(err);
});
setTimeout(function() {
console.log(fileDownloadList);
}, 10000);
}
The done is never getting called!
For debugging purposes, I added a setTimeout that will be called after 10 seconds and what I see is that out of 2 files, the second promise is resolved and the first one is still in pending state.
Any ideas?
Thanks in advance
One way to make your code work
for(var key in config.files) {
(function() {
var deferred = Q.defer();
var saveLocation = __base + config.localDir
downloadHelper(
config.files[key],
saveLocation
).then(function() {
deferred.resolve("downloaded: " + fileUrl);
}).catch(function(err) {
deferred.reject(err);
});
fileDownloadList.push(deferred.promise);
}());
}
But since downloadhelper returns a promise, no need to create yet another one
for (var key in config.files) {
var saveLocation = __base + config.localDir
fileDownloadList.push(downloadHelper(
config.files[key],
saveLocation
).then(function () {
return("downloaded: " + fileUrl);
}));
}
You'll see I removed
.catch(function(err) {
deferred.reject(err);
})
That's redundant, it's the same as not having the catch at all

Understanding chaining of sequential asynchronous operations using jquery deferred and then

I have been trying to wrap my head around jquery deferred and then functions. As I gather from jQuery then documentation, the then function sends the return value of the callback to the next then handler if they are so chained. Given that, why is my code not working as expected?
function log(message) {
var d = new Date();
$('#output').append('<div>' + d.getSeconds() + '.' + d.getMilliseconds() + ': ' + message + '</div>');
}
function asyncWait(millis) {
var dfd = $.Deferred();
setTimeout(function () {
var d = new Date();
log('done waiting for ' + millis + 'ms');
dfd.resolve(millis);
}, millis);
return dfd.promise();
}
function startTest0() {
return asyncWait(1000).then(asyncWait).then(asyncWait).then(asyncWait).done(function () {
log('all done, 4 times');
});
}
function startTest() {
asyncWait(500).then(function () {
return asyncwait(1000);
}).then(function () {
return asyncWait(1500);
}).then(function () {
return asyncWait(2000);
}).done(function () {
log('all done');
});
}
log('welcome');
log('starting test ...');
startTest0().done(function() { log('starting the second test'); startTest(); });
JS Fiddle here: Sample code. I was expecting a similar behavior in both tests but something eludes me. What am I missing?
Thanks in advance!
EDIT: See an updated DEMO where I am trying to chain the async operations to start after the previous one is done.
Except for one typo (asyncwait instead of asyncWait) your code works. Check below.
function log(message) {
var d = new Date();
$('#output').append('<div>' + d.getSeconds() + '.' + d.getMilliseconds() + ': ' + message + '</div>');
}
function asyncWait(millis) {
var dfd = $.Deferred();
setTimeout(function () {
var d = new Date();
log('done waiting for ' + millis + 'ms');
dfd.resolve(millis);
}, millis);
return dfd.promise();
}
function startTest0() {
return asyncWait(1000).then(asyncWait).then(asyncWait).then(asyncWait).done(function () {
log('all done, 4 times');
});
}
function startTest() {
asyncWait(500).then(function () {
return asyncWait(1000);
}).then(function () {
return asyncWait(1500);
}).then(function () {
return asyncWait(2000);
}).done(function () {
log('all done');
});
}
log('welcome');
log('starting test ...');
startTest0().done(function() { log('starting the second test'); startTest(); });
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<div id="output"></div>
Lesson to learn: Put any JS code through jshint before and after you fix bugs.
As i can see here, you are calling startTest0 function returning its promise object and calling then callback without returning new times into next then callback. I modified your startTest() into this :
function startTest() {
return asyncWait(500).then(function () {
asyncWait(1000);
return 1500; // here we pass to the next then
}).then(function (ms) { // ms here we got 1500
asyncWait(ms);
return 2000; // here we pass to the next then
}).then(function (ms) { // ms here we got 2000
asyncWait(ms)
return asyncWait(2500);
}).done(function () {
log('all done');
});
}
DEMO

asynchronous execution in protractor end to end tests

I have written a function , which is returning a value. In my main i am calling the function like this:
var fn_return_val = lesson.select_lesson(lesson1_text);
console.log("returned value is : " + fn_return_val);
And my function implementation is like(other file.js) :
module.exports = {
select_lesson:
function select_lesson(lesson_name) {
console.log('\n ************************* Lessson name: ' + lesson_name);
var desiredOption, status;
var repeter = element.all(by.repeater('item in items'));
repeter.then(function (items) {
items.forEach(function (icon) {
console.log('\n ************************* item');
icon.getText().then(function (txt) {
if (txt == lesson_name) {
desiredOption = icon;
}
})
}).then(function clickOption() {
if (desiredOption) {
var el = desiredOption.all(by.css('[ng-click="launchActivity()"]'));
var el_progress = desiredOption.all(by.css('.pna-progress'));
var abc = el.getAttribute('value').then(function (txt) {
status = txt;
return status
});
el_progress.getAttribute('style').then(function (progress) {
console.log('\n ************************* Lessson progress : ' + progress);
});
el.click();
}
});
});
}
};
The problem is function is returning "undefined" value, and the print statement console.log("returned value is : " + fn_return_val);
is executing before the function implementation
Can anyone help me on resolving this?
This is all about promises and protractor's Control Flow.
You need to resolve the promise and log the results inside then:
lesson.select_lesson(lesson1_text).then(function(fn_return_val) {
console.log("returned value is : " + fn_return_val);
});
And you also need to return from a function:
function select_lesson(lesson_name) {
...
// return here
return repeter.then(function (items) {
...
}).then(function clickOption() {
...
});
});
}

Categories