I know of this question, but my need is a little different.
I'm trying to download multiple images but cannot prevent the app from exiting before the process is complete. I've gone through quite a few iterations, this is where I'm at currently:
router
.post('/', koaBody(), function *() {
var images = [];
//console.log(this.request.body.fields);
if (this.request.body.fields) {
images = this.request.body.fields['images'];
imageCount = images.length;
} else {
this.status = 500;
return;
}
var url = 'https://www.google.ca/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png';
var filepath = '../images/image_1.png';
var pipeRequest = function (url, filepath) {
return new Promise(function (resolve, reject) {
request.get(url)
.pipe(fs.createWriteStream(filepath))
.on('finish', function () {
console.log('piped');
});
});
};
co(function*() {
yield pipeRequest(url, filepath);
}).then(function () {
console.log('done co-func');
}).catch(function (err) {
return console.error(err);
});
console.log('App exiting...');
});
It's not the most elegant code but it works, if I wrap the co-function in a loop I can download as many as I want (the end-goal is to have this API receive a JSON list of image URLs). What I cannot do however is return the results of the piping to the caller, the app will always exit before that process is complete. I thought wrapping the function call in a yield would be the answer...but no.
When you're inside a generator function (a Koa handler), you use yield to wait.
The general pattern is to extract your async logic into a function that returns a promise, and then you yield it inside your route.
The first problem with your code is that you never use reject and resolve in the promise to transition it to a finished/errored state. The promise never knows when it's finished. Yielding it will never return control back to the route, which is probably why you nested it in another co process.
Here, I fix it so that it resolve()s on completion and reject(err)s on error. Note: yielding a promise that hits a reject(err) path will throw an exception (err) that you can try/catch at the call site.
var pipeRequest = function(url, filepath) {
return new Promise(function(resolve, reject) {
request.get(url)
.pipe(fs.createWriteStream(filepath))
.on('finish', function() { resolve() })
.on('error', function(err) { reject(err) })
});
};
The second problem I see is that you're nesting a co process in your route which is going to ensure that the route never waits on the yield inside it.
A Koa handler is already run inside a co process. Just replace that whole co block with a top-level yield pipeRequest(url, filepath) once you've implemented my resolve/reject changes above.
Related
I'm trying to test the following "worker" method in a Class without success as it is "an infinite loop"/recursive:
Class TheClassYaDaYaDa {
constructor(var1) {
(...)
}
async worker() {
return new Promise(async (resolve, reject) => {
try {
await this.doStuff();
return resolve();
} catch (err) {
return reject(err);
} finally {
this.worker();
}
});
}
}
And this is the test I'm building:
it('It should not throw error', async function () {
let error = false;
const var1 = 'parameterTest';
stub1 = sinon.stub(TheClassYaDaYaDa.prototype, 'doStuff').resolves();
// if I use the following stub logically the test will not succeed. If I don't I get an infinte loop
// stub2 = sinon.stub(TheClassYaDaYaDa.prototype, 'worker').resolves();
let classToTest;
try {
classToTest = new TheClassYaDaYaDa(limitedInstance);
result = await classToTest.worker();
} catch (err) {
error = err;
}
expect(error).to.be.false;
sinon.assert.calledOnce(stub1);
//also assert that the finally statement run at least once!
});
Is there a way to test this scenario?
I would approach this from a different angle, recursion is always tricky but one of the popular designs is to split the scheduling and running logic into two.
Consider the following:
async worker() {
await this.scheduleWorker();
}
private scheduleWorker() {
// return new Promise ...
}
It's now possible to test that the scheduleWorker function calls worker, and also that worker calls scheduleWorker (if private methods are accessible).
If not, you could mock the worker method to only return the worker logic on the first call and noops on subsequent calls. This would also work for your current design allowing you to assert the recursive call was correctly triggered.
I am not well versed on sinon but you want something similar to:
var originalWorker = classToTest.originalWorker;
classToTest.originalWorker = function() {
originalWorker();
classToTest.originalWorker = function() { }
}
Most mocking frameworks have similar controls to enable mocking methods multiple times (for subsequent calls).
I'm trying to get my head around promises, I think I can see how they work in the way that you can say do Step 1, Step 2 and then Step 3 for example.
I have created this download function using node-fetch (which uses native Promises)
## FileDownload.js
const fetch = require('node-fetch');
const fs = require('fs');
module.exports = function(url, target) {
fetch(url)
.then(function(res) {
var dest = fs.createWriteStream(target);
res.body.pipe(dest);
}).then(function(){
console.log(`File saved at ${target}`)
}).catch(function(err){
console.log(err)
});
}
So this all executes in order and I can see how that works.
I have another method that then converts a CSV file to JSON (again using a promise)
## CSVToJson.js
const csvjson = require('csvjson');
const fs = require('fs');
const write_file = require('../helpers/WriteToFile');
function csvToJson(csv_file, json_path) {
return new Promise(function(resolve, reject) {
fs.readFile(csv_file, function(err, data){
if (err)
reject(err);
else
var data = data.toString();
var options = {
delimiter : ',',
quote : '"'
};
const json_data = csvjson.toObject(data, options);
write_file(json_path, json_data)
resolve(data);
});
});
}
module.exports = {
csvToJson: csvToJson
}
When I call these functions one after another the second function fails as the first has not completed.
Do I need to wrap these two function calls inside another promise, even though on their own they each have promises implemented?
Please advise if I am totally misunderstanding this
When I call these functions one after another the second function fails as the first has not completed.
There are two issues with the first:
It doesn't wait for the file to be written; all it does is set up the pipe, without waiting for the process to complete
It doesn't provide any way for the caller to know when the process is complete
To deal with the first issue, you have to wait for the finish event on the destination stream (which pipe returns). To deal with the second, you need to return a promise that won't be fulfilled until that happens. Something along these lines (see ** comments):
module.exports = function(url, target) {
// ** Return the end of the chain
return fetch(url)
.then(function(res) {
// ** Unfortunately, `pipe` is not Promise-enabled, so we have to resort
// to creating a promise here
return new Promise((resolve, reject) => {
var dest = fs.createWriteStream(target);
res.body.pipe(dest)
.on('finish', () => resolve()) // ** Resolve on success
.on('error', reject); // ** Reject on error
});
}).then(result => {
console.log(`File saved at ${target}`);
return result;
});
// ** Don't `catch` here, let the caller handle it
}
Then you can use then and catch on the result to proceeed to the next step:
theFunctionAbove("/some/url", "some-target")
.then(() = {
// It worked, do the next thing
})
.catch(err => {
// It failed
});
(Or async/await.)
Side note: I haven't code-reviewed it, but a serious issue in csvToJson jumped out, a minor issue as well, and #Bergi has highlighted a second one:
It's missing { and } around the else logic
The minor issue is that you have var data = data.toString(); but data was a parameter of that function, so the var is misleading (but harmless)
It doesn't properly handle errors in the part of the code in the else part of the readFile callback
We can fix both by doing a resolve in the else and performing the rest of the logic in a then handler:
function csvToJson(csv_file, json_path) {
return new Promise(function(resolve, reject) {
fs.readFile(csv_file, function(err, data){
if (err)
reject(err);
else
resolve(data);
});
})
.then(data => {
data = data.toString();
var options = {
delimiter : ',',
quote : '"'
};
const json_data = csvjson.toObject(data, options);
write_file(json_path, json_data);
return data;
});
}
I have a sequence of function calls, connected with ES6 promises. Apparently, there is something wrong with this implementation, as API calls to the endpoint are not returning anything and the browser is stuck waiting for a response.
Please advise.
module.exports.insertTreatmentDetails = function (req, res) {
var doctorId = 10000
var departmentId = 10000
var procedureid = 10000
var hospitalSchema = new hospitalModel();
var p = new Promise(function (resolve, reject) {
counterSchema.getNext('Treatment.doctor.doctorId', collection, function (doctorId) {
doctorId = doctorId;
})
counterSchema.getNext('Treatment.departmentId', collection, function (departmentId) {
departmentId = departmentId
})
counterSchema.getNext('Treatment.procedureid', collection, function (procedureid) {
procedureid = procedureid
})
}).then(function () {
setData()
}).then(function (){
hospitalSchema.save(function (error, data) {
if (error) {
logger.error("Error while inserting record : - " + error)
return res.json({ "Message": error.message.split(":")[2].trim() });
}
else {
return res.json({ "Message": "Data got inserted successfully" });
}
});
});
};
The short answer is that you aren't calling resolve or reject inside the first promise in your chain. The promise remains in a pending state. Mozilla has a good basic explanation of promises.
How to Fix
It appears that you want to retrieve doctorId, departmentId, and procedureId before calling setData. You could try to wrap all three calls in one promise, checking whether all three have returned something in each callback, but the ideal is to have one promise per asynchronous task.
If it's feasible to alter counterSchema.getNext, you could have that function return a promise instead of accepting a callback. If not, I would recommend wrapping each call in its own promise. To keep most true to what your code currently looks like, that could look like this:
const doctorPromise = new Promise((resolve, reject) =>
counterSchema.getNext('Treatment.doctor.doctorId', collection, id => {
doctorId = id;
resolve();
}));
Then you could replace the first promise with a call to Promise.all:
var p = Promise.all([doctorPromise, departmentPromise, procedurePromise])
.then(setData)
.then(/* ... */);
Promises allow you to pass a value through to the next step, so if you wanted to get rid of your broadly-scoped variables (or set them in the same step where you call setData), you could just pass resolve as your callback to counterSchema.getNext and collect the values in the next step (also how you'd want to do it if you have counterSchema.getNext return a promise:
Promise.all([/* ... */])
.then(([doctorID, departmentID, procedureID]) => {
// If you aren't changing `setData`
doctorId = doctorID;
departmentId = departmentID;
procedureid = procedureID;
setData();
// If you are changing `setData`
setData(doctorID, departmentID, procedureID);
}).then(/* ... */).catch(/* I would recommend adding error handling */);
I have the following function. It calls itself repeatedly and iterates through ftp servers checking for new files. I'm trying to make it a promise so that I can operate().then(function(newFilesObject), but I can't get the .then on operate to activate. It does attempt to resolve it but doesn't send through. By the way, newFiles is a global variable that gets the files per server appended to it. If more code is wanted I can post or github it.
function operate(){
return new Promise(function(resolve, reject){
if(servers[i]){
if(i!== 0) ftp = new JSFtp(servers[i].server)
local = servers[i].local
remote = servers[i].remote
localFiles = fs.readdirSync(local)
}else{
console.log('trying to resolve')
console.log(newFiles)
resolve(newFiles)
}
gatherFiles(remote).then(function(files){
if(files.length>0){
downloadNew(files).then(function(){
console.log('Done: ' + servers[i].server.host)
i++
operate()
})
}else{
console.log('No updates: ' + servers[i].server.host)
i++
operate()
}
})
})
}
operate().then(function(files){
console.log('files: ' + files)
})
The promises in the code sample do not return as their resolvers or rejectors are not always invoked. In fact, resolve is only invoked when i === 0. According to the Promises/A+ specification, promises may only be transitioned to a fulfilled state by invoking resolve. It also can only be transitioned to a rejected state by invoking reject or throwing an exception from within the executor. Therefore, reaching the end of the executor without invoking either or passing one as a callback ensures the promise remains in pending state indefinitely.
The goal you seek may be achieved with a little refactoring. Considering the following as your goal:
Sequentially through each FTP server...
Read a given directory for a list of files
Compare list of files to one stored locally to determine new ones
If there are new ones, download them sequentially
Return a list of all newly downloaded files
Data
var knownFTPServers = [{
'localDirectory': 'sub/',
'localFilepaths': ['docA.json', 'docB.json'],
'remoteDirectory': 'remsub/',
'remoteFilepaths': [],
'jsftpHandle': undefined,
'host': 'example.com'
},
{
'localDirectory': 'root/',
'localFilepaths': ['file1.txt', 'file2.txt'],
'remoteDirectory': 'remroot/',
'remoteFilepaths': [],
'jsftpHandle': undefined,
'host': 'geocities.com'
}];
Logic
function pullNewFilesFromFTPServer(ftpServer) {
return new Promise(function (resolve, reject) {
var handle = new JSFtp(ftpServer);
ftpServer.jsftpHandle = new JSFtp(ftpServer);
// Returns a promise for reading a directory from JSFtp server
// resolves with file list
// rejects with FTP error
function readdir(directory) {
return new Promise(function (resolve, reject) {
handle.ls(ftpServer.remoteDirectory, function (err, res) {
if (err) return reject(err);
resolve(res);
});
});
}
// Returns a promise for downloading a file from a remote JSFtp server
// resolves with the filepath of the downloaded filepath
// rejects with FTP error
function downloadFile(path) {
return new Promise(function (resolve, reject) {
handle.get(path, path, function (err) {
if (err) return reject(err);
resolve(path);
});
});
}
// get all remote filepaths on server
readdir(ftpServer.remoteDirectory)
// filter out filepaths already present locally
.then(function (remoteFilepaths) {
return remoteFilepaths.filter(function (path) {
return ftpServer.localFilepaths.indexOf(path) < 0;
});
})
// download new filepaths sequentially
// reduce turns the array of new filepaths into a promise chain
// return new filepaths after completing the promise chain
.then(function (newFilepaths) {
return newFilepaths.reduce(function (previousDownloadPromise, newPath) {
return previousDownloadPromise.then(function () {
return downloadFile(newPath);
});
}, Promise.resolve())
.then(function () { return newFilepaths; });
})
// resolve server promise with new filepaths or reject with errors
.then(resolve, reject);
});
}
var allFilesDownloaded = [];
knownFTPServers.reduce(function (previousServerPromise, server) {
return previousServerPromise.then(function (filesDownloaded) {
allFilesDownloaded = allFilesDownloaded.concat(filesDownloaded);
return pullNewFilesFromFTPServer(server);
});
}, Promise.resolve([]))
.then(function () {
console.log(allFilesDownloaded);
}, function (err) {
console.err(err);
});
Though it may seem a little more complicated in some places, the actions of each function are more modular. The idea that is somewhat unintuitive is using Array.prototype.reduce to turn an array of data into an array of promises executed sequentially.
Since creating a promise to download a file attempts to download the file immediately, one can't create all the promises at once if one intends to download them one at a time. Otherwise, the sequence might look a somewhat simpler.
Please look at the code below. the request module is this one(https://www.npmjs.com/package/request)
var urlArray = []; // URL in this level
foo ("xyz.com", false);
function crawl (url, finished) {
request(url, function (error, response, body) {
if (finished == true) { return; }
// do some work on body (including getting getting n number of new URLS from
// the body) and set finished = true if we find what we are looking for.
// for each new url urlArray.push(newURL);
// for each new url call crawl(newurl, finished);
// Now How can I know when ALL these requests have finished?
// so that I can have a urlArray corresponding to this level of crawling tree and
// do some work before starting next level of crawl.
});
}
use Promises.
Check out the Q library (specifically I pointed to the methods you need):
Promise creation:
https://github.com/kriskowal/q/wiki/API-Reference#qdefer
var promise = Q.defer();
doAsyncStuff(callbackOfAsync);
return promise.promise;
functioncallbackOfAsync(isSuccess){
if(isSuccess){
promise.resolve();
}
else{
promise.reject();
}
}
Wait for multiple promises:
https://github.com/kriskowal/q/wiki/API-Reference#promise-for-array-methods
Q.all([getFromDisk(), getFromCloud()]).done(function (values) {
assert(values[0] === values[1]); // values[0] is fromDisk and values[1] is fromCloud
});
I don't really understand your question, but I guess you will need Promises. I assume you are using NodeJS.
function makeRequest (url) {
return new Promise(function (resolve, reject) {
request(url, function (err, response, body) {
if (err || response.statusCode !== 200)
reject(err || body);
else
resolve(body);
}
})
}
this function returns a promise. You can use it this way:
var request = makeRequest('url.com');
request.then(function (urls) {
// this is called if no error occured
urls.forEach(function (url) {
console.log (url);
});
}, function (error) {
// this is called on error
console.log (error);
});
If you want to wait for multiple requests to be answered to perform an action, use Promise.all:
var requests = [makeRequest(url1), makeRequest(url2), makeRequest(url3)];
Promise.all(requests).then(function (data) {
// everything is done
console.log(data);
});
I didn't test the code, but I hope you get the idea.
To answer your question specifically, the following flow of logic should work for you, I have added comments to help it make sense to you:
var urlArray = []; // URL in this level
var finished = false;
foo("xyz.com", false);
function start() {
while (urlArray.length) {
crawl(urlArray.pop());
}
}
function crawl(url) {
request(url, function (error, response, body) {
if (finished) {
return;
}
// 1. at this point, a given batch of crawls have all started
// AND urlArray is empty.
// 2. do some work on body (including getting getting n number of new URLS from
// the body) and set finished = true if we find what we are looking for.
// 3. for each new url urlArray.push(newURL);
// 4. start crawling on new batch of URL set
start();
});
}
All the request callbacks will be executed after start() completes, this guarantees that urlArray will be empty then.
If processing of one crawl response indicates (by setting finished = true;) that what you're looking for has been found all other processing of responses will terminate as soon as they begin.
Otherwise, reponse is processed and a new batch of urls are set for crawling. You call start() to begin crawling each.
It would help you also (as suggested in the other answer) if you acquainted yourself with the concept of Promises.