Objective
Disclaimer: I am new to node world and having tough time wrapping head around node asynchronous behaviour.
I am trying to write a wrapper function to do a https.get on a given url and return json output.
Code
const https = require('https');
// Get the user details
var myUrl = <valid-url>;
const getJson = function(url) {
// https get request
const req = https.get(url, (res) => {
// get the status code
const { statusCode } = res;
const contentType = res.headers['content-type'];
// check for the errors
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
//parse json
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
console.log(parsedData);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
}
console.log(getJson(myUrl));
Output
undefined
{ user_id: <user-id>,
name: 'Ajay Krishna Teja',
email: <my-email> }
Issue
So the https.get is able to hit end point and get data but not able to return the json. Constantly returning Undefined.
Things I tried
Returning parsedData on res.on(end) block
Defining a var and copying parsedData
Copying to a global variable (although I knew it's very bad practice)
Places I looked up
Node.js variable declaration and scope
How to get data out of a Node.js http get request
Javascript function returning undefined value in node js
Updated: Working code
const getJson = function(url,callback) {
// https get request
const req = https.get(url, (res) => {
// get the status code
const { statusCode } = res;
const contentType = res.headers['content-type'];
// check for the errors
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
//parse json
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
callback(parsedData);
} catch (e) {
callback(false);
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
return req;
}
// calling
getJson(amznProfileURL,(res) => {
console.log(res);
});
Short answer: You are not returning anything in your getJson function and undefined is the default Node/Javascript return value.
function getJson(){
callAsyncFunction(param1, param2, param3)
// there is no return value!
}
Longer answer: Javascript (and Node as a result) is a single threaded language that uses callbacks as it's mechanism to return async results back to the callee. To do this, you pass a function into asynchronous functions as a parameter and then that function gets called at some point in the future whenever the asynchronous function is ready to send back it's result. Calling return from this "anonymous function" is actually just returning from the "callback" function you are sending into the async function.
function getJson(){
console.log('A')
// request is started, but getJson continues execution!
http.get(url, (res)=> {
console.log('C') // by the time I'm called, 'B' has already been printed and the function has returned!
return true // this won't return getJson! It will only return the callback function which doesn't do anything!
})
console.log('B')
// end of function without return value, return undefined!
}
// Will print 'A', 'B', 'C'
There are a couple different ways you can handle this. Callbacks have been used traditionally but Javascript also natively supports Promises which are a little easier to manage and are used in many popular frameworks by default.
You can implement your function with callbacks by providing your own callback parameter to call as soon as http.get returns itself.
// define getJson with second callback parameter
const getJson = function(url, callback) {
http.get(url, (res) => {
if(res){
callback(res) // result came back, send to your own callback function
} else {
callback(false) // request failed, send back false to signify failure
}
})
}
// now I can use getJson and get the result!
getJson('http://getjson.com', (res) => {
console.log('got result!', res)
})
This is a pretty common hump to get over with async functions in node (and javascript in general).
What's happening is that your console.log(getJson(myUrl)) is called before the http request has returned anything. Basically, things like this won't work with async functions.
If you put your console.log() inside res.on('end) it will work. The way you need to deal with this if either put all your logic in the res.on('end) which kind of sucks, or pass a callback to your getJson() function which you call in res.on('end'), or wrap everything in a promise, which you can return from getJson().
To use a callback you would do something like this:
const getJson = function(url, callback) {
// a bunch of code
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
callback(null, parsedDate) // callbacks in node traditionaly pass an error as the first arg
}
//finish
}
The you call it with a function:
getJson(url, function(err, return_val) {
if (err) // handle error
console.log(return_val)
}
You can also look at other HTTP libraries like Axios that will return a promise without much work. With axios and similar libraries you can simply:
axios.get(url)
.then(response => {
console.log(response);
})
.catch(function (error) {
console.log(error);
});
Which is one of the reasons people use these libraries. More here: https://github.com/axios/axios
Because it runs asynchronously, it does not wait for the function call to end.
You can fix it with promise pattern.
Try something like this:
/**
* Created by bagjeongtae on 2017. 10. 2..
*/
function parseData(url) {
return new Promise((resolve, reject) => {
https.get(url, (res) => {
// get the status code
const {statusCode} = res;
const contentType = res.headers['content-type'];
// check for the errors
let error;
if (statusCode !== 200) {
reject('Request Failed.\n' + `Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
reject('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
reject(error.messag);
}
res.resume();
//parse json
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
console.log(parsedData);
resolve(parseData);
} catch (e) {
console.error(e.message);
reject(e.messag);
}
});
});
});
};
parseData('http://www.example.com').then( result =>{
console.log(result);
}, err => {
console.log(err);
})
Running getJson from console.log is asynchronous, so it does not wait for getJson to finish.
Asynchronous can be used like a synchronous.
I think the output is correct.The getJson(myUrl) is return undefined since you not set a return in the getJson function,the javascript return undefined by default and the
{ user_id: <user-id>,
name: 'Ajay Krishna Teja',
email: <my-email> }
is the output by console.log(parsedData) in you code.
Related
I am currently trying to do a get request in my NodeJS API, get some data and return the modified value.
From what I read in other similar questions is that you cannot just return the modified object but you need to use a callback function or a promise in order to return it. I have a standard MVC pattern where I use a controller, service.
Here is my service:
const rp = require('request-promise');
exports.RequestUserPermissions = async function(role, next) {
try {
await rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
});
} catch (error) {
console.log(error);
next(error);
}
};
Here is my controller:
const UserPermissionsService = require('../services/userPermissions.service');
exports.getUserPermissions = async function(req, res) {
try {
const role = req.headers.role; // console.log(req.headers.role);
const loggedInUserPermissions = await UserPermissionsService.RequestUserPermissions(role);
return res.status(200).json({ status: 200, data: loggedInUserPermissions, message: 'User permissions retrieved.' });
} catch (error) {
throw Error(error, 'error inside the get.user.permissions function');
}
};
So my issue is that I'm trying to return the value of filteredPermissions to my controller but I keep getting undefined. Which I guess it's a async - await issue. Meaning that the function ends before I make my calculations.
I originally had my service as:
await request.get('https://api.myjson.com/bins/7jau8', (error, response, body) => {
if (!error && response.statusCode === 200) {
const permissionsResponse = JSON.parse(body);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
return permissionsResponse;
} else {
console.log('Got an error:', error);
}
});
but I changed it to use the request-promise module, so that I can return my response. What am I doing wrong ? How can I pass my calculations properly??
Change this:
await rp('https://api.myjson.com/bins/7jau8')
to this:
return rp('https://api.myjson.com/bins/7jau8')
You need to be returning something useful from your exports.RequestUserPermissions function. As it stands now, there's no return value from that function which means the promise it returns will just have an undefined resolved value which is apparently what you are experiencing.
Then, I'd suggest using a .catch() for the error condition. And, you need to allow the caller to see the error (probably as a rejected promise) so it can know when there's an error.
I would suggest this:
const rp = require('request-promise');
exports.RequestUserPermissions = function(role, next) {
return rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
}).catch(error => {
console.log(error);
next(error);
throw error;
});
};
The spec for exactly what you want is a bit confused. To be able to test things with the URL you gave me, I created a simple stand-alone node program here. This looks for one matching role and returns that. If no matching role is found, it resolves to null. You could also make that reject, depending upon how the caller wants no matching role to work.
const rp = require('request-promise');
function getRole(role) {
return rp({uri: "https://api.myjson.com/bins/7jau8", json: true}).then(data => {
// need to find the matching role
// apparently role can be a string or an array of strings
for (let item of data) {
if (item[role]) {
return item[role];
}
}
return null;
});
}
getRole("admin").then(data => {
console.log(data);
}).catch(err => {
console.log(err);
});
When, I run this, I get this output:
{ static:
[ 'posts:list',
'posts:create',
'posts:edit',
'posts:delete',
'users:get',
'users:getSelf',
'home-page:visit',
'dashboard-page:visit' ]
}
Hopefully, can you take this an modify to fit your other needs.
Note: I'm using the json:true option with rp() so it will parse the JSON response for me automatically.
If you are using async/await with request-promise then you don't need to call .then(), you can just assign your rp call directly to a variable. For example this:
await rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
});
Would become this:
const response = await rp('https://api.myjson.com/bins/7jau8');
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
I have this function
function getJsonObjectFromURL(url, onData) {
let chunks = [];
return require('https').get(url, res => {
res.setEncoding('utf8')
.on('data', (chunk) => {
chunks.push(chunk);
})
.on('end', () => {
onData(JSON.parse(chunks.join('')));
});
}).on('error', function(e) {
console.log("Got an error: ", e);
});
}
Also I have this script that converts url's data to json array.
url = https://pu.vk.com/c824502/upload.php?act=do_add&mid=213468131&aid=-14&gid=156603484&hash=7ab9a7e723425f4a6ca08709cbd5ebd0&rhash=ba8f0ec6580a6eafce38349b12ed3789&swfupload=1&api=1&wallphoto=1
getJsonObjectFromURL(url, data => {
console.log(data.server, data.photo, data.hash);
});
It goes well when console.log. But when I want to make from this script variable, it gives me huge collection
var xx = getJsonObjectFromURL(url, data => {
return data.server;
});
console.log(xx);
Your function getJsonObjectFromURL() doesn't return the object returned by the URL. It returns the object responsible for the https request code, which is something you don't want.
I see that you are using ES6, so the best solution for you is to probably create an async function that returns a promise, which will give you great flexibility. Here is an improved version of your code:
const https = require('https');
async function getJsonObjectFromURL(url) {
return new Promise((resolve, reject) => {
const chunks = [];
try {
https.get(url, res => {
res.setEncoding('utf8')
.on('data', (chunk) => {
chunks.push(chunk);
})
.on('end', () => {
resolve(JSON.parse(chunks.join('')));
});
}).on('error', e => reject(e));
} catch (err) {
reject(err);
}
});
};
This code allows you to retrieve the remote contents of the HTTPS url synchronously or asynchronously.
Asynchronous Call
As you have already done in your code, you can use a lambda callback that handles the response when it is ready.
const url = 'https://pu.vk.com/c824502/upload.php?act=do_add&mid=213468131&aid=-14&gid=156603484&hash=7ab9a7e723425f4a6ca08709cbd5ebd0&rhash=ba8f0ec6580a6eafce38349b12ed3789&swfupload=1&api=1&wallphoto=1';
// here we use a lambda callback that handles the response
getJsonObjectFromURL(url)
.then(data => {
console.log(data.server, data.photo, data.hash);
})
.catch(err => console.error(err));
Synchronous Call
The synchronous call forces the function to wait for the result. This is how you can do it:
async function getSync() {
try {
// wait for the result
const data = await getJsonObjectFromURL(url);
console.log(data.server);
} catch(err) {
console.error(err);
}
}
getSync();
Please note that we can only use the await keyword when we are inside an async function. This is why I had to wrap the synchronous call with a function.
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I'm stuck with a small problem I think, but I can't find a way to solve it.
I want to load a remote JSON in a variable, using Node.js HTTPS GET request. It's a success, but I can't use it anywhere else in my code. My function is the following (from Node.js doc):
function getMyFile() {
var https = require('https');
https.get('URL_I_am_targeting/file.json', (res) => {
var { statusCode } = res;
var contentType = res.headers['content-type'];
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
}
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
console.log(parsedData);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
}
The request works fine, as I am able to log my .json in the console correctly. But my problem is that I can't use the constant "parsedData" anywhere else in my code. I tried to write my
return parsedData;
at several places in my function, but when I use the function in my code and try f. e.
var fileContent = getMyFile();
console.log(fileContent);
fileContent is undefined. It's like parsedData can't go out of the request. There is no error logged, the request goes fine, but I just can't use the content out of it.
I am not a Javascript professional, and it's probably something I get wrong with returning value from this request.
If somebody knows where I'm wrong and what could do the trick, it will be greatly appreciated !
Thanks !
You could write a promise-based wrapper for your function:
function getMyFile() {
var https = require('https');
return new Promise((resolve, reject) => {
https.get('URL_I_am_targeting/file.json', (res) => {
var { statusCode } = res;
var contentType = res.headers['content-type'];
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
}
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
resolve(parsedData);
} catch (e) {
reject(e.message);
}
});
}).on('error', (e) => {
reject(`Got error: ${e.message}`);
});
});
}
So that, you can use the function like this:
getMyFile()
.then(response => {
// handle success response here
})
.catch(error => {
// handle error here
});
function getMyFile() {
return new Promise((resolve,reject)=>{
//your logic and data manipulations here and finally resolve the variable
resolve(parsedData)
})
}
call it like this
getMyFile().then((parsedData)=>{
//parsedData is the returned value u needed
}).catch((error)=>{
//if any error handle it
})
In the project, I have a loop going through a list of urls. It downloads file from every url and do some post process over the downloaded file.
After the all the process done (both download process and post process), I want to execute a callback function. Because post process includes some streaming task, it has close event. If the last item can be identified, I can pass the callback function to the close event. However, since the loop is async, I can't track which item is done at last.
For now, I use a 5 second timeout to make sure the callback is executed after the whole process. Obviously, this is not sustainable. What's a good way to handle this?
loop code:
exports.processArray = (items, process, callback) => {
var todo = items.concat();
setTimeout(function() {
process(todo.shift());
if(todo.length > 0) {
// execute download and post process each second
// however it doesn't guarantee one start after previous one done
setTimeout(arguments.callee, 1000);
} else {
setTimeout(() => {callback();}, 5000);
}
}, 1000);
};
processArray(
// First param, the array
urlList,
// Second param, download and post process
(url) => {
if(url.startsWith('http')) {
getDataReg(url, uid);
}
else if(url.startsWith('ftp')) {
getDataFtp(url, uid);
}
else {
console.log('not a valid resource');
}
},
// Third param, callback to be executed after all done
() => {
Request.get(`${config.demouri}bound=${request.query.boundary};uid=${uid}`, {
method: 'GET',
auth: auth
})
.on('response', (response) => {
console.log('response event emmits');
zipFiles(uid)
.then((path) => {
reply.file(path, { confine: false, filename: uid + '.zip', mode: 'inline'}).header('Content-Disposition');
});
});
}
);
Download and post process:
exports.getDataFtp = (url, uid) => {
console.log('get into ftp');
var usefulUrl = url.split('//')[1];
var spliter = usefulUrl.indexOf('/');
var host = usefulUrl.substring(0, spliter);
var dir = usefulUrl.substring(spliter+1, usefulUrl.length);
var client = new ftp();
var connection = {
host: host
};
var fileNameStart = dir.lastIndexOf('/') + 1;
var fileNameEnd = dir.length;
var fileName = dir.substring(fileNameStart, fileNameEnd);
console.log('filename: ', fileName);
client.on('ready', () => {
console.log('get into ftp ready');
client.get(dir, (err, stream) => {
if (err) {
console.log('get file err:', err);
return;
} else{
console.log('get into ftp get');
stream.pipe(fs.createWriteStream(datadir + `download/${uid}/${fileName}`));
stream.on('end', () => {
console.log('get into ftp close');
unzipData(datadir + `download/${uid}/`, fileName, uid);
client.end();
});
}
});
});
client.connect(connection);
};
exports.getDataReg = (url, uid) => {
console.log('get into http');
var fileNameStart = url.lastIndexOf('/') + 1;
var fileNameEnd = url.length;
var fileName = url.substring(fileNameStart, fileNameEnd);
var file = fs.createWriteStream(datadir + `download/${uid}/${fileName}`);
if (url.startsWith('https')) {
https.get(url, (response) => {
console.log('start piping file');
response.pipe(file);
file.on('finish', () => {
console.log('get into http finish');
unzipData(datadir + `download/${uid}/`, fileName, uid);
});
}).on('error', (err) => { // Handle errors
fs.unlink(datadir + `download/${uid}/${fileName}`);
console.log('download file err: ', err);
});
} else {
http.get(url, (response) => {
console.log('start piping file');
response.pipe(file);
file.on('finish', () => {
unzipData(datadir + `download/${uid}/`, fileName, uid);
});
}).on('error', (err) => {
fs.unlink(datadir + `download/${uid}/${fileName}`);
console.log('download file err: ', err);
});
}
};
function unzipData(path, fileName, uid) {
console.log('get into unzip');
console.log('creating: ', path + fileName);
fs.createReadStream(path + fileName)
.pipe(unzip.Extract({path: path}))
.on('close', () => {
console.log('get into unzip close');
var filelist = listFile(path);
filelist.forEach((filePath) => {
if (!filePath.endsWith('.zip')) {
var components = filePath.split('/');
var component = components[components.length-1];
mv(filePath, datadir + `processing/${uid}/${component}`, (err) => {
if(err) {
console.log('move file err: ');
} else {
console.log('move file done');
}
});
}
});
fs.unlink(path + fileName, (err) => {});
});
}
After the all the process done (both download process and post process), I want to execute a callback function.
The interesting thing about a series of asynchronous processes is that you can never know when exactly all processes will complete. So setting a timeout for the callback is quick&dirty way to do it, but it's not reliable for sure.
You can instead use a counter to solve this problem.
Let's say you have 10 operations to perform. At the beginning you set your counter to ten counter = 10 And after each process is completed, regardless how (it can either succeed or fail), you can decrement the counter by 1 like counter -= 1 and right after it you can check if the counter is 0, if so that means all processes are completed and we reached the end. You can now safely run your callback function, like if(counter === 0) callback();
If I were you, I would do something like this:
*Notice that the called process should return a promise, so that I can know when it finishes (again regardless how)
*If you need help about promises, this useful article might help you: https://howtonode.org/promises
*Oh and one more thing, you should avoid using arguments.callee, because it's deprecated. Here is why Why was the arguments.callee.caller property deprecated in JavaScript?
exports.processArray = (items, process, callback) => {
var todo = [].concat(items);
var counter = todo.length;
runProcess();
function runProcess() {
// Check if the counter already reached 0
if(checkCounter() === false) {
// Nope. Counter is still > 0, which means we got work to do.
var processPromise = process(todo.shift());
processPromise
.then(function() {
// success
})
.catch(function() {
// failure
})
.finally(function() {
// The previous process is done.
// Now we can go with the next one.
--counter;
runProcess();
})
}
};
function checkCounter() {
if(counter === 0) {
callback();
return true;
} else {
return false;
}
}
};
What you want to do is to make all your asynchronous processes converge into a single promise that you can use to execute the callback at the correct moment.
Lets start at the point each process is complete, which I assume is in the callback passed to the mv() function in unzipData(). You want to wrap each of these asynchronous actions in a Promise that resolves in the callback and you also want to use these promises later and for that you use the .map() method to collect the promises in an array (instead of .forEach()).
Here's the code:
var promises = filelist.map((filePath) => {
if (!filePath.endsWith('.zip')) {
var components = filePath.split('/');
var component = components[components.length-1];
return new Promise((resolve, reject) =>
mv(filePath, datadir + `processing/${uid}/${component}`, (err) => {
if(err) {
console.log('move file err: ');
reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
} else {
console.log('move file done');
resolve();
}
}));
}
return Promise.resolve();
});
(if the asynchronous action is not to be executed, a Promise that resolves immediately is returned instead)
Now, we can turn this list of Promises into a single Promise that resolves when all of the promises in the list has resolved:
var allPromise = Promise.all(promises);
Next, we need to look further up in the code. We can see that the code we've just been looking at is itself part of an event handler of an asynchronous action, i.e. fs.createReadStream(). You need to wrap that in a promise that gets resolved when the inner promises resolve and this is the promise that the unzipData() function shall return:
function unzipData(path, fileName, uid) {
console.log('get into unzip');
console.log('creating: ', path + fileName);
return new Promise((outerResolve) =>
fs.createReadStream(path + fileName)
.pipe(unzip.Extract({path: path}))
.on('close', () => {
console.log('get into unzip close');
var filelist = listFile(path);
// Code from previous examples
allPromise.then(outerResolve);
}));
}
Next, we look at the functions that use unzipData(): getDataReg() and getDataFtp(). They only perform one asynchronous action so all you need to do is to make them return a promise that resolves when the promise returned by unzipData() resolves.
Simplified example:
exports.getDataReg = (url, uid) => {
return new Promise((resolve, reject) => {
// ...
https.get(url, (response) => {
response.pipe(file);
file.on('finish', () => {
unzipData(datadir + `download/${uid}/`, fileName, uid)
.then(resolve);
});
}).on('error', (err) => { // Handle errors
fs.unlink(datadir + `download/${uid}/${fileName}`);
reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
});
// ...
});
}
Finally, we get to the processArray() function and here you need to do the same thing we did to begin with: map the processes into a list of promises. First, the process function passed needs to return the promises returned by getDataReg() and getDataFtp():
// Second param, download and post process
(url) => {
if(url.startsWith('http')) {
return getDataReg(url, uid);
}
else if(url.startsWith('ftp')) {
return getDataFtp(url, uid);
}
else {
console.log('not a valid resource');
}
return Promise.reject(); // or Promise.resolve() if you want invalid resources to be ignored and not prevent the callback from executing later
}
Now, your processArray() function can look like this:
exports.processArray = (items, process, callback) =>
Promise.all(items.map(process))
.then(callback)
.catch(() => console.log('Something went wrong somewhere'));
Your callback will get invoked when all asynchronous actions have completed, regardless of in which order they do. If any one of the promises rejects, the callback will never be executed so manage your promise rejections accordingly.
Here's a JSFiddle with the complete code: https://jsfiddle.net/upn4yqsw/
In general, since nodejs does not appear to have implemented Streams Standard to be Promise based, at least from what can gather; but rather, uses an event based or callback mechanism, you can use Promise constructor within function call, to return a fulfilled Promise object when a specific event has been dispatched
const doStuff = (...args) => new Promise((resolve, reject)) => {
/* define and do stream stuff */
doStreamStuff.on(/* "close", "end" */, => {
// do stuff
resolve(/* value */)
})
});
doStuff(/* args */)
.then(data => {})
.catch(err => {})
Basically I am trying to scrap some data from website and perform the DOM extraction, deletion and updation on a callback function binded to the 'end' event of http.request.
I have returned the data from the 'end' event callback too but it is not receiving in my route callback function. I get undefined there.
Below is the code block:
var scraper = {
extractEmail: function (directoryName) {
var result = getDirectory(directoryName);
if (result !== 404) {
var protocol = result.https ? https : http;
protocol.request({
host: 'somevalue.net',
method: "GET"
}, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
return data;
});
})
.on('error', function (err) {
return err;
})
.end();
//return data;
}
else {
//return "Failed";
}
}
};
And here is the Routes.js function:
app.get('/:directory', function (req, res) {
var n = scraper.extractEmail(req.params.directory);
console.log(n);
res.send(n);
});
In here also I don't get the value of n.
Is your 'var scraper' also in the route.js file?
I guess it's not and you are unable to access that other js file,
for doing so use module.exports.
eg.
// module.js
var name = "foobar";
// export it
exports.name = name;
Then, in route.js...
> //route.js
> // get a reference to your required module
> var myModule = require('./module');
> //correct path to folder where your above file is
> // name is a member of myModule due to the export above
> var name = myModule.name;
You cannot return a value from an asynchronous callback. Well, you can, but it most likely will get ignored and most certainly won't do what you want.
You cannot even return a promise in that place. You can only resolve a promise where you now use the return statements. You need to return a promise from the main function and then resolve or reject the promise in your event handlers where you use returns now.
For more info see those answers:
Return Promise result instead of Promise in Nodejs
Return value in function from a promise block
jQuery: Return data after ajax call success
The simplest modification that will make this work is to pass a callback function to extractEmail to receive the data once it's ready.
var scraper = {
extractEmail: function (directoryName, cb) {
var result = getDirectory(directoryName);
if (result !== 404) {
var protocol = result.https ? https : http;
protocol.request({
host: 'somevalue.net',
method: "GET"
}, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
cb(null, data);
});
})
.on('error', function (err) {
cb(err);
})
.end();
}
else {
cb(new Error('Failed'));
}
}
};
And use it like this:
app.get('/:directory', function (req, res) {
scraper.extractEmail(req.params.directory, function(err, n) {
if (err) return console.error(err);
console.log(n);
res.send(n);
});
});