I have a list from the request body that I want to parse all links in request async.
When all tasks are finished, I then want to return a response, but my response is always empty because none of the async tasks are finishing.
var index = 0;
let items = [];
exports.feed = async (req, res) => {
(async () => {
await getSiteData(req.body.sites)
})();
if(typeof items !== 'undefined' && items.length > 0){
res.status(200).json(items);
} else {
res.status(404).json({ error: "Something went wrong", success: false });
}
}
async function getSiteData(list){
try{
if(index == list.length-1){
return items;
}
await parser.parseURL(list[index], async function(err, feed){
if(err != null){
console.log(err);
return false;
}
items.push(feed.items)
index++
await getSiteData(list);
});
} catch(err){
console.log(err);
return false;
}
}
Firstly, there's no recursion required. It (almost?) never makes sense to recursively process a flat list (Array)
Your main issue is that parser.parseURL does NOT return a Promise, therefore awaiting it makes no sense, since await only waits for Promises to settle
So, let's fix that by creating a "promisified" parser.parseURL, which is easy with nodes util.promisify
const { promisify } = require('util');
const parseURL = promisify(parser.parseURL);
now you can await parseURL(url)
if you need to parse.parseURL in series, i.e. one at a time
async function getSiteData(list) {
const result = [];
for (const url of list) {
const { items } = await parseURL(url);
result.push(items);
}
return result;
}
to parse.parseURL in parallel
function getSiteData(list) {
return Promise.all(list.map(parseURL));
}
Some people think that this should be
async function getSiteData(list) {
return await Promise.all(list.map(parseURL));
}
I'm not one of those people. Their argument is, getSiteData returns a Promise, so should be marked as such, some malarkey about IDE hints or some such garbage. While the code is equivalent (not identical of course), it really isn't necessary in my opinion.
Note: now getSiteData (in either series or parallel form) returns a Promise that will resolve to an array of results, and, further more, neither version does any error handling. This is deliberate, so the function that calls getSiteData determine what to do with errors.
The other issus is in your feed export
(async () => {
await getSiteData(req.body.sites)
})();
//
// the code down here runs without waiting for getSiteData to complete
this runs that asynchronous IIFE, but the code afterwards does not wait for it to complete
You could
await (async () => {
await getSiteData(req.body.sites)
})();
But then, that's just
await getSiteData(req.body.sites);
Without the IIFE
Since the getSiteData now returns the list (rather than using a "global" or external array to hold the list), the code is now:
exports.feed = async (req, res) => {
try {
const items = await getSiteData(req.body.sites);
res.status(200).json(items);
} catch (err) {
res.status(404).json({error: "Something went wrong", success: false});
}
};
I fixed my issue with using for loop and promise.
exports.feed = async (req,res) => {
var items = [];
for(const elem of req.body.sites){
const newElement = await getSiteData(elem)
items.push(newElement)
}
if(typeof items !== 'undefined' && items.length > 0){
res.status(200).json(items);
}else{
res.status(404).json({ error: "Something went wrong", success: false });
}
}
function getSiteData(url) {
return new Promise((resolve) => {
setTimeout(()=>{
parser.parseURL(url, function(err, feed) {
if(err != null){
return false;
}
resolve(feed.items)
})
});
});
}
Related
COIN LIST is an array of crypto coins(["BTCUSDT",...]). I try to get the price using getPriceAction and RSI from getRSI and these two functions are working when I try to console DATA. But when I try to print the response after the completion of the loop. It prints the empty array and the length is 0 of this array. I want to store the DATA object (consisting of SYMBOL, closing price and RSI) as an element in the response array
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
let response = await [];
await COIN_LIST.forEach((element, i) => {
setTimeout(() => {
let data = { symbol: element };
getPriceAction(element, "4h").then((res) => {
data.closingPrice = res;
getRSI(res).then((res) => {
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data)
});
});
}, i * 1000);
});
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
If you want to use async/await properly for your code, then use async/await, don't use .then/.catch as well
Some notable changes
there is no setTimeout of increasing seconds ... just waiting 1 second after one result before getting the next - far cleaner, and if one request happens to take a lot longer, you won't end up with two requests at once (which may be an issue if the API is rate limited)
no .then ... use async/await OR .then/.catch - very rare to need both in the one function
don't use forEach with async/await ... it never does what you want, and creating an array of Promises inside a .forEach is extremely naive, you may as well use .map instead! then you can await Promise.all(xxx.map(.....)) - but that's useful for concurrent requests, not so much for serial requests like your code does
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
const wait = (ms) => new Promise(resolve => setTimeout(resolve, 1000));
let response = []; //---> don't need that `await`
for (let element of COIN_LIST) {
let data = { symbol: element };
data.closingPrice = await getPriceAction(element, "4h");
const res = await getRSI(data.closingPrice);
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data);
await wait(1000);
}
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
the await wait(1000) could be tweaked depending on the rate limiting of the API ... if the rate limit applies to when the request is made, you could make a function that is smart about the delay between requests.
The code this way assumes the rate limit is based on the period between previous response to next request.
After the completion of the loop, the promises didn't get resolved yet, that's why it print an empty array.
One way to achieve what you need is using await for(...), or wait for all promises to be resolved, and then print the results.
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
let response = []; //---> don't need that `await`
const promises = []; //---> array of promises
COIN_LIST.forEach((element, i) => {
setTimeout(() => {
let data = { symbol: element };
const promise = getPriceAction(element, "4h").then((res) => {
data.closingPrice = res;
getRSI(res).then((res) => {
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data)
});
});
promises.push(promise) //---> save the reference to a promise
}, i * 1000);
});
await Promise.all(promises) //---> await for all promises to be resolved, then print the result
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
I'm tryng to upgrade this code for a better maintenance, this code uploads two images to a server, i know it's possible to get rid of those .catch, by applying async await functions, and try catch blocks, but it's pretty confusing for me, any help will be apreciated.
this._uploadService.makeFileRequest(Global.url + "/upload-image1/" + response.product._id, [], this.filesToUpload1, 'image')
.then((result: Product) => {
this.filesToUpload1 = null;
this._uploadService.makeFileRequest(Global.url + "/upload-image/" + response.product._id, [], this.filesToUpload, 'image')
.then((result: Product) => {
this.filesToUpload = null;
setTimeout( () => this._router.navigate(['/detail', this.saveProduct._id]), 800 );
})
.catch(err => {
console.log(err);
this._router.navigate(['/detail', this.saveProduct._id]);
})
})
.catch(err => {
console.log(err);
this._router.navigate(['/detail', this.saveProduct._id]);
})
I suggest using a pen and paper to draw a block diagram for the logic involved, i.e. which api gets called first, with what kind of data, then which api comes afterwards; also include any logical conditionals through branching.
After that, you should attempt to write something like
const aggregateFunction = async() => {
try {
const someResponse = await callFirstApi(); // return response
await callSecondApi(someResponse); // use the response of the first api for the second api
if (someConditional) {
await callThirdApi(); // response not returned (i.e. when not required)
}
} catch (error) { // catch all errors from all await if they're not within another try-catch
console.log(error);
}
}
This pattern should eliminate all then and catch blocks. If you need more specific error handling for calling say a specific api, wrap function call inside another try-catch block, but everything should still be within the outer try-catch so that all errors will be caught regardless.
this._uploadService.makeFileRequest = function(){
return new Promise(resolve => {
// do logic of file request
resolve(true);
})
}
var waitForTime = function() {
return new Promise(resolve => {
setTimeout( () => {
this._router.navigate(['/detail', this.saveProduct._id]),
resolve(true)
}, 800 );
})
}
var f = async function(){
try {
await this._uploadService.makeFileRequest(Global.url + "/upload-image1/" + response.product._id, [], this.filesToUpload1, 'image');
await this.fileToUpload1 = null;
await this._uploadService.makeFileRequest(Global.url + "/upload-image/" + response.product._id, [], this.filesToUpload, 'image')
await this.fileToUpload = null;
await waitForTime();
}
catch(e) {
// error logic
}
}
if (this.filesToUpload1 && this.filesToUpload) {
f()
}
this might be another cleaner approach with async,await and promise
I'd like some help please as I'm quite new in node.js and working with node packages.
I'm having the following script which makes a GET http request running on node using request which is deprecated now
const foo = (bar, callback) => {
const url = 'https://some.api.com?key=abc123';
request({url: url, json: true}, (error, response) => {
if (error) {
callback('Oops, there is an error!', undefined);
} else if(response.body.foobarArray.length === 0) {
callback('No data found', undefined);
} else {
callback(undefined, {
foobar1: response.body.foobar1,
foobar2: response.body.foobar2,
})
}
});
}
console.log(foo('Hello')); // this logs {foobar1: 'Hello', foobar2: 'World'}
I'm trying to rewrite it using axios instead, so this is my code
const foo = async (bar) => {
const url = 'https://some.api.com?key=abc123';
try {
const response = await axios.get(url);
if (response.body.foobarArray.length === 0) {
return 'No data found';
} else {
return {
foobar1: response.body.foobar1,
foobar2: response.body.foobar2,
};
}
} catch (error) {
return 'Ooops! Something went wrong :(';
}
};
console.log(foo('Hello')); // This logs `Promise { <pending> }`
I'm not sure what I'm doing wrong here as I'm not very familiar how promises work exactly, but how can I fix this?
const foo = async (bar) => {
const url = 'https://some.api.com?key=abc123';
try {
return await axios.get(url).then(response => {
return new Promise((resolve, reject) => {
if (response.body.foobarArray.length === 0) {
return reject('No data found');
} else {
return resolve({
foobar1: response.body.foobar1,
foobar2: response.body.foobar2,
});
}
})
}).catch(err => {
return Promise.reject(err);
});
} catch (error) {
// return 'Ooops! Something went wrong :(';
return Promise.reject(`an error occurred : ${error}`);
}
};
foo('hello').then(result => {
console.log(result);
}).catch(err => {
console.log(`error ! : ${err}`);
});
async functions returns a promise. async functions use an implicit Promise to return its result. Even if you don't return a promise explicitly async function makes sure that your code is passed through a promise
as you are using axios asynchronous , it's response is a promise which must be handled inside .then().catch() functions .
if no error occurs you can access the response inside your .then() , else you will have access to your error on .catch()
inside your .then() you can now do what you want with data , returning a new Promise , using resolve() for success and reject() for failure .
You have 2 options here:
Option 1
Any async function returns a Promise (behind the scenes) so:
foo('Hello').then(console.log).error(console.error);
Option 2
You need to await for the result of foo function but, at the moment, you can't use await out of function scope level. So:
async function main() {
try {
const result = await foo('Hello');
console.log(result);
} catch (err) {
console.error(err);
}
}
main();
In future Node.js releases, using await at global scope will be allowed.
I am currently trying to do a get request in my NodeJS API, get some data and return the modified value.
From what I read in other similar questions is that you cannot just return the modified object but you need to use a callback function or a promise in order to return it. I have a standard MVC pattern where I use a controller, service.
Here is my service:
const rp = require('request-promise');
exports.RequestUserPermissions = async function(role, next) {
try {
await rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
});
} catch (error) {
console.log(error);
next(error);
}
};
Here is my controller:
const UserPermissionsService = require('../services/userPermissions.service');
exports.getUserPermissions = async function(req, res) {
try {
const role = req.headers.role; // console.log(req.headers.role);
const loggedInUserPermissions = await UserPermissionsService.RequestUserPermissions(role);
return res.status(200).json({ status: 200, data: loggedInUserPermissions, message: 'User permissions retrieved.' });
} catch (error) {
throw Error(error, 'error inside the get.user.permissions function');
}
};
So my issue is that I'm trying to return the value of filteredPermissions to my controller but I keep getting undefined. Which I guess it's a async - await issue. Meaning that the function ends before I make my calculations.
I originally had my service as:
await request.get('https://api.myjson.com/bins/7jau8', (error, response, body) => {
if (!error && response.statusCode === 200) {
const permissionsResponse = JSON.parse(body);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
return permissionsResponse;
} else {
console.log('Got an error:', error);
}
});
but I changed it to use the request-promise module, so that I can return my response. What am I doing wrong ? How can I pass my calculations properly??
Change this:
await rp('https://api.myjson.com/bins/7jau8')
to this:
return rp('https://api.myjson.com/bins/7jau8')
You need to be returning something useful from your exports.RequestUserPermissions function. As it stands now, there's no return value from that function which means the promise it returns will just have an undefined resolved value which is apparently what you are experiencing.
Then, I'd suggest using a .catch() for the error condition. And, you need to allow the caller to see the error (probably as a rejected promise) so it can know when there's an error.
I would suggest this:
const rp = require('request-promise');
exports.RequestUserPermissions = function(role, next) {
return rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
}).catch(error => {
console.log(error);
next(error);
throw error;
});
};
The spec for exactly what you want is a bit confused. To be able to test things with the URL you gave me, I created a simple stand-alone node program here. This looks for one matching role and returns that. If no matching role is found, it resolves to null. You could also make that reject, depending upon how the caller wants no matching role to work.
const rp = require('request-promise');
function getRole(role) {
return rp({uri: "https://api.myjson.com/bins/7jau8", json: true}).then(data => {
// need to find the matching role
// apparently role can be a string or an array of strings
for (let item of data) {
if (item[role]) {
return item[role];
}
}
return null;
});
}
getRole("admin").then(data => {
console.log(data);
}).catch(err => {
console.log(err);
});
When, I run this, I get this output:
{ static:
[ 'posts:list',
'posts:create',
'posts:edit',
'posts:delete',
'users:get',
'users:getSelf',
'home-page:visit',
'dashboard-page:visit' ]
}
Hopefully, can you take this an modify to fit your other needs.
Note: I'm using the json:true option with rp() so it will parse the JSON response for me automatically.
If you are using async/await with request-promise then you don't need to call .then(), you can just assign your rp call directly to a variable. For example this:
await rp('https://api.myjson.com/bins/7jau8').then(response => {
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
});
Would become this:
const response = await rp('https://api.myjson.com/bins/7jau8');
const permissionsResponse = JSON.parse(response);
const filteredPermissions = permissionsResponse.find(function(x) {
return Object.keys(x).indexOf(role) > -1;
});
console.log(filteredPermissions); // I GET UNDEFINED HERE.
return filteredPermissions;
async function count() {
let nedb = <<functional nedb database>>
let count = 0
await iterate(nedb, {}, g=>{
count++
})
console.log(count)
iterate(xx, query, callback) {
return new Promise((res, rej)=> {
pkgs.find(query).exec((err, res)=>{
if(err!==null) {
this.err(err)
} else {
res.forEach((pkg)=>{
callback(pkg)
})
}
})
res()
})
I would like to write something, after the iteration is over, is there way to do it in some normal way?
You need to put the res()olve call inside the find callback. Immediately fulfilling the promise to undefined won't help.
But you really shouldn't pass an iteration callback at all. Instead, write
async function count() {
let nedb = …;
let count = 0;
const pkgs = await query(nedb, {});
pkgs.forEach(pkg => count++);
// or maybe better just:
const count = pkgs.length;
console.log(count);
}
function query(pkgs, query) {
return new Promise((resolve, reject)=> {
pkgs.find(query).exec((error, result)=>{
if (error !== null) reject(error);
else resolve(result);
});
});
}