I need some help with this helper I'm writing. For some reason using reduction within an async on a readFile, when trying to write results to a file it won't advance to the next item of the array. However, if I use a console.log, it works just fine.
const neatCsv = require('neat-csv');
const fetch = require('node-fetch');
const fs = require('fs');
fs.readFile('./codes.csv', async (err, data) => {
if (err) { throw err; }
let baseUrl = 'https://hostname/orders?from=2019-10-21T00:00:00.001Z&to=2019-12-31T23:59:59.000Z&promo=';
const starterPromise = Promise.resolve(null);
const promos = await neatCsv(data);
const logger = (item, result) => console.log(item, result);
function write (item, result) {
return new Promise((resolve, reject) => {
fs.writeFile(`./output/${item.PROMO}.json`, JSON.stringify(result), (err) => {
if (err) { throw err; }
console.log(`Wrote file ${item.PROMO}`);
});
})
}
function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
return fetch(`${baseUrl}${item.PROMO}`, { headers: { 'x-apikey': 'xyz' }})
.then(res => (res.json())
.then(json => json))
}
await promos.reduce(
(p, item) => p.then(() => asyncFetch(item).then(result => write(item, result))),
starterPromise
)
});
The csv file is just a basic layout like so..
PROMO
12345
56789
98765
...
The goal is to iterate over these, make a REST call to get the json results and write those to a file with the name of the current promo, then move to the next, making a new call and saving that one into a different file with its respective code.
In the reduce, if you call logger instead of write, it works fine. Calling write, it just makes the same call over and over and overwriting to the same file, forcing me to kill it. Please help, I'm losing my mind here...
You might have a better time using async functions everywhere, the fs promises API and a simple while loop to consume the CSV items. Dry-coded, naturally, since I don't have your CSV or API.
(Your original problem is probably due to the fact you don't resolve/reject in the write function, but the reduce hell isn't needed either...)
const neatCsv = require("neat-csv");
const fetch = require("node-fetch");
const fsp = require("fs").promises;
const logger = (item, result) => console.log(item, result);
const baseUrl = "https://hostname/orders?from=2019-10-21T00:00:00.001Z&to=2019-12-31T23:59:59.000Z&promo=";
async function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
const res = await fetch(`${baseUrl}${item.PROMO}`, { headers: { "x-apikey": "xyz" } });
const json = await res.json();
return json;
}
async function write(item, result) {
await fsp.writeFile(`./output/${item.PROMO}.json`, JSON.stringify(result));
console.log(`Wrote file ${item.PROMO}`);
}
async function process() {
const data = await fsp.readFile("./codes.csv");
const promos = await neatCsv(data);
while (promos.length) {
const item = promos.shift();
const result = await asyncFetch(item);
await write(item, result);
}
}
process().then(() => {
console.log("done!");
});
A version that uses mock data and the JSON Placeholder service, works just fine:
const fetch = require("node-fetch");
const fsp = require("fs").promises;
const baseUrl = "https://jsonplaceholder.typicode.com/comments/";
async function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
const res = await fetch(`${baseUrl}${item.PROMO}`);
return await res.json();
}
async function write(item, result) {
const data = JSON.stringify(result);
await fsp.writeFile(`./output/${item.PROMO}.json`, data);
console.log(`Wrote file ${item.PROMO}: ${data}`);
}
async function getItemList() {
return [
{PROMO: '193'},
{PROMO: '197'},
{PROMO: '256'},
];
}
async function process() {
const promos = await getItemList();
while (promos.length) {
const item = promos.shift();
const result = await asyncFetch(item);
await write(item, result);
}
}
process().then(() => {
console.log("done!");
});
Related
How to read files asynchronously in node js, here is a simple function.
There are a lot of convoluted answers on the internet, does anyone agree if this is the simplest?
export default async function handler(req, res) {
let data = await readFiles('data/companies/');
res.status(200).json(data);
}
// async file reader
function readFiles(dirname) {
return new Promise(function (resolve, reject) {
let data = {}
fs.readdir(dirname, async function(err, filenames) {
filenames.forEach(function(filename) {
fs.readFile(dirname + filename, 'utf-8', function(err, content) {
if (err) {
reject(err)
}
data[filename] = content;
if (filenames.length === Object.keys(data).length) {
resolve(data)
}
});
});
});
})
}
A bit cleaner and easier using the built in promise support in fs.promises:
const fs = require('fs');
const fsp = fs.promises;
const path = require('path');
// async file reader
async function readFiles(dirname) {
const data = {};
const files = await fsp.readdir(dirname);
for (const filename of files) {
const full = path.join(dirname, filename);
const content = await fsp.readFile(full, {encoding: 'utf8'});
data[filename] = content;
}
return data;
}
Or, if you want to run your file operations in parallel (at least to the limit of the thread pool), you might get slightly faster end-to-end performance like this:
// async file reader
async function readFiles(dirname) {
const data = {};
const files = await fsp.readdir(dirname);
await Promise.all(files.map(async filename => {
const full = path.join(dirname, filename);
const content = await fsp.readFile(full, {encoding: 'utf8'});
data[filename] = content;
}));
return data;
}
Also, this:
res.status(200).json(data);
can be replaced with:
res.json(data);
200 is already the default status so there is no reason to specify it.
I want to pass multiple query objects with res.render() inside my route.js. I have a select.js which contains the SQL statements and delivers the objects to my route.js. This works fine until I want to pass multiple query objects with res.render().
Any ideas on how I can pass multiple objects at once?
snippet route.js (I need to pass get_PriceData here as well)
I already query get_KategorieData but I have no clue how to handle multiple queries in one route.
router.get('/edit', (req, res, next) => {
var speisenEintragenData = {};
db.get_KategorieData()
.then(({ data: kategorie_data }) => {
res.render('speiseEintragen', { kategorie_data }); //maybe putting res.render() after the db.get?
})
.catch((error) => {
console.log(error);
});
});
select.js
const db = require('./config');
//KATEGORIEN LADEN
const get_KategorieData=()=>{
var sql = 'SELECT * FROM Kategorie';
return new Promise((resolve,reject) => {
db.query(sql, function (err, data, fields) {
if (err) reject(err);
resolve({data});
});
})
}
//PREISE LADEN
const get_PriceData=()=>{
var sql = 'SELECT * FROM preise';
return new Promise((resolve,reject) => {
db.query(sql, function (err, data, fields) {
if (err) reject(err);
resolve({data});
});
})
}
module.exports={
get_KategorieData,
get_PriceData
}
There are two ways to go about this. One is to stick with promises and other is to use async/await.
Using promise
Create a new function to query database. This is if the module you are using does not support async/await and requires a callback.
const query = ( sql ) => {
return new Promise(( resolve, reject) => {
db.query(sql, function (err, data, fields) {
if (err) reject(err);
resolve(data);
});
})
}
// and then you can write an async/await function to call n queries like
const get_data = async () => {
const sql1 = '...';
const a = await query( sql1 );
const sql2 = '...';
const b = await query( sql2 );
....
....
....
const sqln = '...';
const n = await query( sqln );
return { a ,b,... ,n};
}
Or with async/await you can directly call db.query and use the response
const get_data = async () => {
const sql1 = '...';
const res_1 = await db.query(sql1);
const sql2 = '...';
const res_2 = await db.query(sql2);
return { a: res_1 ,b: res_2 };
}
router.js can rewritten as
router.get('/edit', async (req, res, next) => {
const {a:rename_a,b:rename_b and so on}=await db.get_data();
res.render('view', { rename_a,rename_b and so on })
});
I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.
Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results
You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);
async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.
Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.
Im trying to return a value from a Promise in async-await form and use it in another function in another file, but I do have problem because my Promise doesnt return any value.
When im trying to console.log('website') it returns me undefined immediately (it's like the value is not being fetched at all from API services). I dont know what im doing wrong, I really love to learn about Promises and Async-Await but each time im trying to work with them im getting more confused.
const dns = require('dns')
const iplocation = require("iplocation").default;
const emojiFlags = require('emoji-flags');
const getServerIPAddress = async (server) => {
return new Promise((resolve, reject) => {
dns.lookup(server, (err, address) => {
if (err) throw reject(err);
resolve(address);
});
});
};
const getServerLocation = async (server) => {
const ip = await getServerIPAddress(server)
iplocation(ip).then((res) => {
const country = emojiFlags.countryCode(res.countryCode)
const result = `Location: ${country.emoji} ${country.name}`
return result
})
.catch(err => {
return `Location: Unknown`
});
}
(async function() {
console.log(await getServerLocation('www.google.com'))
})()
module.exports = {
getServerLocation
}
It is really important for me to get result from this function first, then use its value in another function. I wish you could give me tips on how to do tasks asynchronously.
You're clearly using async so it's not apparent why you're using then as well. If you use then then you must return the promise as well in order to preserve the promise chain:
const getServerLocation = async (server) => {
const ip = await getServerIPAddress(server)
return iplocation(ip).then((res) => {
const country = emojiFlags.countryCode(res.countryCode)
const result = `Location: ${country.emoji} ${country.name}`
return result
})
.catch(err => {
return `Location: Unknown`
});
}
Otherwise just async this:
const getServerLocation = async (server) => {
const ip = await getServerIPAddress(server)
let res = await iplocation(ip);
const country = emojiFlags.countryCode(res.countryCode)
const result = `Location: ${country.emoji} ${country.name}`
return result
}
const getServerLocation = async (server) => {
const ip = await getServerIPAddress(server)
//you need to return
return iplocation(ip).then((res) => {
const country = emojiFlags.countryCode(res.countryCode)
const result = `Location: ${country.emoji} ${country.name}`
return result
})
.catch(err => {
return `Location: Unknown`
});
}
I have a function, startSurvey, which, when run, checks if there are questions in a .json file. If there are no questions, it fetches some questions from Typeform and writes them to the .json file using saveForm. After it writes, I would like to continue executing some code that reads the .json file and logs its contents. Right now, await saveForm() never resolves.
I have promisified the fs.readFile and fs.writeFile functions.
//typeform-getter.js
const fs = require('fs')
const util = require('util')
const fetch = require('cross-fetch')
require('dotenv').config()
const conf = require('../../private/conf.json')
const typeformToken = conf.tokens.typeform
const writeFile = util.promisify(fs.writeFile)
const getForm = async () => {
const form = await fetch(`https://api.typeform.com/forms/${process.env.FORM_ID}`, {
headers: {
"Authorization": `bearer ${typeformToken}`
}
}).then(res => res.json())
const fields = form.fields
return fields
}
const saveForm = async () => {
const form = await getForm()
return writeFile(__dirname + '/../data/questions.json', JSON.stringify(form))
.then((e) => {
if (e) console.error(e)
else console.log('questions saved')
return
})
}
module.exports = saveForm
//controller.js
const fs = require('fs')
const util = require('util')
const request = require('request')
require('dotenv').config()
const typeformGetter = require('./functions/typeform-getter')
const readFile = util.promisify(fs.readFile)
const saveForm = util.promisify(typeformGetter)
let counter = 1
const data = []
const getQuestions = async() => {
console.log('called')
try {
let data = await readFile(__dirname + '/data/questions.json')
data = JSON.parse(data)
return data
} catch (e) {
console.error('error getting questions from read file', e)
}
}
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions()
if (!questions) await saveForm()
console.log(questions) //NEVER LOGS
} catch (error) {
console.error('error: ', error)
}
}
startSurvey() //function called
I don't know your exact error, but there are multiple things wrong with your code:
You're using incorrectly the promisified version of fs.writeFile, if an error occurs, the promise will be rejected, you won't get a resolved promise with an error as the resolved value, which is what you're doing.
Use path.join instead of concatenating paths.
In startSurvey, you're using console.log(questions) but that wont have any data if questions.json doesn't exists, which should happen the first time you run the program, since it's filled by saveForm, so you probably want to return the questions in saveForm
So saveForm should look something like this:
const saveForm = async () => {
const form = await getForm();
const filePath = path.join(path.__dirname, '..', 'data', 'questions.json');
await writeFile(filePath, JSON.stringify(form));
console.log('questions saved');
return form;
}
And startSurvey
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions() || await saveForm();
// This will be logged, unless saveForm rejects
// In your code getQuestions always resolves
console.log(questions);
} catch (error) {
console.error('error: ', error)
}
}
In your controller.js you're using util.promisify on saveForm when it is already a promise.
So it should be:
const saveForm = require('./functions/typeform-getter')