I'm developing a Node CLI app to use locally. It takes a CSV file as input, and based on the values in its userId column, it makes a GET request to an API using one of those values at a time as input. I've created a dummy example of this below.
Here is the axios request wrapped in an async function, which returns a Promise:
const axios = require("axios");
const utils = require("./utils");
const fs = require("fs").promises;
async function getTitleGivenId(id) {
try {
return await axios.get(`https://jsonplaceholder.typicode.com/posts/${id}`);
} catch (error) {
console.error(error);
}
}
// This works fine
getTitleGivenId(1).then(res => {
console.log(res.data.title);
});
I've come up with this in order to write a CSV, but the allData string doesn't get updated inside the map function:
async function saveTitles(inCsv, outCsv) {
try {
const arrOfObj = utils.readCsv(inCsv);
// [
// { userId: '1', color: 'green' },
// { userId: '2', color: 'blue' },
// { userId: '3', color: 'red' }
// ]
let allData = "color,title\n";
arrOfObj.map(o => {
let title;
getTitleGivenId(o["userId"]).then(res => {
title = res.data.title;
allData += `${o["color"]},${title}\n`;
});
});
await fs.writeFile(outCsv, allData);
} catch (err) {
console.error(err);
}
}
// This writes only "color,title" to "outCsv.csv"
saveTitles("./inputCsv.csv", "./outCsv.csv");
Any suggestions/alternative ways to proceed would be much appreciated.
It gets updated. You're just not waiting for it to be finished. The map() function is executed, but it will not wait for the promises inside to be finished. So one option is to make the map function async as well and just wait for all iterations to be finished:
let allData = "color,title\n";
await Promise.all( arrOfObj.map( async (o) => {
const res = await getTitleGivenId(o["userId"])
const title = res.data.title;
allData += `${o["color"]},${title}\n`;
}) );
await fs.writeFile(outCsv, allData);
Related
SOLVED: SOLUTION AT THE BOTTOM
I have the following Code where I am updating element by element:
//registerCustomers.js
const CustomerRegistrationCode = require("../models/CustomerRegistrationCode");
const setRegCodesToUsed = async (regCodes) => {
for (let regCode of regCodes) {
await setRegCodeToUsed(regCode._id);
}
};
const setRegCodeToUsed = async (id) => {
await CustomerRegistrationCode.findByIdAndUpdate(id, { used: true });
};
The Code works fine but is to slow and i want to update many (1000) CustomerRegistrationCodes at once.
I had a look at the updateMany middleware function but found not much info online and on the official docs. I changed my code to the following but don't know how further.
//registerCustomers.js
const setRegCodesToUsed = async (regCodes) => {
await CustomerRegistrationCode.updateMany(regCodes);
}
//CustomerRegistrationCode.js
CustomerRegistrationCodeSchema.pre('updateMany', async function (next, a) {
console.log('amount arguments: ', arguments.length); //is 2
console.log(arguments); //both parameters are functions.
next();
});
What would be the best way to update many CustomerRegistrationCodes with 1000 different id's?
SOLUTION, thanks to Murat Colyaran
const setRegCodesToUsed = async (regCodes) => {
const ids = [];
regCodes.map(code => ids.push(code._id));
await setRegCodeToUsed(ids);
};
const setRegCodeToUsed = async (ids) => {
await CustomerRegistrationCode.updateMany(
{ _id: { $in: ids } },
{ used: true }
);
};
This should work:
//registerCustomers.js
const CustomerRegistrationCode = require("../models/CustomerRegistrationCode");
const setRegCodesToUsed = async (regCodes) => {
let ids = [];
regCodes.map((code) => ids.push(code._id.toString()));
await setRegCodeToUsed(ids);
};
const setRegCodeToUsed = async (ids) => {
await CustomerRegistrationCode.updateMany(
{
id : { $in: ids }
},
{
used: true
}
);
};
Instead of sending a query for every records, we just parse the id and send a bulk request with $in
I'm creating a JS function that will make a call to an API, loop through the returned data and perform another call to retrieve more information about the initial data (for example where the first call return an ID, the second call would return the name/address/number the ID corresponds to). Positioning the async and await keywords though, have proven to be way more challenging than I imagined:
useEffect(() => {
const getAppointments = async () => {
try {
const { data } = await fetchContext.authAxios.get('/appointments/' + auth.authState.id);
const updatedData = await data.map(value => {
const { data } = fetchContext.authAxios.get('/customerID/' + value.customerID);
return {
...value, // de-structuring
customerID: data
}
}
);
setAppointments(updatedData);
} catch (err) {
console.log(err);
}
};
getAppointments();
}, [fetchContext]);
Everything get displayed besides the customerID, that results undefined. I tried to position and add the async/await keywords in different places, nothing works. What am I missing?
map returns an array, not a promise. You need to get an array of promises and then solve it (also, if your way worked, it would be inefficient waitting for a request to then start the next one.)
const promises = data.map(async (value) => {
const { data } = await fetchContext.authAxios.get('/customerID/' + value.customerID);
return {
...value,
customerID: data
};
});
const updatedData = await Promise.all(promises);
I have two main functions. The first one gets the SOAP data from an API. The second one parses it to json and then formats it into the object I need to store. I have been logging and seeing the return values, and as of so far until now, it's been fine. However, I am calling exports.createReturnLabelfrom my test file and all I can see is Promise { <pending> } . I am returning a value at the last function. Does this code look weird to your? How can I clean it up?
const soapRequest = require('easy-soap-request');
const xmlParser = require('xml2json')
exports.createReturnLabel = async () => {
const xml = hidden
const url = 'https://ws.paketomat.at/service-1.0.4.php';
const sampleHeaders = {
'Content-Type': 'text/xml;charset=UTF-8',
};
const auth = async () => {
const {
response
} = await soapRequest({
url: url,
headers: sampleHeaders,
xml: xml,
timeout: 2000
});
const {
body,
statusCode
} = response;
return body
}
const fetchLabel = async () => {
const soapData = await auth();
try {
const json = xmlParser.toJson(soapData)
const labelData = JSON.parse(json)["SOAP-ENV:Envelope"]["SOAP-ENV:Body"]["ns1:getLabelResponse"]
return {
courier: 'dpd',
tracking_number: labelData["return"]["paknr"],
barCode: labelData["return"]["barcodecontent"],
printLabel: labelData["return"]["label"],
_ref: null
}
} catch (e) {
return (e)
}
}
return fetchLabel()
}
calling from my test file return console.log(file.createReturnLabel())
There's an async function call inside your code.
Should be: return await fetchLabel(), so that it awaits for completion before going on its merry way.
I'm pushing files to amazon using pre-signed URLs, and modifying the files array with the file name reference inside the newData object. (The files array are inside an array of objects called items)
// Add job
const addJob = async(data, user) => {
const newData = { ...data };
data.items.map((item, itemIndex) => {
if (item.files !== []) {
item.files.map(async(file, fileIndex) => {
const uploadConfig = await axios.get(`/api/s3upload`, {
params: {
name: file.name,
},
});
console.log(uploadConfig.data.key);
newData.items[itemIndex].files[fileIndex] = uploadConfig.data.key;
await axios.put(uploadConfig.data.url, file);
});
}
});
console.log(newData);
try {
const res = await axios.post('/api/jobs', newData);
dispatch({
type: ADD_JOB,
payload: res.data,
});
} catch (error) {
console.log(error);
}
};
The file references comes in the uploadConfig.data.key and are being save into the newData object.
When this function is executed, something peculiar happens:
the console log of newData returns the correct array of references to the files
the files are uploaded just fine
the request made to /api/jobs, which is passing newData, sends an array of objects that contains { path: ... }
console.log(newData):
Post request:
JavaScript does this because forEach and map are not promise-aware. It cannot support async and await. You cannot use await in forEach or map.
for loops are promise-aware, thus replacing the loops with for loops and marking them as await returns the expected behaviour.
source: zellwk article
Corrected (functioning) code:
const addJob = async (data, user) => {
const newData = { ...data };
const { items } = data;
const loop = async () => {
for (let outer in items) {
if (items[outer].files !== []) {
const loop2 = async () => {
for (let inner in items[outer].files) {
const uploadConfig = await axios.get(`/api/s3upload`, {
params: {
name: items[outer].files[inner].name,
},
});
const res = await axios.put(uploadConfig.data.url, items[outer].files[inner])
newData.items[outer].files[inner] = uploadConfig.data.key;
}
};
await loop2();
}
}
};
await loop();
try {
const res = await axios.post('/api/jobs', newData);
dispatch({
type: ADD_JOB,
payload: res.data,
});
} catch (error) {
console.log(error);
}
};
I'm trying to only make one http call at time but when I log the response from getUrl they are piling up and I start to get 409s (Too many requests)
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
fetch(fetchUrl).then(async res => {
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
cb(url);
});
}
let requests = urls.map((url, i) => {
return new Promise(resolve => {
getUrl(url, i, resolve);
});
});
const all = await requests.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult => [...chainResults, currentResult]),
);
}, Promise.resolve([]));
Basically I don't want the next http to start until the previous one has finished. Otherwise I hammer their server.
BONUS POINTS: Make this work with 5 at a time in parallel.
Since you're using await, it would be a lot easier to use that everywhere instead of using confusing .thens with reduce. It'd also be good to avoid the explicit Promise construction antipattern. This should do what you want:
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
Then your results variable will contain an array of response texts (or an error will have been thrown, and the code won't reach the bottom).
The syntax for an async function is an async keyword before the argument list, just like you're doing in your original code:
const fn = async () => {
const results = [];
for (const url of urls) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
}
// do something with results
};
To have a limited number of requests at a time, make a queue system - when a request completes, recursively call a function that makes another request, something like:
const results = [];
const queueNext = async () => {
if (!urls.length) return;
const url = urls.shift();
const response = await fetch(url);
if (!response.ok) {
throw new Error(response); // or whatever logic you need with errors
}
results.push(await response.text());
await queueNext();
}
await Promise.all(Array.from({ length: 5 }, queueNext));
// do something with results
You cannot use Array methods to sequentually run async operations because array methods are all synchronous.
The easiest way to achieve sequential async tasks is through a loop. Otherwise, you will need to write a custom function to imitate a loop and run .then after a async task ends, which is quite troublesome and unnecessary.
Also, fetch is already returning a Promise, so you don't have to create a Promise yourself to contain that promise returned by fetch.
The code below is a working example, with small changes to your original code (see comments).
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
// To imitate actual fetching
const fetch = (url) => new Promise(resolve => {
setTimeout(() => {
resolve({
ok: true,
text: () => new Promise(res => setTimeout(() => res(url), 500))
});
}, 1000);
});
function getUrl(url, i, cb) {
const fetchUrl = `https://api.scraperapi.com?api_key=xxx&url=${url.url}`;
return fetch(fetchUrl).then(async res => { // <-- changes here
console.log(fetchUrl, 'fetched!');
if (!res.ok) {
const err = await res.text();
throw err.message || res.statusText;
}
url.data = await res.text();
return url; // <--- changes here
});
}
async function getAllUrls(urls){
const result = [];
for (const url of urls){
const response = await getUrl(url);
result.push(response);
}
return result;
}
getAllUrls(urls)
.then(console.log);
async/await is perfect for this.
Assuming you have an array of URLs as strings:
let urls = ["https://example.org/", "https://google.com/", "https://stackoverflow.com/"];
You simply need to do:
for (let u of urls) {
await fetch(u).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
The loop will not iterate until the current fetch() has resolved, which will serialise things.
The reason array.map doesn't work is as follows:
async function doFetch(url) {
return await fetch(url).then(res => {
// Handle response
}).catch(e => {
// Handle error
});
}
let mapped = urls.map(doFetch);
is equivalent to:
let mapped;
for (u of urls) {
mapped.push(doFetch(u));
}
This will populate mapped with a bunch of Promises immediately, which is not what you want. The following is what you want:
let mapped;
for (u of urls) {
mapped.push(await doFetch(u));
}
But this is not what array.map() does. Therefore using an explicit for loop is necessary.
Many people provided answers using for loop. But in some situation await in for loop is not welcome, for example, if you are using Airbnb style guide.
Here is a solution using recursion.
// Fake urls for example purpose
const urls = [{ url: 'abc' }, { url: 'def', }, { url: 'ghi' }];
async function serialFetch(urls) {
return await doSerialRecursion(
async (url) => {
return result = await fetch(url)
.then((response) => {
// handle response
})
.catch((err) => {
// handle error
});
},
urls,
0
);
}
async function doSerialRecursion(fn, array, startIndex) {
if (!array[startIndex]) return [];
const currResult = await fn(array[startIndex]);
return [currResult, ...(await doSerialRecursion(array, fn, startIndex + 1))];
}
const yourResult = await serialFetch(urls);
The doSerialRecursion function will serially execute the function you passed in, which is fetch(url) in this example.