chained async rest calls feeding follow on functions - javascript

** Update: The answer below works great up to the merge. I had to add some iterations into getL2 and getL3 to make all the calls necessary. The issue seemed to be that some of the tasks from L1 did not have a child at L2, and some L2 did not have an L3. So the final results has some empty arrays. the final result before the merge looks like below. right now, the merge function returns the L1 data with no children added.
var result = [
[{
"ID": 1,
"title": "Task 1",
"org": "A"
}, {
"ID": 2,
"title": "Task 2",
"org": "B"
}],
[{}, {},
{
"ID": 10,
"ParentID": 2
}, {
"ID": 11,
"ParentID": 2
}
],
[{}, {}, {
"ID": 20,
"ParentID": 10
}, {
"ID": 21,
"ParentID": 10
}]
]
I continue to struggle with async and promises, despite the amazing help on stack. I tried to reverse engineer some similar posts with no success. I am trying to chain 3 functions that will make multiple REST calls in SharePoint 2013. the first call grabs all items in list taskL1 that match the organization. it uses the ID's to create a set of unique entries. then a second call against taskL2 is made using the the set to find all entries that have those id's in the parentID column. It then creates a set of those IDs to make a final call against taskL3 and matches those IDs to the parentID column in that list. I need to pass all three results along until a final function will merge the data into a nested object.Bascially, put all the L3 tasks under the correct L2 and all the L2 under the correct L1. I can't seem to keep all 3 results passing along until the end and available to a follow on function. https://jsfiddle.net/75ghvms8/
var setL1 = new Set();
var setL2 = new Set();
var taskL1 = [{"id":1,"title":"Task 1","org":"A"},{"id":2,"title":"Task 2","org":"B"},{"id":3,"title":"Task 3","org":"A"}]
var taskL2 = [{"id":20,"parentID":1},{"id":21,"parentID":1},{"id":22,"parentID":2},{"id":23,"parentID":2}]
var taskL3 = [{"id":100,"parentID":20},{"id":111,"parentID":21},{"id":120,"parentID":22},{"id":220,"parentID":23}]
getL1(taskL1,'A').then(()=>{getL2(taskL2,setL1)}).then(()=>{getL3(taskL3,setL2)});
async function getL1(srcList,org){
const l1List = await getData(srcList,org);
console.log(l1List);
return l1List
}
async function getL2(srcList,set){;
const l2List = await getData2(srcList,set);
console.log(l2List);
return l2List
}
async function getL3(srcList,set){
const l3List = await getData3(srcList,set);
console.log(l3List);
return l3List
}
async function getData(srcList,org,result={}) {
const listItems = await getTaskItems(srcList,org);
result = listItems;
return result;
}
async function getData2(srcList,item,result={}) {
let j = 0;
for(let i of item) {
const listItems = await getTaskItems2(srcList,i)
result[j] = listItems;
j++
}
return result
}
async function getData3(srcList,item,result={}) {
let j = 0;
for(let i of item) {
const listItems = await getTaskItems3(srcList,i)
result[j] = listItems;
j++
}
return result
}
function getTaskItems(srcList,org) {
const arrData = srcList.filter(obj=> {
return obj.org === org;
});
for (let i = 0; i < arrData.length; i++) {
setL1.add(arrData[i].id);
}
return {arrData,setL1}
}
function getTaskItems2(srcList,id) {
const arrData = srcList.filter(obj=> {
return obj.parentID === id;
});
for (let i = 0; i < arrData.length; i++) {
setL2.add(arrData[i].id);
}
return {arrData,setL2}
}
function getTaskItems3(srcList,id) {
const arrData = srcList.filter(obj=> {
return obj.parentID === id;
});
return arrData;
}
// real functions are spRest-Lib functions
/*
function getTaskItems(srcList, org) {
return new Promise((resolve,reject) =>{
sprLib.list(srcList).getItems({
listCols: {
columns here
}
})
.then(function(arrData){
for (let i = 0; i < arrData.length; i++) {
setL1.add(arrData[i].ID);
}
resolve(arrData);
})
.catch(function(errMsg) {console.log(errMsg);});
})
}
*/

You'll need to tweak some key names. Promise chaining is achieved by returning custom objects {l1: [], l2: [], l3: []}. Merge can probably be done as you go along each step, but instead here is one way to do it with explicit merge step.
var taskL1 = [{"id":1,"title":"Task 1","org":"A"},{"id":2,"title":"Task 2","org":"B"},{"id":3,"title":"Task 3","org":"A"}]
var taskL2 = [{"id":20,"parentID":1},{"id":21,"parentID":1},{"id":22,"parentID":2},{"id":23,"parentID":2}]
var taskL3 = [{"id":100,"parentID":20},{"id":111,"parentID":21},{"id":120,"parentID":22},{"id":220,"parentID":23}]
async function getL1(org) {
return new Promise((resolve, reject) => {
// assuming some external reqeust
resolve({ l1:
taskL1.filter((item) => item.org === org)
});
})
}
async function getL2(l1Result) {
return new Promise((resolve, reject) => {
const allL1Ids = Array.from(new Set(Object.values(l1Result.map((item) => item.id))));
// assuming some external request
const filteredList = taskL2.filter((item) => allL1Ids.indexOf(item.parentID !== -1));
resolve({ l1: l1Result, l2: filteredList});
})
}
async function getL3(l1Result, l2Result) {
return new Promise((resolve, reject) => {
const allL2Ids = Array.from(new Set(Object.values(l2Result.map((item) => item.id))));
// assuming some external request
const filteredList = taskL3.filter((item) => allL2Ids.indexOf(item.parentID !== -1));
resolve({l1: l1Result, l2: l2Result, l3: filteredList})
})
}
function groupByKey(items, key) {
return items.reduce(function(results, item) {
(results[item[key]] = results[item[key]] || []).push(item);
return results;
}, {});
}
function merge(l1, l2, l3) {
// we want to put l3.parentID into l2.id.children
let l3ByParentId = groupByKey(l3, "parentID");
let l2ById = groupByKey(l2, "id");
Object.keys(l3ByParentId).forEach((parentId) => {
if (l2ById[parentId]) {
l2ById[parentId][0]['l3children'] = l3ByParentId[parentId];
}
});
let l2ByParentId = groupByKey(Object.values(l2ById).map((item) => item[0]), "parentID");
let l1ById = groupByKey(l1, "id");
Object.keys(l2ByParentId).forEach((parentId) => {
if (l1ById[parentId]) {
l1ById[parentId][0]['l2children'] = l2ByParentId[parentId];
}
});
return Object.values(l1ById).map(item => item[0]);
}
getL1("A")
.then((result) => {
return getL2(result.l1)
})
.then((result) => {
return getL3(result.l1, result.l2)
})
.then((result) => {
return merge(result.l1, result.l2, result.l3)
})
.then((result) => {
console.log(JSON.stringify(result, null, 2));
})

Related

Extracting values out of an array of objects?

I am trying to extract id from the below array of objects and so far I am able to give it a go with the below code but it is showing undefined and cannot get id , would you please check the code and adjust to get id out?
const array = [{
contact: {
id: 1,
email: 'roar#gmail.com',
},
date: '2/4/22'
},
{
contact: {
id: 2,
email: 'grr#gmail.com',
},
date: '2/4/22'
}
]
function extractValue(arr, prop) {
let extractedValue = [];
for (let i = 0; i < arr.length; ++i) {
// extract value from property
extractedValue.push(arr[i][prop]);
}
return extractedValue;
}
const result = extractValue(array, 'contact.id');
console.log(result);
A good way to do this is the Array Map method
This will get all the id's from your array
const result = array.map((val) => val.contact.id)
const extractValue = (array, path) => {
const [first, second] = path.split('.')
if (second) return array.map(obj => obj[first][second])
return array.map(obj => obj[first])
}
const res = extractValue(array, 'contact.id')
console.log(res)
// => [ 1, 2 ]
this will support single and double level nested results
function find(val, arr) {
for (let x of arr)
val = val[x];
return val;
}
function extractValue(arr, prop) {
return array.map(x => find(x, prop.split(".")))
}

How can you resolve promises in multiple nested forEach loops?

For a "flights" portfolio app, I am trying to generate tickets for each seat in a plane (so that flights are prepopulated with some people). This is part of a function that also prepopulates flights to the database to set up the data. There are calls to the database to check to see if the Flight Number, as well as Confirmation Numbers are unique, and not pre-existing. In the case of flights, if the flight number already exists, the flight just gets dropped. However, I am running into some issues with the asynchronous aspect of regenerating a confirmation number if one already exists when it is in loops. So essentially, the issues I'm having:
generateTickets returns an empty tickets array
I am not sure how to structure my nested forEach loops to return promises when generating the tickets for each seat, in each row. (In my searches on Stack Overflow, I have found multiple solutions referencing Promise.all with array.map. I've tried to implement these solutions in a variety of ways, but I'm still getting an empty array, and I think I'm just not understanding it well enough.)
I am getting: SyntaxError: Unexpected reserved word on await generateConfirmationNumber() - even though generateConfirmationNumber is an asynchronous function
I don't know what to say about what I've tried. It's a lot. I've searched through every link for the first 2-3 pages of Google, I've scoured Stack Overflow trying to find solutions, and then trying to implement them. I think I'm just having a hard time interpreting how their code works, when this has additional complexity. The solutions I have on my browser tab currently are:
Why doesn't the code after await run right away? Isn't it supposed to be non-blocking?
Pushing Elements into an Array That Is Inside a Callback Function
How to wait a Promise inside a forEach loop
Why does my async function return an empty array
Best way to wait for .forEach() to complete
Using async/await with a forEach loop
How to return values from async functions using async-await from function?
I've been googling and trying to get this to work for 3 days now, and I figured it was time to ask a question. :P So on to the code! (If you see crazy all caps comments, it just helps me notice them easily in the console.)
createFlight - Called by router, starts everything.
async function createFlight(req, res) {
create_flights().then(() => {
console.log('Done')
res.redirect('/')
})
}
create_flights - Async Function that generates a random flight.
const create_flights = async() => {
let flights_array = []
let flights_with_tickets = []
for(let i = 0; i < 1; i++) {
let airline = randomAirline();
let origin = randomAirport();
let destination = randomDestination(origin.code);
let duration = generateDuration()
let departure = generateDeparture()
let departure_date = dateString(departure)
let flight_data = {
number: generateFlightNumber(airline.code),
airline: airline.name,
plane: {
name: randomPlane()
},
origin: origin,
destination: destination,
departure: departure,
departure_date: departure_date,
duration: duration,
gate: generateGate(),
amenities: randomAmenities(duration),
tickets: []
}
const flight = new Flight(flight_data);
flights_array.push(flight)
}
console.log("FLIGHTS ARRAY")
console.log(flights_array)
for(let flight of flights_array) {
console.log(flight)
const flight_with_ticket = await returnTicketsAsync(flight)
console.log(flight_with_ticket)
if(flight_with_ticket) {
const record = await returnFlight(flight_with_ticket)
console.log(record)
if(record) {
console.log("Created Flight")
console.log(record.tickets[0])
}
}
}
}
returnTicketsAsync - Waits for the tickets to be generated. This is always console logging an empty tickets value.
async function returnTicketsAsync(flight) {
console.log("ASYNC TICKETS?!?!?!!?")
let tickets = await generateTickets(flight)
console.log(tickets)
}
returnFlight - This checks to see if the flight's confirmation number is already in the database. If it is, we just skip this flight and keep moving on. Otherwise, we save it to the database.
const returnFlight = flight => {
return new Promise((resolve, reject) => {
Flight.find({$or:[{number: flight.number}]}, function(error, records) {
if(records.length) {
console.log("Similar flight already exists")
resolve()
} else {
flight.save(function (error, result) {
if(error) {
console.log("ERROR:" + error)
resolve()
} else {
resolve(result)
console.log("Flight " + result.number + " successfully saved.")
}
})
}
})
})
}
generateTickets(flight) - The source of all my troubles. The Flight has a plane map, with sections (first class, preferred class, economy). Each section has rows with seats.
So I need to loop through first class, preferred class, and economy class, generate the tickets for all the seats. There is one function, generateConfirmationNumber() that generates the number, and checks the database for a result. If there's no result, the confirmation number is good to be used, and it returns that. I need to await for that to be done, but I'm getting a SyntaxError here.
async function generateTickets(flight) {
let tickets = []
let prices = generatePrice()
const first_class = await Promise.all(_.map(flight.plane.sections.first_class.rows, async function (row, i, rows) {
let first_class_tickets = await _.map(row, function(seat) {
if(!seat.isle) {
let passenger = generatePassenger()
let confirmation_number = await generateConfirmationNumber()
let tickets = {
confirmation_number: '',
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: '1ST CL',
flight_class: 'first',
seat: {
isle: seat.letter,
number: i + 1
},
boarding_zone: generateBoardingZone(),
price: prices.first,
passenger: passenger
}
if(passenger.first_name !== '' || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number
}
console.log(ticket)
tickets.push(ticket)
}
})
}))
const preferred_class = await Promise.all(_.map(flight.plane.sections.preferred_class.rows, async function (row, i, rows) {
let preferred_class_tickets = await _.map(row, function(seat) {
if(!seat.isle) {
let passenger = generatePassenger()
let confirmation_number = await generateConfirmationNumber()
let tickets = {
confirmation_number: '',
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: 'PREF PLUS',
flight_class: 'preferred',
seat: {
isle: seat.letter,
number: i + flight.plane.sections.first_class.total_rows + 1
},
boarding_zone: generateBoardingZone(),
price: prices.preferred,
passenger: passenger
}
if(passenger.first_name !== '' || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number
}
console.log(ticket)
tickets.push(ticket)
}
})
}))
const economy_class = await Promise.all(_.map(flight.plane.sections.economy_class.rows, async function (row, i, rows) {
let economy_class_tickets = await _.map(row, function(seat) {
if(!seat.isle) {
let passenger = generatePassenger()
let confirmation_number = await generateConfirmationNumber()
let tickets = {
confirmation_number: '',
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: 'PREF PLUS',
flight_class: 'preferred',
seat: {
isle: seat.letter,
number: i + flight.plane.sections.first_class.total_rows + flight.plane.sections.preferred_class.total_rows + 1
},
boarding_zone: generateBoardingZone(),
price: prices.economy,
passenger: passenger
}
if(passenger.first_name !== '' || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number
}
console.log(ticket)
tickets.push(ticket)
}
})
}))
console.log("Tickets")
console.log(ticekts)
}
generateConfirmationNumber - Just throwing this here, too.
async function generateConfirmationNumber() {
let letters = 'ABCDEFGHJKLMNPRSTUVWXYZ'
let number = Math.floor(Math.random() * 10000)
let confirmation = letters[Math.floor(Math.random() * letters.length)] + letters[Math.floor(Math.random() * letters.length)] + letters[Math.floor(Math.random() * letters.length)] + number
return new Promise((resolve, reject) => {
Flight.findOne({'tickets.confirmation_number': confirmation}, (err, result) => {
if (err) console.log(error)
else if (result) return resolve(generateConfirmationNumber())
else return resolve(confirmation)
})
})
}
I'm at a loss for how to get this working. The goal is simply to generate the tickets, wait for the generateConfirmationNumber() to finish, and push all the tickets into a "tickets" array for that flight. Then, return that array so it can be $set to the flight, and so then, the flight can be pushed into the database.
The main issue your codes doesn't work because you try to Promise.all() to a array of the ticket below.
Promise.all only working on promises object, but you can see the ticket itself is not a purely Promise object. Also, the map() function will not await the promise in the loop. If you want to await a promise in a loop, you need to use for loop.
let confirmation_number = await generateConfirmationNumber();
let ticket = {
confirmation_number: confirmation_number,
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: "1ST CL",
flight_class: "first",
seat: {
isle: seat.letter,
number: i + 1,
},
boarding_zone: generateBoardingZone(),
price: prices.first,
passenger: generatePassenger(),
};
The below codes doesn't work because ticket is not a promise object.
Promise.all([ticket1,ticket2,ticket3]);
To make it work, you have two options.
Promises in Parallel: Promise.all all the promises object and use array#map() to create a array of ticket objects.
Promises in Series: await each promise and construct the ticket one by one.
If you want to do a Promises in Parallel way, you can do like below:
const arrayOfPromises = flight.plane.sections.first_class.rows.flatMap(row =>
row.map(seat => generateConfirmationNumber())
);
const confirmationNumbers = await Promise.all(arrayOfPromises);
const tickets = confirmationNumbers.map(cfmNumber => ({
confirmation_number: cfmNumber,
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: "1ST CL",
flight_class: "first",
seat: {
isle: seat.letter,
number: i + 1,
},
boarding_zone: generateBoardingZone(),
price: prices.first,
passenger: generatePassenger(),
}));
The answer I use is 2nd option(Promises in Series).
I use for of loop to await each promise and create a ticket from it.
In the end, I add ticket to a array tickets.
async function generateTickets(flight) {
let tickets = [];
let prices = generatePrice();
const firstClass = [];
for (const row of flight.plane.sections.first_class.rows) {
for (const seat of row) {
if (!seat.isle) {
const passenger = generatePassenger();
const confirmation_number = await generateConfirmationNumber();
const ticket = {
confirmation_number: "",
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: "1ST CL",
flight_class: "first",
seat: {
isle: seat.letter,
number: flight.plane.sections.first_class.rows.indexOf(row) + 1,
},
boarding_zone: generateBoardingZone(),
price: prices.first,
passenger: passenger,
};
if (passenger.first_name !== "" || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number;
}
firstClass.push(ticket);
}
}
}
console.log(firstClass);
const preferredClassTickets = [];
for (const row of flight.plane.sections.preferred_class.rows) {
for (const seat of row) {
if (!seat.isle) {
let passenger = generatePassenger();
let confirmation_number = await generateConfirmationNumber();
let ticket = {
confirmation_number: "",
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: "PREF PLUS",
flight_class: "preferred",
seat: {
isle: seat.letter,
number: flight.plane.sections.preferred_class.rows.indexOf(row) + flight.plane.sections.first_class.total_rows + 1,
},
boarding_zone: generateBoardingZone(),
price: prices.preferred,
passenger: passenger,
};
if (passenger.first_name !== "" || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number;
}
preferredClassTickets.push(ticket);
}
}
}
const economyClass = [];
for (const row of flight.plane.sections.economy_class.rows) {
for (const seat of row) {
if (!seat.isle) {
let passenger = generatePassenger();
let confirmation_number = await generateConfirmationNumber();
let ticket = {
confirmation_number: "",
tracking_number: generateTrackingNumber(),
data: generateDataNumber(),
options: "PREF PLUS",
flight_class: "preferred",
seat: {
isle: seat.letter,
number:
flight.plane.sections.economy_class.rows.indexOf(row) +
flight.plane.sections.first_class.total_rows +
flight.plane.sections.preferred_class.total_rows +
1,
},
boarding_zone: generateBoardingZone(),
price: prices.economy,
passenger: passenger,
};
if (passenger.first_name !== "" || passenger.first_name.length > 1) {
ticket.confirmation_number = confirmation_number;
}
economyClass.push(ticket);
}
}
}
tickets.push(...firstClass, ...preferredClassTickets, ...economyClass);
return tickets;
}
async function generateConfirmationNumber() {
let letters = "ABCDEFGHJKLMNPRSTUVWXYZ";
let number = Math.floor(Math.random() * 10000);
let confirmation =
letters[Math.floor(Math.random() * letters.length)] +
letters[Math.floor(Math.random() * letters.length)] +
letters[Math.floor(Math.random() * letters.length)] +
number;
return new Promise((resolve, reject) => {
Flight.findOne(
{ "tickets.confirmation_number": confirmation },
(err, result) => {
if (err) reject(error);
// else if (result) return resolve(generateConfirmationNumber());
else if (result) return resolve(result);
else return resolve(confirmation);
}
);
});
}
const returnFlight = flight => {
return new Promise((resolve, reject) => {
Flight.find(
{ $or: [{ number: flight.number }] },
function (error, records) {
if (error) reject(error);
if (records.length) {
console.log("Similar flight already exists");
resolve(records);
} else {
flight.save(function (error, result) {
if (error) {
console.log("ERROR:" + error);
reject(error);
} else {
console.log("Flight " + result.number + " successfully saved.");
resolve(result);
}
});
}
}
);
});
};
const create_flights = async () => {
let flights_array = [];
// let flights_with_tickets = [];
for (let i = 0; i < 1; i++) {
let airline = randomAirline();
let origin = randomAirport();
let destination = randomDestination(origin.code);
let duration = generateDuration();
let departure = generateDeparture();
let departure_date = dateString(departure);
let flight_data = {
number: generateFlightNumber(airline.code),
airline: airline.name,
plane: {
name: randomPlane(),
},
origin: origin,
destination: destination,
departure: departure,
departure_date: departure_date,
duration: duration,
gate: generateGate(),
amenities: randomAmenities(duration),
tickets: [],
};
const flight = new Flight(flight_data);
flights_array.push(flight);
}
console.log("FLIGHTS ARRAY");
console.log(flights_array);
for (let flight of flights_array) {
const tickets = await generateTickets(flight);
if (tickets) {
flight.set({
tickets: tickets
})
const record = await returnFlight(flight);
console.log(record);
if (record) {
console.log("Created Flight");
console.log(record.tickets);
}
}
}
};
async function createFlight(req, res) {
try {
await create_flights();
console.log("Done");
res.redirect("/");
} catch (err) {
console.log(err);
}
}

async/await loop works as expected with axios.get but not with a required local .json file

I really want to know what concept(s) of javascript (regarding async/await) I am missing here. I am pretty sure this is not a duplicate question.
My code is too involved to show as an example so I will attempt to describe it the best I can and show the issue in its simplest form.
The main purpose of this code is to make a set of network requests in parallel and do some actions after they all are completed. I have that working properly, the loop execution 'appears' to pause until the awaited value is returned and this is desired.
However when I use a local .json file (loaded via require) rather than using axios.get, the loop runs all the way through before the awaited value is returned. This is problematic since I am mutatating the awaited value based on the premise of the loop pausing.
/*
Simplified as much as possible.
Note: The code works as desired when globalOptions.useNetworkStub = false
*/
const axios = require('axios').default;
const globalOptions = {
useNetworkStub: true
}
const getSearchByTerm = async(term) => {
if (globalOptions.useNetworkStub) {
const networkStub = require('./fake-response.json')
return Promise.resolve(networkStub)
}
return axios.get('https://jsonplaceholder.typicode.com/todos/1')
}
const getSearchesByTerms = async(terms = ['cats','dogs']) => {
const results = []
let result
try {
for (let i = 0; i < terms.length; i++) {
const term = terms[i]
result = await getSearchByTerm(term)
result.data && (result.data.searchTerm = term) // The issue is here!
results.push(result)
}
} catch (err) {
return Promise.reject(`getSearchesByTerms() Failed: ${err}`)
}
// ... code truncated here to keep things simple ...
return Promise.resolve(results)
}
getSearchesByTerms()
.then((responses) => {
let merged = {responses: []}
for (const response of responses) {
merged.responses.push(response.data)
}
console.log(`SUCCESS, data: ${JSON.stringify(merged, null, 2)}`)
})
.catch(e=>console.log(e))
fake-response.json
{
"data": {
"userId": 1,
"id": 1,
"title": "delectus aut autem",
"completed": false
}
}
As I metioned earlier, when axios is used the end result is correct. The first response has a key value pair of searchTerm: 'cats' and the second response has a key value pair of searchTerm: 'dogs'
When the local .json file is used both the first and second reponses have the same key value pair of searchTerm: 'dogs'. This is the problem.
EDIT: changed const term = terms[i].term to const term = terms[i]
ANOTHER EDIT: fixed typos in code, added data for fake-response.json and posted a working example of this issue here on repl.it
Minus a few typos, your code runs -- with some issues.
// example.js
const axios = require('axios').default;
const globalOptions = {
useNetworkStub: true
}
const getSearchByTerm = async function(term) {
if (globalOptions.useNetworkStub) {
const networkStub = require('./fake-response.json')
return Promise.resolve(networkStub)
}
return axios.get('https://some-live-endpoint.com/', {params:{q: term}})
}
const getSearchesByTerms = async function(terms = ['cats','dogs']) {
const results = []
let result
try {
for (let i = 0; i < terms.length; i++) {
const term = terms[i]
result = await getSearchByTerm(term)
console.log("Term: ", term, "Result: ", result); // added this log to clarify your issue
result.data && (result.data.searchTerm = term) // The issue is here!
results.push(result)
}
} catch (err) {
return Promise.reject(`getSearchesByTerms() Failed: ${err}`)
}
// ... code truncated here to keep things simple ...
return Promise.resolve(results)
}
getSearchesByTerms()
.then((responses) => {
let merged = {responses: []}
for (const response of responses) {
merged.responses.push(response.data)
}
console.log(`SUCCESS, data: ${JSON.stringify(merged, null, 2)}`)
})
.catch(e=>console.log(e))
And here's and example json file
{
"data": {
"payload": "Success"
}
}
Here's the output you'll get:
Term: cats Result: { data: { payload: 'Success' } }
Term: dogs Result: { data: { payload: 'Success', searchTerm: 'cats' } }
SUCCESS, data: {
"responses": [
{
"payload": "Success",
"searchTerm": "dogs"
},
{
"payload": "Success",
"searchTerm": "dogs"
}
]
}
The thing to notice is that your issue isn't async it's using a reference to the same object for both results. This is a lesson that can burn you in a lot of subtle but important ways in Javascript -- and many other languages that hide the complexity of pointers from the programmer. You should generally avoid mutating objects.
Here's a version that uses JS spread syntax to copy the object instead of mutating.
const axios = require('axios').default;
const globalOptions = {
useNetworkStub: true
}
const getSearchByTerm = async function(term) {
if (globalOptions.useNetworkStub) {
const networkStub = require('./fake-response.json')
return Promise.resolve(networkStub)
}
return axios.get('https://some-live-endpoint.com/', {params:{q: term}})
}
const getSearchesByTerms = async function(terms = ['cats','dogs']) {
const results = []
let result
try {
for (let i = 0; i < terms.length; i++) {
const term = terms[i]
result = await getSearchByTerm(term)
console.log("Term: ", term, "Result: ", result); // added this log to clarify your issue
if (result && "data" in result) {
results.push({ data: { ...result.data, term }}) // copies instead of mutating original object
}
}
} catch (err) {
return Promise.reject(`getSearchesByTerms() Failed: ${err}`)
}
// ... code truncated here to keep things simple ...
return Promise.resolve(results)
}
getSearchesByTerms()
.then((responses) => {
let merged = {responses: []}
for (const response of responses) {
merged.responses.push(response.data)
}
console.log(`SUCCESS, data: ${JSON.stringify(merged, null, 2)}`)
})
.catch(e=>console.log(e))
Here's a version that mutates in the way you were hoping it would work. The important change being that the stub has more than one objects that you can query:
// newExample.js
// gets a new object each time, so mutation doesn't break
const axios = require('axios').default;
const globalOptions = {
useNetworkStub: true
}
const getSearchByTerm = async function(term) {
if (globalOptions.useNetworkStub) {
const networkStub = require('./fake-response.json')[term]
return Promise.resolve(networkStub)
}
return axios.get('https://some-live-endpoint.com/', {params:{q: term}})
}
const getSearchesByTerms = async function(terms = ['cats','dogs']) {
const results = []
let result
try {
for (let i = 0; i < terms.length; i++) {
const term = terms[i]
result = await getSearchByTerm(term)
console.log("Term: ", term, "Result: ", result); // added this log to clarify your issue
result.data && (result.data.searchTerm = term) // no longer mutating same object
results.push(result)
}
} catch (err) {
return Promise.reject(`getSearchesByTerms() Failed: ${err}`)
}
// ... code truncated here to keep things simple ...
return Promise.resolve(results)
}
getSearchesByTerms()
.then((responses) => {
let merged = {responses: []}
for (const response of responses) {
merged.responses.push(response.data)
}
console.log(`SUCCESS, data: ${JSON.stringify(merged, null, 2)}`)
})
.catch(e=>console.log(e))
// fake-response.json
{
"cats": {
"data": {
"payload": "Success for cats!"
}
},
"dogs": {
"data": {
"payload": "Success for dogs!"
}
}
}
Still. If you're worried about deep cloning -- I recommend you plan your output in a way where you don't have to mutate or clone the value:
// better.js
// plans output to not mutate or copy
const axios = require('axios').default;
const globalOptions = {
useNetworkStub: true
}
const getSearchByTerm = async function(term) {
if (globalOptions.useNetworkStub) {
const networkStub = require('./fake-response.json')[term]
return Promise.resolve(networkStub)
}
return axios.get('https://some-live-endpoint.com/', {params:{q: term}})
}
const getSearchesByTerms = async function(terms = ['cats','dogs']) {
const results = []
let result
try {
for (let i = 0; i < terms.length; i++) {
const term = terms[i]
result = await getSearchByTerm(term)
console.log("Term: ", term, "Result: ", result); // added this log to clarify your issue
if (result && "data" in result) {
results.push({ term, data: result.data }) // doesn't copy or mutate result
}
}
} catch (err) {
return Promise.reject(`getSearchesByTerms() Failed: ${err}`)
}
// ... code truncated here to keep things simple ...
return Promise.resolve(results)
}
getSearchesByTerms()
.then((responses) => {
let merged = {responses: []}
for (const response of responses) {
merged.responses.push(response) // grabbing while response
}
console.log(`SUCCESS, data: ${JSON.stringify(merged, null, 2)}`)
})
.catch(e=>console.log(e))

Object.assign({}, item) to merge objects into one doesn't work

In first API call I'm getting an array of IDs to be able to make another API call to grab revenue key/value pairs, then push it back to the main object(as it doesn't have it by default).
const mainURL =
"https://api.themoviedb.org/3/discover/movie?api_key=2339027b84839948cd9be5de8b2b36da&language=en-US&sort_by=revenue.desc&include_adult=false&include_video=false&page=";
const movieBaseURL =
"https://api.themoviedb.org/3/movie/335?api_key=2339027b84839948cd9be5de8b2b36da&language=en-US";
const apiKey = "2339027b84839948cd9be5de8b2b36da";
let getRevenueByID,revenue, data, response, obj;
let Each_ID_Array = [],
All_ID_Array = [],
All_ID_Array_merged = [],
pages_Array = [1, 2],
keyValueArr = [],
allKeyValueArr = [],
objectArr = [];
responses = [];
newData = [];
let text = "";
//fetch function
let getSearchResultsForOnePage = url => {
//fetch
return fetch(url);
};
//Function to cool down requests
let pause = time => {
// handy pause function to await
return new Promise(resolve => setTimeout(resolve, time));
};
//Get main object for d3 visualisation
getAllSearchResultProfiles = async searchURL => {
let URLs = [];
for (let i = 0; i < pages_Array.length; i++) {
URLs.push(searchURL + pages_Array[i]);
}
console.log(URLs);
for (let url of URLs) {
console.log("sending request");
response = await getSearchResultsForOnePage(url);
console.log("received", response);
responses.push(await response.json());
await pause(500);
}
console.log('responses', responses);
return responses;
};
//Function to get revenue by ID
getRevenueByID = async (arr) => {
let resObj = {}; // Is an complete result
for (let i = 0; i < arr.length; i++) {
console.log("ID is: ", arr[i]);
getRevenueURL = await fetch(
"https://api.themoviedb.org/3/movie/" +
arr[i] +
"?api_key=" +
apiKey +
"&language=en-US"
);
let data = await getRevenueURL.json();
console.log("data received: ", data);
await pause(200);
//return key/value array
keyValueArr = Object.entries(data)[16];
console.table(keyValueArr);
allKeyValueArr.push(keyValueArr);
//convert key/value array to object
obj = Object.assign(
...allKeyValueArr.map(d => ({
[d[0]]: d[1]
}))
);
console.log("object is: ", obj);
resObj[arr[i]] = obj;
console.log("resObj: ", resObj[arr[i]]);
}
console.log("OUTPUT - getRevenueByID: ", resObj);
return resObj;
};
(getFinalObject = async () => {
getAllSearchResultProfiles(mainURL).then(async (data) => {
//get movie ID's
for (let i = 0; i < data.length; i++) {
Each_ID_Array = data[i].results.map(d => d.id);
All_ID_Array.push(Each_ID_Array);
console.table('All ID Array', All_ID_Array);
All_ID_Array_merged = [].concat.apply([], All_ID_Array);
console.table('All ID Array merged',All_ID_Array_merged);
console.log('data[i].results: ',data[i].results);
obj = await getRevenueByID(All_ID_Array_merged);
console.log('obj is', obj);
//inject revenue key/value
Object.keys(data[i].results).forEach(key => {
console.log("The name of the current key: ", key);
console.log("The value of the current key: ", data[i].results[key]);
data[i].results[key]["revenue"] = obj[data[i].results[key].id]["revenue"];
console.log('data[i].results[key]["revenue"]: ', data[i].results[key]["revenue"]);
});
}
console.log('data', data);
Data I'm getting is in the following format [{…}, {…}]. So instead of an array of objects I'm trying to get an array of one object [{...}] inside
newData = data.map(item =>
Object.assign({}, item)
);
console.log('data for visualisation is:',newData);
}).catch(error => console.log(error));
})();
I think you have to reduce the array:
const data = [{a:1}, {b:2}];
const newData = data.reduce((acc,curr) => ({...acc, ...curr}), {});
console.log(newData); // { a: 1, b: 2 }

Node JS: How to Promise For Loop

I'm trying to create a NodeJS app. Below is code that is supposed to be called when an admin creates a new product. Most of the code works, but I'm having trouble rendering the view only after the rest of the code has executed (the DB functions are asynchronous). I've wrapped much of the code in promises (to make certain blocks execute in the right order) and console logs (to pinpoint problems).
I'd like to point out that the console.dir(rejProducts)just below console.log(111) logs and empty array. Also, adding console.dir(rejProducts) just before the end bracket of the for loop logs an empty array. Thanks! Please let me know if you need more information.
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
Category.find(function(err, categories) {
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
var promiseLoopProducts = new Promise(function(resolve, reject) {
var promiseProducts = [];
// Loop through all addded products
for (let i = 0; i < numProducts; i++) {
var promiseProductCheck = new Promise(function(resolve, reject) {
var name = validate.sanitize(req.body[`name_${i}`]);
var category = validate.sanitize(req.body[`category_${i}`]);
var price = validate.sanitize(req.body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(req.body[`stock_${i}`]);
var image = validate.sanitize(req.body[`image_${i}`]);
var description = validate.sanitize(req.body[`description_${i}`]);
var rejProduct;
var rejFields = { 'name': name, 'category': category, 'price': price,
'stock': stock, 'image': image,
'description': description };
var rejErrors = {};
var productData = {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
var promiseCategoryCheck = new Promise(function(resolve, reject) {
if (ObjectId.isValid(category)) {
var promiseCategoryCount = new Promise(function(resolve, reject) {
Category.count({'_id': category}, function(error, count) {
rejErrors['name'] = validate.isEmpty(name);
if (count == 0) rejErrors['category'] = true;
rejErrors['price'] = !validate.isPrice(price);
rejErrors['stock'] = !validate.isInt(stock);
if( validate.isEmpty(name) || !validate.isPrice(price) ||
count == 0 || !validate.isInt(stock) ) {
rejProduct = { 'fields': rejFields, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
console.log(count);
return resolve();
}
else {
Product.create(productData, function (error, product) {
if (error) return next(error);
console.log(77);
console.dir(rejProducts);
return resolve();
});
}
if (error) return next(error);
});
});
promiseCategoryCount.then(function() {
console.dir(rejProducts);
return resolve();
});
} else {
rejErrors['category'] = true;
rejProduct = { 'fields': rejFields, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
}
});
promiseCategoryCheck.then(function() {
console.dir(rejProducts);
promiseProducts.push(promiseProductCheck);
console.log(promiseProductCheck);
console.log(promiseProducts);
return resolve();
});
});
promiseProductCheck.then(function() {
console.log(106);
console.dir(rejProducts);
});
}
Promise.all(promiseProducts).then(function() {
console.log(111);
console.dir(rejProducts); // Empty array!
return resolve();
});
});
promiseLoopProducts.then(function() {
console.log(118);
console.dir(rejProducts); // Empty array!
res.render('products/new', { categories: categories, rejProducts: rejProducts });
});
});
});
Edit: I've made some changes to the code. There it seems like util.promisify is not being recognized as a function. I am using Node 9.4.
module.exports = function(app){
const util = require('util');
require('util.promisify').shim();
const validate = require('../functions/validate');
const Category = require('../db/categories');
const Product = require('../db/products');
var ObjectId = require('mongoose').Types.ObjectId;
//Category.find as function that returns a promise
const findCategories = util.promisify(Category.find);
const countCategories = (categoryId) => {
util.promisify(Category.count({'_id': categoryId}));
};
const bodyToProduct = (body, i) => {
var name = validate.sanitize(body[`name_${i}`]);
var category = validate.sanitize(body[`category_${i}`]);
var price = validate.sanitize(body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(body[`stock_${i}`]);
var image = validate.sanitize(body[`image_${i}`]);
var description = validate.sanitize(body[`description_${i}`]);
return {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
};
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
return findCategories()
.then(
categories=>{
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
return Promise.all(
Array.from(new Array(numProducts),(v,i)=>i)
.map(//map [0...numProducts] to product object
i=>bodyToProduct(req.body,i)
)
.map(
product => {
var rejErrors;
var rejName = validate.isEmpty(name);
var rejCategory;
var rejPrice = !validate.isPrice(price);
var rejStock = !validate.isInt(stock);
if (ObjectId.isValid(product.category)) {
return countCategories()
.then(
count=> {
if (count == 0) rejCategory = true;
if(rejName || rejCategory || rejPrice || rejStock ) {
rejErrors = {
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
rejProduct = { 'fields': product, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
console.log(count);
} else {
Product.create(productData, function (error, product) {
if (error) return next(error);
console.log(77);
console.dir(rejProducts);
});
}
}
).catch(function() {
console.log("Count function failed.");
});
} else {
rejCategory = true;
rejErrors = {
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
rejProduct = { 'fields': product, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
}
}
)
).then(function() {
res.render('products/new', { categories: categories, rejProducts: rejProducts });
}).catch(function() {
console.log("Promise all products failed.");
});
}
).catch(function() {
console.log("Find categories failed.");
})
});
}
Some tips: if your function is over 100 lines you may be doing to much in the function.
If you have to get data from your request the way you get products of it then write better client side code (products should be an array of product objects that should not need to be sanitized). Validation is needed on the server because you an never trust what the client is sending you. But looking at the sanitize you don't even trust what your client script is sending you.
Try to write small functions that do a small thing and try to use those.
Use .map to map type a to type b (for example req.body to array of products as in the example code).
Use the result of .map as an argument to Promise.all
Use util.promisify to change a callback function into a function that returns a promise, since you are using an old version of node I've added an implementation of promisify:
var promisify = function(fn) {
return function(){
var args = [].slice.apply(arguments);
return new Promise(
function(resolve,reject){
fn.apply(
null,
args.concat([
function(){
var results = [].slice.apply(arguments);
(results[0])//first argument of callback is error
? reject(results[0])//reject with error
: resolve(results.slice(1,results.length)[0])//resolve with single result
}
])
)
}
);
}
};
module.exports = function(app){
const validate = require('../functions/validate');
const Category = require('../db/categories');
const Product = require('../db/products');
var ObjectId = require('mongoose').Types.ObjectId;
//Category.find as function that returns a promise
const findCategories = promisify(Category.find.bind(Category));
const countCategories = (categoryId) => {
promisify(Category.count.bind(Category))({'_id': categoryId});
};
const createProduct = promisify(Product.create.bind(Product));
const REJECTED = {};
const bodyToProduct = (body, i) => {
var name = validate.sanitize(body[`name_${i}`]);
var category = validate.sanitize(body[`category_${i}`]);
var price = validate.sanitize(body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(body[`stock_${i}`]);
var image = validate.sanitize(body[`image_${i}`]);
var description = validate.sanitize(body[`description_${i}`]);
return {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
};
const setReject = product => {
var rejErrors;
var rejName = validate.isEmpty(product.name);
var rejCategory;
var rejPrice = !validate.isPrice(product.price);
var rejStock = !validate.isInt(product.stock);
const countPromise = (ObjectId.isValid(product.category))
? countCategories()
: Promise.resolve(0);
return countPromise
.then(
count => {
if (count == 0) rejCategory = true;
if (rejName || rejCategory || rejPrice || rejStock) {
rejErrors = {
type:REJECTED,
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
return rejErrors;
}
return false;
}
);
};
const productWithReject = product =>
Promise.all([
product,
setReject(product)
]);
const saveProductIfNoRejected = ([product,reject]) =>
(reject===false)
? Product.create(product)
.catch(
err=>({
type:REJECTED,
error:err
})
)
: reject;
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
return findCategories()
.then(
categories => {
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
return Promise.all(
Array.from(new Array(numProducts), (v, i) => i)
.map(//map [0...numProducts] to product object
i => bodyToProduct(req.body, i)
)
.map(
product=>
productWithReject(product)
.then(saveProductIfNoRejected)
)
).then(
results =>
res.render(
'products/new',
{
categories: categories,
rejProducts: results.filter(x=>(x&&x.type)===REJECTED)
}
)
).catch(
error=>
console.log("Promise all products failed.",error)
);
}
).catch(
error=>
console.log("Find categories failed.",error)
)
});
}

Categories