We Keep Coding

Get all returning values from a transaction containing multiple queries - knex.js

I'm trying to update some rows at once.
I'm waiting for all queries to be written before executing them all in one query. However, I need some values to be returned by the query after the update. I'm using the returning() method for that.
I can't manage to get all the returned values at once as an array after the query has been executed. If I use a then() directly in the transaction function it returns the returned values one by one and my promise won't work. It also considerably increases the execution time.
How do I get all the returning values from my update request at once in a list.
return await knex.transaction(trx => {
const queries = [];
data.forEach(pair => {
const query = knex('pair')
.where('symbol', pair.s)
.update({
last_price: pair.c,
})
.returning(['id'])
.transacting(trx)
//.then(function (updated) {
// ... doing it one by one here
//})
queries.push(query);
});
return Promise.all(queries) // Once all queries are written
.then(() => {
trx.commit // We try to execute them all
.catch((e) => {
trx.rollback // And rollback in case any of them goes wrong
console.error(e)
});
})

I found a solution without using trx.commit():
async function updateDB(data) {
return await knex.transaction(async (trx) => {
const queries = [];
data.forEach(item => {
const query = knex('core_exchangepair')
.where('symbol_slug', item.s)
.transacting(trx)
.update({
last_price: item.c,
})
.returning(['id'])
queries.push(query);
});
try {
return await Promise.all(queries);
} catch (e) {
console.error(e);
return e;
}
})
}

The await after the return is unnecessary. You need to await your Promise.all before returning it, so your trx function should be async.

Fetching data from mongo via express, build object, and send to React

I am currently stuck in asynchronous hell.
In my React, I have a page /menu, that would load data from my mongo instance via expressjs api.
In my database, called menu, i have collections which represent a meal-type eg "breakfast", "lunch" etc. In those collections, the documents for every item looks like this bread collection example:
{
_id: 2398jcs9dn2f9f,
name: "Ciabatta",
desc: "Italian bread",
imageURI: "image01.jpg",
reviews: []
}
This is my api that would be called when the page loads
exports.getAllFoods = (req, res, next) => {
const db = mongoose.connection
const allCollections = {}
try {
db.db.listCollections().toArray((err, collections) => {
collections.forEach((k) => {
allCollections[k.name] = []
})
Object.keys(allCollections).map(k => {
let Meal = mongoose.model(k, MealSchema)
meal = Meal.find((err, docs) => {
allCollections[k] = docs
console.log(allCollections)
})
})
res.send(allCollections)
})
} catch (error) {
console.log(error)
res.send('unable to get all collections')
}
}
The last output of the console.log(allCollections) produces this:
{ snacks:
[ { review: [],
tags: [],
_id: 5fcec3fc4bc5d81917c9c1fe,
name: 'Simosa',
description: 'Indian food',
imageURI: 'image02.jpg',
__v: 0 } ],
breads:
[ { review: [],
tags: [],
_id: 5fcec41a4bc5d81917c9c1ff,
name: 'Ciabatta',
description: 'Italian bread',
imageURI: 'image02.jpg',
__v: 0 } ],
}
This is exactly what I need, but I am stuck in figuring out how to send to React. What am I to do to send the above json? The res.send(allCollections) gives me this:
{
"snacks": [],
"breads": [],
"drinks": []
}
I understand why the above is being sent, but I dont know what I need to do to address it.
This is my React on page load
useEffect(() => {
axios
.get('http://localhost:8888/api/allFoods')
.then((res) => {
setMealTypes(res.data)
})
.catch((err) => [
console.log(err)
])
}, [])
Ultimately, I need the json outputted in console as I wanted to loop through that data and use the key as a title, and then list the values from the value array eg
<div>
<h2>Breads</h2>
<img src=image01.jpg/>
<h3>Ciabatta</h3>
<p>Italian bread</p>
...
</div>
...
I'd appreciate any help, and any docs I should read to help and improve my javascript understandings

I'd prefer to solve this using async/await and Promise.all, replacing most callbacks.
Because you're calling the DB when you're iterating through an array, you have the most annoying callback situation: how do you issue a bunch of async things and then get the results after? You'll need something else to ensure all callbacks are called before sending the results.
Async/await means we can declare a function is async, and await the results of an async operation. async/await is annoying in JS because it abstracts away callbacks and is actually creating a Promise underneath. Complicating things further, async/await doesn't solve issuing multiple async functions, so again we have to rely on this fancy Promise.all() function combined with map-ing the desired input array to async functions.
Original:
Object.keys(allCollections).map(k => {
let Meal = mongoose.model(k, MealSchema)
meal = Meal.find((err, docs) => {
allCollections[k] = docs
console.log(allCollections)
})
});
Suggested async/await:
await Promise.all(Object.keys(allCollections).map(async k => {
let Meal = mongoose.model(k, MealSchema)
let docs = await Meal.find();
allCollections[k] = docs;
console.log(allCollections);
}));
Another advantage is error handling. If any errors happen in the callback of the original example, they won't be caught in this try/catch block.
async/await handles errors like you'd expect, and errors will end up in the catch block.
...
// Now that we have awaited all async calls above, this should be executed _after_ the async calls instead of before them.
res.send(allCollections);
})
} catch (error) {
console.log(error)
res.send('unable to get all collections')
}
}
Technically Promise.all() returns an array of results, but we can ignore that since you're formatting an Object anyway.
There is plenty of room to optimize this further. I might write the whole function as something like:
exports.getAllFoods = async (req, res, next) => {
const db = mongoose.connection.db;
try {
let collections = await db.listCollections().toArray();
let allCollections = {};
collections.forEach((k) => {
allCollections[k.name] = [];
})
// For each collection key name, find docs from the database
// await completion of this block before proceeding to the next block
await Promise.all(Object.keys(allCollections).map(async k => {
let Meal = mongoose.model(k, MealSchema)
let docs = await Meal.find();
allCollections[k] = docs;
}));
// allCollections should be populated if no errors occurred
console.log(allCollections);
res.send(allCollections);
} catch (error) {
console.log(error)
res.send('unable to get all collections')
}
}
Completely untested.
You might find these links more helpful than my explanation:
https://javascript.info/async-await
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
https://medium.com/dailyjs/the-pitfalls-of-async-await-in-array-loops-cf9cf713bfeb

I hope this will help you : You need to first use the stringify method before sending the collections from the express api and then use JSON.parse on the React front end to restore the object.
PS: can you do a console.log(allCollections) one line above res.send(allCollections)?

You need to send it to the front-end in a JSON format.
replace res.send(allCollections) with res.json(allCollections)

Making multiple web api calls synchronously without nesting in Node.js with Axios

Is there any way I can make the below code run synchronously in a way where I can get all of the productLine ids and then loop through and delete all of them, then once all of this is complete, get all of the productIds and then loop through and delete all of them?
I really want to be able to delete each set of items in batch, but the next section can't run until the first section is complete or there will be referential integrity issues.
// Delete Product Lines
axios.get('https://myapi.com/ProductLine?select=id')
.then(function (response) {
const ids = response.data.value
ids.forEach(id => {
axios.delete('https://myapi.com/ProductLine/' + id)
})
})
.catch(function (error) {
})
// Delete Products (I want to ensure this runs after the above code)
axios.get('https://myapi.com/Product?select=id')
.then(function (response) {
const ids = response.data.value
ids.forEach(id => {
axios.delete('https://myapi.com/Product/' + id)
})
})
.catch(function (error) {
})

There's a lot of duplication in your code. To reduce code duplication, you can create a helper function that can be called with appropriate arguments and this helper function will contain code to delete product lines and products.
async function deleteHelper(getURL, deleteURL) {
const response = await axios.get(getURL);
const ids = response.data.value;
return Promise.all(ids.map(id => (
axios.delete(deleteURL + id)
)));
}
With this helper function, now your code will be simplified and will be without code duplication.
Now to achieve the desired result, you could use one of the following ways:
Instead of two separate promise chains, use only one promise chain that deletes product lines and then deletes products.
const prodLineGetURL = 'https://myapi.com/ProductLine?select=id';
const prodLineDeleteURL = 'https://myapi.com/ProductLine/';
deleteHelper(prodLineGetURL, prodLineDeleteURL)
.then(function() {
const prodGetURL = 'https://myapi.com/Product?select=id';
const prodDeleteURL = 'https://myapi.com/Product/';
deleteHelper(prodGetURL, prodDeleteURL);
})
.catch(function (error) {
// handle error
});
Use async-await syntax.
async function delete() {
try {
const urls = [
[ prodLineGetURL, prodLineDeleteURL ],
[ prodGetURL, prodDeleteURL ]
];
for (const [getURL, deleteURL] of urls) {
await deleteHelper(getURL, deleteURL);
}
} catch (error) {
// handle error
}
}
One other thing that you could improve in your code is to use Promise.all instead of forEach() method to make delete requests, above code uses Promise.all inside deleteHelper function.

Your code (and all other answers) are executing delete requests sequentially, which is huge waste of time. You should use Promise.all() and execute in parallel...
// Delete Product Lines
axios.get('https://myapi.com/ProductLine?select=id')
.then(function (response) {
const ids = response.data.value
// execute all delete requests in parallel
Promise.all(
ids.map(id => axios.delete('https://myapi.com/ProductLine/' + id))
).then(
// all delete request are finished
);
})
.catch(function (error) {
})

All HTTP request are asynchronous but you can make it sync-like. How? Using async-await
Suppose you have a function called retrieveProducts, you need to make that function async and then await for the response to keep processing.
Leaving it to:
const retrieveProducts = async () => {
// Delete Product Lines
const response = await axios.get('https://myapi.com/ProductLine?select=id')
const ids = response.data.value
ids.forEach(id => {
axios.delete('https://myapi.com/ProductLine/' + id)
})
// Delete Products (I want to ensure this runs after the above code)
const otherResponse = await axios.get('https://myapi.com/Product?select=id') // use proper var name
const otherIds = response.data.value //same here
otherIds.forEach(id => {
axios.delete('https://myapi.com/Product/' + id)
})
}
But just keep in mind that it's not synchronous, it keeps being async

NodeJS with MariaDB, Cannot read property 'threadId' of undefined

I'm quite desperate, because of this annoying error. I can't find any information about that on the internet.
Actually my application works fine, but if I start multiple queries and wait for them with Promise.all(), the server crashes. But step by step:
In my index.js, I'm initializing the connection pool with 60 max. connections, and exporting the variable.
const pool = mariadb.createPool(config.mariaDB);
module.exports.pool = pool
Later I'm importing the index file as "app" and using app.pool.query(...) to query my database.
Next, I have a post request /getUsersGroups. A User can be a member of multiple groups, so I expect the response to be an array of objects. Each object contains information about the group. Moreover there is a members field inside this object. This field also is an array containing information about the members, so s.th. like that:
[
{
"groupId": 125758,
"title": "test",
"members": [
{userId: 5, name:Max, }, ...]
}, ...
]
Therefore I have a post request, which looks like that. I'm getting all group ids the user is a member. Then for each groupId, I call the method Group.groupInfo.
This returns a promise, therefore I'm collecting all promises in an array and wait for all of them, before sending the response.
router.post("/getUsersGroups", (req, res) => {
let userId = req.body.userId;
let result = []
let promises = []
Group.getGroupsByUser(userId) // this gets all group ids the user is a member
.then((item) => {
let groups = item.map(x => objectParser.parseUnderscoreToCamel(x))
for (let i = 0; i < item.length; i++) {
let groupId = item[i].groupId
promises.push( //adding promises
Group.groupInfo(groupId, userId)
.then((item) => {
result.push(item)
})
.catch((err)=>{console.log(err)}))
}
})
.then(() => {
// waiting for all promises to finish
return Promise.all(promises)
.then(() => {
res.status(200).json({groups: result})
})
})
.catch((err) => {
console.log(err)
}
})
}
So the Promise Group.groupInfo looks like this. It first selects the group information and then for each member inside the group it calls another Promise User.userInfo. The pattern is the same like in the method above. So I push the Promise to an array and wait for them to finish:
module.exports.groupInfo = (groupId, userId)=>{
let groupInfo = {}
let promises = []
return new Promise(((resolve, reject) => {
app.pool.query("SELECT* FROM Groups WHERE group_id=?", [groupId])
.then((item) =>{
groupInfo = item[0]
groupInfo["members"] = [];
return app.pool.query("SELECT user_id FROM Users_groups WHERE group_id=?", [groupId])
.then((item) =>{
let members = [] //contains all user ids
for(let i=0; i<item.length;i++){
members.push(item[i].userId)
}
members.forEach((memberId)=>{
// for each memberID call User.userInfo
promises.push(User.userInfo(userId,memberId)
.then((item)=>{
groupInfo.members.push(item)}))
}
})
})
})
.then(()=>{
// wait for all Promises
return Promise.all(promises)
})
.then(() =>{
return resolve(groupInfo)
})
.catch((err)=>{
return reject(err)
})
}))
}
User.userInfo itself, makes a query to the Database to collect the information about the user.
If I'm calling this, the server crashes and giving me this error:
activeConnections[conn.threadId] = conn;
TypeError: Cannot read property 'threadId' of undefined
at Pool.handleTaskQueue (****\node_modules\mariadb\lib\pool.js:431:30)
at _combinedTickCallback (internal/process/next_tick.js:132:7)
at process._tickCallback (internal/process/next_tick.js:181:9)
I guess the problem is somehow with the Promise.all() right?
But I cannot find any information about this error. I'm thankful for every hint!
My solution was not to use Promise.all but to call the Promises sequentially, but seems not right to me.
return groupsId.reduce((promise, groupId) => {
return promise.then(() => {
return Group.groupInfo(groupId, userId)
.then((item)=>{
result.push(item)
console.log(result)
})
.catch((err)=>{
throw err
})
})
}, Promise.resolve())
I cannot rely on this, as long as I don't understand the error.
Thanks!

"Iterating" throw promises does not let to generate different ids

Reading some amazing tutorials about promises, I've discovered that, if I need to interate throw some promises, I can't use forEach or some other "traditional" iteration mechanisms, I have to use Q library for node, I've to "iterate" using Q.all.
I've written a simple example in Nodejs Express in order to be sure I've understood promises:
var form = [
{'name':'FORM_NAME_1.1',
'label2':'FORM_LABEL_1.2'
},
{'name':'FORM_NAME_2.1',
'label2':'FORM_LABEL_2.2'
}
];
var params = ['params1','params2'];
var meta = ['meta1','meta2'];
app.get('/', (req,res) => {
return Q.all([
form.map((currentValue,index,arr) => {
req.id = Math.random(); //Random ID to be used in the next promises
console.log(currentValue);
return Form.insert(currentValue,req);
}),
params.map((currentValue,index,arr) => {
console.log(req.id);
return Field.insert(currentValue,req.id);
}),
meta.map((currentValue,index,arr) => {
console.log(req.id);
return Meta.insert(currentValue,req.id);
})
])
.catch((err) => next(err))
.done(() => console.log('It\'s done'));
});
Form.insert code simply is a promise with a console.log call, the same for Field.insert and Meta.insert
var Form = {
insert: (param1,req) => {
var deferred = Q.defer();
console.log('Goes throw Form');
deferred.resolve();
return deferred.promise;
}
}
The problem is that seems to iterate right but the dynamicly generated id does not change along the promises, this is the console output:
Listening at port 3000...
{ name: 'FORM_NAME_1.1', label2: 'FORM_LABEL_1.2' }
Goes throw Form
{ name: 'FORM_NAME_2.1', label2: 'FORM_LABEL_2.2' }
Goes throw Form
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Meta
0.3757301066790548
Goes throw Meta
It's done
Any ideas about what is going wrong? Thanks!!

the reason it is not working is because in first for loop, the req.id is set multiple times before other promises are started and and all of them end up using the last randomly generated value, change your code to:
app.get('/', (req,res) => {
let process = (currentValue,index,arr) => {
let reqCopy = {id: Math.random()}
for(let attr in req) // copy all the request attributes
if(attr && attr!='id')
reqCopy[attr] = req[attr]
return Q.all([
Form.insert(form[index],reqCopy),
Field.insert(params[index],reqCopy),
Meta.insert(meta[index],reqCopy)
])
}
return Q.all(form.map(process))
.catch(next)
.done(() => console.log('It\'s done'));
})
you would notice that I am copying all the attributes of req to clone reqCopy for I am not sure what attributes of req are required by the subsequent methods, but at the same time, single req.id would not work thanks to the async nature of code