I'm quite desperate, because of this annoying error. I can't find any information about that on the internet.
Actually my application works fine, but if I start multiple queries and wait for them with Promise.all(), the server crashes. But step by step:
In my index.js, I'm initializing the connection pool with 60 max. connections, and exporting the variable.
const pool = mariadb.createPool(config.mariaDB);
module.exports.pool = pool
Later I'm importing the index file as "app" and using app.pool.query(...) to query my database.
Next, I have a post request /getUsersGroups. A User can be a member of multiple groups, so I expect the response to be an array of objects. Each object contains information about the group. Moreover there is a members field inside this object. This field also is an array containing information about the members, so s.th. like that:
[
{
"groupId": 125758,
"title": "test",
"members": [
{userId: 5, name:Max, }, ...]
}, ...
]
Therefore I have a post request, which looks like that. I'm getting all group ids the user is a member. Then for each groupId, I call the method Group.groupInfo.
This returns a promise, therefore I'm collecting all promises in an array and wait for all of them, before sending the response.
router.post("/getUsersGroups", (req, res) => {
let userId = req.body.userId;
let result = []
let promises = []
Group.getGroupsByUser(userId) // this gets all group ids the user is a member
.then((item) => {
let groups = item.map(x => objectParser.parseUnderscoreToCamel(x))
for (let i = 0; i < item.length; i++) {
let groupId = item[i].groupId
promises.push( //adding promises
Group.groupInfo(groupId, userId)
.then((item) => {
result.push(item)
})
.catch((err)=>{console.log(err)}))
}
})
.then(() => {
// waiting for all promises to finish
return Promise.all(promises)
.then(() => {
res.status(200).json({groups: result})
})
})
.catch((err) => {
console.log(err)
}
})
}
So the Promise Group.groupInfo looks like this. It first selects the group information and then for each member inside the group it calls another Promise User.userInfo. The pattern is the same like in the method above. So I push the Promise to an array and wait for them to finish:
module.exports.groupInfo = (groupId, userId)=>{
let groupInfo = {}
let promises = []
return new Promise(((resolve, reject) => {
app.pool.query("SELECT* FROM Groups WHERE group_id=?", [groupId])
.then((item) =>{
groupInfo = item[0]
groupInfo["members"] = [];
return app.pool.query("SELECT user_id FROM Users_groups WHERE group_id=?", [groupId])
.then((item) =>{
let members = [] //contains all user ids
for(let i=0; i<item.length;i++){
members.push(item[i].userId)
}
members.forEach((memberId)=>{
// for each memberID call User.userInfo
promises.push(User.userInfo(userId,memberId)
.then((item)=>{
groupInfo.members.push(item)}))
}
})
})
})
.then(()=>{
// wait for all Promises
return Promise.all(promises)
})
.then(() =>{
return resolve(groupInfo)
})
.catch((err)=>{
return reject(err)
})
}))
}
User.userInfo itself, makes a query to the Database to collect the information about the user.
If I'm calling this, the server crashes and giving me this error:
activeConnections[conn.threadId] = conn;
TypeError: Cannot read property 'threadId' of undefined
at Pool.handleTaskQueue (****\node_modules\mariadb\lib\pool.js:431:30)
at _combinedTickCallback (internal/process/next_tick.js:132:7)
at process._tickCallback (internal/process/next_tick.js:181:9)
I guess the problem is somehow with the Promise.all() right?
But I cannot find any information about this error. I'm thankful for every hint!
My solution was not to use Promise.all but to call the Promises sequentially, but seems not right to me.
return groupsId.reduce((promise, groupId) => {
return promise.then(() => {
return Group.groupInfo(groupId, userId)
.then((item)=>{
result.push(item)
console.log(result)
})
.catch((err)=>{
throw err
})
})
}, Promise.resolve())
I cannot rely on this, as long as I don't understand the error.
Thanks!
Related
i'm building a crud app for a project,
people can add view and delete entries, i'm using nodejs, js and fireabse for this.
i have such a firestore database structure:
entries:
--entry id
--entry data
users:
--user id
--user email (query)
-- my entries (collection)
-- entries id
--entry id
now i want to display all users entries,
so i created this module.exports function:
module.exports = {
//get all the users entries from database with passed in uid ('/dashboard')
getUserEntries: async uid => {
//get all his entries docs id from "myEntries" collection in an, array
const usersEntriesId = await db
.collection("users")
.doc(uid)
.collection("myEntries")
.get()
.then(entries => {
return entries.docs.map(entry => entry.data().entry);
})
.catch(err => console.log(err));
console.log(usersEntriesId); //works fine, logs array with ids
const userEntriesDocs = usersEntriesId.map(async id => {
const entry = await db
.collection("entries")
.doc(id)
.get()
.then(entry => {
return entry.data();
});
console.log("hello:", entry); //works fine returns logs entry data
return entry;
});
console.log("hello1: ", userEntriesDocs); //doesnt work i get "hello1: [ Promise { <pending> },
// Promise { <pending> },
//Promise { <pending> } ]"
//i got three entries, that's why i get 3 times "Promise { <pending> }"
}
};
so how do i resolve that?
Thanks
well, async function returns Promise, it how they works under the hood. If there was no .map you could just await on that function or use it as arbitrary Promise with .then() and .catch. But since there is array of Promise you need Promise.all to wait until all are resolved.
const userEntriesDocs = await Promise.all(usersEntriesId.map(async id => {
....
);
Beware: unlike .allSettled, .all() will fail immediately if any of subsequent Promise fails. So if for any reason you want to have data from those requests that succeeded, you need more complex logic.
As an alternative you may go through loop manually:
const userEntriesDocs = [];
for(const docPromise of userEntriesId.map(.....)) {
userEntriesDocs.push(await docPromise);
}
But to me await Promise.all[...] is more readable.
Also I highlight there is array of Promises(requests have already been sent). If you try sending requests inside the loop like
const userEntriesDocs = [];
for(const id of userEntriesId) {
userEntriesDocs.push(await db
.collection("entries")
.doc(id)
.get()
.then(entry => {
return entry.data();
})
);
}
you will find that requests are going strictly one-by-one, not in parallel. That will require much more time to process the list.
I have a Firebase function that is attempting to take an array of UIDs and return an array of User Objects. I am trying to use Promise.all() to return all of the async results, but I am getting an empty array returned. I am, however, getting the logged out results after the fact.
const fetchUserObjects = function(uids){
let promises = []
uids.forEach((uid) => {
admin.database().ref(`/users/${uid}`).once("value")
.then(function(dataSnapshot) {
const userDataAll = dataSnapshot.val()
const userData = {}
userData.id = userDataAll.id
userData.username = userDataAll.username
userData.first = userDataAll.first
userData.last = userDataAll.last
promises.push(userData)
console.log(userData)
})
.catch((error) => {
// Re-throwing the error as an HttpsError so that the client gets the error details.
throw new functions.https.HttpsError('unknown', error.message, error);
});
})
return Promise.all(promises);
}
return fetchUserObjects(uids)
fetchUserObjects is always returning an empty array. Nothing is ensuring that the async work started by once() is copmlete before you push values into the array. Also notice that you're not actually pushing promises into that array. You're pushing plain old JavaScript objects. You need to push actual promises into the array instead, and you need to do it without having to wait for other promises to resolve. Here's what it should look like instead:
const fetchUserObjects = function(uids){
let promises = []
uids.forEach((uid) => {
const promise = admin.database().ref(`/users/${uid}`).once("value")
.then(function(dataSnapshot) {
const userDataAll = dataSnapshot.val()
const userData = {}
userData.id = userDataAll.id
userData.username = userDataAll.username
userData.first = userDataAll.first
userData.last = userDataAll.last
console.log(userData)
return userData
})
.catch((error) => {
// Re-throwing the error as an HttpsError so that the client gets the error details.
throw new functions.https.HttpsError('unknown', error.message, error);
});
promises.push(promise)
})
return Promise.all(promises);
}
Notice that a promise is immediately pushed into the promises array, and it will resolve with a userData object that was returned by the then callback.
Created an Rest Post-type API in nodeJS,in which :
I am executing two queries here.
1. Firstly Executing the query on answers table to fetch user-Id and answer detail as well in that table. // i have check in the my console they show me two user-Id
2.Second ,executing the query on users table to fetch users detail on the basis of user-Id that i pass in find function. // check my console they show me two answer object because i have two user-Id.
------------------ above process work fine ---------------------
Now, i got stuck because , i have two merge both result into one object.
I'm doing this but isn't work perfect.Help me out here..!!
My code : -
app.post('/getTopAns', function(req, res) {
console.log("inside getTopAns ");
var questionId=req.body.questionId;
mongoose.model('answers').find({
questionId:questionId,
compliance:"Y"
}, function(err, ansResult){
for (var i = 0;i<ansResult.length;i++) {
mongoose.model('users').findOne({
userId:ansResult[i].userId,
}, function(err,usrResult){
var obj = {
followerLength : usrResult.follower.length,
upvote : ansResult[i].upvote
}
})
console.log(obj);
}
});
})
Maybe you should first log ansResult to see if db is returning this. If it is, then check usrResult. If both are returned, I usually use lodash's assign or merge to merge two objects into one
I recommend using promises instead of callback to handle async easier with node.js. The flow is to get the first call result, map the array of result into a list of promises which we can wait for the result by using Promise.all. Finally map the second results which the first list.
app.post("/getTopAns", function(req, res) {
console.log("inside getTopAns ");
var questionId = req.body.questionId;
mongoose.model("answers")
.find({
questionId: questionId,
compliance: "Y"
})
.then(ansResult => {
const promises = ansResult.map(ans => {
return mongoose.model("users")
.findOne({ userId: ansResult[i].userId })
})
return Promise.all(promises)
.then(usrResults => {
return usrResults.map((usrResult, i) => {
followerLength: usrResult.follower.length,
upvote: ansResult[i].upvote
})
})
})
.then(results => {
console.log('got all results here', results)
})
});
It can be even better using async/await.
app.post("/getTopAns", async (req, res) => {
console.log("inside getTopAns ");
var questionId = req.body.questionId;
const ansResult = mongoose.model("answers")
.find({
questionId: questionId,
compliance: "Y"
})
const promises = ansResult.map(ans => {
return mongoose.model("users")
.findOne({ userId: ansResult[i].userId })
})
const usrResults = await Promise.all(promises)
const results = usrResults.map((usrResult, i) => {
followerLength: usrResult.follower.length,
upvote: ansResult[i].upvote
})
console.log('got all results here', results)
});
More information can be found here: https://medium.com/#ThatGuyTinus/callbacks-vs-promises-vs-async-await-f65ed7c2b9b4
Kind of a sequel to this question, I need to accept multiple objects in a POST request and then for each object process it, save it, and then return the saved object to the frontend (so that the client can see which columns were successfully edited).
When I use .map, it does save to the database and I can confirm this. However, I have two problems:
It does not execute res.locals.retval.addData(dtoObject); correctly, and my returning payload has no data transfer objects inside of it.
My object validation cannot be done inside of the callback of map. I initially tried reduce, but that didn't work at all and just saved all the same values to each database object. How can I exclude invalid JSON objects while I'm mapping them?
var jsonObjects = req.body;
//for (var n in req.body) {
var promises = jsonObjects.map((jsonObject) => {
var transform = new Transform();
// VALIDATION OF jsonObject VARIABLE IS HERE
if (jsonObject.id == 0) {
var databaseObject = Database.getInstance().getModel(objectName).build(jsonObject);
transform.setNew(true);
transform.setJsonObject(jsonObject);
transform.setDatabaseObject(databaseObject);
transform.baseExtract()
.then(() => transform.extract())
.then(() => transform.clean())
.then(() => transform.getDatabaseObject().save())
.then(function(data) {
// PROCESSING DATA
}).catch((e) => {
// ERROR
});
} else {
var queryParameters = {
where: {id: jsonObject.id}
};
console.log("Query parameters: ");
console.log(queryParameters);
Database.getInstance().getModel(objectName).findOne(queryParameters).then((databaseObject) => {
transform.setJsonObject(jsonObject);
transform.setDatabaseObject(databaseObject);
})
.then(() => transform.baseExtract())
.then(() => transform.extract())
.then(() => transform.clean())
.then(() => transform.getDatabaseObject().save())
.then((data) => {
// PROCESSING DATA
}).catch((e) => {
// ERROR
});
}
});
Promise.all(promises)
.then((results) => {
return next();
}).catch((e) => {
throw e;
});
Here's the resulting payload:
{
"errors": [],
"warnings": [],
"data": []
}
As #KevinB said in the comments, you are missing the return calls inside of your arrow functions so the database saves are going through because they are part of the Promise chain, but pushes to the response are stalled waiting for the return, and then the Express.js call resolves before the Promises do. Add return Database.getInstance() and return transform.baseExtract() to your code to fix this.
Use Array.prototype.filter() to remove elements you want to ignore since you won't ever need to execute Promises on them, then call Array.prototype.map() on the resulting array. If you don't want to use the arrow functions, you can specify this as a parameter to filter and map:
jsonObjects.filter(function(jsonObject) {
}, this);
var promises = jsonObjects.map(function(jsonObject) {
}, this);
Reading some amazing tutorials about promises, I've discovered that, if I need to interate throw some promises, I can't use forEach or some other "traditional" iteration mechanisms, I have to use Q library for node, I've to "iterate" using Q.all.
I've written a simple example in Nodejs Express in order to be sure I've understood promises:
var form = [
{'name':'FORM_NAME_1.1',
'label2':'FORM_LABEL_1.2'
},
{'name':'FORM_NAME_2.1',
'label2':'FORM_LABEL_2.2'
}
];
var params = ['params1','params2'];
var meta = ['meta1','meta2'];
app.get('/', (req,res) => {
return Q.all([
form.map((currentValue,index,arr) => {
req.id = Math.random(); //Random ID to be used in the next promises
console.log(currentValue);
return Form.insert(currentValue,req);
}),
params.map((currentValue,index,arr) => {
console.log(req.id);
return Field.insert(currentValue,req.id);
}),
meta.map((currentValue,index,arr) => {
console.log(req.id);
return Meta.insert(currentValue,req.id);
})
])
.catch((err) => next(err))
.done(() => console.log('It\'s done'));
});
Form.insert code simply is a promise with a console.log call, the same for Field.insert and Meta.insert
var Form = {
insert: (param1,req) => {
var deferred = Q.defer();
console.log('Goes throw Form');
deferred.resolve();
return deferred.promise;
}
}
The problem is that seems to iterate right but the dynamicly generated id does not change along the promises, this is the console output:
Listening at port 3000...
{ name: 'FORM_NAME_1.1', label2: 'FORM_LABEL_1.2' }
Goes throw Form
{ name: 'FORM_NAME_2.1', label2: 'FORM_LABEL_2.2' }
Goes throw Form
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Meta
0.3757301066790548
Goes throw Meta
It's done
Any ideas about what is going wrong? Thanks!!
the reason it is not working is because in first for loop, the req.id is set multiple times before other promises are started and and all of them end up using the last randomly generated value, change your code to:
app.get('/', (req,res) => {
let process = (currentValue,index,arr) => {
let reqCopy = {id: Math.random()}
for(let attr in req) // copy all the request attributes
if(attr && attr!='id')
reqCopy[attr] = req[attr]
return Q.all([
Form.insert(form[index],reqCopy),
Field.insert(params[index],reqCopy),
Meta.insert(meta[index],reqCopy)
])
}
return Q.all(form.map(process))
.catch(next)
.done(() => console.log('It\'s done'));
})
you would notice that I am copying all the attributes of req to clone reqCopy for I am not sure what attributes of req are required by the subsequent methods, but at the same time, single req.id would not work thanks to the async nature of code