We Keep Coding

Await sql and push to array before executing the result

I'm using mysql npm package to get data from the database.
My goal is to add every list(id) from lists Table and push it into lists array.
I'm getting the correct data from the database but when I result the query lists array is empty.
I think that I have to add async and await to the code to make it work. Tried in several places but I didn't make it work. Any idea what I'm doing wrong?
// GET - get all grocery_lists
Grocery_list.getAll = (result) => {
let lists = []; // <--- List Array
sql.query("SELECT id FROM lists", (err, res) => { // <--- Get all id from 'lists' table
if (err) {
console.log(err);
}
res.forEach(list => { // <--- Loop thru all lists
sql.query(`
SELECT items.id, items.name, items_x_lists.purchased
FROM items_x_lists
INNER JOIN items ON items_x_lists.itemId = items.id
WHERE items_x_lists.listId = ${list.id};
`, (err, res) => { // <--- Get all items for ${list.id} from 'items' table
if (err) {
console.log(err);
}
const list = {};
list.id = res.id;
console.log(list); // <--- { id: 1 } ... { id: 2 }
lists.push(list);
});
});
result(null, lists); // <--- returning empty array instead of [{ id: 1 }, { id: 2 }]
});
};

I think you can do it simply like this (not the exact code) but I think you've got the idea:
// GET - get all grocery_lists
Grocery_list.getAll = (result) => {
getData().then(data => {
// call result(data);
}).catch(err => {
// handle error for no data or any other errors
})
}
const getData = async () => {
try {
var res = await sql.query("SELECT id FROM lists");
// Handle error if no data found
if (!res) { Promise.reject("No data found"); }
} catch (error) {
return Promise.reject(error);
}
const listIds = res.map(id => id); // array of ids
try {
var data = await sql.query(`
SELECT items.id, items.name, items_x_lists.purchased
FROM items_x_lists
INNER JOIN items ON items_x_lists.itemId = items.id
WHERE items_x_lists.listId in ${listIds};`);
// Handle error if no data found
if (!data) { return Promise.reject("No data found") }
} catch (error) {
return Promise.reject(error);
}
return Promise.resolve(data)
}

this mighht be the solution
Grocery_list.getAll = async (result) => { return sql.query("SELECT id FROM lists", (err, res) => { // <--- Get all id from 'lists' table if (err) { console.log(err); } }); };

I’m outside, so I can only edit with my mobile phone. If the layout is strange, I apologize for it.
First of all, if it is me, I will adjust a few places
1.Create an async function and use Promise.all technology
async function queryAll(querys){
return await Promise.all(querys);
}
2.Create a Promise function to execute each sql
const queryPromise = id => {
return new Promise((resolve, reject) => {
sql.query(
'SELECT items.id, items.name, items_x_lists.purchased FROM items_x_lists INNER JOIN items ON items_x_lists.itemId = items.id WHERE items_x_lists.listId = ? ;',
id,
(err, rows, fields) => {
console.log(rows);
if (err) reject(err);
else resolve(rows);
}
);
});
};
3.Adjust the internal logic of the getAll event
const querys = []
res.forEach(list => {
querys.push(queryPromise(list.id));
});
const querysResults = await queryAll(querys);// remember to use async for function
querysResults.forEach(querysResult => {
lists.push({id:querysResult['id']});
});
Because there is no computer on hand, there may be some situations that need to be judged, but roughly this will work normally.
Hope this helps you :)

NodeJS + ExpressJS: How to wait for forEach to finish with SQL queries inside

I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone

You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}

To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.

Javascript how the better way to code nested callback?

I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks

You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});

You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);

What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.

You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed

javascript promise after foreach loop with multiple mongoose find

I'm trying to have a loop with some db calls, and once their all done ill send the result. - Using a promise, but if i have my promise after the callback it dosent work.
let notuser = [];
let promise = new Promise((resolve, reject) => {
users.forEach((x) => {
User.find({
/* query here */
}, function(err, results) {
if(err) throw err
if(results.length) {
notuser.push(x);
/* resolve(notuser) works here - but were not done yet*/
}
})
});
resolve(notuser); /*not giving me the array */
}).then((notuser) => {
return res.json(notuser)
})
how can i handle this ?

Below is a function called findManyUsers which does what you're looking for. Mongo find will return a promise to you, so just collect those promises in a loop and run them together with Promise.all(). So you can see it in action, I've added a mock User class with a promise-returning find method...
// User class pretends to be the mongo user. The find() method
// returns a promise to 'find" a user with a given id
class User {
static find(id) {
return new Promise(r => {
setTimeout(() => r({ id: `user-${id}` }), 500);
});
}
}
// return a promise to find all of the users with the given ids
async function findManyUsers(ids) {
let promises = ids.map(id => User.find(id));
return Promise.all(promises);
}
findManyUsers(['A', 'B', 'C']).then(result => console.log(result));

I suggest you take a look at async it's a great library for this sort of things and more, I really think you should get used to implement it.
I would solve your problem using the following
const async = require('async')
let notuser = [];
async.forEach(users, (user, callback)=>{
User.find({}, (err, results) => {
if (err) callback(err)
if(results.length) {
notUser.push(x)
callback(null)
}
})
}, (err) => {
err ? throw err : return(notuser)
})
However, if you don't want to use a 3rd party library, you are better off using promise.all and await for it to finish.
EDIT: Remember to install async using npm or yarn something similar to yarn add async -- npm install async

I used #danh solution for the basis of fixing in my scenario (so credit goes there), but thought my code may be relevant to someone else, looking to use standard mongoose without async. I want to gets a summary of how many reports for a certain status and return the last 5 for each, combined into one response.
const { Report } = require('../../models/report');
const Workspace = require('../../models/workspace');
// GET request to return page of items from users report
module.exports = (req, res, next) => {
const workspaceId = req.params.workspaceId || req.workspaceId;
let summary = [];
// returns a mongoose like promise
function addStatusSummary(status) {
let totalItems;
let $regex = `^${status}$`;
let query = {
$and: [{ workspace: workspaceId }, { status: { $regex, $options: 'i' } }],
};
return Report.find(query)
.countDocuments()
.then((numberOfItems) => {
totalItems = numberOfItems;
return Report.find(query)
.sort({ updatedAt: -1 })
.skip(0)
.limit(5);
})
.then((reports) => {
const items = reports.map((r) => r.displayForMember());
summary.push({
status,
items,
totalItems,
});
})
.catch((err) => {
if (!err.statusCode) {
err.statusCode = 500;
}
next(err);
});
}
Workspace.findById(workspaceId)
.then((workspace) => {
let promises = workspace.custom.statusList.map((status) =>
addStatusSummary(status)
);
return Promise.all(promises);
})
.then(() => {
res.status(200).json({
summary,
});
})
.catch((err) => {
if (!err.statusCode) {
err.statusCode = 500;
}
next(err);
});
};

Please break down the event path of a Callback Function into an ORM file

I was tasked with creating a small application using Object Relational Mapping (ORM) for class. It's a fairly simple application, that takes client side input, places it into the database and also displays it back to the client.
However, I am having a hard time understanding the breakdown in what is happening as we pass code between 3 files. There is a burgers-controller.js file which requires a burgers.js file which requires the orm.js file. For anyone who wants to see the full code you can find it on my Github.
I get the reasoning and benefits of using an ORM and modularizing everything but I need someone to break down what is happening inside the code and in what order.
So in burgers-controller.js file we have the following code.
router.get("/", (req, res)=>{
burgers.selectAll((result)=>{
let handlebarsObj = {
burgers: result
}
res.render("index", handlebarsObj)
});
})
Then inside burgers.js we have the following code.
let burgers = {
selectAll: (callback)=>{
orm.selectAll("burgers", (result)=>{
callback(result);
})
}
}
And finally in orm.js we have.
let orm = {
selectAll: (tableName, callback)=>{
let queryString = "SELECT * FROM ??"
connection.query(queryString, tableName, (err, data)=>{
if (err){console.error("ERROR: " + err.stack)}
callback(data)
})
}
}
The ORM file is connected to a MySQL database.
Would you explain what is happening here, and in what order?

I think I got it, although its a shot in the dark since there are no indication of what the actual error is, in your description. Please forgive me if I mistakes.
The callback from burgers-controller.js which is sent to burgers.selectAll( is executed inside burgers.js. When its executed inside burgers.js and not within burgers-controller.js it wont have access to res object, which is only present in burgers-controller.js.
We can solve the problem using promises. I am going to use native one here, but you are free to use any other package (eg: bluebird):
// burgers-controller.js
router.get('/', (req, res)=> {
burgers
.selectAll()
.then((result) => {
let handlebarsObj = {
burgers: result
}
res.render('index', handlebarsObj)
})
})
// burgers.js
let burgers = {
selectAll: () => {
return new Promise((resolve, reject) => {
orm.selectAll('burgers', (result) => {
resolve(result)
})
})
}
}
You can use promises everywhere are deal with failures more gracefully:
// burgers-controller.js
router.get('/', (req, res)=> {
burgers
.selectAll()
.then((result) => {
let handlebarsObj = {
burgers: result
}
res.render('index', handlebarsObj)
})
.catch((err) => {
let errMsg = {
message: err.message,
error: err
}
res.render('index', errMsg)
})
})
// burgers.js
let burgers = {
selectAll: () => {
return new Promise((resolve, reject) => {
orm
.selectAll('burgers')
.then((result) => {
resolve(result)
})
.catch((err) => {
reject(err)
})
})
}
}
// orm.js
let orm = {
selectAll: (tableName) => {
return new Promise((resolve, reject) => {
let queryString = "SELECT * FROM ??"
connection.query(queryString, tableName, (err, data)=>{
if (err) {
console.error("ERROR: " + err.stack)
reject(err)
} else {
resolve(data)
}
})
})
}
}