I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks
You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});
You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);
What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.
You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed
Related
I have an application that it has nodejs as backend and some scripts in Python
The problem is to make the 'PythonShell' (function to access the scripts) as a async function. I do not know why but it is not working.
I'll put the code from my router.js file and inside of it I put three 'console.log('steps')' to check the sequences.
It should be Step01 > Step02 > Step03, but as it is not working, It always prints Step01 > Step03 > Step02
Everything is working fine, except for this async problem! For me it should work as it is.
How can I edit my functions to execute first the 'goToscript/PythonShell' and then execute 'res.json(responseScript)'?
Thanks
router.put("/uploads/script-03", async (req, res) => {
let options = {
scriptPath: "scripts",
args: JSON.stringify(req.body)
};
const goToScript = async () => {
await PythonShell.run("script-01.py", options, (err, res) => {
if (err) {
}
if (res) {
responseScript = JSON.parse(res)
console.log('Step 02')
}
});
}
console.log('Step 01')
goToScript()
console.log('Step 03')
res.json(responseScript)
});
module.exports = router
A couple things:
1. Your goToScript is not actually async/returning a Promise
From what I can tell, PythonShell doesn't support async, only callbacks, so you can rewrite your gotToScript like so:
const goToScript = () => {
return new Promise((resolve, reject) => {
PythonShell.run("script-01.py", options, (err, res) => {
if (err) {
reject(err)
}
if (res) {
responseScript = JSON.parse(res)
console.log('Step 02')
resolve(responseScript)
}
})
})
}
const scriptResult = await goToScript()
This code will work like a regular async function, where the promise will resolve to the parsed JSON, and reject with the error if it meets one.
2. You are not awaiting your call to goToScript
When you want to make an async call that finishes in sequence with everything else, you need to await it. Take these two examples:
In this first chunk of code, waitFn waits before 100ms before logging "Step 2!":
const waitFn = () => {
return new Promise((resolve) => {
setTimeout(() => {
console.log('Step 2!')
resolve()
}, 100)
})
}
console.log('Step 1!')
waitFn()
console.log('Step 3!')
Because you do not await the result of the Promise, your code doesn't care that is has not finished, and will print:
Step 1!
Step 3!
Step 2!
Instead, however, if you await the result of the Promise returned in waitFn, it will execute in order:
const waitFn = () => {
return new Promise((resolve) => {
setTimeout(() => {
console.log('Step 2!')
resolve()
}, 100)
})
}
console.log('Step 1!')
await waitFn() // Finishes before moving on
console.log('Step 3!')
You can read a bunch more about Promises and async/await here :)
To be able to await function - this function needs to return a promise. In the npm page of your lib there is a description, about what PythonShell.run returns. It does not return a promise. So it is asynchronous, but not awaitable, it is callback based.
All you need to do - to promisify this function. Additionaly - you need to await the call to goToScript();.
router.put("/uploads/script-03", async (req, res) => {
let options = {
scriptPath: "scripts",
args: JSON.stringify(req.body)
};
const goToScript = async () => {
return new Promise((resolve, reject) => {
PythonShell.run("script-01.py", options, (err, res) => {
console.log("Step 02");
if (err) return reject(err);
return resolve(JSON.parse(res));
});
});
};
console.log("Step 01");
const responseScript = await goToScript();
console.log("Step 03");
res.json(responseScript);
});
module.exports = router;
This question already has answers here:
How do I convert an existing callback API to promises?
(24 answers)
Closed 2 years ago.
I have the following code on a server:
let tmpFileName;
// GET
app.get('/clicked', async (req, res) => {
let nullOutput = writeTmpFile("hello, world!");
await deleteTmpFile();
console.log("Hurray, finished!");
res.send({result:nullOutput});
})
function writeTmpFile(content){
tmpFileName = "tmp" + Math.random().toString() + "tsl";
return new Promise(resolve => {
fs.writeFile(tmpFileName, content, function (err) {
if (err) throw err;
console.log('Temp file creation successful.');
});
})
}
function deleteTmpFile(spec){
return new Promise(resolve => {
fs.unlink(tmpFileName, function (err) {
if (err) throw err;
console.log('Temp file deletion successful.');
});
})
}
However, in my console output, I only get
Temp file creation successful.
Temp file deletion successful.
However, if I delete await deleteTempFile(), then Hurray, finished! shows up on the console.
And more generally, how do I debug these patterns of problems?
Why is this happening?
I have rewritten your code, to showcase how to use promises.
Promise callback gets two functions as arguments: resolve and reject.
You should call resolve when operation finishes with success, and reject when it fails.
// I moved `tmpFileName` variable from here into the request handler,
// because it was "global" and would be shared between requests.
app.get('/clicked', async (req, res) => {
let tmpFileName = "tmp" + Math.random().toString() + "tsl"
let writingResult = await writeTmpFile(tmpFileName, "hello, world!")
let deletionResult = await deleteTmpFile(tmpFileName)
res.send({ writingResult, deletionResult })
console.log("Hurray, finished!")
})
function writeTmpFile (filename, content) {
return new Promise((resolve, reject) => {
fs.writeFile(filename, content, function (err) {
// on error reject promise with value of your choice
if (err) reject(err)
// on success resolve promise with value of your choice
resolve('Temp file creation successful.')
})
})
}
function deleteTmpFile (filename) {
return new Promise((resolve, reject) => {
fs.unlink(filename, function (err) {
if (err) reject(err)
resolve('Temp file deletion successful.')
})
})
}
For working with the file you can use writeFileSync instead writeFile. (Reference).
For multiple Promise you can use the Promise.all method.
const promise1 = Promise.resolve(3);
const promise2 = 42;
const promise3 = new Promise((resolve, reject) => {
setTimeout(resolve, 100, 'foo');
});
Promise.all([promise1, promise2, promise3]).then((values) => {
console.log(values);
});
from MDN
In the 1st function 'get_files()' I can log the file_list variable, it is correct here, however when I log it again in my 2nd function 'get_diffs()' it is undefined..
// Get files
async function get_files() {
await fs.readdir(dirPath, function (err, files) {
(async () => {
if (await err) {
console.log("Error getting directory information.", err)
} else {
var file_list = []; // Reset
await files.forEach(function (file) {
file_list.push(file);
});
console.log('1st Call = ' + file_list); // Correct
return await file_list;
}
})();
});
}
// Get Diffs
async function get_diffs() {
const file_list = await get_files();
console.log('2nd Call = ' + file_list); // Undefined
const dates = await get_dates();
return await files.filter(x => !dates.includes(x));
}
You have misunderstood async/await. Learn the basics here
function get_files() {
return new Promise((resolve, reject) => {
fs.readdir(dirPath, function (err, files) {
if (err) {
reject(err);
} else {
var file_list = []; // Reset
files.forEach(function (file) {
file_list.push(file);
});
console.log('1st Call = ' + file_list); // Correct
resolve(file_list);
}
});
})
}
fs.readdir does not return a promise. Use the promise based function fs.promise.readdir instead.
async function get_diffs() {
const file_list = await fs.promise.readdir(dirPath);
// ...
}
So you don't really need the other function. It had many problems anyway. await makes not much sense when used with an expression that is not a promise. All the places where you have used await in get_files, the expression that follows it does not represent a promise.
I'm trying to wait for a forEach to finish, and the forEach loop has two nested requests inside.
I need to wait untill the forEach finish beacuse I fill an array with the queries results and then, when the forEach is finish, then call another function, but I cannot do it well because sometimes, the array is fully filled, but othertimes the array is incomplete.
Here is my code:
readAllClientsAndInvoices: function(request, response) {
let clientsInvoices = [];
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) {
console.log(err);
} else {
clients.forEach(function (client, idx, array) {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
clientsInvoices.push(pair);
}
});
}
if (idx === array.length - 1) {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices, function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
}
});
});
}
});
}
This is how I do it now, but I need to wait untill the array is fully filled with all the clients and its invoices and then call to createclientPDFReportWOCommentsV2 function but I don't know how to do it.
Thanks everyone
You can try to use a map instead of forEach in order to accept a return value from every call of the callback function, that return value will have to be a Promise, resolving after particular call has been completed. Since I don't see any particular error handling in your example I just made it so that in case of error Promise resolves undefined which is filtered afterwards in the createClientPDFReportWOCommentsV2 call.
function readAllClientsAndInvoices(request, response) {
DAOClients.readAllClientesById(request.session.id, function (err, clients) {
if (err) return console.log(err);
Promise.all(clients.map(client => {
return new Promise(resolve => {
DAOClients.readClientDataById(client.id, function (err, data) {
if (err) {
console.log(err)
resolve();
} else {
DAOClients.readAllclientInvoices(data.id, function (err, invoices) {
if (err) {
console.log(err);
resolve();
} else {
let pair = {
clientData: data,
invoicesList: invoices
};
resolve(pair);
}
});
}
});
});
})).then(clientsInvoices => {
DAOClients.createClientPDFReportWOCommentsV2(clientsInvoices.filter(Boolean), function (err, res) {
if (err) {
console.log(err);
} else {
response.redirect(307, '/client/searchClient');
}
});
});
});
}
To solve these problems i would use Async/Await https://javascript.info/async-await. Make sure all the methods you're calling on DAOClients returns a Promise https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
For example
function readAllClientesById() {
return new Promise((resolve, reject) => {
// Wait for some data to get fetched from SQL
// and call resolve instead of callback function
resolve(data)
// Or of there was an error
reject(err)
})
}
This is natively supported in the latest versions of Node.js.
Example of Async/Await if promises is implemented:
async function readAllClientsAndInvoices(req, res) {
try {
const clientInvoices = []
const clients = await DAOClients.readAllClientesById(req.session.id)
for (const client of clients) {
const clientData = await DAOClients.readClientDataById(client.id)
const clientInvoices = await DAOClients.readAllclientInvoices(clientData.id)
clientInvoices.push({
clientData: data,
invoicesList: invoices
})
}
// This code won't be executed until the for loop is completed
await DAOClients.createClientPDFReportWOCommentsV2(clientInvoices)
} catch (err) {
return res.status(err.code).send(err)
}
res.redirect(307, '/client/searchClient');
}
I haven't tested the code, it's just an example of how I approach these type of problems.
I am trying to compile a list of all customers using the Stripe Node API. I need to make continual fetches 100 customers at a time. I believe that I need to use a Promise within the API call to use async await, but I can't for the life of me figure out where to put it. Hoping to make this gist public use and I want to get it right, thanks.
getAllCustomers()
function getMoreCustomers(customers, offset, moreCustomers, limit) {
if (moreCustomers) {
stripe.customers.list({limit, offset},
(err, res) => {
offset += res.data.length
moreCustomers = res.has_more
customers.push(res.data)
return getMoreCustomers(customers, offset, moreCustomers, limit)
}
)
}
return customers
}
async function getAllCustomers() {
const customers = await getMoreCustomers([], 0, true, 100)
const content = JSON.stringify(customers)
fs.writeFile("/data/stripe-customers.json", content, 'utf8', function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
}
IF res in stripe.customers.list({limit, offset}).then(res => ...) is the same as res in the "callback" version of stripe.customers.list({limit, offset}, (err, res) - then you probably could rewrite your code like
const getMoreCustomers = limit => {
const getThem = offset => stripe.customers.list({limit, offset})
.then(res => res.has_more ?
getThem(offset + res.data.length).then(result => res.data.concat(...result)) :
res.data
);
return getThem(0);
};
async function getAllCustomers() {
const customers = await getMoreCustomers(100);
const content = JSON.stringify(customers);
fs.writeFile("/data/stripe-customers.json", content, 'utf8', function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
}
additional to Jaromanda X's answer, it seems there is no offset option to customers api, but starting_after https://stripe.com/docs/api/node#list_customers
So, getMoreCustomers may be fixed like
const getMoreCustomers = starting_after => {
const getThem = starting_after => stripe.customers.list({limit: 100, starting_after: starting_after})
.then(res => res.has_more ?
getThem(res.data[res.data.length - 1]).then(result => res.data.concat(...result)) :
res.data
);
return getThem(starting_after);
};
async function getAllCustomers() {
const customers = await getMoreCustomers(null);
const content = JSON.stringify(customers);
fs.writeFile("/data/stripe-customers.json", content, 'utf8', function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
}