We Keep Coding

Refactoring Complicated Nested Node.js Function

I have the following snippet of code below. It currently works, but I'm hoping to optimize/refactor it a bit.
Basically, it fetches JSON data, extracts the urls for a number of PDFs from the response, and then downloads those PDFs into a folder.
I'm hoping to refactor this code in order to process the PDFs once they are all downloaded. Currently, I'm not sure how to do that. There are a lot of nested asynchronous functions going on.
How might I refactor this to allow me to tack on another .then call before my error handler, so that I can then process the PDFs that are downloaded?
const axios = require("axios");
const moment = require("moment");
const fs = require("fs");
const download = require("download");
const mkdirp = require("mkdirp"); // Makes nested files...
const getDirName = require("path").dirname; // Current directory name...
const today = moment().format("YYYY-MM-DD");
function writeFile(path, contents, cb){
mkdirp(getDirName(path), function(err){
if (err) return cb(err)
fs.writeFile(path, contents, cb)
})
};
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
.then((res) => {
res.data.results.forEach((item) => {
download(item.pdf_url).then((data) => {
writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
if(err){
console.log(err);
} else {
console.log("FILE WRITTEN: ", item.pdf_file_name);
}
})
})
})
})
.catch((err) => {
console.log("COULD NOT DOWNLOAD FILES: \n", err);
})
Thanks for any help you all can provide.
P.S. –– When I simply tack on the .then call right now, it fires immediately. This means that my forEach loop is non-blocking? I thought that forEach loops were blocking.

The current forEach will run synchronously, and will not wait for the asynchronous operations to complete. You should use .map instead of forEach so you can map each item to its Promise from download. Then, you can use Promise.all on the resulting array, which will resolve once all downloads are complete:
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
.then(processResults)
.catch((err) => {
console.log("COULD NOT DOWNLOAD FILES: \n", err);
});
function processResults(res) {
const downloadPromises = res.data.results.map((item) => (
download(item.pdf_url).then(data => new Promise((resolve, reject) => {
writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
if(err) reject(err);
else resolve(console.log("FILE WRITTEN: ", item.pdf_file_name));
});
}))
));
return Promise.all(downloadPromises)
.then(() => {
console.log('all done');
});
}
If you wanted to essentially block the function on each iteration, you would want to use an async function in combination with await instead.

Javascript Promises : Can I know which part of promise chain caused error?

(Please excuse my English)
I am learning about javascript promises, now.
Below sample code is a simple javascript code for node.js(my node.js version is v10.0.0), which asynchronously reads and parses a JSON file using promise chain.
const fs = require("fs");
function readFileAsync(filename) {
return new Promise((resolve, reject) => {
fs.readFile(filename, 'utf8', (error, result) => {
if (error)
reject(error);
else
resolve(result);
});
});
}
readFileAsync('test.json')
.then(res => JSON.parse(res))
.then(res => { console.log('JSON=', res); })
.catch(err => { console.log(err.message); });
I found that this sample code generates different formats of error messages.
For example, if it cannot find 'test.json', the error message is...
ENOENT: no such file or directory, open '/home/node/test.json'
If it cannot parse 'test.json', the error message is...
Unexpected token / in JSON at position 31
I want to modify the sample code to generate same format of error message always containing JSON file name.
To do so, firstly I should know which part of promise chain caused error. How can I know?

There are two ways to arrived what you want.
Promise.then has two arguments, see below code and you can get more information here
readFileAsync('test.json')
.then(res => JSON.parse(res))
.then(res => { console.log('JSON=', res); }, error => {
// here can catch error of previous then function
});
Another way is modify the function readFileAsync
function readFileAsync(filename) {
return new Promise(resolve => {
fs.readFile(filename, (error, result) => {
if (error)
resolve(null); // you can resolve whatever you want
else
resolve(result);
});
});
}
And .catch() will not catch any error of readFileAsync.

Below sample code is a my solution. Thank you, Bergi and Stephen.
I choose this solution because I want to know exactly where in the chain the error occurred and what is the error.
const fs = require("fs");
function readFileAsync(filename) {
return new Promise((resolve, reject) => {
fs.readFile(filename, 'utf8', (error, result) => {
if (error)
reject(error);
else
resolve(result);
});
});
}
function readJsonAsync(filename, fnCallBack) {
function fnMessage(n, str) {
console.log(`[${n}:${filename}]`, str);
}
readFileAsync(filename)
.then(
res => JSON.parse(res),
err => { fnMessage(-1, err.message); }
).then(
res => {
// if some errors occured at the previous step, res === undefined
if (res !== undefined)
fnCallBack(filename, res);
},
err => { fnMessage(-2, err.message); }
);
}
function printJSON(filename, json) {
console.log(`JSON[${filename}]:`, json);
}
readJsonAsync('test.json', printJSON);
My solution has a prerequisite. The prerequisite is...
There is no simple way to break a promise chain even if some errors
occured at previous steps of the chain.
Is this prerequisite right?

Node JS + AWS Promise Triggered Twice

AWS = require('aws-sdk');
AWS.config.region = 'eu-west-1';
ses = new AWS.SES();
var params = {};
return ses.sendEmail(params, function (err, data) {
console.log('----->sending email')
}).promise().then((data) => {
console.log('---->sending promise')
}).catch((err) => {
console.log('----->am in error')
console.log(err)
})
Can someone help my above code promise is triggered twice.
I should get below
----->sending email
---->sending promise
But i got
----->sending email
---->sending promise
----->sending email

It looks like you are providing both a callback function and using the promise approach. Effectively, this means that you have two different functions that execute when the request is done. Choose one of the following:
You can use the promise approach, with then and catch.
function send(params) {
ses.sendEmail(params).promise().then((data) => {
console.log('Email was sent')
}).catch((err) => {
console.log('There was an error')
})
}
You can use promise with async/await. Make sure you add the async keyword to your function header. Make sure your Node runtime supports async/await or that you code is transpiled to whatever Node version you are using.
async function send(params) {
try {
const data = await ses.sendEmail(params).promise();
console.log('Email was sent')
} catch(err) {
console.log('There was an error')
}
}
Finally, you could use the callback approach:
function send(params) {
ses.sendEmail(params, function(err, data) {
if (err) {
console.log('There was an error')
return
}
console.log('Email was sent')
})
}

Error : Callback was already called when using pg-promise with async series

I'm having trouble understanding the output printed why executing this code :
1
2
Unhandled rejection Error: Callback was already called.
It seems like both then and catch are executed when the query is successful.
Any idea ?
Cheers
async.series([
function(callback) {
db.none(query)
.then(function () {
return callback(null, true);
})
.catch(function (err) {
return callback(err, null);
});
},
function(callback) {
db.any(query)
.then(function (data) {
console.log('1')
return callback(null, data);
})
.catch(function (err) {
console.log('2')
console.log(err);
return callback(err, null);
});
}
],
function(err, results) {
if (results && !results[1].isEmpty()) {
// do something
}
});
EDIT :
TypeError: results[1].isEmpty is not a function
It seems like the problem come from the rest of the code and was just a simple undefined function error, thanks.
But i still don't understand something : why is this error catched inside the second query instead of outside the async queries ?

I'm the author of pg-promise.
You should never use async library with pg-promise, it goes against the concept of shared/reusable connections.
Implementation with proper use of the same connection, via a task:
db.task(t => {
return t.batch([
t.none(query1),
t.any(query2)
]);
})
.then(data => {
// data[0] = null - result of the first query
// data[1] = [rows...] - result of the second query
callback(null, data); // this will work, but ill-advised
})
.catch(error => {
callback(error, null); // this will work, but ill-advised
});
See also: Chaining Queries.
However, in your case it looks like when you call the successful callback(null, data), it throws an error, which in turn results in it being caught in the following .catch section. To test this, you can change your promise handler like this:
.then(data => {
callback(null, data);
}, error => {
callback(error, null);
});
It should normally throw an error about Promise missing .catch because you threw an error while in .then and there is no corresponding .catch chained below, which you can also check through this code:
.then(data => {
callback(null, data);
}, error => {
callback(error, null);
})
.catch(error => {
// if we are here, it means our callback(null, data) threw an error
});
P.S. You really should learn to use promises properly, and avoid any callbacks altogether. I only provided an example consistent with your own, but in general, converting promises into callbacks is a very bad coding technique.

This is what happens:
callback(null, data) is called within the context of the .then();
async notices that this was the last item of the series, so it calls the final handler (still within the context of the .then());
the final handler throws an error;
because the code runs in the context of .then(), the promise implementation catches the error and calls the .catch();
this calls the callback again;
PoC:
const async = require('async');
async.series([
callback => {
Promise.resolve().then(() => {
callback(null);
}).catch(e => {
callback(e);
});
}
], err => {
throw Error();
})

Have you try to define your function externally:
function onYourFunction() {
console.log('Hi function');
}
and than do:
.then(onYourFunction) //-->(onYourFunction without parentheses )
Unfortunately i don't use pg-promise but i can advise promise
at this point i create all promises that are necessary:
function createPromise(currObj) {
return new Promise(function (resolve, reject) {
currObj.save(function (errSaving, savedObj) {
if(errSaving){
console.log("reject!");
return reject(errSaving, response);
}
console.log('currObj:' + currObj);
return resolve(savedObj);
});
});
}
and then in cascades:
var allPromiseOs = Promise.all(promise1, promise2, promise3);

Understanding when to use Events and when to use Callback

With events, the initiator raises an event that will be received by those routines that have elected to receive that event. The receiver specifies what events it will receive from what initiators.
With callbacks, the routine after completion notifies the caller of the completion.
So I am confused where should I use events or where should I use the callbacks as I can accomplish what a callback do with events but resulting in a lot of events created in the application.
What should be a good approach to follow while coding, to use events or callbacks?

Events - for things that can happen many times.
Callbacks (or promises) - for things that can happen once.
So for example, when you have a function that you call because you need a current temperature taken from some API, that function should either return a promise or take a callback that can later be called with the right value (or error).
If, on the other hand, you have a function that you call because you need to get a new temperature every time when it changes, then this function should return an event emitter (or take an event handler to attach to some internal event emitter).
Now, the question on when to use callbacks and when to use promises is a little bit more tricky because they are good for the same kinds of situations - when you want to know a result of some asynchronous operation (some data or error). Since both work for the same situations let's consider two examples of reading the contents of a file.
First, with callbacks:
let fs = require('fs');
fs.readFile('a.txt', 'utf-8', (err, data) => {
if (err) {
console.log('Error:', err.message);
} else {
console.log('Data:', data.trim());
}
});
If there is no file it will print:
Error: ENOENT: no such file or directory, open 'a.txt'
If there is a file it will print:
Data: Contents of a.txt
Now, the same with promises:
let fs = require('mz/fs');
fs.readFile('b.txt', 'utf-8')
.then(data => {
console.log('Data:', data.trim());
})
.catch(err => {
console.log('Error:', err.message);
});
It works exactly the same as the previous example.
For that simple example the difference may not be very obvious but what if you wanted to have a function that abstracts some of that logic away.
For example this, with callbacks:
let fs = require('fs');
function a(cb) {
fs.readFile('b.txt', 'utf-8', (err, data) => {
if (err) {
return cb('a() error: ' + err.message);
}
cb(null, 'a() data: ' + data.trim());
});
}
a((err, data) => {
if (err) {
console.log('Error:', err);
} else {
console.log('Data:', data);
}
});
It will print either this
Error: a() error: ENOENT: no such file or directory, open 'a.txt'
or something like this:
Data: a() data: Contents of a.txt
Now, what is different with promises is that you can store it in a variable, return it from a function or pass it as an argument to some other function before attaching the success/error handlers. For example:
let fs = require('mz/fs');
function a() {
return fs.readFile('a.txt', 'utf-8')
.then(data => 'a() data: ' + data.trim())
.catch(err => Promise.reject('a() error: ' + err.message));
}
let promise = a();
promise.then(data => console.log('Data:', data))
.catch(err => console.log('Error:', err));
It works the same, it is written in a different style that you may or may not find more readable, but the difference is that now you don't have to attach a callback at the time of calling the a() function. You can do it somewhere else.
If you didn't want to change the error message, it would be this with callbacks:
function a(cb) {
fs.readFile('a.txt', 'utf-8', (err, data) => {
if (err) {
return cb(err);
}
cb(null, 'a() data: ' + data.trim());
});
and this with promises:
function a() {
return fs.readFile('a.txt', 'utf-8')
.then(data => 'a() data: ' + data.trim());
}
Another difference is that if you have a function that returns a promise, you can use a new await keyword inside of a async function like this:
async function x() {
try {
console.log('Data:', await a());
} catch (err) {
console.log('Error:', err);
}
}
You cannot use await with a function that doesn't return a promise.
It gets very convenient for example when you need to read file a.txt to get another filename that it contains, and then read that other file and print its contents while handling all errors in more complex situations.
To use async and await with Node v7.x you need to use the --harmony flag, see:
http://node.green/#async-functions