We Keep Coding

JavaScript NodeJS How to use stream/promises with async functions?

I have a JS async function in Node. Say, it downloads a file from a URL and does something with it, eg. unzip it. I wrote it like this, it works, but eslint showed me there is a thick code smell: error Promise executor functions should not be async no-async-promise-executor. Async was required because of await fetch in body function.
I am not skilled enough with streams nor async/await to correct it by myself. I would like to get rid of the Promise and fully use async/await. The module stream/promises seems the way to go from Node-15 on as commented here how-to-use-es8-async-await-with-streams. How to use await pipeline(...) in this context? Maybe there's a better and shorter way?
Here's the function:
function doSomething(url) {
return new Promise(async (resolve, reject) => {
try {
const fileWriteStream = fs.createWriteStream(someFile, {
autoClose: true,
flags: 'w',
});
const res = await fetch(url);
const body = res.body;
body
.pipe(fileWriteStream)
.on('error', (err) => {
reject(err);
})
.on('finish', async () => {
await doWhatever();
resolve('DONE');
});
} catch (err) {
reject(err);
}
});
}

You could simply perform the await before getting to the executor:
async function doSomething(url) {
const fileWriteStream = fs.createWriteStream(someFile, { autoClose: true, flags: 'w' });
let { body } = await fetch(url);
body.pipe(fileWriteStream);
return new Promise((resolve, reject) => {
body.on('error', reject);
body.on('finish', resolve);
});
};
My advice in general is to remove as much code from within your promise executors as possible. In this case the Promise is only necessary to capture resolution/rejection.
Note that I've also removed doWhatever from within doSomething - this makes doSomething much more robust. You can simply do:
doSomething('http://example.com').then(doWhatever);
Lastly I recommend you set someFile as a parameter of doSomething instead of referencing it from some broader context!

To use the pipeline function you're looking for, it would be
const { pipeline } = require('stream/promises');
async function doSomething(url) {
const fileWriteStream = fs.createWriteStream(someFile, {
autoClose: true,
flags: 'w',
});
const res = await fetch(url);
await pipeline(res.body, fileWriteStream);
await doWhatever();
return 'DONE';
}

You can use the fs/promises in NodeJS and trim your code down to the following:
import { writeFile } from 'fs/promises'
async function doSomething(url) {
const res = await fetch(url);
if (!res.ok) throw new Error('Response not ok');
await writeFile(someFile, res.body, { encoding: 'utf-8'})
await doWhatever();
return 'DONE';
});
}

Use of Async await does not wait for the return of function [duplicate]

I would like to use async/await with some filesystem operations. Normally async/await works fine because I use babel-plugin-syntax-async-functions.
But with this code I run into the if case where names is undefined:
import fs from 'fs';
async function myF() {
let names;
try {
names = await fs.readdir('path/to/dir');
} catch (e) {
console.log('e', e);
}
if (names === undefined) {
console.log('undefined');
} else {
console.log('First Name', names[0]);
}
}
myF();
When I rebuild the code into the callback hell version everything is OK and I get the filenames.
Thanks for your hints.

Native support for async/await fs functions since Node 11
Since Node.JS 11.0.0 (stable), and version 10.0.0 (experimental), you have access to file system methods that are already promisify'd and you can use them with try catch exception handling rather than checking if the callback's returned value contains an error.
The API is very clean and elegant! Simply use the .promises member of fs object:
import fs from 'fs';
async function listDir() {
try {
return await fs.promises.readdir('path/to/dir');
} catch (err) {
console.error('Error occurred while reading directory!', err);
}
}
listDir();

Starting with node 8.0.0, you can use this:
const fs = require('fs');
const util = require('util');
const readdir = util.promisify(fs.readdir);
async function myF() {
let names;
try {
names = await readdir('path/to/dir');
} catch (err) {
console.log(err);
}
if (names === undefined) {
console.log('undefined');
} else {
console.log('First Name', names[0]);
}
}
myF();
See https://nodejs.org/dist/latest-v8.x/docs/api/util.html#util_util_promisify_original

Node.js 8.0.0
Native async / await
Promisify
From this version, you can use native Node.js function from util library.
const fs = require('fs')
const { promisify } = require('util')
const readFileAsync = promisify(fs.readFile)
const writeFileAsync = promisify(fs.writeFile)
const run = async () => {
const res = await readFileAsync('./data.json')
console.log(res)
}
run()
Promise Wrapping
const fs = require('fs')
const readFile = (path, opts = 'utf8') =>
new Promise((resolve, reject) => {
fs.readFile(path, opts, (err, data) => {
if (err) reject(err)
else resolve(data)
})
})
const writeFile = (path, data, opts = 'utf8') =>
new Promise((resolve, reject) => {
fs.writeFile(path, data, opts, (err) => {
if (err) reject(err)
else resolve()
})
})
module.exports = {
readFile,
writeFile
}
...
// in some file, with imported functions above
// in async block
const run = async () => {
const res = await readFile('./data.json')
console.log(res)
}
run()
Advice
Always use try..catch for await blocks, if you don't want to rethrow exception upper.

As of v10.0, you can use fs.Promises
Example using readdir
const { promises: fs } = require("fs");
async function myF() {
let names;
try {
names = await fs.readdir("path/to/dir");
} catch (e) {
console.log("e", e);
}
if (names === undefined) {
console.log("undefined");
} else {
console.log("First Name", names[0]);
}
}
myF();
Example using readFile
const { promises: fs } = require("fs");
async function getContent(filePath, encoding = "utf-8") {
if (!filePath) {
throw new Error("filePath required");
}
return fs.readFile(filePath, { encoding });
}
(async () => {
const content = await getContent("./package.json");
console.log(content);
})();

You might produce the wrong behavior because the File-Api fs.readdir does not return a promise. It only takes a callback. If you want to go with the async-await syntax you could 'promisify' the function like this:
function readdirAsync(path) {
return new Promise(function (resolve, reject) {
fs.readdir(path, function (error, result) {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
and call it instead:
names = await readdirAsync('path/to/dir');

This is the TypeScript version to the question. It is usable after Node 11.0:
import { promises as fs } from 'fs';
async function loadMonoCounter() {
const data = await fs.readFile('monolitic.txt', 'binary');
return Buffer.from(data);
}

Node v14.0.0 and above
you can just do:
import { readdir } from "fs/promises";
just like you would import from "fs"
see this PR for more details: https://github.com/nodejs/node/pull/31553

I have this little helping module that exports promisified versions of fs functions
const fs = require("fs");
const {promisify} = require("util")
module.exports = {
readdir: promisify(fs.readdir),
readFile: promisify(fs.readFile),
writeFile: promisify(fs.writeFile)
// etc...
};

Here is what worked for me:
const fsp = require('fs-promise');
(async () => {
try {
const names = await fsp.readdir('path/to/dir');
console.log(names[0]);
} catch (e) {
console.log('error: ', e);
}
})();
This code works in node 7.6 without babel when harmony flag is enabled: node --harmony my-script.js. And starting with node 7.7, you don't even need this flag!
The fsp library included in the beginning is just a promisified wrapper for fs (and fs-ext).
I’m really exited about what you can do in node without babel these days! Native async/await make writing code such a pleasure!
UPDATE 2017-06: fs-promise module was deprecated. Use fs-extra instead with the same API.

Recommend using an npm package such as https://github.com/davetemplin/async-file, as compared to custom functions. For example:
import * as fs from 'async-file';
await fs.rename('/tmp/hello', '/tmp/world');
await fs.appendFile('message.txt', 'data to append');
await fs.access('/etc/passd', fs.constants.R_OK | fs.constants.W_OK);
var stats = await fs.stat('/tmp/hello', '/tmp/world');
Other answers are outdated

You can use the simple and lightweight module https://github.com/nacholibre/nwc-l it supports both async and sync methods.
Note: this module was created by me.

Testing file read wtih fs using JEST

I am trying to test the following function:
async function read () {
return new Promise(function(resolve, reject) {
fs.readFile("INVALID_PATH", (err, contents) => {
if (err) {
reject(new Error('ERROR'));
}
resolve(contents);
});
});
}
through the test:
try {
await read();
} catch (e) {
expect(e).toMatch('ERROR');
}
However, I can't catch the rejection on catch, it gives me timeout.
Any suggestions?
EDIT:
I managed to get it working by using a mock, this way:
try {
fs.readFile.mockReset()
fs.readFile.mockImplementation((path, options, cb) => {
cb('ERROR')
})
await readFile()
} catch (err) {
expect(err).toBe('ERROR')
}
However, I still don't understand why this is necessary...

Have you tried declaring the test function as async in Jest? await cannot be used outside async functions (or in top level code), you have to do something like this:
it('works with async/await', async () => {
expect.assertions(1);
const data = await user.getUserName(4);
expect(data).toEqual('Mark');
});
// async/await can also be used with `.resolves`.
it('works with async/await and resolves', async () => {
expect.assertions(1);
await expect(user.getUserName(5)).resolves.toEqual('Paul');
});
More info for testing async/await in jest https://jestjs.io/docs/en/tutorial-async.html

How to handle multiple args of callback function in async/await format?

In shelljs, the exec function has 3 arguments in callback (err, stdout, stderr). When using this in async/await by promisify(shelljs.exec), I'm not able to catch the stderr?
const { promisify } = require('util'),
shellExec = promisify(shelljs.exec);
....
// in the function
try {
variableName = await shellExec('some valid shell command', {});
return variableName;
}
catch (err) {
console.log(err);
}
If shelljs exec return code 0 i.e. valid response it works fine, but when the the command is invalid, it returns 1.
I'm not able to get the stderr.

I am assuming you want the value of stderr to output while also using async/await. Something like this might be of use:
var shelljs = require('shelljs');
async function promiseExec(input) {
return new Promise((resolve, reject) => {
let { err, stdout, stderr } = shelljs.exec(input);
shelljs.exit(1);
if(stdout === "") reject(stderr);
resolve(stdout);
})
}
async function main () {
let result = await promiseExec('dir');
console.log(result);
}
main();

Node.JS Async / Await Dealing With Callbacks? [duplicate]

This question already has answers here:
How do I convert an existing callback API to promises?
(24 answers)
Closed 5 years ago.
Is there a way to deal with callback functions inside an async function() other than mixing in bluebird or return new Promise()?
Examples are fun...
Problem
async function bindClient () {
client.bind(LDAP_USER, LDAP_PASS, (err) => {
if (err) return log.fatal('LDAP Master Could Not Bind', err);
});
}
Solution
function bindClient () {
return new Promise((resolve, reject) => {
client.bind(LDAP_USER, LDAP_PASS, (err, bindInstance) => {
if (err) {
log.fatal('LDAP Master Could Not Bind', err);
return reject(err);
}
return resolve(bindInstance);
});
});
}
Is there a more elegant solution?

NodeJS v.8.x.x natively supports promisifying and async-await, so it's time to enjoy the stuff (:
const
promisify = require('util').promisify,
bindClient = promisify(client.bind);
let clientInstance; // defining variable in global scope
(async () => { // wrapping routine below to tell interpreter that it must pause (wait) for result
try {
clientInstance = await bindClient(LDAP_USER, LDAP_PASS);
}
catch(error) {
console.log('LDAP Master Could Not Bind. Error:', error);
}
})();
or just simply use co package and wait for native support of async-await:
const co = require('co');
co(function*() { // wrapping routine below to tell interpreter that it must pause (wait) for result
clientInstance = yield bindClient(LDAP_USER, LDAP_PASS);
if (!clientInstance) {
console.log('LDAP Master Could Not Bind');
}
});
P.S. async-await is syntactic sugar for generator-yield language construction.

Use modules! pify for instance
const pify = require('pify');
async function bindClient () {
let err = await pify(client.bind)(LDAP_USER, LDAP_PASS)
if (err) return log.fatal('LDAP Master Could Not Bind', err);
}
Haven't tested it yet