I want to understand a mechanism of callbacks, so i try to compute a summary size for files in one directory. Here is my code:
import path from 'path';
import fs from 'fs';
import async from 'async';
export const getDirectorySize = (directory) => {
fs.readdir(directory, (error, files) => {
if (error) throw error;
const counter = (el, callback) => {
fs.stat(path.join(directory, el), (error2, result) => {
if (error2) throw error2;
if (result.isFile()) callback(null, result.size);
});
}
async.map(files, counter, (error3, results) => {
if (error3) throw error3;
console.log(results.reduce((prev, current) => prev + current, 0));
});
});
}
getDirectorySize('/usr/src/app');
So when i run code above, the third argument (callback) of async.map doesn't call. But function "counter" is called (There were no errors)! So how to see the results of async.map?
Related
i am trying to pass the sql query parameters to data access js file. i have imported the function in current file, but still am getting the below error.
current file
const tcount = async (value) => {
const sql = 'trainingcount';
const result = await query.findAll(sql);
return result;
}
data access file
const query = (results) => {
findAll: async (sql, result) => {
connection.query(`SELECT * FROM trainingcount`, (err, rows) => {
if (err) {
return results(null, err);
} else {
return results(rows);
}
});
};
};
export { query };
(node:11132) UnhandledPromiseRejectionWarning: TypeError:
query.findAll is not a function
EDIT: Check #rid solutions on the comments for the specific problem of calling the proper function. My answer solves a different problem in OP code.
you call return inside the callback function, so you are returning THAT function, not findAll. You need to return a Promise:
const query = (results) => {
findAll: (sql, result) => {
return new Promise((resolve, reject) => {
connection.query(`SELECT * FROM trainingcount`, (err, rows) => {
if (err) {
reject(err);
} else {
resolve(rows);
}
});
});
};
};
export { query };
I am trying to rename files asynchronously in Node.js only if destination files don't exist.
I made a quick test like follows:
const fs = require('fs')
const files = [ 'file1', 'file2', 'file3' ]
const new_name = 'new-name' // same destination name for all
fs.exists() - DEPRECATED
for (let file of files)
fs.exists(new_name, (exists) => {
if (!exists) fs.rename(file, new_name, (err) => {})
})
fs.access() - RECOMMENDED
for (let file of files)
fs.access(new_name, fs.constants.F_OK, (err) => {
if (err) fs.rename(file, new_name, (err) => {})
})
fs.move() - from fs-extra
const fs_extra = require('fs-extra')
for (let file of files)
fs_extra.move(file, new_name, { overwrite: false }, (err) => {})
Each time all 3 files were overwriten and renamed to one file.
I believe this is happens because all exists checks fire sooner than any rename happens.
I know how to accomplish this task synchronously, but want to be sure that there is no proper async way to do so.
You can create Promise which resolve's when file is renamed
fs.rename(file, new_name, (err) => {
resolve(); <------
});
or when renaming is skipped
fs.access(new_name, fs.constants.F_OK, (err) => {
if (err) {
return fs.rename(file, new_name, (err) => {
resolve();
});
}
resolve(); <------
});
Full code
(async () => {
for (let file of files) {
await new Promise((resolve) => {
fs.access(new_name, fs.constants.F_OK, (err) => {
if (err) {
return fs.rename(file, new_name, (err) => {
resolve();
});
}
resolve();
});
});
}
})();
and if you don't want to mix async/await with Promise
(async () => {
function rename(file, new_name) {
return new Promise((resolve) => {
fs.access(new_name, fs.constants.F_OK, (err) => {
if (err) {
return fs.rename(file, new_name, (err) => {
resolve();
});
}
resolve();
});
});
}
for (let file of files) {
await rename(file, new_name);
}
})();
#ponury-kostek solution works brilliantly and marked as accepted answer.
I ended up with the following code since it's a bit shorter:
async function rename_files() {
for (let file of files)
await fs.move(file, new_name)
}
rename_files()
Instead of wrapping fs library in promises.
I like to import the promise implementation of the fs library.
Then call the fs methods with await.
.
import {promises as fs_promises} from 'fs'; // The promise implmentation of fs library.
async function renameFile() {
const fileFullPath = '1234.txt';
const newFileFullPath = '5678.txt';
await fs_promises.rename(fileFullPath, newFileFullPath, (error) => {
if (error) {
console.log(error);
} else {
console.log("\nFile Renamed\n");
}
});
}
await renameFile(); // Call the async method.
In data.task package, I could resolve or reject a api call as following:
import Task from 'data.task';
import fs from 'fs';
const readFile = (filename, enc) => {
return new Task((rej, res) =>
fs.readFile(filename, enc, (err, contents) => {
err ? rej(err) : res(contents);
})
);
};
How would I accomplish that in the new folktale version of Task? I can resolve requests, but how do I reject? I have tried the following:
const {task, rejected} = require('folktale/concurrency/task');
import fs from 'fs';
const readFile = (filename, enc) => {
return task(res => {
fs.readFile(filename, enc, (err, contents) => {
err ? rejected(err) : res.resolve(contents);
});
});
};
const writeFile = (filename, contents) => {
return task(res => {
fs.writeFile(filename, contents, (err, success) => {
err ? rejected(err) : res.resolve(success);
});
});
};
const app = readFile('FILE_DOESNOT_EXIST.json', 'utf-8')
.map(contents => contents.replace(/8/g, '6'))
.chain(contents => writeFile('config1.json', contents));
app.run().listen({
onCancelled: () => {
console.log('the task was cancelled');
},
onRejected: () => {
console.log('something went wrong');
},
onResolved: value => {
console.log(`The value is Good`);
},
});
When I gave a file that doesn't exist, the onRejected handler does not get called.
What do I expect to see:
Since I have the program read a file that does not exist, it should run onRejected which should log something went wrong.
What do I see now:
Nothing. The program does not bug out, but it also does not produce anything, it simply runs as normal.
When using data.task(the older version of Task), I can use reject which is why it stills works there. How do I do it now with the new version of Task?
Ok this is really silly! For some reason I could not find this solution right away on the doc.That's why I imported the rejected from task...
Basically resolver function coming from task has not only resolve, but also reject, which should have been obvious, but it was not on the doc.
So here is working code:
import {task} from 'folktale/concurrency/task';
import fs from 'fs';
const readFile = (filename, enc) => {
return task(res => {
fs.readFile(filename, enc, (err, contents) => {
err ? res.reject() : res.resolve(contents);
});
});
};
const writeFile = (filename, contents) => {
return task(res => {
fs.writeFile(filename, contents, (err, success) => {
err ? res.reject() : res.resolve(success);
});
});
};
const app = readFile('confg.json', 'utf-8')
.map(contents => contents.replace(/8/g, '6'))
.chain(contents => writeFile('config1.json', contents));
app.run().listen({
onCancelled: () => {
console.log('the task was cancelled');
},
onRejected: () => {
console.log('something went wrong');
},
onResolved: value => {
console.log(`The value is Good`);
},
});
i am writing a small utility using ramda and data.task that reads image files out of a directory and outputs their size. I got it working like so:
const getImagePath = assetsPath => item => `${assetsPath}${item}`
function readImages(path) {
return new Task(function(reject, resolve) {
fs.readdir(path, (err, images) => {
if (err) reject(err)
else resolve(images)
})
})
}
const withPath = path => task => {
return task.map(function(images) {
return images.map(getImagePath(path))
})
}
function getSize(task) {
return task.map(function(images) {
return images.map(sizeOf)
})
}
const getImageSize = dirPath => compose(getSize, withPath(dirPath), readImages)
The problem is with the withPath function that adds the correct image path to the image file name but forces my api to pass in the directoryName twice: once for reading the files and second time for reading the path. This means I have to call the getImageSize function like so:
const portfolioPath = `${__dirname}/assets/`
getImageSize(portfolioPath)(portfolioPath).fork(
function(error) {
throw error
},
function(data) {
console.log(data)
}
)
Is there any way to pass the dirname as a parameter only once? I want the api to work like this:
getImageSize(portfolioPath).fork(
function(error) {
throw error
},
function(data) {
console.log(data)
}
)
You shouldn't be building paths manually like that
One of Node's better APIs is the Path module – I would recommend that your readImages wrapper is made a generic readdir wrapper, and instead resolve an Array of path.resolve'd file paths
const readdir = dir =>
new Task ((reject, resolve) =>
fs.readdir (dir, (err, files) =>
err
? reject (err)
: resolve (files.map (f => path.resolve (dir, f)))
const getImagesSizes = dir =>
readdir (dir) .map (R.map (sizeOf))
Wrapping the Node continuation-passing style APIs just to return a Task gets to be a bother, doesn't it?
const taskify = f => (...args) =>
Task ((reject, resolve) =>
f (...args, (err, x) =>
err ? reject (err) : resolve (x)))
const readdir = (dir, ...args) =>
taskify (fs.readdir) (dir, ...args)
.map (R.map (f => path.resolve (dir, f)))
const getImagesSizes = dir =>
readdir (dir) .map (R.map (sizeOf))
You should probably also take care to file out file paths that are directories – unless your sizeOf implementation handles that
I managed to solve this by passing the Task resolution a single object like so:
function readImages(path) {
return new Task(function(reject, resolve) {
fs.readdir(path, (err, images) => {
if (err) reject(err)
else resolve({ images, path })
})
})
}
const withPath = task => {
return task.map(function({ images, path }) {
return images.map(getImagePath(path))
})
}
...and then destructing it out of the task payload and now my compose function looks like this:
module.exports = (function getImageSize(dirPath) {
return compose(getSize, withPath, readImages)
})()
And my api call looks like this:
getImageSize(portfolioPath).fork(
function(error) {
throw error
},
function(data) {
console.log(data)
}
)
I'm taking a look at the async library but I can't seem to find a control flow for handling pipelines. I'm just wondering if I'm missing something here.
I want to implement a pipeline. Example:
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next(null, input); });
var pipelineResult = pipelineRunner.run(pipeline, 'sample input', (error, result) => {});
Explanation: A series of functions is called. Each function receives an input and a next function. Each function processes the input and passes it as a parameter to the next function. As a result of the pipeline execution, I get the processed input, or, if any function calls next with an error, the pipeline stops and the callback is called.
I guess this is a pretty common use case so I think async can do it, but I'm not being able to find it. If you know of any other library that can achieve such result, that would be acceptable too.
You are looking for the async.waterfall function.
Alternatively you can apply asyc.seq or async.compose with multiple arguments if you need a function that you can pass an initial input to.
I ended up implementing it myself even though, as #Bergi just showed, async does have support for it.
/**
* Runs asynchronous pipelines
*/
class PipelineRunner {
/**
* Runs the given pipeline
* #param pipeline - The array of functions that should be executed (middleware)
* #param middlewareArgs - The array of arguments that should be passed in to the middleware
* #param input
* #param next
*/
run(pipeline, middlewareArgs, input, next) {
if (!pipeline) throw Error('\'pipeline\' should be truthy');
if (!context) throw Error('\'context\' should be truthy');
if (!input) throw Error('\'input\' should be truthy');
if (!next) throw Error('\'next\' should be truthy');
if (!pipeline.length) throw Error('\'pipeline.length\' should be truthy');
let index = 0;
// the link function "binds" every function in the pipeline array together
let link = (error, result) => {
if (error) {
next(error);
return;
}
let nextIndex = index++;
if (nextIndex < pipeline.length) {
let args = [result].concat(middlewareArgs).concat(link);
pipeline[nextIndex].apply(null, args);
}
else {
next(null, result);
}
};
let args = [input].concat(middlewareArgs).concat(link);
pipeline[index++].apply(null, args);
}
}
export default new PipelineRunner();
Unit tests:
import chai from 'chai';
import pipelineRunner from '../src/server/lib/pipelines/pipelineRunner';
let assert = chai.assert;
describe('PipelineRunner', () => {
describe('run', function() {
it('Happy path', () => {
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next(null, input); });
pipelineRunner.run(pipeline, [], 'happy', (error, result) => {
assert.strictEqual(result, "happy");
});
});
it('Happy path - with arguments', () => {
let pipeline = [];
pipeline.push((input, someArgument, next) => {
assert.strictEqual(someArgument, 'something that should be passed in');
next(null, input);
});
pipeline.push((input, someArgument, next) => { next(null, input); });
pipelineRunner.run(pipeline, ['something that should be passed in'], 'happy', (error, result) => {
assert.strictEqual(result, "happy");
});
});
it('When something goes wrong', () => {
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next('something went wrong'); });
pipelineRunner.run(pipeline, [], 'happy', (error, result) => {
assert.strictEqual(error, 'something went wrong');
});
});
});
});