I'm taking a look at the async library but I can't seem to find a control flow for handling pipelines. I'm just wondering if I'm missing something here.
I want to implement a pipeline. Example:
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next(null, input); });
var pipelineResult = pipelineRunner.run(pipeline, 'sample input', (error, result) => {});
Explanation: A series of functions is called. Each function receives an input and a next function. Each function processes the input and passes it as a parameter to the next function. As a result of the pipeline execution, I get the processed input, or, if any function calls next with an error, the pipeline stops and the callback is called.
I guess this is a pretty common use case so I think async can do it, but I'm not being able to find it. If you know of any other library that can achieve such result, that would be acceptable too.
You are looking for the async.waterfall function.
Alternatively you can apply asyc.seq or async.compose with multiple arguments if you need a function that you can pass an initial input to.
I ended up implementing it myself even though, as #Bergi just showed, async does have support for it.
/**
* Runs asynchronous pipelines
*/
class PipelineRunner {
/**
* Runs the given pipeline
* #param pipeline - The array of functions that should be executed (middleware)
* #param middlewareArgs - The array of arguments that should be passed in to the middleware
* #param input
* #param next
*/
run(pipeline, middlewareArgs, input, next) {
if (!pipeline) throw Error('\'pipeline\' should be truthy');
if (!context) throw Error('\'context\' should be truthy');
if (!input) throw Error('\'input\' should be truthy');
if (!next) throw Error('\'next\' should be truthy');
if (!pipeline.length) throw Error('\'pipeline.length\' should be truthy');
let index = 0;
// the link function "binds" every function in the pipeline array together
let link = (error, result) => {
if (error) {
next(error);
return;
}
let nextIndex = index++;
if (nextIndex < pipeline.length) {
let args = [result].concat(middlewareArgs).concat(link);
pipeline[nextIndex].apply(null, args);
}
else {
next(null, result);
}
};
let args = [input].concat(middlewareArgs).concat(link);
pipeline[index++].apply(null, args);
}
}
export default new PipelineRunner();
Unit tests:
import chai from 'chai';
import pipelineRunner from '../src/server/lib/pipelines/pipelineRunner';
let assert = chai.assert;
describe('PipelineRunner', () => {
describe('run', function() {
it('Happy path', () => {
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next(null, input); });
pipelineRunner.run(pipeline, [], 'happy', (error, result) => {
assert.strictEqual(result, "happy");
});
});
it('Happy path - with arguments', () => {
let pipeline = [];
pipeline.push((input, someArgument, next) => {
assert.strictEqual(someArgument, 'something that should be passed in');
next(null, input);
});
pipeline.push((input, someArgument, next) => { next(null, input); });
pipelineRunner.run(pipeline, ['something that should be passed in'], 'happy', (error, result) => {
assert.strictEqual(result, "happy");
});
});
it('When something goes wrong', () => {
let pipeline = [];
pipeline.push((input, next) => { next(null, input); });
pipeline.push((input, next) => { next('something went wrong'); });
pipelineRunner.run(pipeline, [], 'happy', (error, result) => {
assert.strictEqual(error, 'something went wrong');
});
});
});
});
Related
I'm putting together some Node.js code for querying LDAP that uses promises. When I run it, I get Unexpected reserved word concerning the await on line 43. This part:
let connection = await connect(ldapURL).catch((err) => {
console.error('LDAP server error:', err);
reject(err);
});
The entire code is shown below.
I have a promise returned in the connect() function and that's working fine. In fact, if I remove the promise from the listObjects() function, the console.debug(results); line prints exactly what I'm expecting.
So why is the await connect() causing an error in my listObjects() function? My searching has yielded a lot of answers saying, "You need to use async," but I already have my listObjects() declared as async.
Where have I gone wrong?
Full Code:
#!/usr/bin/env node
import ldapjs from 'ldapjs';
const ldapURL = [ 'ldap://127.0.0.1:389' ];
const bindDN = 'uid=search,dc=home';
const bindPassword = 'P#ssw0rd';
function connect(serverURL) {
return new Promise((resolve, reject) => {
const client = ldapjs.createClient({
url: serverURL
});
client.on('connect', () => {
console.debug('Connected to:', ldapURL);
console.debug('Binding as:', bindDN);
client.bind(bindDN, bindPassword, (err) => {
if (err) {
console.debug(err.message);
reject('Bind credentials rejected.');
}
else {
resolve(client);
}
});
});
client.on('error', (err) => {
reject('Unable to connect to ' + serverURL);
});
});
}
/**
* Search LDAP and return objects.
* #baseDN {string} Where to start, like 'ou=People,dc=example,dc=com'
* #filter {string} Optional LDAP query to limit results, like '(objectClass=posixAccount)'
* #returns {promise} ... Eventually.
*/
async function listObjects(baseDN, filter) {
return new Promise((resolve, reject) => {
let connection = await connect(ldapURL).catch((err) => {
console.error('LDAP server error:', err);
reject(err);
});
let opts = {
filter: filter,
scope: 'sub'
};
let results = [];
connection.search(`${baseDN}`, opts, (err, res) => {
res.on('searchEntry', (entry) => {
results.push(entry);
});
res.on('end', () => {
connection.unbind(() => {
console.debug(results);
resolve(results);
});
});
});
});
}
let ldapObjects = await listObjects('dc=home', '(objectClass=posixAccount)');
console.log(ldapObjects);
After helpful suggestions in the comments, the solution was to Move the line return new Promise((resolve, reject) => { down so that it only wraps the connection.search(…) part as suggested by Bergi
Here is the code after that modification:
#!/usr/bin/env node
import ldapjs from 'ldapjs';
const ldapURL = [ 'ldap://127.0.0.1:389' ];
const bindDN = 'uid=search,dc=home';
const bindPassword = 'P#ssw0rd';
function connect(serverURL) {
return new Promise((resolve, reject) => {
const client = ldapjs.createClient({
url: serverURL
});
client.on('connect', () => {
console.debug('Connected to:', ldapURL);
console.debug('Binding as:', bindDN);
client.bind(bindDN, bindPassword, (err) => {
if (err) {
console.debug(err.message);
reject('Bind credentials rejected.');
}
else {
resolve(client);
}
});
});
client.on('error', (err) => {
reject('Unable to connect to ' + serverURL);
});
});
}
/**
* Search LDAP and return objects.
* #baseDN {string} Where to start, like 'ou=People,dc=example,dc=com'
* #filter {string} Optional LDAP query to limit results, like '(objectClass=posixAccount)'
* #returns {promise} ... Eventually.
*/
async function listObjects(baseDN, filter) {
let connection = await connect(ldapURL).catch((err) => {
console.error('LDAP server error:', err);
reject(err);
});
let opts = {
filter: filter,
scope: 'sub'
};
let results = [];
return new Promise((resolve, reject) => {
connection.search(`${baseDN}`, opts, (err, res) => {
res.on('searchEntry', (entry) => {
results.push(entry);
});
res.on('end', () => {
connection.unbind(() => {
resolve(results);
});
});
});
});
}
let ldapObjects = await listObjects('dc=home', '(objectClass=posixAccount)');
console.log(ldapObjects);
I think you have to remove new Promise because async return the data wraps in the Promise. I think that your ldapObjects store a Promise and listObjects return a Promise wrap inside a Promise.
I want to write some tests for a method which reads from a JSON file (simulating a db) and returns the correct name, given that exists.
This is the code I have written for my method. It does throw an error when the id is not valid.
const getOne = (id, callback) => {
...
fs.readFile('db.json', (err, data) => {
if (err) {
throw new Error('Error reading file');
}
const person = JSON.parse(data)
.filter(el => el.id === id)
.map(el => el.name);
if (person.length === 0) {
throw new Error('It does not match DB entry');
}
callback(person);
});
...
The test I have written is:
it('Should reject an invalid id', (done) => {
api.getOne(100, (person) => {
try {
personFromDB = person;
} catch (error) {
assert.throws(() => {
}, new Error('It does not match DB entry'));
//done();
}
But it doesn't seem to pass the test. When I have the 'done()' uncommented, it passes the test, but I don't think it is because I pass the actual test, but rather because the test gets in the catch and executes the done() callback.
Any help, guidance or recommendation is much appreciated.
You won't be able to catch an Error being thrown in the fs.readFile callback.
Instead, pass any errors to the callback you pass to getOne.
Then you can check if an Error got passed to your callback in your test.
Here is a working example to get you started:
const fs = require('fs');
const assert = require('assert');
const api = {
getOne: (id, callback) => {
// ...
fs.readFile('db.json', (err, data) => {
if (err) return callback(err); // <= pass err to your callback
const person = JSON.parse(data)
.filter(el => el.id === id)
.map(el => el.name);
if (person.length === 0) return callback(new Error('It does not match DB entry')); // <= pass the Error to your callback
callback(null, person); // <= call the callback with person if everything worked
})
}
}
it('Should reject an invalid id', done => {
api.getOne(100, (err, person) => {
assert.strictEqual(err.message, 'It does not match DB entry'); // Success!
done();
});
});
I have 3 layer callbacks like this :
app.post('/', (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
//first
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`, function (err) {
if (err) return res.status(500).send(err);
//second
trainOutput.mv(`inputs/${req.body.caseName}/train_output.csv`, function (err) {
if (err) return res.status(500).send(err);
//third
testInput.mv(`inputs/${req.body.caseName}/test_input.csv`, function (err) {
if (err) return res.status(500).send(err);
res.send('success');
});
});
});
});
In this case, there are only 3 file uploads. In another case, I have more than 10 file uploads, and it makes 10 layer callbacks. I know it because of JavaScript asynchronous.
Is there any way, with this case, to make a beautiful code? This is because when it 10 layer callbacks, the code looks horizontally weird.
Thanks
You can use the following code to make you code look better and avoid callback hell
app.post('/', async (req, res) => {
var filename = `outputs/${Date.now()}_output.json`;
let trainInput = req.files.trainInput;
let trainOutput = req.files.trainInput;
let testInput = req.files.trainInput;
try {
var result1 = await trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`);
var result2 = await trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`);
var result2 = await testInput.mv(`inputs/${req.body.caseName}/test_input.csv`);
res.send('success');
}
catch (error) {
res.status(500).send(error);
}
});
You can make the functions return a Promise
I advice to make one function because you do the same thing 3 times. In this case I called the function 'save' but you can call it what ever you want. The first parameter is the file end the second the output filename.
function save(file, output) = return new Promise((resolve, reject) => {
file.mv(`inputs/${req.body.caseName}/${output}`, err =>
if (err) return reject(err)
resolve()
})
Promise.all([
save(req.files.trainInput, 'train_input.csv'),
save(req.files.trainInput, 'train_output.csv'),
save(req.files.trainInput, 'test_input.csv')
])
.then(_ => res.send(200))
.catch(err => res.send(400);
What version of Node you using? If async/await is available that cleans it up a bunch.
const moveCsv = (file, dest) => {
return new Promise((resolve, reject) => {
//third
file.mv(dest, function (err) {
if (err) reject(err);
resolve();
});
})
}
app.post('/', async(req, res) => {
try {
var filename = `outputs/${Date.now()}_output.json`;
const {
trainInput,
trainOutput,
testInput
} = req.files;
const prefix = `inputs/${req.body.caseName}`;
await moveCsv(trainInput, `${prefix}/train_input.csv`);
await moveCsv(trainOutput, `${prefix}/train_output.csv`);
await moveCsv(testInput, `${prefix}/test_input.csv`);
res.send('success');
} catch(err) {
res.status(500).send(err);
}
});
I'm also assuming here that your trainInput, trainOutput, testOutput weren't all meant to be req.files.trainInput.
Just be careful since the synchronous nature of the await calls are thread blocking. If that writer function takes ages you could also looking at putting those calls onto a worker thread. Won't really matter if your requests to that server endpoint are fast and non-frequent.
You can add RXJS to your project and use Observables.forkJoin()
Solution with Observables(assuming that trainInput.mv() returns Observable):
/* Without a selector */
var source = Rx.Observable.forkJoin(
trainInput.mv(`inputs/${req.body.caseName}/train_input.csv`),
trainInput.mv(`inputs/${req.body.caseName}/train_output.csv`),
trainInput.mv(`inputs/${req.body.caseName}/test_input.csv`)
);
var subscription = source.subscribe(
function (x) {
// On success callback
console.log('Success: %s', x);
},
function (err) {
// Error callback
console.log('Error');
},
function () {
// Completed - runs always
console.log('Completed');
});
// => Success: [result_1, result_2, result_3] or Error
// => Completed
This may be a stupid question but I was not able to find an answer.
My db query (node -> PostgreSQL) is not firing the callback when the query function itself is called from another callback like this:
routes.js
router.post("/getsheet/", (req, res) => {
googleAPI.getSheet(googleToken, req.body.sheetid).then((invoices) => {
templateData.sheetData = invoices;
templateData.sheetData.length = invoices.length;
// When sheet is received, db query runs but it's callback won't
// function takes an array and index
db.query(invoices, 1).then((db_results) => {
console.log(db_results);
res.redirect("/");
}).catch((db_error) => {
console.error(db_error)
res.redirect("/");
});
}).catch((error) => {
console.error(error);
});
});
db.js
const query = (data, index) => {
return new Promise((resolve, reject) => {
// console log fires but not the callback function so it does not resolve or reject
console.log("... querying index ... " + index + " customer: " + data[index].name);
client.query(`SELECT * FROM customer WHERE customer_number=${data[index].customer_id};`, (err, res) => {
// does not run this block
if (!err) {
resolve(res);
} else {
return reject(err);
}
});
});
}
Thanks! :)
Have you already established a connection? You'd need to do that first before trying to run a query. See an example here:
http://mherman.org/blog/2015/02/12/postgresql-and-nodejs/#.WeJpJWhSyUk
pg.connect(connectionString, (err, client, done) => {
// Handle connection errors
if(err) {
done();
console.log(err);
return res.status(500).json({success: false, data: err});
}
// SQL Query > Select Data
const query = client.query('SELECT * FROM items ORDER BY id ASC;');
// Stream results back one row at a time
query.on('row', (row) => {
results.push(row);
});
// After all data is returned, close connection and return results
query.on('end', () => {
done();
return res.json(results);
});
});
Also, I'd highly recommend you use npm pg-promise. Will simplify your life.
Lastly, you're also missing the point of promises. I'd encourage you to use the newer async/await. However if you don't want to and/or can't, at least see why you're just getting back into a callback hell... promises are in place to help reduce all the call back nesting. Really you should reduce the nest level of your promises, like so:
router.post("/getsheet/", (req, res) => {
googleAPI.getSheet(googleToken, req.body.sheetid).then((invoices) => {
templateData.sheetData = invoices;
templateData.sheetData.length = invoices.length;
// When sheet is received, db query runs but it's callback won't
// function takes an array and index
return db.query(invoices, 1);
}).then((db_results) => {
console.log(db_results);
res.redirect("/");
}).catch((error) => {
console.error(error);
res.redirect("/");
});
});
How I should modify the following code, so I can make sure Process3 is triggered after Process2.update or Process2.create completed?
The main purpose for following code is I want to makeProcess1 finished. Then check if id exist, if yes, Process2.update is triggered. if not, Process2.create is triggered.Once Process2 finished, check if cmd existed. if yes,triggered Process3.
run: function (req, res) {
if (req.session) {
const values = req.params.all();
const id = values.id;
const cmd = values.cmd;
const param = _.omit(values, ['cmd', 'id']);
const cb1 = (e, d) => {
if (e) {
console.log(e);
res.status(400).send({ e });
} else {
Process1(values);
res.status(200).send({ d });
}
};
const cd2 = (id, param, cb1) => {
if (id) {
Process2.update({ id }, param, cb1);
} else {
Process2.create(param, cb1);
}
};
if (cmd) {
cd2(id, param, cb1, Process3(values, cmd));
}
else {
cd2(id, param, cb1);
}
} else {
res.status(403).send({ e: 'Forbidden access.' });
}
}
try approach by following, but not sure how I can pass argument id, params to Process2 and process3
let async = require('async');
const Process1 = (value, cb) => {
console.log("Process1()");
console.log(value);
cb(null, value + 1);
};
const Process2 = (value, cb) => {
console.log("value(): wait 5 sec");
console.log(value);
cb(null, value+10);
};
const Process3 = (value, cb) => {
console.log(value);
console.log("Process3(): wait 5 sec");
cb(null, value+100);
};
let Pro_1_2 = async.compose(Process2, Process1);
let Pro_2_3 = async.compose(Process3, Process2);
Pro_1_2(1, (e, r) => {
Pro_2_3(r, (error, result) => {
console.log(result);
});
});
The code you posted in your original question seems pretty twisted up, so I'm not going to attempt to rewrite it, but in general if you want to perform asynchronous calls which depend on each other, async.auto is a good way to go. Rather than declaring variables at the top that you attempt to mutate via some function calls, it's better to make Process1, Process2 and Process3 asynchronous functions that call their callbacks with a new values object. Something like:
async.auto({
doProcess1: function(cb) {
// Assuming Process1 calls `cb(undefined, newValues)` when done.
Process1(values, cb);
return;
},
doProcess2: ['doProcess1', function(results, cb) {
if (results.doProcess1.id) {
Process2.update({id: results.doProcess1.id}, cb);
return;
} else {
Process2.create(_.omit(results.doProcess1, ['cmd', 'id']), cb);
return;
}
}],
doProcess3: ['doProcess2', function(results, cb) {
if (results.doProcess2.cmd) {
Process3(results.doProcess2, cb);
return;
}
else {
cb(undefined, results.process2);
return;
}
}]
}, function afterProcess3(err, results) {
// Handler err or process final results.
});
Note all the return calls. They're not strictly necessary, but good practice to avoid accidentally running more code after calling your asynchronous functions.
Have you considered using "compose", from async.js?
const a = (data, cb) => {
var result = 'a';
cb(null, result);
};
const b = (data, id, cb) => {
var result = 'b';
cb(null, result);
};
const c = (data, cb) => {
// stuff to do with result
};
var aThenC = async.compose(c, a);
var bThenC = async.compose(c, b);
if (useA) {
aThenC(data, (result) => {
// result from c
res.status(200).send(result);
});
} else {
bThenC(data, id, (result) => {
// result from c
res.status(200).send(result);
});
}
In this scenario, a and b are your Process2 create and update, respectively, and c is the callback to Process3, if I understood correctly.
EDIT: You'll only have to enter the initial parameters (e.g. register ID) on the composed function. What composes really do is this: a(b(c(param))). That param is basically everything you need to start the process. The parameters for the following functions will be set inside the function before that.
I'll add code to support it as soon as I'm on a keyboard.