We Keep Coding

Callback is already called using async?

Once fs.readFile loop through all files and get the matching data and push it to results, I want to call callback(results) so i can send response to client. I am getting an error with below code Error: Callback is already called HOw can i resolve this issue using async approach.
app.js
searchFileService.readFile(searchTxt, logFiles, function(lines, err) {
console.log('Logs', lines);
if (err)
return res.send();
res.json(lines);
})
readFile.js
var searchStr;
var results = [];
function readFile(str,logFiles,callback){
searchStr = str;
async.map(logFiles, function(logfile, callback) {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(null,err);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(results,null);
}
});
});
}), function(error, result) {
results.map(result,function (result){
console.log(result);
});
};
}

Note: this answer is an extension to trincot's answer. So if this answers your question, kindly mark his as the answer!
You said: Once fs.readFile loop through all files and get the matching data and push it to results then I don't think .map is the appropriate function for this, to be honest. This is for transforming every element from an array into another which is not what you are doing.
A better method would be .eachSeries to read one file at a time.
It's a good idea to rename your second callback to something else e.g. done to not confuse yourself (and others). Calling done() is for telling that the operation on the file is completed as in we are "done" reading the file.
Lastly, be careful with your typos. The first one may have prevented you from getting into the last part.
var results = [];
var searchStr;
function readFile(str, logFiles, callback) {
searchStr = str;
// loop through each file
async.eachSeries(logFiles, function (logfile, done) {
// read file
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function (err, data) {
if (err) {
return done(err);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
}
});
// when you are done reading the file
done();
});
// wrong: }), function (err) {
}, function (err) {
if (err) {
console.log('error', err);
}
console.log('all done: ', results);
// wrong: results.map(result, function (result){
results.map(function (result){
console.log(result);
});
// send back results
callback(results);
});
}

(Node) Running many if statements at the same time always returns undefined after 6

Edit: I can't believe I forgot what I was asking! I always get typeError Undefined for whatever the sixth if statement is testing. When I swap around the fifth/sixth statements the if(!data.body.main) comes out as undefined which it doesn't when in the fifth position however I can't figure out why.
So I'm trying to test POST data from express/bodyparser for its contents and then fill default information to be used by the rest of my program if the fields aren't set in the post data.
To do this I'm running many if statements in a list like so:
function setChannelOptions(data, channelSettings){
console.log('Setting channel options');
var channelOptions = {
channelName: data.body.name,
mainNumber: data.body.main,
backupNumber: data.body.backup,
music: data.body.music,
spoofNumber: data.body.spoof,
price: data.body.price,
customerId: data.body.customerid,
allowOutbound: data.body.allowoutbound,
welcomeMessage: data.body.welcomemessage,
accountCode: data.body.accountCode
}
if(!data.body.accountCode){
channelOptions.accountCode = findChannelID(data.body.name, function (err, data) {
if(err) throw err;
return data.channelid;
});
}
if(!data.body.allowoutbound){
channelOptions.allowOutbound = 0;
}
if(!data.body.welcomemessage){
channelOptions.welcomeMessage = 'sometext';
}
if(!data.body.price){
channelOptions.price = '-1';
}
if(!data.body.backup){
var data = 'NG';
channelOptions.backupNumber = assignNumber(data, function(err, data){
if (err) throw err;
return data.number;
})
}
if(!data.body.main){
var data = 'VU';
channelOptions.mainNumber = assignNumber(data, function(err, data){
if (err) throw err;
return data.number;
})
}
channelSettings(null, channelOptions);
I feel like it might be caused by the callback at the end returning before all of the if statements are complete but without nesting each of these if's in callback functions I wouldn't know how to stop that.
Thanks for having a look, I'm a bit new to node and JS in general.

Try using other name for your variables. You use data identifier as incoming parameter of your setChannelOptions function, and then redefine it with var keyword. Variables defined with var are functionally, not block-scoped. Refactor your code to something like:
if(!data.body.backup){
var code = 'NG';
channelOptions.backupNumber = assignNumber(code, function(err, data){
if (err) throw err;
return data.number;
})
}
if(!data.body.main){
var code = 'VU';
channelOptions.mainNumber = assignNumber(code, function(err, data){
if (err) throw err;
return data.number;
})
}

Basically you redefined your data arg. You should probably copy data at the top of your function and use that copy instead.
if(!data.body.backup){
var data = 'NG'; // <----- redefinition
channelOptions.backupNumber = assignNumber(data, function(err, data){
if (err) throw err;
return data.number;
})
}
if(!data.body.main){ // data === 'NG'
var data = 'VU';
channelOptions.mainNumber = assignNumber(data, function(err, data){
if (err) throw err;
return data.number;
})
}

Use promise library like bluebird.
require a promise library
var Promise = require('bluebird'); // install it using nom
Declare a array of promises as
var promises = [];
if(!data.body.accountCode){
promises.push(
findChannelID(data.body.name).then(function (data){
channelOptions.accountCode = data.channelid;
})
);
}
//.....rest all other checks
Promise.all(promises).then(function(){
channelSettings(null, channelOptions);
});
you will need to convert your callback function (like findChannelID) to return a Promise. which is very simple as
function findChannelID(name){
return new Promise(function(resolve, reject){
//.. do you math here
// finally return value as
// resolve(the calculated value);
resolve({data:{name: 'something'}})
});
}

Node for-loop callback required

I want to change the Tags format which I am fetching form one of the collections. Tags data contains some KC ids in an array which I am using to get KC data and insert in TagUnit to get final response format.
var newTags = Tags.map(function(TagUnit) {
for (var i = 0; i < TagUnit.kcs.length; i++) {
KCArray = [];
KC.findById(TagUnit.kcs[i], function(error, data) {
KCMap = {};
KCMap['kc_id'] = data._id;
KCMap['kc_title'] = data.title;
KCArray.push(KCMap);
if (KCArray.length == TagUnit.kcs.length) {
TagUnit.kcs = KCArray;
}
});
}
return TagUnit;
});
response.send(JSON.stringify(newTags));
But I am not getting desired result. Response is giving out Tag data in initial for instead of formatted form. I guess it is due to event looping. I will be grateful if someone can help me with this.
**Edit: ** I am using MongoDB as database and mongoose as ORM.

I'd suggest using promises to manage your async operations which is now the standard in ES6. You don't say what database you're using (it may already have a promise-based interface). If it doesn't, then we manually promisify KC.findById():
function findById(key) {
return new Promise(function(resolve, reject) {
KC.findById(key, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
Then, assuming you can do all these find operations in parallel, you can use Promise.all() to both keep track of when they are all done and to order them for you.
var allPromises = Tags.map(function(TagUnit) {
var promises = TagUnit.kcs.map(function(key) {
return findById(key).then(function(data) {
// make resolved value be this object
return {kc_id: data._id, kc_title: data.title};
});
});
// this returns a promise that resolves with an array of the kc_id and kc_title objects
return Promise.all(promises).then(function(results) {
return {
_id: TagUnit._id,
kcs: results
};
});
});
// now see when they are all done
Promise.all(allPromises).then(function(results) {
response.send(JSON.stringify(results));
}).catch(function(err) {
// send error response here
});

You can use promises or Async module
var async = require('async');
...
function getKC (kc, callback) {
KC.findById(kc, function(err, data) {
if (err)
return callback(err);
callback(null, {kc_id: data._id, kc_title: data.title})
});
}
function getKCs (tag, callback) {
async.map(tag.kcs, getKC, callback);
}
async.map(Tags, getKCs, function(err, results){
if (err)
return console.log(err.message);
res.json(results); // or modify and send
});
P.S. Perhaps, code contains errors. I cann't test it.

Callback problems

I am new into javascript, and currently I'm trying to learning callback to my script. This script should return reduced words in array of objects
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = []
list.forEach(function (val) {
readFile(val, function(error,data){
txtObj.push(data)
})
})
function readFile(src, cb){
fs.readFile(src,'utf8', function (error,data) {
if (error) return callback(error,null)
return mapred(data)
})
}
return callback(null,txtObj)
}
But it returns empty array. Any help would be appreciated.
Thanks!

`fs.readFile`
is an asynchronous function, before it's done and result callback is invoked, you are returning the empty txtObj array.
how to fix it ?
call return callback(null,txtObj) after fs.readFile is finished running.
and also, as you are running asynchronous function on an array of items one-by-one, it might not still work the way you want. might want to use modudles like async in nodejs
Here comes an asynchronous version using module async. synchronous file operation is strongly objected :)
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
async = require('async');
async.each(list, readFile, function(err) {
callback(err,txtObj)
});
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}
EDIT : You can do this without async, but it is little bit dirty isn't it ? also its OK if you remove the self invoking function inside the forEach, i included so that you can access the val, even after the callback is done
var fs = require('fs')
var dict = ['corpus.txt','corpus1.txt','corpus2.txt'];
mapping(dict, function(error,data){
if(error) throw error
console.log(data)
})
function mapping(list, callback){
var txtObj = [],
counter = list.length,
start = 0;
list.forEach(function (val) {
(function(val)
readFile(val, function(error,data) {
txtObj.push(data);
start++;
if(error || (start === counter)) {
callback(error,txtObj);
}
}))(val);
})
function readFile(src, cb) {
fs.readFile(src,'utf8', function (error,data) {
if (error) {
cb(error);
}
else {
txtObj.push(mapred(data));
cb(null);
}
})
}
}

The reason you are getting an empty array result is that you are performing the callback before the readFile function has a chance to populate the array. You are performing multiple asynchronous actions but not letting them to complete before continuing.
If there was only one async action, you would call callback() in the callback function of readFile, but as you need to perform multiple async actions before calling callback(), you should consider using fs.readFileSync().
Sometimes sync cannot be avoided.
function mapping(list, callback)
{
var txtObj = []
list.forEach(function(val)
{
try { txtObj.push(mapred(fs.readFileSync(val, 'utf8'))) }
catch(err) { callback(err) }
})
callback(null, txtObj)
}

Correct procedure to call multiple asynchronous functions in node.js

I have a requirement where I need to get the records from table 1 and store in redis cache and once redis cache is finished storing, get table 2 records and store in redis cache. So there are 4 asynchronous functions.
Steps:
Get table 1 records
Store in redis cache
Get table 2 records
Store in redis cache
What is the correct procedure to handle it.
Below is the code which I have written to handle it. Please confirm whether its the right procedure or any other ways to handle it as per node.js
var redis = require("redis");
var client = redis.createClient(6379, 'path', {
auth_pass: 'key'
});
var mysqlConnection = // get the connection from MySQL database
get_Sections1()
function get_Sections1() {
var sql = "select *from employee";
mysqlConnection.query(sql, function (error, results) {
if (error) {
console.log("Error while Sections 1 : " + error);
} else {
client.set("settings1", JSON.stringify(summaryResult), function (err, reply){
if (err) {
console.log("Error during Update of Election : " + err);
} else {
get_Sections2();
}
});
}
});
}
function get_Sections2()
{
var sql = "select *from student";
mysqlConnection.query(sql, function (error, results)
{
if (error)
{
console.log("Error while Sections 2 : " + error);
}
else
{
client.set("settings2", JSON.stringify(summaryResult), function (err, reply)
{
if (err)
{
console.log("Error during Update of Election : " + err);
}
else
{
console.log("Finished the task...");
}
});
}
});
}

Create two parameterised functions. One for retrieval, one for storing.
Then promisify them both.
Then write:
return getTableRecords(1)
.then(storeInRedisCache)
.then(getTableRecords.bind(null,2))
.then(storeInRedisCache)
.then(done);
To promisify a function, something like this might work:
var getEmployees = new Promise(function(resolve, reject) {
var sql = "select *from employee";
mysqlConnection.query(sql, function (error, results) {
if (error) {
return reject();
} else {
return resolve(results);
}
});
});
If you are using an old version of NodeJS you will need a polyfill for Promise.

Here is an alternative to Ben Aston's solution using Promise.coroutine assuming promises:
const doStuff = Promise.coroutine(function*(){
const records = yield getTableRecords(1);
yield storeRecordsInCache(records);
const otherRecords = yield getTableRecords(2);
yield storeRecordsInCache(otherRecords); // you can use loops here too, and try/cath
});
doStuff(); // do all the above, assumes promisification
Alternatively, if you want to use syntax not yet supposed in Node (and use Babel to get support) you can do:
async function doStuff(){
const records = await getTableRecords(1);
await storeRecordsInCache(records);
const otherRecords = await getTableRecords(2);
await storeRecordsInCache(otherRecords); // you can use loops here too, and try/cath
})