I have search function once i have search String from clinet i want to loop through files and match the string from files in fs, I have problem in loop i want to get all match result and send result to client. Below trying to achieve but getting an error pasted in question. New to async library any help will be appreciated.
app.js
app.get('/serverSearch', function (req, res) {
var searchTxt = req.query.searchTxt;
dirDirectory.readDirectory(function(logFiles){
// res.json(logFiles);
if(logFiles){
searchFileService.readFile(searchTxt,logFiles,function(lines,err){
console.log('Logs',lines);
if (err)
return res.send();
res.json(lines);
})
}
});
console.log('Search text', searchTxt);
});
service.js
var fs = require('fs');
var path = require('path');
var async = require('async');
var searchStr;
var result = [];
//Async Method
function readFile(str, logFiles, callback) {
async.series([
//Load user to get `userId` first
function(callback) {
searchStr = str;
for (var i = 0; i < logFiles.length; i++) {
if (logFiles[i].filename !== '.gitignore') {
fs.readFile('logs/dit/' + logFiles[i].filename, 'utf8', function(err, data) {
if (err) {
return console.log(err);
}
inspectFile(data);
});
}
callback(result);
}
},
//Load posts (won't be called before task 1's "task callback" has been called)
function() {
function inspectFile(data, callback) {
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
result.push(line);
// then log it
return line;
}
});
}
}
], function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return err;
});
};
Error
if (fn === null) throw new Error("Callback was already called.");
You should be using async.map instead of series. You are miss understanding what series does, series process request top down. You are attempting to break this chain by accessing a function within the series itself. Which is a no, no.
for example:
async.series([
function() {
let i = 0;
do {
console.log("I'm first in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
},
function() {
let i = 0;
do {
console.log("I'm next in the series: ", i);
i++;
} while (i < 3);
callback(); // This tells us this function has finished.
}
]);
The output of this would be:
I'm next in the series: 0
I'm next in the series: 1
I'm next in the series: 2
until the callback is fired, which then tells async to move to the next function in the series array.
The output then would be:
I'm last in the series: 0
I'm last in the series: 1
I'm last in the series: 2
At no point in this series should you be accessing the function within the series after the current. So you should never be trying to cross access that.
With async.map you can actually perform on operation on each entity within your array, which is essentially what you are trying to do.
var results = [];
async.map(logFiles, function(logfile, callback) {
if (logfile.filename !== '.gitignore') {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(err, null);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(null, results);
}
});
}
}), function(error, result) {
results.map(result => {
console.log(result);
});
});
Also you should use util.inspect instead of console.log, it's much cleaner and has more options.
The documentation on this is a bit rough, but here it is. https://caolan.github.io/async/docs.html#map hope this helps!
You should use async.eachSeries method:
function readFile(str, logFiles, callback) {
async.eachSeries(array, function(item, cb){
//Process item
cb(error,item);
}, function(err){
if (err) {
console.log("Some error in one of the items");
callback(err);
} else {
console.log("All arrays items have been treated successfully");
callback(null);
}
});
}
And I would recommend to load the user and posts before using the async.eachSeries function.
Related
Once fs.readFile loop through all files and get the matching data and push it to results, I want to call callback(results) so i can send response to client. I am getting an error with below code Error: Callback is already called HOw can i resolve this issue using async approach.
app.js
searchFileService.readFile(searchTxt, logFiles, function(lines, err) {
console.log('Logs', lines);
if (err)
return res.send();
res.json(lines);
})
readFile.js
var searchStr;
var results = [];
function readFile(str,logFiles,callback){
searchStr = str;
async.map(logFiles, function(logfile, callback) {
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function(err, data) {
if (err) {
callback(null,err);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
callback(results,null);
}
});
});
}), function(error, result) {
results.map(result,function (result){
console.log(result);
});
};
}
Note: this answer is an extension to trincot's answer. So if this answers your question, kindly mark his as the answer!
You said: Once fs.readFile loop through all files and get the matching data and push it to results then I don't think .map is the appropriate function for this, to be honest. This is for transforming every element from an array into another which is not what you are doing.
A better method would be .eachSeries to read one file at a time.
It's a good idea to rename your second callback to something else e.g. done to not confuse yourself (and others). Calling done() is for telling that the operation on the file is completed as in we are "done" reading the file.
Lastly, be careful with your typos. The first one may have prevented you from getting into the last part.
var results = [];
var searchStr;
function readFile(str, logFiles, callback) {
searchStr = str;
// loop through each file
async.eachSeries(logFiles, function (logfile, done) {
// read file
fs.readFile('logs/dit/' + logfile.filename, 'utf8', function (err, data) {
if (err) {
return done(err);
}
var lines = data.split('\n'); // get the lines
lines.forEach(function(line) { // for each line in lines
if (line.indexOf(searchStr) != -1) { // if the line contain the searchSt
results.push(line);
}
});
// when you are done reading the file
done();
});
// wrong: }), function (err) {
}, function (err) {
if (err) {
console.log('error', err);
}
console.log('all done: ', results);
// wrong: results.map(result, function (result){
results.map(function (result){
console.log(result);
});
// send back results
callback(results);
});
}
Are both my examples the same in terms of functionality considering the fact that in error handeling I'm terminating by res.json(400, err)? Also I would like to know that in my second example the second async.each always run after the first async.each, so using results1 in the second async.each is safe? Sorry I'm new to Node and async!
Example1: where I'm using the results of each async.each in the last block as an input of the other async.each:
var results1 = {};
var results2 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
}
});
or Example2: where I have separate async.each blocks
var results1 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
}
});
var results2 = {};
async.each(results1, function (item, callback) {
//Do something here and add some data to results2
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results2);
}
});
UPDATED:
Since the second approach is not right way and it is not guaranteed that the second async.each runs after the first one the problem is: Does it mean I cannot have a simple for loop like the following example either? If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well and not a for loop, do you know how I can have this recursive functionality here?
var results1 = {};
var results2 = [];
var results3 = {};
async.each(inputs, function (input, callback) {
//Do something here and add some data to results1
callback();
}, function (err) {
if (err) {
//Handeling error
} else {
// So in this case that I need to have nested function, does it mean I cannot have a simple for loop like this as well?
// If yes, it is easy to change this one to async.each, but the problem is this one is recursive and that's make it complicated! If this needs to be async as well, do you know how I can have this recursive functionality here?
for (var input in inputs) {
inferFromUnion(inputs[input], results1);
results2.push(inputs[input]);
}
async.each(results2, function (item, callback) {
//Do something here and add some data to results2
}, function (err) {
if (err) {
//Handeling error
} else {
console.log("Final result", results3);
}
});
}
});
// Here just checking each object schema and if they are missing any fields from results1 we add that field with a value of null
function inferFromUnion(obj, allFields) {
Object.keys(allFields).forEach(function (key) {
if (lodash.isUndefined(obj[key])) {
if (lodash.isPlainObject(allFields[key])) {
obj[key] = {};
inferFromUnion(obj[key], allFields[key]);
} else {
obj[key] = null;
}
}
});
}
The first example is the way to go, if you want to use results of the first bunch of calls in the second bunch. The second example won't work, because the second async.each() is guaranteed to run before the callbacks bound to your asynchronous operations.
Asynchronous recursion with loops is very much possible:
(function doSomeAsyncRecursion (results) {
async.each(someItems, function (item, callback) {
// ...
}, function () {
if (results /* ... (are incomplete) */) {
doSomeAsyncRecursion(results);
} else {
// ... (results are complete now, do something with them)
}
});
})(/* initial value of results */);
These two examples are different in desing. First example will run second async after first async is successful. But second example will run second async everytime, if theres an error or not.
I have a parse cloud code function, in this function I preform a query on some items then using a for loop I save some of those items. But the for loop continues and does not save some of the items before correctly.
Heres a general version of the code:
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
queryMenuURL.find().then(function(resultsMenuURL) {
//********************************************************************************************************
//I want the save to happen before it goes thought this for loop for the second time, and so on
for (var i = 0; i < resultsMenuURL.length; i++) {
var location = resultsMenuURL[i].get("Location");
Parse.Cloud.httpRequest({
url: url,
success: function(httpResponse) {
var xmlData = httpResponse.text;
xmlReader.read(xmlData, function (err, res){
if(err) return console.log(err);
for (var i3 = 0; i3 < res.menu.day.at(dayNumber).meal.count(); i3++) {
var meal = res.menu.day.at(dayNumber).meal.at(i3).attributes().name;
testItem.set("meal", meal);
testItem.set("location", location);
testItem.save().then(function(testItem) {
});
}
}
});
},
error: function(httpResponse) {
console.error('Request failed with response code ' + httpResponse.status);
}
});
}
});
});
I have looked at the parse docs, but I can't make sense of them, the promises section I just can't grasp.
Thanks so much for the help in advance
EDIT 2
When I have your code like this I get the error TypeError: Cannot call method 'reduce' of undefined
Parse.Cloud.define("saveCurrentDayItems23", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
results = res;
}
});
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, location, testItem) {
return meals.reduce(function(promise, meal) {
return promise.then(function() {
testItem.set("meal", meal.attributes().name);
//the line above does not work it cannot get meal, it is undefined
testItem.set("location", location);
return testItem.save();
});
}, Parse.Promise.as());
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
for (var i = 0; i < resultsMenuURL.length; i++) {
var url = resultsMenuURL[i].get('URL');
return resultsMenuURL.reduce(function(promise, item) {
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of item?
}).then(function(httpResponse) {
return readResponse_async(httpResponse.text).then(function() {
var TestItem = Parse.Object.extend("TestItem");
var testItem = new TestItem();
return saveMeals_async(result.menu.day.meal.counter.dish.name.text(),item.get("Location"),
testItem);
//this line above does not work, it sends only one string, not an array, so reduce cannot be called
});
});
});
}, Parse.Promise.as());
}
}).fail(function(err) {
console.error(err);
});
});
To do as the question asks ("I want the save to happen before it goes [through] this for loop for the second time, and so on"), is fairly involved - not really a beginners' problem.
It appears that you have several async operations here, viz :
queryMenuURL.find()
Parse.Cloud.httpRequest()
xmlReader.read()
testItem.save()
These operations need to work in cooperation with each other to give the desired effect.
queryMenuURL.find(), Parse.Cloud.httpRequest() and testItem.save() each appear to return a promise, while xmlReader.read() takes a node style callback, which makes things slightly awkward but not too bad.
You could write the code as one big block but you would end up with patterns within patterns. To make everything readable, you can pull out some of the code as (readabe) worker functions, leaving behind a (readable) master routine.
To convert your current outer for loop to set of sequential operations, you need the following pattern, which exploits Array.prototype.reduce() to build a .then chain, and returns a promise :
function doThings_async(arr) {
return arr.reduce(function(promise, item) {
return promise.then(function(result) {
return doSomething_async(item, result);
});
}, resolvedPromise);
}
You will see below that this pattern is also used for the inner for loop, though other possibilities exist.
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
}
}
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, location, testItem) {
return meals.reduce(function(promise, meal) {
return promise.then(function() {
testItem.set("meal", meal.attributes().name);
testItem.set("location", location);
return testItem.save();
});
}, Parse.Promise.as());
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
...
return resultsMenuURL.reduce(function(promise, item) {
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of item?
}).then(function(httpResponse) {
return readResponse_async(httpResponse).then(function() {
return saveMeals_async(res.menu.day.at(dayNumber).meal, item.get("Location"), testItem);
});
});
});
}, Parse.Promise.as());
}).fail(function(err) {
console.error(err);
});
});
saveMeals_async() could be written to perform its saves in parallel rather than in series, but it depends on what you want. For parallel saves, only saveMeals_async() would need to be rewritten, using a different pattern.
EDIT
Revised code based on the edits in the question.
Due to changes in saveMeals_async(), the arr.reduce(...) pattern is now used only once in the master routine.
Parse.Cloud.define("saveCurrentDayItems", function(request, response) {
// ***
// insert all the Date/dayNumber code here
// ***
var xmlReader = require('cloud/xmlreader.js');
//First worker function, promisify xmlReader.read();
function readResponse_async(xlmString) {
var promise = new Parse.Promise();
xmlReader.read(xlmString, function (err, res) {
if(err) {
promise.reject(err);
} else {
promise.resolve(res);
}
}
return promise;
}
//Second worker function, perform a series of saves
function saveMeals_async(meals, school, diningHallNumber, menuLocation) {
//Declare all vars up front
var i3, i4, i5, m,
TestItem = Parse.Object.extend("TestItem"),//can be reused within the loops?
promise = Parse.Promise.as();//resolved promise to start a long .then() chain
for (i3 = 0; i3 < meals.count(); i3++) {
m = meals.at(i3);
//get number of stations in day
for (i4 = 0; i4 < m.counter.count(); i4++) {
//get number of items at each station
for (i5 = 0; i5 < m.counter.at(i4).dish.count(); i5++) {
//Here a self-executing function is used to form a closure trapping `testItem`.
//Otherwise the `testItem` used in `promise.then(...)` would be the last
//remaining `testItem` created when all iterations are complete.
(function(testItem) {
testItem.set("item", m.counter.at(i4).dish.at(i5).name.text());
testItem.set("meal", m.attributes().name);
testItem.set("school", school);
testItem.set("diningHallNumber", diningHallNumber);
testItem.set("schoolMenu", menuLocation);
//build the .then() chain
promise = promise.then(function() {
return testItem.save();
});
})(new TestItem());
});
}
}
return promise;
}
var MenuURL = Parse.Object.extend("MenuURL");
var queryMenuURL = new Parse.Query(MenuURL);
//Master routine
queryMenuURL.find().then(function(resultsMenuURL) {
return resultsMenuURL.reduce(function(promise, menuItem) {
var url = menuItem.get('URL'),
school = menuItem.get("school"),
diningHallNumber = menuItem.get("diningHallNumber"),
menuLocation = menuItem.get("menuLocation");
return promise.then(function() {
return Parse.Cloud.httpRequest({
url: url,
//data: ... //some properties of menuItem?
}).then(function(httpResponse) {
return readResponse_async(httpResponse).then(function(res) {
if (res.menu.day.at(dayNumber).meal) {
return saveMeals_async(res.menu.day.at(dayNumber).meal, school, diningHallNumber, menuLocation);
} else {
return Parse.Promise.as();//return resolved promise to keep the promise chain going.
}
});
});
});
}, Parse.Promise.as());
}).fail(function(err) {
console.error(err);
});
});
untested so may well need debugging
I have a array of ids:
var ids = ['53asd3','53asd2','53asd5'];
Each id has a corresponding document in the mongodb.
I want to generate a object by populating data from each of them and save in some other document. Like this:
{
person: { /* data from some collection populated with first id */},
company : { /* data from some collection populated with first id */},
employee : {/* data from some collection populated with first id */}
}
WHAT I DID
var document = {}
models.persons.find({_id:'53asd3'},function(err,data){
if(!err) {
document['persons']=data;
models.company.find({_id:'53asd2'},function(err,data){
if(!err) {
document['company']=data;
models.employee.find({_id:'53asd2'},function(err,data){
if(!err) {
document['employee']=data;
document.save(function(err){ });
}
});
}
});
}
});
So I just use nested calls using callbacks and somewhat make it synchronous. Is there a chance where I can execute all these three find queries in parallel and then execute the save command? I actually want to leverage the async nature of node.js. Any suggestions?
You could build something like async.parallel yourself if you don't want to include an external library. Here's what a simple parallel function might look like. It could be a nice exercise to implement the other functions in the async library.
var parallel = function () {
var functs = arguments[0];
var callback = arguments[1];
// Since we're dealing with a sparse array when we insert the results,
// we cannot trust the `length` property of the results.
// Instead we count the results separately
var numResults = 0;
var results = [];
var getCallback = function (i) {
return function (err, res) {
if (err) {
callback(err)
}
else {
results[i] = res;
numResults += 1;
if (numResults === functs.length) {
callback(null, results);
}
}
}
}
functs.forEach(function (fn, i) {
fn(getCallback(i));
});
};
var getTest = function (timeout) {
return function (callback) {
setTimeout(function () {
callback(null, timeout);
}, timeout);
}
};
parallel([getTest(99), getTest(1000), getTest(199)], console.log.bind(console));
>> null [99, 1000, 199]
Then in your case you can do something like
var findItem = function (collection, id) {
return function (callback) {
collection.find({
_id: id
}, callback);
};
};
parallel([
findItem(models.persons, '53asd3'),
findItem(models.company, '53asd2'),
findItem(models.employee, '53dsa2')
], function (err, results) {
document.persons = results[0];
document.company = results[1];
document.employee = results[2];
document.save(function (err) {
// and so on
});
});
Here I'm trying to make an array of functions with arguments to Async.js.
The array consists of instances of RunRequest that are supposed to be set inside the loop in MakeRequest, right before I try pass the function array to Async.
So the request in request[i] is fine when I pass it to RunRequest, but inside RunRequest function its undefined?
// Process Requests
function RunRequest(db, collection, request, requestHandler, callback) {
console.log('this happening?')
// Connect to the database
db.open(function(err, db) {
if(err) callback(err, null);
// Connect to the collection
db.collection(collection, function(err, collection) {
if (err) callback(err, null);
// Process the correct type of command
requestHandler(db, collection, request, callback);
});
});
}
function MakeRequest(request, requestHandler, collection, callback) {
var data = [];
var doneRequest = function(err, results) {
console.log('done was called')
if (err) callback(err, null);
else if(results) data = data.concat(results);
}
// Make Request Array
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
requestArray.push(function() {RunRequest(dbConnection, collection, request[i], requestHandler, doneRequest)});
}
// Make all requests in Parallel then invoke callback
Async.parallel(requestArray, function(err, results) {
console.log('Step WORKS')
if(data) {
var uniqueResults = [];
for(var i = 0; i < data.length; i++) {
if( !uniqueResults[data[i]['_id']] ) {
uniqueResults[uniqueResults.length] = data[i];
uniqueResults[data[i]['_id']] = true;
}
callback (null, uniqueResults);
}
}
});
}
// Request Handlers
var FindHandler = function(db, collection, request, callback) {
console.log('FindHandler was called')
console.log('Request Query' + request);
collection.find(request.query, function(err, cursor) {
if (err) callback(err, null);
cursor.toArray(function(err, docs) {
if (err) callback(err, null);
if(docs.length <= 0) console.log("No documents match your query");
var requestResults = [];
for(var i=0; i<docs.length; i++) {
requestResults[requestResults.length] = docs[i];
}
db.close();
callback(null, requestResults);
});
});
}
This is just a shot in the dark:
I think the problem is how you call RunRequest inside MakeRequest. Inside the first for-loop you are iterating over request and use request[i] inside an anonymous function, but i changes in the next iteration and the current scope gets lost when RunRequest is actually executed.
It's hard to reproduce, but try this:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
console.log('run request was called')
var dbConnection = new Db('KidzpaceDB', new Server(Host, Port, {auto_reconnect: true}))
function wrap(dbConnection, collection, request, requestHandler, doneRequest) {
return function() {
RunRequest(dbConnection, collection, request, requestHandler, doneRequest);
}
}
requestArray.push(wrap(dbConnection, collection, request[i], requestHandler, doneRequest));
}
This is a scoping issue. When the loop finishes variable i is set to request.length, so request[i] is undefined.
Wrap your code with anonymous function like that:
var requestArray = [];
for(var i = 0; i < request.length; i++) {
(function(i) {
console.log('run request was called');
var dbConnection = ...;
requestArray.push( ... );
})(i);
}
or even better ( avoids unnecessary overhead when creating anonymous functions ):
var requestArray = [];
request.forEach( function( el ) {
console.log('run request was called');
// the other code goes here, use el instead of request[i]
});
EDIT The callback is not called, because you don't define functions in arrays correctly. You will have to refactor your code a bit, so let me just show you how it should be:
requestArray.push(function(callback) { // <---- note the additional parameter here
// do some stuff, for example call db
db.open(function(err, db) {
if (err) {
callback( err );
} else {
callback( );
}
});
});
If you want to use RunRequest, then you need to pass callback as an additional parameter to RunRequest ( so use callback instead of doneRequest ).