I am reading all the json files in the directory and performing CRUD operations on them in Couchbase in a cyclic manner for some performance benchmarks.
function readFiles(dirName,crudOps, onError){
fs.readdir(dirName,function(err,filenames){
if (err) {
onError(err);
return;
}
var circularIterator = cyclicIterator(filenames);
while(1){
fname = circularIterator.next().value;
fs.readFile(dirName + fname, function(err, content) {
console.log(fname) // NEVER REACHES HERE
if (err) {
console.log(err);
return;
}
crudOps(fname, content);
});
});
}
})
}
However, it does not seem to be executing the fs.readFile function. How can I make it 'circular' iterate through the list of filenames and use the file content for my crudOps function?
EDIT:
Per the suggestion by Ry, I have used a promise to readFile.
const readFile = util.promisify(fs.readFile);
async function getStuff(filename) {
return await readFile(filename);
}
function readFiles(dirName,onFileContent, onError){
fs.readdir(dirName,function(err,filenames){
if (err) {
onError(err);
return;
}
var iterator = circularIterator(filenames);
//filenames.forEach(function(filename) {
while(1){
fname = iterator.next().value;
//iterator.getNext(function(filename){
//
getStuff(dirName + fname).then(data => {console.log(data); onFileContent(fname, data)})
}
})
}
This is hanging and causing my Chrome browser to crash and with the below error on my console:
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
Is there a way to make this better?
You cannot have limitless amounts of file reads, you can find the maximum your computer can handle and limit the program to that.
To limit activity you can recursively call a function that will (if not maximum active processes are started) read file and do stuff with it and if it's finished then do it again.
function readFiles(dirName, onFileContent, onError) {
fs.readdir(dirName, function (err, filenames) {
if (err) {
onError(err);
return;
}
const recur = (iterator) => {
const fname = iterator.next().value;
getStuff(dirName + fname)
.then(
data => {
console.log(data);
//if the following is an asynchronous function then
// you should wait for it to finish before calling recur
onFileContent(fname, data);
//recursively call itself
recur(iterator);
}
);
};
const iterator = circularIterator(filenames);
var i = 0;
while(i++<1000){//maximum amount of active tasks
recur(iterator);
}
})
}
Working example:
var later = time => value =>
new Promise(
(resolve,reject)=>
setTimeout(
()=>resolve(value),
time
)
)
;
var waitTwoSeconds = later(2000);
function *myIterator(){
var i = 1;
while(i<=15){
yield i++;
}
}
function readFiles() {
const recur = (iterator) => {
const value = iterator.next().value;
if(value===undefined){
console.log("----- Done")
return;
}
waitTwoSeconds(value)
.then(
data => {
console.log("returned",data);
recur(iterator);
}
);
};
const iterator = myIterator();
var i = 0;
while(i++<5){
recur(iterator);
}
}
readFiles();
Related
When the code below runs I expect score to have a value (lets say: {"A": 1, "B": 2}), but when I print it I get an empty dict ({}).
I have tried to use promises but the result is the same.
driversBySeason(season) {
var query = `SELECT raceId FROM races WHERE year = ${season}`;
var score = {};
this.con.query(query, (err, drop) => {
if (err) {
console.error(err);
}
drop.forEach((element) => {
var raceId = element["raceId"];
query = `SELECT driverId, points FROM results WHERE raceId = ${raceId}`;
this.con.query(query, (err, drop) => {
if (err) {
console.error(err);
}
drop.forEach((element) => {
if (score[element["driverId"]] == undefined) {
score[element["driverId"]] = 0;
} else if (score[element["points"]] != undefined) {
score[element["driverId"]] += element["points"];
}
});
});
});
console.log(score);
});
}
First of all you need to change your .forEach loop to for const of loop or just a regular for loop because it just fires the code inside the callback and never waits for it. And then you need to change your this.con.query(...) function which has callback to promises too. Your code should be like this:
const asyncQuery = (query) => new Promise((resolve, reject) => {
this.con.query(query, (err, drop) => {
if (!!err) {
console.log(error);
return reject(err)
}
return resolve(drop);
})
});
async function driversBySeason(season) {
var query = `SELECT raceId FROM races WHERE year = ${season}`;
var score = {};
const drop = await asyncQuery(query).catch(err => err /* some error handling */);
for (const element of drop) {
var raceId = element["raceId"];
query = `SELECT driverId, points FROM results WHERE raceId = ${raceId}`;
const drop2 = await asyncQuery(query).catch(err => err /* some error handling */);
drop2.forEach((element) => {
if (score[element["driverId"]] == undefined) {
score[element["driverId"]] = 0;
} else if (score[element["points"]] != undefined) {
score[element["driverId"]] += element["points"];
}
});
}
console.log(score);
}
You are outside of your callback. When this executes the code continues to run past "this.con.query". You are seeing your empty object that was assigned at the top do to this. Go inside the callback after the drop.forEach where you assign the values, or convert to an async/await approach.
when you call driversBySeason your score variable has value of {}
and that is value for console.log() when you call it because of how closers are works and you updating score with callback function that happen later in time...
you get better understanding if you use promise...not callback
there is a function that I use to read all files in a directory and then sent an object with emitter to the client.
this is my code that works fine,
const getFilesList = (path, emitter) => {
fs.readdir(path, (err, files) => {
emitter('getFileList', files);
});
};
but when I want to filter hidden files with this code, the 'standardFolders' will send empty in the emitter.
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
});
});
} else {
standardFolders = null;
}
emitter('getFileList', standardFolders);
});
};
what is wrong with my code in the second part?
winattr.get(filepath,callback) is asynchronous, so imagine your code "starts" the file.map() line and then immediately skips to emitter('getFileList',standardFolders) --- which standardFolders is empty because it hasn't finished yet!
You can use a library like async.io to handle your callback functions, or you can use a counter and keep track of when all of the callbacks (for each file) has finished yourself.
Example:
// an asynchronous function because setTimeout
function processor(v,cb){
let delay = Math.random()*2000+500;
console.log('delay',delay);
setTimeout(function(){
console.log('val',v);
cb(null,v);
},delay);
}
const main = function(){
const list = ['a','b','c','d'];
let processed = [];
let count = 0;
console.log('starting');
list.map(function(v,i,a){
console.log('calling processor');
processor(v,function(err,value){
processed.push(v);
count+=1;
console.log('count',count);
if(count>=list.length){
// all are finished, continue on here.
console.log('done');
}
})
})
console.log('not done yet!');
};
main();
Similarly, for your code:
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
let count = 0;
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
count+=1;
if(count>=files.length){
// finally done
emitter('getFileList', standardFolders);
}
});
});
} else {
standardFolders = null;
emitter('getFileList', standardFolders);
}
});
};
As already sayed in the other answer winattr.get is async, so the loop finishes before any of the callbacks of winattr.get is called.
You could convert your code using async/await and primitify into a code that looks almost like a sync version, and you can completely get rid of the callbacks or counters
const {promisify} = require('util')
const readdir = promisify(require('fs').readdir)
const winattrget = promisify(require('winattr').get)
const getFilesList = async (path, emitter) => {
let standardFolders = [];
try {
let files = await readdir(path);
for (let file of files) {
try {
let attrs = await winattrget(path + file)
if (attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
} catch (err) {
// do nothing if an error occurs
}
}
} catch (err) {
standardFolders = null;
}
emitter('getFileList', standardFolders);
};
An additional note: In your code you write files.map, but mapping is use to transform the values of a given array and store them in a new one, and this is not done in your current code, so in the given case you should use a forEach loop instead of map.
I need to create a function that runs a 'getFile' function on each item in an array. The getFile function logs 'File contents of x' x being whatever element is in the array.
Currently, I have a working function that runs the getFile on the array and waits for the final response before logging the results.
However, I now need to log the responses as I receive them in order. For example, if my array is [1, 2, 3, 4, 5] currently it logs 'File contents of x' in a random order, so if it was to return the logs, 3 then 4 then 1. As soon as I receive 1, I need to log that, then once I receive 2 logs that and so on.
I will insert my current code below. The problem I'm having is I need to know when the 'empty space' in my array becomes populated so I can log it in real time. Therefore allowing my user to see the result build up rather than just having to wait until all the responses have come back
function fetchContentOfFiles(fileNames, testCB) {
const fileContent = [];
let counter = 0;
fileNames.forEach((file, i) => {
getFile(file, (err, fileName) => {
if (err) console.log(err)
else {
fileContent[i] = fileName;
counter++
if (counter === fileNames.length) {
testCB(null, fileContent)
};
console.log(fileContent)
};
});
});
};
The cleanest way to write this would be to use a for loop inside an async function. Promisify getFile so that it returns a Promise, then await it in every iteration of the loop. At the end of the loop, call the callback:
const getFileProm = file => new Promise((resolve, reject) => {
getFile(file, (err, fileName) => {
if (err) reject(err);
else resolve(fileName);
});
});
async function fetchContentOfFiles(fileNames, testCB) {
const fileContent = [];
try {
for (let i = 0; i < fileNames.length; i++) {
fileContent.push(
await getFileProm(fileNames[i])
);
}
} catch(e) {
// handle errors, if you want, maybe call testCB with an error and return?
}
testCB(null, fileContent);
}
It would probably be even better if fetchContentOfFiles was called and handled as a Promise rather than with callbacks, and then the errors can be handled in the consumer:
async function fetchContentOfFiles(fileNames) {
const fileContent = [];
for (let i = 0; i < fileNames.length; i++) {
fileContent.push(
await getFileProm(fileNames[i])
);
}
return fileContent;
}
fetchContentOfFiles(arr)
.then((fileContent) => {
// do stuff with fileContent
})
.catch((err) => {
// something went wrong
});
While making a link Shortner script using nodejs, i encountered the following problem:
my program went on an infinite loop for a reason i ignore
here is the code:
function makeShort() {
var short = "";
var cond = true;
while(cond){
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for(var i = 0; i < length; i++){
short += possible.charAt(Math.floor(Math.random() * possible.length));
}
let query = {short:short};
Link.findOne(query, (err, link)=>{
if(err) throw err;
if(!link){
console.log("here");
cond = false;
}
});
}
return short;
}
to then use it here :
router.post('/', (req, res)=>{
let short = makeShort();
const newLink = new Link({
url: req.body.url,
short:short
});
newLink.save().then(link => {
res.json(link);
});
});
the idea is that i generate a random string (5 characters), and then, if it exists i create another one and so on.. until i find one that isn't used ( the database is empty btw so there is no reason for it to go infinite loop ).
You can loop over and test for values in your database using async/await. What we do is convert your function to an async function, then create a new function that will return a promise which will resolve true/false.
Next we call that function in the while loop and await for a result which will contain true/false we then set that to the variable cond and continue the loop.
It would look something like this:
async function makeShort(length) {
let cond = true;
while (cond) {
let short = (Math.random() * 1000).toString(32).replace(/\./g, '').substr(0, length);
let query = { short: short };
cond = await findOne(query);
}
return short;
}
function findOne(query) {
return new Promise(resolve => {
Link.findOne(query, (err, link) => {
if (err) resolve(false);
if (!link) {
return resolve(false);
}
return resolve(true);
});
})
}
We then can call it using let short = await makeShort() like this (we also have to the make (req, res) function async):
router.post('/', async (req, res) => {
let short = await makeShort();
const newLink = new Link({
url: req.body.url,
short: short
});
newLink.save().then(link => {
res.json(link);
});
});
Don't mix synchronous looping and asynchronous condition updating. Something like this is guaranteed to run the while body as many times as it can before that DoSomething call returns a result:
while(cond) {
// call something async. don't wait for a result.
DoSomething.asynchronous( () => { cond = false; });
// then immediately restart the iteration
}
So don't do that. Make your makeShort generate a shortform string asynchronously.
const symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
const symbolCount = symbols.length;
function makeShort(howMany) {
howMany = howMany || 5;
let short = "";
while(howMany--) {
short += symbols[(Math.random() * symbolCount)|0];
}
return short;
}
Then, do your verification independently of this:
function assignShortForm(req, res) {
let short = makeShort();
verifyShortIsAvailable(
short,
success => {
// this short form was available
new Link({ url: req.body.url, short }).save().then(link => res.json(link));
}, error => {
// try again. RNG is not your friend, and this COULD run a very long time.
assignShortForm(req, res);
}
);
}
With your router using that function, not inlining it:
router.post('/', assignShortForm);
In this, verifyShortIsAvailable should do its work asynchronously:
verify verifyShortIsAvailable(short, resolve, reject) {
Link.findOne(query, (err, link) => {
if (err) return reject(err);
if (link) return reject("...");
return resolve();
});
}
while loops run synchronously, meaning they block the thread from further execution until they are complete. Because the link shortener is asynchronous, it is being blocked by the while loop.
To handle this code asynchronously, you can return a Promise
function makeShort() {
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
// note length was previously undefined in the comparison. use possible.length or another arbitrary value
for(var i = 0; i < possible.length; i++){
short += possible.charAt(Math.floor(Math.random() * possible.length));
}
let query = {short:short};
return new Promise((resolve, reject) => {
Link.findOne(query, (err, link) => {
if(err) return reject(err);
resolve(link)
});
})
}
Then you can use it like so...
let short = makeShort().then(shortLink => {
// do something with the link
}).catch(err => {
// handle the error
});
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Using_promises
I'm using for loop to iterate over an array of elements and to call the same function with different parameters inside the for loop. Here is my code:
exports.listTopSongs = function(query) {
return new Promise(function(resolve, reject) {
var str = query.split(","), category,
for(var i=0; i<str.length; i++) {
sampleFn(str[i], 'sample', resolve, reject);
}
});
};
function sampleFn(lang, cat, resolve, reject) {
client.on("error", function (err) {
console.log(err);
var err = new Error('Exception in redis client connection')
reject(err);
});
client.keys(lang, function (err, keys){
if (err) return console.log(err);
if(keys.length != 0) {
client.hgetall(keys, function (error, value) {
var objects = Object.keys(value);
result['title'] = lang;
result[cat] = [];
var x =0;
for(x; x<objects.length; x++) {
var val = objects[x];
User.findAll({attributes: ['X', 'Y', 'Z'],
where: {
A: val
}
}).then(data => {
if(data != null) {
//some actions with data and stored it seperately in a Json array
if(result[cat].length == objects.length) {
resolve(result);
}
} else {
console.log(""+cat+" is not avilable for this value "+data.dataValues['X']);
}
});
}
});
});
}
Here it won't wait for completion of first iteration. It just run asyncronously before completing first iteration function. I need to return the result as result:[{ 1, 2}, {3,4}]. but it runs seamlessly and returns empty or only one object before completing all. How to resolve it.
I used node-async-loop. But it uses next and i can't able to send my parameteres while using that package. Please help me
Async provides control flow methods allowing to do so.
Using async.each:
async.each(openFiles, function(file, callback) {
// Perform operation on file here.
console.log('Processing file ' + file);
if( file.length > 32 ) {
console.log('This file name is too long');
callback('File name too long');
} else {
// Do work to process file here
console.log('File processed');
callback();
}
}, function(err) {
// if any of the file processing produced an error, err would equal that error
if( err ) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A file failed to process');
} else {
console.log('All files have been processed successfully');
}
});
If you don't want to use a library, you can code it yourself. It would also be very instructive. I took your issue and coded a dummy async loop :
function listTopSongs(query) {
return new Promise(async(resolve, reject) => { //add async here in order to do asynchronous calls
const str = query.split(",") //str is const, and the other variable was not used anyway
for( let i = 0;i < str.length; i++) {
const planet = await sampleFn(str[i], 'sample', resolve, reject)
console.log(planet)
}
});
};
function sampleFn(a, b, c, d) {
return fetch(`https://swapi.co/api/planets/${a}/`)
.then(r => r.json())
.then(rjson => (a + " : " + rjson.name))
}
listTopSongs("1,2,3,4,5,6,7,8,9")
I used some dummy star wars API to fake a long promise but it should work with your sampleFn. Be careful, it is very, very slow if you have network call like the one in the example.
EDIT: I ran your code and I noticed there are a few mistakes: there is no resolve in your promise so it's not a thenable (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/resolve see thenable )
Here is a fully working code. The nice part : no library needed, no dependencies.
//for node.js, use node-fetch :
//const fetch = require("node-fetch")
function listTopSongs(query) {
return new Promise(async(resolve, reject) => { //add async here in order to do asynchronous calls
const str = query.split(",") //str is const, and the other variable was not used anyway
const planets = []
for (let i = 0; i < str.length; i++) {
const planet = await sampleFn(i + 1, str[i], resolve, reject)
planets[i] = planet
console.log(planet)
}
resolve(planets)
});
};
function sampleFn(a, b, c, d) {
return fetch(`https://swapi.co/api/planets/${a}/`)
.then(r => r.json())
.then(rjson => (a + b + " : " + rjson.name))
}
listTopSongs("a,b,c,d").then(planets => console.log(planets))
Since you are using promise, you can do something like this
exports.listTopSongs = function(query) {
return Promise.resolve(true).then(function(){
var str = query.split(",");
var promises = str.map(function(s){
return sampleFn(str[i], 'sample');
});
return Promise.all(promises);
}).then(function(results){
//whatever you want to do with the result
});
};
For this to work you have to change your sampleFn to not to depend on external resolve and reject functions. I don't see a reason using external resolve and reject. why just not use Promise.Resolve, Promise.Reject;