Assume I have an array with paths to multiple files. I would like to delete these files asynchronously.
var files = ['file1.txt', 'file2.txt'];
fs.unlink(..., callback())
I came across with this solution Delete several files in node.js but I think it violates node.js practices (asynchronous function inside a for loop). Is there any other, better solution for this?
If you want to run a an arbitrary list of tasks (unlinking files) asynchronously, but know when they are all done, you can use the async.js module. It can run tasks in series and parallel.
So push all your unlink function calls into an array, then call async.parallel() and let them fly. Then when they are all done, you land in a single manageble callback.
var files = ['file1.txt', 'file2.txt'];
var myParallelTasks = [];
files.forEach( function( fileName )
{
myParallelTasks.push( fs.unlink( fileName, function(callback)
{
callback();
})
);
}
async.parallel( myParallelTasks, function()
{
// all done
console.log( "all done" );
});
Try the option of recursion (code from your link in the question):
function deleteFiles(files, callback){
var i = files.length;
var file = files.pop();
if ( file == undefined ) {
callback();
} else {
// do the unlinking ...
deleteFiles(files, callback);
}
}
async deleteAll(filePathsList) {
try {
await this.deleteFiles(filePathsList);
logger.log('info', "Following files deleted successfully from EFS --> " + filePathsList.toString());
return true;
} catch (error) {
logger.log('error', error.stack || error);
logger.log('error', "Error occured while deleting files from EFS");
return false;
}
}
async deleteFiles(files) {
return new Promise((resolve, reject) => {
let i = files.length;
files.forEach(function(filepath) {
fs.unlink(filepath, function(err) {
i--;
if (err && err.code == 'ENOENT') {
// file doens't exist
logger.log('info', "Following file doesn't exist, So won't be deleted-->" + (filepath || ''));
} else if (err) {
// other errors, e.g. maybe we don't have enough permission
logger.log('error', "Error occured while deleting the file " + (filepath || '') + " due to error" + err);
reject(err);
return;
} else if (i <= 0) {
resolve();
}
});
});
})
}
Related
OK, so I have a situation where I cannot just fire thousands of requests to an API server.
I have a Node process (no UI) that I need to have process each API response/update sequentially, waiting for completion before sending the next request.
I may be making this more complicated than I think - not sure. I can only figure out how to do this with recursive calls, but this results in a stack overflow as there can be thousands of records. The general process is this:
get rows from SQL table with ID's (result)
formulate and send of an API call to retrieve ID's info
if returned data has image data, write it back to SQL table
wait on this process so not to bombard API server with thousands of requests all at once
repeat until last ID is processed (can be thousands, more than stack space)
Here's sample code (not actual so ignore syntax errors if any)...
UPDATED: actual running code with sensitive items removed
var g_con = null; //...yeah I know, globals are bad
//
// [ found updating ]
//
function getSetImage(result, row, found) {
if(row >= result.length) { //...exit on no row or last row processed
con.end();
return;
}
item = result[row]; //...next SQL row
if((item !== undefined) && (item.autoid !== undefined)) {
//...assemble API and send request
//
let url = 'https://...API header...'
+ item.autoid
+ '...API params...';
request(url, (error, response, body) => {
if(response.statusCode !== 200)
throw('Server is not responding\n' + response.statusMessage);
let imageData = JSON.parse(body);
if((imageData.value[0] !== undefined) &&
(imageData.value[0].DETAIL !== undefined) &&
(imageData.value[0].DETAIL.Value.length) ) {
//...post back to SQL
//
found++;
console.log('\n' + item.autoid + '/['+ item.descr + '], ' + 'Found:' + found);
qry = 'update inventory set image = "'+imageData.value[0].DETAIL.Value+'" where autoid = "'+item.autoid+'";';
g_con.query(qry, (err) => {
if (err) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
});
row++;
setTimeout(()=>{getSetImage(result, row, found)}, 0); //...nested call after SQL
} else {
row++;
process.stdout.write('.'); //...show '.' for record, but no image
setTimeout(()=>{getSetImage(result, row, found)}, 0); //...nested call after SQL
}
}); //...request callback
}
// } else {
// throw '\nERROR! result['+row+'] undefined? Images found: '+found;
// }
}
//
// [ main lines ]
//
(() => {
let params = null;
try {
params = JSON.parse(fs.readFileSync('./config.json'));
//...load autoids array from SQL inventory table - saving autoids
// autoids in INVENTRY join on par_aid's in INVENTRYIMAGES
//
g_con = mysql.createConnection(params.SQLConnection);
g_con.connect((err) => { if(err) {
console.log('ERROR:',err.message);
throw err.message;
}
});
//...do requested query and return data or an error
//
let qry = 'select autoid, descr from inventory order by autoid;';
g_con.query(qry, (err, results, flds) => {
if (err || flds === undefined) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
console.log('Results length:',results.length);
let row = 0;
let found = 0;
getSetImage(results, row, found);
});
}
catch (err) {
console.log('Error parsing config parameters!');
console.log(err);
}
})();
So here's the answer using Promises (except for MySQL):
//
// [ found updating ]
//
async function getSetImage(data) {
for(let item of data) {
if(item && item.autoid) {
//...assemble API and send request
//
let url = g_URLHeader + g_URLPartA + item.autoid + g_URLPartB;
let image = await got(url).json().catch(err => {
console.log(err);
err.message = 'API server is not responding';
throw err;
});
if(image && image.value[0] && image.value[0].DETAIL &&
image.value[0].DETAIL.Value.length ) {
console.log('\nFound: ['+item.autoid+' - '+item.descr
+ '] a total of ' + g_found + ' in ' + g_count + ' rows');
g_found++;
//...post back to SQL
//
let qry = 'update inventory set image = "'
+ image.value[0].DETAIL.Value
+ '" where autoid = "'
+ item.autoid+'";';
await g_con.query(qry, (err) => {
if (err) {
console.log('ERROR:',err.message, '\nSQL:['+err.sql+']\n');
throw err.message;
}
});
} else {
process.stdout.write('.'); //...show '.' for record, but no image
} //...if/else image.value
g_count++;
} //...if item
} //...for()
}
As I've said in all my comments, this would be a ton simpler using promises and async/await. To do that, you need to switch all your asynchronous operations over to equivalents that use promises.
Here's a general outline based on the original pseudo-code you posted:
// use got() for promise version of request
const got = require('got');
// use require("mysql2/promise" for promise version of mysql
async function getSetImage(data) {
for (let item of data) {
if (item && item.id) {
let url = uriHeader + uriPartA + item.id + uriPartB;
let image = await got(url).json().catch(err => {
// log and modify error, then rethrow
console.log(err);
err.msg = 'API Server is not responding\n';
throw err;
});
if (image.value && image.value.length) {
console.log('\nFound image for ' + item.id + '\n');
let qry = 'update inventory set image = "' + image.value + '" where id = "' + item.id + '";';
await con.query(qry).catch(err => {
console.log('ERROR:', err.message, '\nSQL:[' + err.sql + ']\n');
throw err;
});
}
} else {
// no image data found
process.stdout.write('.'); //...show '.' for record, but no image
}
}
}
//...sql query is done, returning "result" - data rows
getSetImage(result).then(() => {
console.log("all done");
}).catch(err => {
console.log(err);
});
Some notes about this code:
The request() library is no longer getting new features and is in maintenance mode and you need to change to a different library to get built-in promise support. You could use request-promise (also in maintenance mode), but I recommend one of the newer libraries such as got() that is more actively being developed. It has some nice features (automatically checks status for you to be 2xx, built-in JSON parsing, etc...) which I've used above to save code.
mysql2/promise has built-in promise support which you get with const mysql = require('mysql2/promise');. I'd recommend you switch to it.
Because of the user of async/await here, you can just loop through your data in a regular for loop. And, no recursion required. And, no stack build-up.
The way promises work by default, any rejected promises will automatically terminate the flow here. The only reason I'm using .catch() in a couple places is just for custom logging and tweaking of the error object. I then rethrow which propagates the error back to the caller for you.
You can tweak the error handling to your desire. The usual convention with promises is to throw an Error object (not a string) and that's often what callers are expecting to see if the promise rejects.
This code can be easily customized to log errors and continue on to subsequent items in the array. Your original code did not appear to do that so I wrote it to abort if it got an error.
This is a piece of code which writes data to a ble device and reads data from it. data is written to the device in the form of a buffer. the value in 'mydata' (AAAD0000) is the command to be written in order to read the data.
function named chara3() consists of write and read function which is a callback function in which the command is passed read back.
My requirement is the 'mydata' value which i said earlier, the last two zeros is the memory address. i need to read the data in different memory addresses starting from zero to 59. That is AAAD0000 to AAAD0059. so of course i need to run a loop. If I'm reading the zeroth location, the code is quite fine and i got the output as well but when i tried to make it inside a loop, the code is all a mess. the read part is not executing.
can any one suggest a better way to read data from zeroth memory location to 59th memory location (AAAD0000 to AAAD0059)???
first command writes to it
then reads data
memory location incremented by 1
this should repeat up to 59
var mydata = 'AAAD0000';
function chara3() {
var buff2 = new Buffer(mydata, 'hex');
SensorCharacteristic.write(buff2, false, function(error) { //[0x002d]
console.log('Writing command SUCCESSFUL',mydata);
if (!error) {
SensorCharacteristic.read((error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
}
else {
console.log('contains null value');
} //else
});
}
function enter()
{
mydata = (parseInt(mydata, 16) + 00000001).toString(16);
}
}); //.write
} //chara3
there's no error. But some part of the code is not executing.
You can use the recursion by wrapping your methods into a promise
async function chara3(mydata = 'AAAD0000') {
if (mydata === 'AAAD0059') {
return;
}
var buff2 = new Buffer(mydata, 'hex');
return new Promise((resolve) => {
SensorCharacteristic.write(buff2, false, function (error) { //[0x002d]
console.log('Writing command SUCCESSFUL', mydata);
if (!error) {
SensorCharacteristic.read(async (error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
let next = await chara3(enter())
return resolve(next);
}
else {
console.log('contains null value');
return resolve();
} //else
});
}
}); //.write
});
} //chara3
function enter() {
return (parseInt(mydata, 16) + 00000001).toString(16);
}
Also if you can convert your methods SensorCharacteristic.write and SensorCharacteristic.read into promises you can simply map
function chara3(mydata) {
var buff2 = new Buffer(mydata, 'hex');
await SensorCharacteristic.write(buff2, false);
console.log('Writing command SUCCESSFUL', mydata);
let data = await SensorCharacteristic.read();
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
} else {
console.log('contains null value');
}
};
let promiseArray = Array(60).fill().map((_, i) => (parseInt('AAAD0000', 16) + i).toString(16)).map(chara3);
Promise.all(promiseArray).then(() => console.log('done'));
I'm writing an Electron program which takes a CSV file as input, and does file operations depending on the CSV content and file existence (it's to manage MAME arcade roms).
In order to have a progress bar on the UI side, I have switched the code from fully synchronous (because it was much easier) to asynchronous.
I just cannot find out how to reliably display a message to the user when all the lines in the CSV file are processed, and all the zip files are copied or removed.
Here is a (simplified) sample method:
fs.readFile(file, { 'encoding': 'utf8' }, (err, fileContents) => {
let fileCsv = csvparse(fileContents);
let lines = fileCsv.length;
fileCsv.forEach((line) => {
lines--;
let zip = line.name + '.zip';
let sourceRom = path.join(romset, zip);
let destRom = path.join(selection, zip);
this.emit('progress.add', fileCsv.length, fileCsv.length - lines, zip);
if (fs.existsSync(sourceRom) && !fs.existsSync(destRom)) {
fs.copy(sourceRom, destRom, (err) => {
let sourceChd = path.join(romset, game);
if (fs.existsSync(sourceChd)) {
fs.copy(sourceChd, path.join(selection, game), (err) => {
if (lines <= 0) { callback(); } // chd is copied
});
} else {
if (lines <= 0) { callback(); } // no chd, rom is copied
}
});
} else {
if (lines <= 0) { callback(); } // no source found or already exists
}
});
});
The problem is that the CSV file is processed really fast, but the file are not copied as fast. So it decrements the lines counter to 0, then after each file copy, it finds that it's zero and triggers the callback.
How do I reliably trigger the callback at the end of all these nested callbacks and conditions?
Thanks
I tried to change the code without massively overwriting your style - assuming there is a reason to avoid things like bluebird, async/await & native Promises, and the async lib.
You need to decrement lines after a line is processed. I pulled the processing logic out into a function to make this clearer:
function processLine({
sourceRom, destRom, romset, game, callback
}) {
if (fs.existsSync(sourceRom) && !fs.existsSync(destRom)) {
fs.copy(sourceRom, destRom, (err) => {
// *really* should handle this error
let sourceChd = path.join(romset, game);
if (fs.existsSync(sourceChd)) {
fs.copy(sourceChd, path.join(selection, game), (err) => {
// *really* should handle this error
callback();
});
} else {
callback();
}
});
} else {
callback() // no source found or already exists
}
}
fs.readFile(file, { 'encoding': 'utf8' }, (err, fileContents) => {
let fileCsv = csvparse(fileContents);
let lines = fileCsv.length;
fileCsv.forEach((line) => {
let zip = line.name + '.zip';
let sourceRom = path.join(romset, zip);
let destRom = path.join(selection, zip);
this.emit('progress.add', fileCsv.length, fileCsv.length - lines, zip);
processLine({ sourceRom, destRom, game, romset, callback: () => {
lines--;
if (lines <= 0) {
callback();
}
}})
});
});
I've been trying to diagnose this bug for some time now but can't figure out why my completed() function executes before all my asynch functions are done. I'm using the async library:
async.forEach(data.DBInstances, function (dbInstance, fcallback) {
let dbtype = dbInstance.Engine;
let logFilename = log[dbtype].log();
let instanceId = dbInstance.DBInstanceIdentifier;
if (tagFilter) {
let arn = dbInstance.DBInstanceArn;
checkRDSTag(arn, tagFilter, function (err, found) {
if (!err) {
//tag was found, continue processing and check other filters...
if (found) {
if (noFilter || (instanceTypes && instanceTypes.indexOf(dbtype))) {
//console.log('db type is: ' + dbtype);
processOrCreateLog(instanceId, dbType, function (err, data) {
if (!err) {
console.log("Data: " + JSON.stringify(data));
completed.push(data);
fcallback(null);
} else {
cb(err, null);
}
});
}
} else {
//tag wasn't found but was specified, don't process anything...
console.log("tag specified was not found on instance: " + instanceId);
}
} else {
console.log("Error checking RDS Tag");
cb(err, null);
}
});
}
//only process filtered types...
else if (noFilter || (instanceTypes && instanceTypes.indexOf(dbtype))) {
console.log('db type is: ' + dbtype);
processOrCreateLog(instanceId, dbtype, fcallback, function (err, data, fcallback) {
if (!err) {
console.log("Data: " + JSON.stringify(data));
completed.push(data);
fcallback(null);
} else {
cb(err, null);
}
});
}
}, testme(completed));
My async functions are running correctly and each completing correctly but my testme(completed) runs immediately before any of my asynch functions ever finish. Not sure why..
my testme(completed) is simply:
function testme(completed) {
console.log("Completed: " + JSON.stringify(completed));
}
One note, my function to execution on each element itself has asynch functions inside of it (checkRDSTag(), processOrCreateLog(), etc). I'm guessing its something to do with the callback() that async is expecting / tracking executing out of place or something? Not really sure..
Retrun callback only when last item iterating
var index=0;
async.forEach(data.DBInstances, function (dbInstance, fcallback) {
let dbtype = dbInstance.Engine;
let logFilename = log[dbtype].log();
let instanceId = dbInstance.DBInstanceIdentifier;
if (tagFilter) {
let arn = dbInstance.DBInstanceArn;
checkRDSTag(arn, tagFilter, function (err, found) {
if (!err) {
//increament index here
index++;
//tag was found, continue processing and check other filters...
if (found) {
if (noFilter || (instanceTypes && instanceTypes.indexOf(dbtype))) {
//console.log('db type is: ' + dbtype);
processOrCreateLog(instanceId, dbType, function (err, data) {
if (!err) {
console.log("Data: " + JSON.stringify(data));
completed.push(data);
//check if last item running
if(index===data.DBInstances.length){
return fcallback(null);
}else{
fcallback()
}
} else {
cb(err, null);
}
});
}
} else {
//tag wasn't found but was specified, don't process anything...
console.log("tag specified was not found on instance: " + instanceId);
}
} else {
console.log("Error checking RDS Tag");
cb(err, null);
}
});
}
//only process filtered types...
else if (noFilter || (instanceTypes && instanceTypes.indexOf(dbtype))) {
console.log('db type is: ' + dbtype);
processOrCreateLog(instanceId, dbtype, fcallback, function (err, data, fcallback) {
if (!err) {
console.log("Data: " + JSON.stringify(data));
completed.push(data);
//check if last item running
if(index===data.DBInstances.length){
return fcallback(null);
}else{
fcallback()
}
} else {
cb(err, null);
}
});
}
}, testme(completed));
My problem ended up being in my other asynchronous call (processOrCreateLog()) within my iteratee. There was flow control logic in my asynchronous calls that didn't callback so fcallback() never ran.
Also to clarify, async is the async node.js library: https://caolan.github.io/async/docs.html#each
As long as you execute the callback on the iteratee for each element with either an error or null it can track all executions and will then run your final callback properly.
I'll start by saying I've found several similar issues posted on this site. None of them apply to my situation though.
I have a server and client (as is the norm with node.js/socket.io) and call emit a socket event when a button is pressed. This works fine... Except it seems to emit three times (at least the server runs the function three times). I've been staring at the code for way too long at this point and need another set of eyes.
Hopefully someone has an idea.
client code:
importJS('/js/pages/admin_base.js',function(){
var restartLMC = function(io){
toggleLoad();
var user = localStorage.getItem('User');
io.emit('restart_request',{session: user});
};
AdminIO = new io('http://localhost:26266');
AdminIO.on('restart_success',function(dat){
toggleLoad();
dropInfo(dat);
});
AdminIO.on('sendError',function(dat){
dropInfo(dat,{level: 'error'});
});
AdminIO.on('restart_fail',function(dat){
toggleLoad();
dropInfo(dat,{level: 'error'});
});
$('#restart').on('click',function(){
restartLMC(AdminIO);
});
});
Admin code:
process.stdout.write('\033c');
console.log('\x1b[36m', "Admin server starting...", '\x1b[0m');
var
ini = require('node-ini')
, conf = ini.parseSync('../config.ini')
, CS = require('../lm_modules/CoreSync.js')
, CoreSync = new CS()
, checkSession = function (session, callback) {
var res;
if (!CoreSync) { throw "Fatal error, there is no connection to the Core service!"; }
if (CoreSync.sessions) {
if (CoreSync.sessions[session]) {
res = CoreSync.sessions[session];
callback(res);
}
else {
CoreSync.sync('session', function (err, dat) {
if (CoreSync.sessions[session]) {
res = CoreSync.sessions[session];
callback(res);
} else { res = false; callback(res); }
});
}
} else {
res = false; callback(res);
}
if (res === "undefined") { callback(false); }
}
, runCMD = function(cmd,errCB,callback){
var
command
, args;
if(cmd.cmd){ command = cmd.cmd; } else { command = cmd; }
if(cmd.args){ args = cmd.args; }
const spawn = require('child_process').spawn;
const ex = spawn(command, args);
ex.stdout.on('data', (data) => {
callback(data);
});
ex.stderr.on('data', (data) => {
errCB(data);
});
ex.on('close', (code) => {
});
}
, executeCMD = function(cmd,callback){
const exec = require('child_process').exec
, cdw = (__dirname + '/../');
exec(cmd, {cwd: cdw}, (err, stdout, stderr) => {
if (err) {
callback(err,null);
return;
}
callback(stderr,stdout);
});
}
, io = require('socket.io').listen(26266) // can use up to 26485
console.log('\x1b[32m', "Admin server started.", '\x1b[0m');
console.log("Admin server listening at " + "http://" + conf["Server"]["binding"] + ":26266");
io.on('connection', function (socket) {
socket.on('restart_request', function(req){
console.log('Recieved restart request');
var success = false
, session = JSON.parse(req.session)
, sessionID = session.sessionID;
checkSession(sessionID, function (ses) {
if (ses === false) { console.error('CheckSession failed: No session exists'); return; }
if (ses.user.uuid !== session.uuid) { console.error('CheckSession failed: UUID mismatched'); return; }
if (ses.user.role < conf['Permissions']['lm_restart']){ socket.emit('restart_fail','Insufficient permissions.'); return; }
if(process.platform === 'win32'){
executeCMD('cd',function(err,res){
var errSent = false;
if(err){
console.error(err);
if(!errSent){ socket.emit('sendError','Restart failed'); }
errSent = true;
if(res === null){return;}
}
console.log(res);
socket.emit('restart_success','LM successfully restarted.');
});
}
else if(process.platform === 'linux'){
}
});
});
});
For those of you who may have seen this and found it a curious question/situation... I found two parts to this.
The first part is the $().on binding. For some reason (even though it's by no means called multiple times in the js code) adding unbind() in front of the binding resolved the issue in part... it cut the extra emits down from 3 to two (until I started another server app, then it went back up to three...)
The other part I found was that (for some reason) the socket.io connection is being duplicated as many times as there are socket servers running. More details on this issue here... I believe that once the cause for this is found, my issue will be resolved.