We Keep Coding

Problem using async functions in NodeJs middleware

I am running into an issue when I try to load the initial data for my blacklist from a Redis DB in my middleware code. Since the DB request takes some time it starts to fail.
Below is my code which gets fired when app starts via app.use(blacklist.blockRequests());.
When I try to make the function async I get the error that new TypeError('app.use() requires a middleware function').
One of the side effects is also that my array is empty when it's called again.
blockRequests: function() {
this.read();
this.logEvent('info', 'There are ' + this.blacklist.length + ' address(es) on the blacklist');
var self = this;
var interceptor = function(request, response, next) {
var ip = request.headers['x-forwarded-for'] || request.connection.remoteAddress;
if (self.isInBlacklist(ip)) {
self.logEvent('warn', 'Rejecting request from ' + ip + ', path and query was ' + request.originalUrl);
response.status(403).send();
} else {
next();
}
}
return interceptor;
},
And here is my read() function code:
read: function() {
try {
// get all records with prefix block:: from redis
redis.redis.keys('block::*', function (err, reply) {
// reply is null when the key is missing
if(err){}
else {
this.blacklist = []
for (let i = 0; i < reply.length; i++) {
let ipInt = reply[i].substring(7)
let ipStr = ipToInt(ipInt).toIP()
this.blacklist.push(ipStr)
}
}
});
} catch (error) {
if (error) {
this.blacklist = [];
}
}
}

If you're trying to make blockRequests() async, then it will start returning a promise and you can't use its return value directly in app.use(). Because then you'd be doing app.use(somePromise) and Express will balk because you have to pass it a function reference, not a promise.
Instead, you will have to use .then() or await to get the return value which is the function which you could then use with app.use().
If you show the larger calling context here (like where you're calling blockRequests() from), then we could offer more ideas on a fuller solution.
Here's a conceptual idea for how you could do this:
blockRequests: function() {
const self = this;
const interceptor = function(request, response, next) {
const ip = request.headers['x-forwarded-for'] || request.connection.remoteAddress;
if (self.isInBlacklist(ip)) {
self.logEvent('warn', 'Rejecting request from ' + ip + ', path and query was ' + request.originalUrl);
response.status(403).send();
} else {
next();
}
}
return interceptor;
},
read: function() {
// get all records with prefix block:: from redis
return new Promise((resolve, reject) => {
redis.redis.keys('block::*', (err, reply) => {
if (err) {
this.blacklist = [];
reject(err);
} else {
this.blacklist = [];
for (let i = 0; i < reply.length; i++) {
let ipInt = reply[i].substring(7)
let ipStr = ipToInt(ipInt).toIP()
this.blacklist.push(ipStr)
}
}
this.logEvent('info', 'There are ' + this.blacklist.length + ' address(es) on the blacklist');
resolve();
});
});
}
// register middleware for using blacklist
app.use(blacklist.blockRequests());
// now read the blacklist and when that is in place, then start the server
blacklist.read().then(() => {
// now we know that blacklist.blacklist is up-to-date
// start your server here
}).catch(err => {
console.log("Unable to start server - error in reading blacklist");
process.exit(1);
});

Is it neccessary to call app.delete() at the end of calling admin.initializeApp() in firebase cloud function

I am writing a cloud function in which i am creating or initializing firebase app in a for loop with new names, my question is that , is it necessary to call app.delete() function on every instance or not?
snapshot.forEach(doc => {
counter++;
// console.log(counter," in first");
// console.log(doc.id, " is the doc ID");
var home = doc.get('home');//e.g: hyperoffice
var switches = doc.get('switches');
var action = doc.get('action');//ON or OFF
var repeat=doc.get('repeat');// true or false
try {
// console.log("home ", home, " switches ", switches);
let defapp = admin.initializeApp({ databaseURL: databaseUrl }, `${home}${counter}`);
var databaseUrl = `https://${home}.firebaseio.com/`;
switches.forEach(s => {
let dbRef = admin.database(defapp).ref(`Controls/${s}`);
dbRef.once("value").then(val => {
var data = val.val();
// console.log("this is data", data);
var payload_on = data.payload_on;
var payload_off = data.payload_off;
let valRef = admin.database(defapp).ref(`Controls/${s}/value`);
if (action === "ON") {
// on the device
return valRef.set(payload_on);
}
else if (action === "OFF") {
//off the device
return valRef.set(payload_off);
} else {
console.log("Undefined action field in firestore");
}
return null;
}).catch(err => {
console.log("Error", err);
});
});
} catch (error) {
console.log("Eroor: ", error);
}
finally {
console.log("in finally block 67");
}
});

Strictly speaking, you don't have to, but you will leak a lot of memory over time, and your function might crash in a future invocation. Your function should always clean up unused memory before it terminates. This means that you should delete any initialized apps, unless you want to use that exact same app instance in a future invocation.

Javascript: How to check if async operation is still pending / In progress?

I would like to know if it is somehow possible to check if an asynchronous operation in Javascript is still pending..
Because I am doing a database request on calling a specific URL... While the db call is still in progress, I want to stop any other incoming db-calls (which means, stop any further calls to that URL in case the db-request is still pending).
Is that somehow possible?
Because the database call takes up to minutes, and I don't want to launch another database-call while the first is still in progress.. The problem is, I somehow cannot figure out how to check if the call has started and is still in progress, because the response comes only after the .then() clause when the process has already finished.
this is my db-call function:
const getWriteIndex = async () => {
return Promise.all(someFunction1, someFunction2...).then(...) {
writeMessageObject = checkDocuments(...);
return Promise.resolve(writeMessageObject);
})).catch((err) => {
return Promise.reject(err);
});
}
This is my URL/Route Call function with express:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
//HOW CAN I CHECK HERE IF THE DB-CALL IS ALREADY IN PROGRESS?
readmsg_obj.message = '';
getReadIndex().then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});

hmm I found a workaround here:
https://ourcodeworld.com/articles/read/317/how-to-check-if-a-javascript-promise-has-been-fulfilled-rejected-or-resolved
so I wrote that function to check for promise stati, but I am still wondering if it's not somehow possible to query for static promise properties to get their actual state ;) (but weirdly, I didn't find any on the web).
const checkPendingRequest= (promise) => {
if (promise.isResolved) return promise;
// Set initial state
var isPending = true;
var isRejected = false;
var isFulfilled = false;
// Observe the promise, saving the fulfillment in a closure scope.
var result = promise.then(
function(v) {
isFulfilled = true;
isPending = false;
return v;
},
function(e) {
isRejected = true;
isPending = false;
throw e;
}
);
result.isFulfilled = function() { return isFulfilled; };
result.isPending = function() { return isPending; };
result.isRejected = function() { return isRejected; };
return result;
}
So I fixed my function for the request:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
readmsg_obj.message = '';
if ((dbQueryPromiseRead != null) && dbQueryPromiseRead.isPending()) {
logger.info(`Database request for Index-read is still pending!`);
return;
}
dbQueryPromiseRead = checkPendingRequest(getReadIndex());
dbQueryPromiseRead.then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});

You need to try add in node.js like global.dbCallState flag if operation is still running.
This global var one for all modules.
Do not change this object like global = new Object();, but you can use child field's.
https://nodejs.org/api/globals.html
You can change it in another module like global.dbCallState = false.
It not best solution, but it can help.
But i don't know, why you want only one connection. Its not good solution to block I/O in node.js

Async write after select Pouchdb

I'm trying to check if an element exist before inserting it in my bdd.
I have to do this in order to (in the future) modify this existing element.
I'm using PouchDb and PouchDb-find with Node 6.9.1.
Actually I'm doing this:
for(var i = 0; i < 10;i++ ){
(function(_count, _pdb){
var count = _count;
var db = _pdb;
db.find({
selector: {numeroCandidat: parseInt(results[count].no_apb)}
}).then((result) => {
if(result.docs.length != 0){
console.log("l'étudiant existe");
}else{
console.log("l'étudiant n'existe pas");
var etudiant = {
"numeroCandidat": results[count].no_apb,
"nom": results[count].nom,
"numeroGroupe": "gr" + results[count].groupe,
"filiere": results[count].libelle,
};
db.post(etudiant).then((response) =>{
// handle response
console.log("STUDENT CREATED");
}).catch(function (err) {
console.log(err);
});
}
}).catch(function (err) {
});
})(i, this.pdb);
};
But the problem is : Due to the asynchronous version of my select query... if an element exists two times it appends that the second select occurred BEFORE the insertion of the first element, and I have this element two times in my database. I don't know how to deal with this one.

SO.. I'v found a workaround !
Simply create a function that I call recursivly after writting into my database.
Goodbye for loop.
var createStudents = function(_count, _pdb, _students){
if(_count >= 10) return;
console.log(_count);
var count = _count;
var db = _pdb;
var students = _students.slice(0);
db.find({
selector: {numeroCandidat: parseInt(students[count].no_apb)}
}).then((result) => {
if(result.docs.length != 0){
console.log("l'étudiant existe");
createStudents(++count,db,results);
}else{
var etudiant = {
"numeroCandidat": students[count].no_apb,
"nom": students[count].nom,
"numeroGroupe": "gr" + students[count].groupe,
"filiere": students[count].libelle,
"etudiantComms": [
{"commentaire": students[count].commentaire}
]
};
db.post(etudiant).then((response) =>{
// handle response
console.log("STUDENT CREATED");
createStudents(++count,db,results);
}).catch(function (err) {
console.log(err);
});
}
}).catch(function (err) {
});
}
createStudents(0,this.pdb,results);

How can I delete multiple files asynchronously?

Assume I have an array with paths to multiple files. I would like to delete these files asynchronously.
var files = ['file1.txt', 'file2.txt'];
fs.unlink(..., callback())
I came across with this solution Delete several files in node.js but I think it violates node.js practices (asynchronous function inside a for loop). Is there any other, better solution for this?

If you want to run a an arbitrary list of tasks (unlinking files) asynchronously, but know when they are all done, you can use the async.js module. It can run tasks in series and parallel.
So push all your unlink function calls into an array, then call async.parallel() and let them fly. Then when they are all done, you land in a single manageble callback.
var files = ['file1.txt', 'file2.txt'];
var myParallelTasks = [];
files.forEach( function( fileName )
{
myParallelTasks.push( fs.unlink( fileName, function(callback)
{
callback();
})
);
}
async.parallel( myParallelTasks, function()
{
// all done
console.log( "all done" );
});

Try the option of recursion (code from your link in the question):
function deleteFiles(files, callback){
var i = files.length;
var file = files.pop();
if ( file == undefined ) {
callback();
} else {
// do the unlinking ...
deleteFiles(files, callback);
}
}

async deleteAll(filePathsList) {
try {
await this.deleteFiles(filePathsList);
logger.log('info', "Following files deleted successfully from EFS --> " + filePathsList.toString());
return true;
} catch (error) {
logger.log('error', error.stack || error);
logger.log('error', "Error occured while deleting files from EFS");
return false;
}
}
async deleteFiles(files) {
return new Promise((resolve, reject) => {
let i = files.length;
files.forEach(function(filepath) {
fs.unlink(filepath, function(err) {
i--;
if (err && err.code == 'ENOENT') {
// file doens't exist
logger.log('info', "Following file doesn't exist, So won't be deleted-->" + (filepath || ''));
} else if (err) {
// other errors, e.g. maybe we don't have enough permission
logger.log('error', "Error occured while deleting the file " + (filepath || '') + " due to error" + err);
reject(err);
return;
} else if (i <= 0) {
resolve();
}
});
});
})
}