I am running into an issue when I try to load the initial data for my blacklist from a Redis DB in my middleware code. Since the DB request takes some time it starts to fail.
Below is my code which gets fired when app starts via app.use(blacklist.blockRequests());.
When I try to make the function async I get the error that new TypeError('app.use() requires a middleware function').
One of the side effects is also that my array is empty when it's called again.
blockRequests: function() {
this.read();
this.logEvent('info', 'There are ' + this.blacklist.length + ' address(es) on the blacklist');
var self = this;
var interceptor = function(request, response, next) {
var ip = request.headers['x-forwarded-for'] || request.connection.remoteAddress;
if (self.isInBlacklist(ip)) {
self.logEvent('warn', 'Rejecting request from ' + ip + ', path and query was ' + request.originalUrl);
response.status(403).send();
} else {
next();
}
}
return interceptor;
},
And here is my read() function code:
read: function() {
try {
// get all records with prefix block:: from redis
redis.redis.keys('block::*', function (err, reply) {
// reply is null when the key is missing
if(err){}
else {
this.blacklist = []
for (let i = 0; i < reply.length; i++) {
let ipInt = reply[i].substring(7)
let ipStr = ipToInt(ipInt).toIP()
this.blacklist.push(ipStr)
}
}
});
} catch (error) {
if (error) {
this.blacklist = [];
}
}
}
If you're trying to make blockRequests() async, then it will start returning a promise and you can't use its return value directly in app.use(). Because then you'd be doing app.use(somePromise) and Express will balk because you have to pass it a function reference, not a promise.
Instead, you will have to use .then() or await to get the return value which is the function which you could then use with app.use().
If you show the larger calling context here (like where you're calling blockRequests() from), then we could offer more ideas on a fuller solution.
Here's a conceptual idea for how you could do this:
blockRequests: function() {
const self = this;
const interceptor = function(request, response, next) {
const ip = request.headers['x-forwarded-for'] || request.connection.remoteAddress;
if (self.isInBlacklist(ip)) {
self.logEvent('warn', 'Rejecting request from ' + ip + ', path and query was ' + request.originalUrl);
response.status(403).send();
} else {
next();
}
}
return interceptor;
},
read: function() {
// get all records with prefix block:: from redis
return new Promise((resolve, reject) => {
redis.redis.keys('block::*', (err, reply) => {
if (err) {
this.blacklist = [];
reject(err);
} else {
this.blacklist = [];
for (let i = 0; i < reply.length; i++) {
let ipInt = reply[i].substring(7)
let ipStr = ipToInt(ipInt).toIP()
this.blacklist.push(ipStr)
}
}
this.logEvent('info', 'There are ' + this.blacklist.length + ' address(es) on the blacklist');
resolve();
});
});
}
// register middleware for using blacklist
app.use(blacklist.blockRequests());
// now read the blacklist and when that is in place, then start the server
blacklist.read().then(() => {
// now we know that blacklist.blacklist is up-to-date
// start your server here
}).catch(err => {
console.log("Unable to start server - error in reading blacklist");
process.exit(1);
});
Related
I'm new to learning Node.js, so I'm still getting used to asynchronous programming and callbacks. I'm trying to insert a record into a MS SQL Server database and return the new row's ID to my view.
The mssql query is working correctly when printed to console.log. My problem is not knowing how to properly return the data.
Here is my mssql query - in addJob.js:
var config = require('../../db/config');
async function addJob(title) {
var sql = require('mssql');
const pool = new sql.ConnectionPool(config);
var conn = pool;
let sqlResult = '';
let jobID = '';
conn.connect().then(function () {
var req = new sql.Request(conn);
req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
jobID = result['recordset'][0]['JobID'];
conn.close();
//This prints the correct value
console.log('jobID: ' + jobID);
}).catch(function (err) {
console.log('Unable to add job: ' + err);
conn.close();
});
}).catch(function (err) {
console.log('Unable to connect to SQL: ' + err);
});
// This prints a blank
console.log('jobID second test: ' + jobID)
return jobID;
}
module.exports = addJob;
This is my front end where a modal box is taking in a string and passing it to the above query. I want it to then receive the query's returned value and redirect to another page.
// ADD NEW JOB
$("#navButton_new").on(ace.click_event, function() {
bootbox.prompt("New Job Title", function(result) {
if (result != null) {
var job = {};
job.title = result;
$.ajax({
type: 'POST',
data: JSON.stringify(job),
contentType: 'application/json',
url: 'jds/addJob',
success: function(data) {
// this just prints that data is an object. Is that because I'm returning a promise? How would I unpack that here?
console.log('in success:' + data);
// I want to use the returned value here for a page redirect
//window.location.href = "jds/edit/?jobID=" + data;
return false;
},
error: function(err){
console.log('Unable to add job: ' + err);
}
});
} else {
}
});
});
And finally here is the express router code calling the function:
const express = require('express');
//....
const app = express();
//....
app.post('/jds/addJob', function(req, res){
let dataJSON = JSON.stringify(req.body)
let parsedData = JSON.parse(dataJSON);
const addJob = require("../models/jds/addJob");
let statusResult = addJob(parsedData.title);
statusResult.then(result => {
res.send(req.body);
});
});
I've been reading up on promises and trying to figure out what needs to change here, but I'm having no luck. Can anyone provide any tips?
You need to actually return a value from your function for things to work. Due to having nested Promises you need a couple returns here. One of the core features of promises is if you return a Promise it participates in the calling Promise chain.
So change the following lines
jobID = result['recordset'][0]['JobID'];
to
return result['recordset'][0]['JobID']
and
req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
to
return req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
and
conn.connect().then(function () {
to
return conn.connect().then(function () {
You may need to move code around that is now after the return. You would also be well served moving conn.close() into a single .finally on the end of the connect chain.
I recommend writing a test that you can use to play around with things until you get it right.
const jobId = await addJob(...)
console.log(jobId)
Alternatively rewrite the code to use await instead of .then() calls.
How can I end a request in koa.js using another request. Lets say I'm keeping the active request contexts in an object. Assume request A is started and takes a long time. How can I make another request, that tells request A to end.
var requests = {};
// middleware to track requests
app.use(function*(next) {
var reqId = crypto.randomBytes(32).toString('hex');
requests[reqId] = {
context: this
}
yield next;
delete requests[reqId];
}
);
// route to kill request using ID generated from middleware above
router.get('/kill/:reqId', function *(next) {
var req = requests[this.params.reqId];
if (req) {
// abort request here
} else {
this.body = {
error: 'Request not found'
};
}
});
You should implement a cancellation token that you check regularly.
Example:
// Factory to create a token
const cancellationToken = () => {
let _cancelled = false;
function check() {
if (_cancelled == true) {
throw new Error('Request cancelled');
}
}
function cancel() {
_cancelled = true;
}
return {
check: check,
cancel: cancel
};
}
const reqs = {};
// Middleware to create tokens.
app.use(function *(next) {
const reqId = crypto.randomBytes(32).toString('hex');
const ct = cancellationToken();
reqs[reqId] = ct;
this.cancellationToken = ct;
yield next;
delete reqs[reqId];
});
// route to kill request using ID generated from middleware above
router.get('/kill/:reqId', function *(next) {
const ct = requests[this.params.reqId];
if (ct) {
ct.cancel();
} else {
this.body = {
error: 'Request not found'
};
}
});
// A request checking for cancellation.
router.get('/longrunningtask', function *(next) {
for (let i = 0; i < 1000; i++) {
yield someLongRunningTask(i);
// This is where you check to see if you're done.
// The method will throw and abort the request.
this.cancellationToken.check();
}
});
You could even pass along the cancellation token to the someLongRunningTask function so you can control cancellation there.
I'll start by saying I've found several similar issues posted on this site. None of them apply to my situation though.
I have a server and client (as is the norm with node.js/socket.io) and call emit a socket event when a button is pressed. This works fine... Except it seems to emit three times (at least the server runs the function three times). I've been staring at the code for way too long at this point and need another set of eyes.
Hopefully someone has an idea.
client code:
importJS('/js/pages/admin_base.js',function(){
var restartLMC = function(io){
toggleLoad();
var user = localStorage.getItem('User');
io.emit('restart_request',{session: user});
};
AdminIO = new io('http://localhost:26266');
AdminIO.on('restart_success',function(dat){
toggleLoad();
dropInfo(dat);
});
AdminIO.on('sendError',function(dat){
dropInfo(dat,{level: 'error'});
});
AdminIO.on('restart_fail',function(dat){
toggleLoad();
dropInfo(dat,{level: 'error'});
});
$('#restart').on('click',function(){
restartLMC(AdminIO);
});
});
Admin code:
process.stdout.write('\033c');
console.log('\x1b[36m', "Admin server starting...", '\x1b[0m');
var
ini = require('node-ini')
, conf = ini.parseSync('../config.ini')
, CS = require('../lm_modules/CoreSync.js')
, CoreSync = new CS()
, checkSession = function (session, callback) {
var res;
if (!CoreSync) { throw "Fatal error, there is no connection to the Core service!"; }
if (CoreSync.sessions) {
if (CoreSync.sessions[session]) {
res = CoreSync.sessions[session];
callback(res);
}
else {
CoreSync.sync('session', function (err, dat) {
if (CoreSync.sessions[session]) {
res = CoreSync.sessions[session];
callback(res);
} else { res = false; callback(res); }
});
}
} else {
res = false; callback(res);
}
if (res === "undefined") { callback(false); }
}
, runCMD = function(cmd,errCB,callback){
var
command
, args;
if(cmd.cmd){ command = cmd.cmd; } else { command = cmd; }
if(cmd.args){ args = cmd.args; }
const spawn = require('child_process').spawn;
const ex = spawn(command, args);
ex.stdout.on('data', (data) => {
callback(data);
});
ex.stderr.on('data', (data) => {
errCB(data);
});
ex.on('close', (code) => {
});
}
, executeCMD = function(cmd,callback){
const exec = require('child_process').exec
, cdw = (__dirname + '/../');
exec(cmd, {cwd: cdw}, (err, stdout, stderr) => {
if (err) {
callback(err,null);
return;
}
callback(stderr,stdout);
});
}
, io = require('socket.io').listen(26266) // can use up to 26485
console.log('\x1b[32m', "Admin server started.", '\x1b[0m');
console.log("Admin server listening at " + "http://" + conf["Server"]["binding"] + ":26266");
io.on('connection', function (socket) {
socket.on('restart_request', function(req){
console.log('Recieved restart request');
var success = false
, session = JSON.parse(req.session)
, sessionID = session.sessionID;
checkSession(sessionID, function (ses) {
if (ses === false) { console.error('CheckSession failed: No session exists'); return; }
if (ses.user.uuid !== session.uuid) { console.error('CheckSession failed: UUID mismatched'); return; }
if (ses.user.role < conf['Permissions']['lm_restart']){ socket.emit('restart_fail','Insufficient permissions.'); return; }
if(process.platform === 'win32'){
executeCMD('cd',function(err,res){
var errSent = false;
if(err){
console.error(err);
if(!errSent){ socket.emit('sendError','Restart failed'); }
errSent = true;
if(res === null){return;}
}
console.log(res);
socket.emit('restart_success','LM successfully restarted.');
});
}
else if(process.platform === 'linux'){
}
});
});
});
For those of you who may have seen this and found it a curious question/situation... I found two parts to this.
The first part is the $().on binding. For some reason (even though it's by no means called multiple times in the js code) adding unbind() in front of the binding resolved the issue in part... it cut the extra emits down from 3 to two (until I started another server app, then it went back up to three...)
The other part I found was that (for some reason) the socket.io connection is being duplicated as many times as there are socket servers running. More details on this issue here... I believe that once the cause for this is found, my issue will be resolved.
/* GET home page. */
router.get('/home/', function(req, res, next) {
var code = req.query.code;
req.SC.authorize(code, function(err, accessToken) {
if ( err ) {
throw err;
} else {
req.session.oauth_token = accessToken;
// Client is now authorized and able to make API calls
//res.render('home', { token: accessToken });
var url = 'https://api.soundcloud.com/me?oauth_token=' + accessToken;
requestify.get(url).then(function(response){
var user = response.getBody();
req.session.user = user;
var user_url = config.base_url + '/api/users/add';
var options = { user: user };
requestify.post(user_url, options).then(function(response){
console.log("done with users/add")
var href = 'https://api.soundcloud.com/users/' + user.id
+ '/favorites?client_id=' + config.auth.client_id + '&linked_partitioning=1&limit=200';
soundcloud.getCollection(req, res, [], href, function(collection){
console.log("can't get here...");
//console.log(collection);
res.json(collection);
//return collection;
});
/*
var collection_url = config.base_url + '/api/collections/add';
requestify.post(collection_url, options).then(function(response){
console.log("done with collections/add")
res.json(response);
})
*/
});
});
}
});
});
function getCollection(req, res, collection, next_href, done){
console.log("here");
requestify.get(next_href).then(function(response){
var updatedCollection = collection.concat(response.getBody().collection);
if (next_href && updatedCollection.length < 500){
var href = response.getBody().next_href;
getCollection(req, res, updatedCollection, href);
}
else {
console.log("done");
done(updatedCollection);
}
//res.json(response.getBody());
});
}
Behavior I'm seeing is, the collection is properly built up, the console.log("done") is showing up in the console, but after I call done(updatedCollection), the callback function I pass in does not get executed. No print statement, no json rendering. Do you guys see what the issue is?
You're recursively calling the getCollection function without the callback, so the next time it's called, done is undefined.
Pass on the callback to the recursive calls as well
function getCollection(req, res, collection, next_href, done) {
requestify.get(next_href).then(function(response){
var updatedCollection = collection.concat(response.getBody().collection);
if (next_href && updatedCollection.length < 500){
var href = response.getBody().next_href;
getCollection(req, res, updatedCollection, href, done); // <- HERE
} else {
console.log("done");
done(updatedCollection);
}
//res.json(response.getBody());
});
}
I have a function in my express app that makes multiple queries within a For Loop and I need to design a callback that responds with JSON when the loop is finished. But, I'm not sure how to do this in Node yet. Here is what I have so far, but it's not yet working...
exports.contacts_create = function(req, res) {
var contacts = req.body;
(function(res, contacts) {
for (var property in contacts) { // for each contact, save to db
if( !isNaN(property) ) {
contact = contacts[property];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err) { console.log(err) };
}); // .save
}; // if !isNAN
}; // for
self.response();
})(); // function
}; // contacts_create
exports.response = function(req, res, success) {
res.json('finished');
};
There are a few problems with your code besides just the callback structure.
var contacts = req.body;
(function(res, contacts) {
...
})(); // function
^ you are redefining contacts and res in the parameter list, but not passing in any arguments, so inside your function res and contacts will be undefined.
Also, not sure where your self variable is coming from, but maybe you defined that elsewhere.
As to the callback structure, you're looking for something like this (assuming contacts is an Array):
exports.contacts_create = function(req, res) {
var contacts = req.body;
var iterator = function (i) {
if (i >= contacts.length) {
res.json('finished'); // or call self.response() or whatever
return;
}
contact = contacts[i];
var newContact = new Contact(contact);
newContact.user = req.user.id
newContact.save(function(err) {
if (err)
console.log(err); //if this is really a failure, you should call response here and return
iterator(i + 1); //re-call this function with the next index
});
};
iterator(0); //start the async "for" loop
};
However, you may want to consider performing your database saves in parallel. Something like this:
var savesPending = contacts.length;
var saveCallback = function (i, err) {
if (err)
console.log('Saving contact ' + i + ' failed.');
if (--savesPending === 0)
res.json('finished');
};
for (var i in contacts) {
...
newContact.save(saveCallback.bind(null, i));
}
This way you don't have to wait for each save to complete before starting the next round-trip to the database.
If you're unfamiliar with why I used saveCallback.bind(null, i), it's basically so the callback can know which contact failed in the event of an error. See Function.prototype.bind if you need a reference.