The problem is that the rate limit is not enforced for the amount of time I specify. Instead of lasting 35 minutes, it lasts for only about 20 seconds. Also, if I keep making the request, the limit is always enforced, so that seems to refresh the time limit, which I think is also unexpected.
Apart from these issues, it works as expected, limiting the number of requests I specify in "max", as long as I make them quickly enough. I have tested locally, and on a Heroku server.
Here is the relevant code:
app.js
var express = require('express');
var dbRouter = require('./routes/db');
var limiter = require('express-rate-limit');
var app = express();
app.set('trust proxy', 1);
// This is a global limiter, not the one I'm having issues with.
// I've tried removing it, but the issue remained.
app.use(limiter({
windowMs: 10000,
max: 9
}));
app.use('/db', dbRouter);
module.exports = app;
db.js
var express = require('express');
var router = express.Router();
var level_controller = require('../controllers/levelController');
var limiter = require('express-rate-limit');
var level_upload_limiter = limiter({
windowMS: 35 * 60 * 1000,
max: 1,
message: 'Too many level uploads. Please try again in about 30 minutes.'
});
router.post('/level/create', level_upload_limiter, level_controller.level_create_post);
module.exports = router;
levelController.js
exports.level_create_post = [
(req, res, next) => {
// ...
}
];
It's the typo you made in your settings: windowMS -> windowMs
Related
I am trying to show Nodejs is not good for CPU Intensive tasks.
In this code, I created the async function mySlowFunction that accepts baseNumber.
When I hit http://localhost:8080 with query params 12 it takes 5 sec.
What If 5 subsequent request is being made to this api for query params 11,89,23,12,23? It would be very slow.
In this kind of scenario is it really good to use NodeJs?[i.e for CPU Intensive Task]. If yes, then how can I make this fast?
async function mySlowFunction(baseNumber) {
let result = 0;
for (var i = Math.pow(baseNumber, 7); i >= 0; i--) {
result += Math.atan(i) * Math.tan(i);
};
return result;
}
const express = require('express');
const cookieparser = require('cookie-parser');
const csrf = require('csurf');
const dotenv = require("dotenv").config();
const app = express();
app.use(express.urlencoded({extended:true}));
app.use(express.json());
app.use(cookieparser());
app.get("/",async(req,res)=>{
let data = await mySlowFunction(req.query.base);
res.json({data})
})
app.listen(process.env.PORT, () => {
console.log(`Listening on http://localhost:${process.env.PORT}`);
});
I have a very general answer.
You can precalculate the results if query params are limited.
You can store results in files or database query them.
You can cache results.
If there is no result for query param in db, calculate it and save it to db.
I have a router.post in my index.js and I am trying to get an input from the user and then direct the user to a different page called question.ejs. Question.ejs has a js called question.js that scraps user's relevant articles from the web and present them in html. The problem is that my question.ejs loads before my request is finished and I have no content to show for. Another odd thing, I have to press enter twice to make everything work... It's all very hard to explain but I made a video of my problem and I hope you can help me.
I've made several console logs and I have listed what they said and numbers next to blank indicates the order of console logs in my terminal.
Here is my code for index.js:
var express = require('express');
var router = express.Router();
var news = require('../news.json');
var newsQuestion = require('../newsQuestion.json');
var bodyParser = require('body-parser');
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index', {
title: 'Express',
news: news
});
});
var parse = require('./parse');
router.use(bodyParser.urlencoded({ extended: true }));
router.post('/questionUser', function(req,res){
newsQuestion.question.txt = req.body.id;
//var question = require('./question');
question = require('./question');
router.use('/question',question);
console.log("this is after router.use: ", newsQuestion.bbc.title); =>
3**undefined**
res.redirect('question');
});
module.exports = router;
Here is my code for question.js
var express = require('express');
var router = express.Router();
var news = require('../news.json');
var newsQuestion = require('../newsQuestion.json');
var bodyParser = require('body-parser');
var request = require('request');
var cheerio = require('cheerio');
var linkName = 'http://www.bbc.co.uk/search?
q='+newsQuestion.question.txt+'&sa_f=search-product&filter=news&suggid=';
console.log('this is before request: ', newsQuestion.bbc.title); =>
1**undefined**
request(linkName, function (error, response, html) {
console.log('This is inside of request but before the actual
scrapping'); => 4**undefined**
var titleArray = [];
var linkArray = [];
var img = [];
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
$('a', 'h1').each(function () {
var title = $(this).text().trim();
var link = $(this).attr('href');
if (title.length != 0) {
titleArray.push(title);
linkArray.push(link);
}
});
$('img').each(function () { //#siteTable is a the context and a.title is within the context.
img.push($(this).attr('src'));
});
console.log('This is inside request: ', newsQuestion.bbc.title); => 5*undefined*
newsQuestion.bbc.imgUrl = img[0];
newsQuestion.bbc.title = titleArray[0];
newsQuestion.bbc.url = linkArray[0];
console.log(newsQuestion.bbc.title); => 6*Chile moves towards legalizing abortion in limited cases*
}
});
console.log('this is after request: ',newsQuestion.bbc.title); => 2**undefined**
router.get('/', function(req, res, next) {
res.render('question');
});
Here is my terminal:
"C:\Program Files\JetBrains\WebStorm 2017.1.4\bin\runnerw.exe" "C:\Program
Files\nodejs\node.exe" "C:\Users\Archie
Jugdersuren\WebstormProjects\summaproject_trial3 - Copy (3)\bin\www"
this is inside parse
GET / 304 9.783 ms - -
GET /stylesheets/style.css 304 1.505 ms - -
this is before request: undefined
this is after request: undefined
this is after router.use: undefined
POST /questionUser 302 14.087 ms - 60
GET /question 304 1.673 ms - -
GET /stylesheets/style.css 304 0.417 ms - -
This is inside of request but before the actual scrapping
This is inside request: undefined
Chile moves towards legalising abortion in limited cases
Here is a video of me describing my problem (HD):
https://www.youtube.com/watch?v=y1VSu1DylFQ
Thank you.
Yeah I figured it out.. it's sort of bootleg, I just made two separate functions.. one holding the request and the other holding the res.render, and I made sure that res.render just waits like 3 seconds when request is called. I figured that's enough time for request to finish.... If there is any better way please let me know.
I am trying to set API rate limit on my app using express-rate-limit. It works if it is from the same IP address. I have an error message once it reaches a max of 5. However, it fails when it is tried from different IP address/computer. Any idea how I can fix this? I tried using 127.0.0.1 to generate a key regardless of which IP address but that failed as well.
Below is my code:
// Rate Limit
var RateLimit = require('express-rate-limit');
app.enable('trust proxy');
var limiter = new RateLimit({
windowMs: 365*24*60*60*1000, // 1 year
max: 5, // limit each IP to 1 requests per windowMs
delayMs: 365*24*60*60*1000, // delaying - 365 days until the max limit is reached
message: "Sorry, the maximum limit of 50 letters sent has been reached. Thank you for participating!",
keyGenerator: function (req) {
req.ip = "127.0.0.1";
// req.ip = "ip address";
return req.ip;
}
});
app.use('/api/letter', limiter);
The memory store implementation used by express-rate-limit uses setTimeout() to clear the store after windowMs milliseconds.
According to the Node.js documentation for setTimeout(),
When delay is larger than 2147483647 or less than 1, the delay will be set to 1.
In your case, the delay is larger than that amount, namely 31536000000 milliseconds. This results in the store never storing any data for more than 1ms.
To solve this, you probably have to implement your own store (see the store option), or perhaps look for an alternative rate limiter that doesn't have this limit (it seems to me that with such large expiry times, you'll need some sort of persistent storage anyway).
I think it's perfectly reasonable to call this "rate limiting". Just because the time period is big (yearly) doesn't mean it's not a limit per time period.
https://www.ratelim.it/documentation/once_and_only_once Takes this even further and let's you do N times per "infinite" which is super useful.
You should be able to use this service to do 5 per year. (I run ratelim.it).
rate-limiter-flexible package with Mongo can help set up rate limits for 1 year
const { RateLimiterMongo } = require('rate-limiter-flexible');
const { MongoClient } = require('mongodb');
const mongoOpts = {
useNewUrlParser: true,
reconnectTries: Number.MAX_VALUE, // Never stop trying to reconnect
reconnectInterval: 100, // Reconnect every 100ms
};
const mongoConn = MongoClient.connect(
'mongodb://localhost:27017',
mongoOpts
);
const opts = {
mongo: mongoConn,
points: 5, // Number of points
duration: 365*24*60*60, // Per 1 year
};
const rateLimiter = new RateLimiterMongo(opts);
app.use('/api/letter', (req, res, next) => {
rateLimiter.consume(req.ip)
.then(() => {
next();
})
.catch((rejRes) => {
res.status(429).send('Too Many Requests');
});
);
It is also recommended to set up insuranceLimiter and block strategy. Read more here
I use express.js and 'cron' module for auto db updating, so I dont know where should I add my cron init code, so I added it to my 'bin/www' , but after server started it spams like every seconds(but I need every 2 minutes), but if I changed 2 on 5 , its never started.
here is my cron update js:
var catalogUpdater = require('../utils/catalogUpdater');
var descriptionDownloader = require('../utils/descriptionDownloader');
var CronJob = require('cron').CronJob;
var job = new CronJob('* */2 * * * *', function(){
console.log('started');
},
function(){console.log('stop')},
true);
module.exports = job;
here is my 'bin/www' code:
var app = require('../app');
var debug = require('debug')('shopnagby:server');
var http = require('http');
var config = require('../config');
var job = require('../cron/updateCron'); // include job updateCron to server startup;
/**
* Get port from environment and store in Express.
*/
var port = normalizePort(config.get("port"));
app.set('port', port);
/**
* Create HTTP server.
*/
var server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
.....
Where should I put my updateCron script?
You have incorrect cron syntax. Correct syntax have only 5 fields:
field allowed values
----- --------------
minute 0-59
hour 0-23
day of month 0-31
month 0-12 (or names, see below)
day of week 0-7 (0 or 7 is Sun, or use names)
To run in every two minutes:
var job = new CronJob('*/2 * * * *', function(){
console.log('started');
},
function(){console.log('stop')},
true);
I'm working on an app that uses Node.js as the backend. Currently, I have a web server setup like this:
var express = require('express');
var http = require('http');
var app = module.exports.app = express();
http.createServer(app).listen(appConfig.port, function () {
var logger = app.get('logger');
logger.info('**** STARTING SERVER ****');
});
My app is working just fine. Except, I have now added a request that is takes ~5 minutes. From my understanding, Node defaults to a 2 minute timeout window. I can read the documentation here. However, I don't understand how to set the timeout to a greater value.
Can someone show me how to increase the timeout to 10 minutes?
this should set it. you need a reference to the server object then set the timeout
var express = require('express');
var http = require('http');
var app = module.exports.app = express();
var server = http.createServer(app);
server.setTimeout(10*60*1000); // 10 * 60 seconds * 1000 msecs
server.listen(appConfig.port, function () {
var logger = app.get('logger');
logger.info('**** STARTING SERVER ****');
});