We Keep Coding

How to limit size of chunk incoming from rest server on nodejs http module?

I am requesting a rest server from nodejs by using nodejs request module.
I want to cancel stream if incoming data size is out of allowed limit.the purpose here is to ensure that my network is not locked.
My code example is as follows;
var http = require("http");
async function httpRequest({
host,
method,
port,
path
} = params, data) {
if (method.toUpperCase() === "GET") {
let query = "";
data = JSON.parse(data);
for (var key in data) {
if (data.hasOwnProperty(key)) {
let value = data[key];
console.log(key + " -> " + value);
query = query
.concat("&")
.concat(key)
.concat("=")
.concat(value);
}
}
if (query) {
query = "?".concat(query.substring(1));
}
path = encodeURI(path.concat(query));
console.log("path : " + path);
}
var opts = {
hostname: host,
port: port,
path: path,
method: method,
timeout: 30 * 1000,
headers: {
"Content-Type": "application/json"
}
};
return new Promise(function (resolve, reject) {
const req = http.request(opts, function (response) {
console.log("Status Code : " + response.statusCode);
if (response.statusCode < 200 || response.statusCode >= 300) {
req.end();
return reject("Fetch data failed = " + response.statusCode);
}
var str = "";
response.on("data", function (chunk) {
console.log("chunk : " + chunk);
str += chunk;
if (str.length > 256) {
req.abort();
reject(
new Error(
"The size of the incoming data is larger than the allowable limit."
)
);
}
});
response.on("end", function () {
console.log("\n Result from web service : ", str);
try {
let jsonData = JSON.parse(str);
if (jsonData.status) {
if (jsonData.status.toLowerCase === "success") {
if (!("result" in jsonData)) {
reject("Json Structure Error");
}
} else if (jsonData.status.toLowerCase === "error") {
if (!jsonData.error) {
reject("Json Structure Error");
}
}
resolve(jsonData);
} else {
reject("Json Structure Error");
}
} catch (error) {
reject("Response json error : " + error);
}
});
});
if (method.toUpperCase() !== "GET" && data) {
req.write(data);
}
//req bitti
req.on("timeout", function () {
console.log("timeout! " + opts.timeout / 1000 + " seconds expired");
req.abort();
});
req.on("error", function (err) {
console.log("Error : " + err);
if (err.code === "ECONNRESET") {
req.abort();
console.log("Timeout occurs : " + err);
reject(new Error("Timeout occurs : " + err));
} else if (err.code === "ENOTFOUND") {
req.abort();
console.log("Address cannot be reachable : " + err);
reject(new Error("Address cannot be reachable : " + err));
} else {
req.abort();
reject(new Error(err));
}
});
req.end();
});
}
let data = JSON.stringify({
username: "monetrum",
password: "123456",
name: "Loremipsumdolorsitamet,consecteturadipiscingelit" +
".Aeneaninaliquamodio,egetfac"
});
let params = {
host: "127.0.0.1",
method: "GET",
port: 3010,
path: "/login"
};
httpRequest(params, data);
So farr so good.But There is a problem.I am controlling incoming data.Size of data I allowed must not greater than 256 Bytes.But first fetch of chunk is larger than allowed size.So my size control is nonsense.Is there a way to handle it.Is it possible to limit size of chunk. Thanks in advance.

The 'readable' event
You want to use the readable event instead of the data event:
var byteCount = 0;
var chunkSize = 32;
var maxBytes = 256;
req.on('readable', function () {
var chunks = [];
var data;
while(true) {
data = this.read(chunkSize);
if (!data) { break; }
byteCount += data.byteLength;
if (byteCount > maxBytes) {
req.abort();
break;
}
chunks.push(data);
}
// do something with chunks
});
req.on('abort', function () {
// do something to handle the error
});
Since your question is very specific I made the example a little more generic so that hopefully others will be able to glean from it as well.
See https://nodejs.org/api/stream.html#stream_event_readable
However...
The Network Does't Care
However, you're going to get more data than that. TCP packet size is 64k. Over non-gigabit ethernet that gets MTU truncated to 1500 bytes (1.5k).
There's nothing that you can do to prevent the network activity from happening other than closing the connection, and you can't get less than 1.5k of data per data event unless there is less than 1.5k of data being sent (or crazy network issues happen, which you have no control over).t
P.S.
I'd recommend that you use a code editor, like VSCode. It's very difficult to read code that has mixes of tabs and spaces and different blocks at different levels. It will suggest plugins that can help you catch mistakes earlier and reformat your code so that it's easier for others (and yourself) to read it.
https://code.visualstudio.com/
https://code.visualstudio.com/docs/languages/javascript
https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode

How to pass the value of the request.get to the callback using async.series node.js

I wanna show my code:
request({url: API_URL}, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("Items received with success.");
var data = JSON.parse(body);
// catch duration from video
for(var list in data) {
var item = data[list];
async.series([
function(callback){
console.log("I'm inside fileExists function");
var result = '';
request.get('http://tobeymotos.com.br/video.mp4', function (error, response, body) {
if (!error && response.statusCode == 200) {
var video = body;
console.log("File exists and is OK");
var result = true;
console.log("Valor da result", result);
}
else{
console.log("This file do not exists, check error: "+ error);
var result = "xixi";
}
});
// usar aqui
callback(null, result);
},
function(callback){
// catch video duration
file = "http://tobeymotos.com.br/video.mp4"; // used as e.g.
console.log("I'm inside getVideoDuration function");
getDuration(file).then(function (duration) {
console.log("Duração do vídeo: " +duration);
return duration;
});
callback(null, 'b');
},
function(callback){
// code c
callback(null, 'c');
},
function(callback){
// code d
callback(null, 'd');
}],
// optional callback
function(err, results){
console.log(results);
}
)
return 0;
}
} else {
console.log("Error: "+error);
}
});
I'm using async.series to perform some functions in sequence, because I depend on the result of the first one, to continue with the next.
As you can see I have a "request.get" in the first function, where:
request.get('http://tobeymotos.com.br/video.mp4', function (error, response, body) {
if (!error && response.statusCode == 200) {
var video = body;
console.log("File exists and is OK");
var result = true;
console.log("Valor da result", result);
}
else{
console.log("This file do not exists, check error: "+ error);
var result = "xixi";
}
});
// usar aqui
callback(null, result);
The problem is that I can not call result in the callback, it just does not "exit" the request.get
Some help?

request({url: API_URL}, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log("Items received with success.");
var data = JSON.parse(body);
// catch duration from video
for(var list in data) {
var item = data[list];
async.series([
function(callback){
console.log("I'm inside fileExists function");
var result = '';
request.get('http://tobeymotos.com.br/video.mp4', function (error, response, body) {
if (!error && response.statusCode == 200) {
var video = body;
console.log("File exists and is OK");
var result = true;
console.log("Valor da result", result);
callback(null, result);
}
else{
console.log("This file do not exists, check error: "+ error);
var result = "xixi";
callback(null, result);
}
});
// usar aqui
},
function(callback){
// catch video duration
file = "http://tobeymotos.com.br/video.mp4"; // used as e.g.
console.log("I'm inside getVideoDuration function");
getDuration(file).then(function (duration) {
console.log("Duração do vídeo: " +duration);
return duration;
});
callback(null, 'b');
},
function(callback){
// code c
callback(null, 'c');
},
function(callback){
// code d
callback(null, 'd');
}],
// optional callback
function(err, results){
console.log(results);
}
)
return 0;
}
} else {
console.log("Error: "+error);
}
});
Your callback is outside of the request.get function, this is possibly causing your callback to call before the request has returned due to the nature of Node JS. Node JS is non-blocking and so does not wait for a function to be carried out before moving on to the next unless explicitly told to. Putting the callback inside of the request.get response function will force it to wait for the request to return and then carry out the callback with the result

MeteorJS: Can not assign a variable

I have such code, and want to assign one variable to another in function, code example:
Meteor.methods({
parsing:function(){
var aa ;
request("https://google.com/", function(error, response, body) {
if (!error && response.statusCode == 200) {
var k=1;
aa = k;
}
});
console.log(aa);
}
});
It's loged undefined, can someone explain me why ?
EDIT :
Meteor.methods({
parsing:function(){
var aa ;
var tmp;
request("https://google.com/", function(error, response, body) {
if (!error && response.statusCode == 200) {
var k=1;
aa = k;
console.log(aa);
request("https://google.com/xyz", function(error, response, body) {
tmp = response.request.uri.href;
});
}
console.log(tmp);
});
}
});
For example i need console.log(tmp); only when my second request will be call (ended)

The function that you give as parameter of request will only be executed when the request will have an answer from Google. But the next part of the code (your console.log line) is still executed without waiting anything.
If you want to wait the request result before your log, add it to your function:
parsing:function(){
var aa ;
request("https://google.com/", function(error, response, body) {
if (!error && response.statusCode == 200) {
var k=1;
aa = k;
console.log(aa);
}
});
}
EDIT :
Well, you can use the same technique as many times you want:
parsing:function(){
var aa ;
var tmp;
request("https://google.com/", function(error, response, body) {
if (!error && response.statusCode == 200) {
var k=1;
aa = k;
console.log(aa); // Waited the first answer.
request("https://google.com/xyz", function(error, response, body) {
tmp = response.request.uri.href;
console.log(tmp); // Waited the second one.
});
}
});
}

Node.js : multiple callbacks from one function

I've run into a problem in my application's code and I would like to know the best way to handle it: I have a function that applies 5 values on callback and I would like to know the best way to use it.
Here is my function code :
var someFunc = function(callback) {
var http = require('http');
var id;
var url = 'http://somesite.com/json';
// First request to get an array of 5 elements
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var jsonResult = JSON.parse(body);
// 5 requests with a value from each of the 5 elements
for (var i=0;i<5;i++)
{
(function(idx) {
gameId = jsonResult.gameList[idx].id;
url = 'http://somesite.com' + id + '/token';
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
jsonRes = JSON.parse(body);
callback.apply(null, [idx, jsonRes.interestScore]);
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
})(i);
}
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
};
exports.someFunc = someFunc;
When I call the function to retrieve the 5 values I do it like this :
exports.featured = function(req, res){
getSome.someFunc(function callback(result) {
var variables = {};
var variableName = result;
variables[variableName] = jsonRes.interestScore;
res.render('featured', { score0: variables[0], score1: variables[1], score2: variables[2], score3: variables[3], score4: variables[4] });
});
};
Unfortunately 'res.render' is called after the function retrieved only 1 value, so I want to know how to do it proprely, or make a proper callback.
Thanks.

The function you call is async, response end events can be happening anytime. And your code causes res.render to execute 5 times, but you only need it to execute 1 time with 5 values. You should be using a module like async which will help you to fire multiple tasks, and callback when all of them is finished.
Example:
var jsonResult = JSON.parse(body);
var arr = [];
for(var i = 0; i < 5; i++){
arr.push(jsonResult[0].interestScore);
}
async.map(arr, myAsyncFunction, function(err, results){
// results[0] => response of first index
// results[4] => response of last index
});

The first problem which I'm seeing is that you are assigning listener for the end event five times. You should do that only once. You can collect the result five times and after that call the callback. Here is an example which uses request module:
var request = require('request');
var makeRequests = function(callback) {
var result = [],
done = 0;
request('http://www.google.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
// read the body here
var searchFor = [
'nodejs', // 1
'http request', // 2
'npm', // 3
'express', // 4
'javascript' // 5
];
for(var i=0; keyword = searchFor[i]; i++) {
request('https://www.google.bg/search?q=' + keyword, function (error, response, body) {
if (!error && response.statusCode == 200) {
result.push(body);
++done;
if(done == searchFor.length) {
callback(result);
}
}
});
}
}
});
}
makeRequests(function(result) {
console.log("result=" + result.length);
})

Making the while loop synchronous

I've got a following piece of code
var page = 2;
var last_page = 100;
while(page <= last_page) {
request("http://some_json_server.com/data?page=" + page, function (error, response, body) {
if (!error && response.statusCode == 200) {
store_data(body)
}
page++;
});
}
I've done the following, but it is actually not retrieving anything. Am I doing this correctly?
var page = 2;
var last_page = 100;
while(page <= last_page) {
var async_arr = [];
async_arr.push(
function(next) {
request("http://some_api_url?page=" + page, function (error, response, body) {
if (!error && response.statusCode == 200) {
store_data(body);
}
});
}
);
async.series(
async_arr, done
);

With while you get a busy loop, which is counter-purpose in Node.
Make it a recursive function instead. Each call will be done in a separate tick.
var page = 2;
var last_page = 100;
(function loop() {
if (page <= last_page) {
request("/data?page=" + page, function (error, response, body) {
if (!error && response.statusCode == 200) {
store_data(body)
}
page++;
loop();
});
}
}());

You're looking for async.whilst(). This solution is assuming you actually want to do each request after the other. As #UpTheCreek mentions (edit: the comment I referred to was edited) it would likely be possible to do it asynchronously and keep track of each result using async.parallel.
var page = 2,
lastPage = 100;
async.whilst(function () {
return page <= lastPage;
},
function (next) {
request("http://some_json_server.com/data?page=" + page, function (error, response, body) {
if (!error && response.statusCode == 200) {
store_data(body)
}
page++;
next();
});
},
function (err) {
// All things are done!
});

You can also wrap your while loop in async and break after your promise resolves/conditions have been met...
const request = require("request")
;(async()=>{
let results = []
while(true){
await new Promise(resolve => {
request('http://www.seanbehan.com/', (err, resp, body)=>{
console.log(new Date, 'Downloading..')
results.push(body)
resolve(body)
})
})
if(results.length >= 5){
break
}
}
console.log(results)
})()