We Keep Coding

How to limit size of chunk incoming from rest server on nodejs http module?

I am requesting a rest server from nodejs by using nodejs request module.
I want to cancel stream if incoming data size is out of allowed limit.the purpose here is to ensure that my network is not locked.
My code example is as follows;
var http = require("http");
async function httpRequest({
host,
method,
port,
path
} = params, data) {
if (method.toUpperCase() === "GET") {
let query = "";
data = JSON.parse(data);
for (var key in data) {
if (data.hasOwnProperty(key)) {
let value = data[key];
console.log(key + " -> " + value);
query = query
.concat("&")
.concat(key)
.concat("=")
.concat(value);
}
}
if (query) {
query = "?".concat(query.substring(1));
}
path = encodeURI(path.concat(query));
console.log("path : " + path);
}
var opts = {
hostname: host,
port: port,
path: path,
method: method,
timeout: 30 * 1000,
headers: {
"Content-Type": "application/json"
}
};
return new Promise(function (resolve, reject) {
const req = http.request(opts, function (response) {
console.log("Status Code : " + response.statusCode);
if (response.statusCode < 200 || response.statusCode >= 300) {
req.end();
return reject("Fetch data failed = " + response.statusCode);
}
var str = "";
response.on("data", function (chunk) {
console.log("chunk : " + chunk);
str += chunk;
if (str.length > 256) {
req.abort();
reject(
new Error(
"The size of the incoming data is larger than the allowable limit."
)
);
}
});
response.on("end", function () {
console.log("\n Result from web service : ", str);
try {
let jsonData = JSON.parse(str);
if (jsonData.status) {
if (jsonData.status.toLowerCase === "success") {
if (!("result" in jsonData)) {
reject("Json Structure Error");
}
} else if (jsonData.status.toLowerCase === "error") {
if (!jsonData.error) {
reject("Json Structure Error");
}
}
resolve(jsonData);
} else {
reject("Json Structure Error");
}
} catch (error) {
reject("Response json error : " + error);
}
});
});
if (method.toUpperCase() !== "GET" && data) {
req.write(data);
}
//req bitti
req.on("timeout", function () {
console.log("timeout! " + opts.timeout / 1000 + " seconds expired");
req.abort();
});
req.on("error", function (err) {
console.log("Error : " + err);
if (err.code === "ECONNRESET") {
req.abort();
console.log("Timeout occurs : " + err);
reject(new Error("Timeout occurs : " + err));
} else if (err.code === "ENOTFOUND") {
req.abort();
console.log("Address cannot be reachable : " + err);
reject(new Error("Address cannot be reachable : " + err));
} else {
req.abort();
reject(new Error(err));
}
});
req.end();
});
}
let data = JSON.stringify({
username: "monetrum",
password: "123456",
name: "Loremipsumdolorsitamet,consecteturadipiscingelit" +
".Aeneaninaliquamodio,egetfac"
});
let params = {
host: "127.0.0.1",
method: "GET",
port: 3010,
path: "/login"
};
httpRequest(params, data);
So farr so good.But There is a problem.I am controlling incoming data.Size of data I allowed must not greater than 256 Bytes.But first fetch of chunk is larger than allowed size.So my size control is nonsense.Is there a way to handle it.Is it possible to limit size of chunk. Thanks in advance.

The 'readable' event
You want to use the readable event instead of the data event:
var byteCount = 0;
var chunkSize = 32;
var maxBytes = 256;
req.on('readable', function () {
var chunks = [];
var data;
while(true) {
data = this.read(chunkSize);
if (!data) { break; }
byteCount += data.byteLength;
if (byteCount > maxBytes) {
req.abort();
break;
}
chunks.push(data);
}
// do something with chunks
});
req.on('abort', function () {
// do something to handle the error
});
Since your question is very specific I made the example a little more generic so that hopefully others will be able to glean from it as well.
See https://nodejs.org/api/stream.html#stream_event_readable
However...
The Network Does't Care
However, you're going to get more data than that. TCP packet size is 64k. Over non-gigabit ethernet that gets MTU truncated to 1500 bytes (1.5k).
There's nothing that you can do to prevent the network activity from happening other than closing the connection, and you can't get less than 1.5k of data per data event unless there is less than 1.5k of data being sent (or crazy network issues happen, which you have no control over).t
P.S.
I'd recommend that you use a code editor, like VSCode. It's very difficult to read code that has mixes of tabs and spaces and different blocks at different levels. It will suggest plugins that can help you catch mistakes earlier and reformat your code so that it's easier for others (and yourself) to read it.
https://code.visualstudio.com/
https://code.visualstudio.com/docs/languages/javascript
https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode

Confused with the callback

I have got a node js program as below.
msg = "Generate a number......";
sessionAttributes = {};
console.log("Session arreubs are :" + sessionAttributes);
console.log("end of flow");
callback(close(intentRequest.sessionAttributes, "Fulfilled", buildMessage(msg)));
here is my close callback
function close(sessionAttributes, fulfillmentState, message) {
return {
sessionAttributes,
dialogAction: {
type: "Close",
fulfillmentState,
message,
},
};
}
when I run this, I get the output as Generate a number....... But now I need to integrate this with an API call. and the updated code is as below.
var service = require("./service.js");
service.createIncident(enterpriseID, shortDesc,
function (incidentNo) {
if (incidentNo) {
console.log("Index.js ServiceNow Incident:" + incidentNo);
msg = "Thank you! Your " + TicketType + " Number is " + incidentNo;
console.log("end of flow");
callback(close(intentRequest.sessionAttributes, "Fulfilled", buildMessage(msg)));
} else {
console.log("Index.js ServiceNow Incident:" + incidentNo);
msg = "Err";
console.log("end of flow");
callback(close(intentRequest.sessionAttributes, "Fulfilled", buildMessage(msg)));
}
});
console.log("process is done");
my service.js
var request = require("request");
var servicenow = require("./configfile.json");
var snowURL = servicenow.url;
var snowUsername = servicenow.username;
var snowPassword = servicenow.password;
var ticketNo = "00000";
console.log("Service Now URL:" + snowURL + " Username:" + snowUsername + " Password:" + snowPassword);
module.exports.createIncident = function (caller_id, short_description, callback) {
var snowdetails = {
uri: snowURL,
method: "POST",
"content-type": "application/json",
"auth": {
"username": snowUsername,
"password": snowPassword
}
};
request(snowdetails, function (error, resp, body) {
console.log("Status code " + resp.statusCode);
if (!error && (resp.statusCode == 200 || resp.statusCode == 201)) {
if (body) {
var data = JSON.parse(body);
ticketNo = data.result.number;
console.log("Service Now Incident No:" + ticketNo);
callback(ticketNo);
return;
} else {
console.log("I am unable to authenticate you. please disable the skill and re link your account");
callback("I am unable to authenticate you. please disable the skill and re link your account");
}
} else {
console.log(error);
callback(error);
}
});
};
Here when I run this, I get the output as
process is done
Service Now Incident No:INC0010035
Index.js ServiceNow Incident:INC0010035
Thank you! Your incident Number is INC0010035
and in my window, earlier, when there was no callback, it used to print the msg content, i.e. Generating ticket......, but now it is not printing anything.
Where am I going wrong and how can I fix it?

NodeJS if string contains execute script

I have a the following script:
function execute(){
require("fs").readFile("sometextfile.txt", function(err, cont) {
if (err)
throw err;
console.log("ALERT:"+(cont.indexOf("mystring")>-1 ? " " : " not ")+"found!");
});
}
setInterval(execute,9000);
I want to execute a javascript only if the string contains "Alert: found!"
The script:
var Prowl = require('node-prowl');
var prowl = new Prowl('API');
prowl.push('ALERT', 'ALERT2', function( err, remaining ){
if( err ) throw err;
console.log( 'I have ' + remaining + ' calls to the api during current hour. BOOM!' );
});
Help!

Are you asking how to combine the two?
const fs = require('fs');
const Prowl = require('node-prowl');
const prowl = new Prowl('API');
function alert() {
prowl.push('ALERT', 'ALERT2', function(err, remaining) {
if (err) throw err;
console.log('I have ' + remaining + ' calls to the API during current hour. BOOM!');
});
}
function poll() {
fs.readFile('sometextfile.txt', function(err, cont) {
if (err) throw err;
if (cont.indexOf('mystring') !== -1) {
alert();
}
});
}
setInterval(poll, 9000);

receive and delete message in a while loop on aws

I have a code like this written in node.js for aws applications. I'm familiar with java and python but not with javascript.
I need to check if i have any messages left in my queue and if so i need to proccess them then delete. But as far as i understand the while loop doesn't wait for my queue processes and just run. After some time it exhausts my memory.
If i do it with for loop then no problem but i must do this with while loop so is there any way to use while loop for this?
message_count = true;
while (message_count === true)
{
queue.getQueueAttributes(params_queue, function (err, data)
{
if (err)
console.log(err, err.stack);
else
console.log(data);
if (data.Attributes.ApproximateNumberOfMessages == "0")
{
message_count = false;
}
queue.receiveMessage(function (err, data)
{
if (data)
{
message = data.Messages[0].Body
receipthandle = data.Messages[0].ReceiptHandle;
params.ReceiptHandle = receipthandle
queue.deleteMessage(params, function (err, data)
{
if (err)
console.log(err, err.stack);
else
console.log(data);
});
}
});
});
}

Here is some sample code I wrote sometime back to consume messages from queue. And when there are no messages try again after 1 minute delay.
var AWS = require('aws-sdk');
AWS.config.loadFromPath('./config.json');
AWS.config.update({region: 'us-east-1'});
var sqs = new AWS.SQS();
var sqsQueueURl = "<queueurl>";
var receiveMessageParams = {
QueueUrl : sqsQueueURl,
MaxNumberOfMessages : 10,
VisibilityTimeout : 10,
WaitTimeSeconds : 10
};
var receiveMessage = function() {
sqs.receiveMessage(receiveMessageParams, function(err, data) {
if(err){
console.log(err);
}
if (data.Messages) {
for (var i = 0; i < data.Messages.length; i++) {
var message = data.Messages[i];
var body = JSON.parse(message.Body);
// execute logic
removeFromQueue(message);
}
receiveMessage();
} else {
setTimeout(function() {
receiveMessage()
}, 60 * 1000);
}
});
};
var removeFromQueue = function(message) {
sqs.deleteMessage({
QueueUrl : sqsQueueURl,
ReceiptHandle : message.ReceiptHandle
}, function(err, data) {
err && console.log(err);
});
};
receiveMessage();

Missing ) after argument list (Node)

I'm using Node to run this program, where is my error(s)? It's saying I'm missing ) after argument list. I can't find where this error is, I've tried putting the ) in various places. I'm using Node v5
var Twit = require('twit');
var T = new Twit(require('./config.js'));
var stream = T.stream('statuses/filter', {
track: 'xoxo, oi, i\m fine,'
});
(stream.on('tweet', function(tweet) {
console.log('#' + tweet.user.screen_name + ': ' + tweet.text);
if (tweet.text.indexOf('RT') > -1) {
return;
}
var replyString;
if (tweet.user.utc_offset === null) {
replyString = ' Ok';
} else {
replyString = ' Okay';
}
})
(T.post('statuses/update', {
status: '#' + tweet.user.screen_name + replyString,
in_reply_to_status_id: tweet.id_str
}, function(err, data, response) {
if (err) {
console.log(err);
return;
}
}
tweet.botReplyId = data.id_str);
db.tweets.insert(tweet);
});
(end)
})
setInterval(stream, 60000);

The code seems to be a bit all over the place with regards to scope and it makes it a bit difficult to follow.
Try using something like the following which annotates it a bit and should help avoid issues like this (as it seems to validate without any errors) :
// Define your variables
var Twit = require('twit');
var T = new Twit(require('./config.js'));
var stream = T.stream('statuses/filter', { track: 'xoxo, oi, i\'m fine,'});
// When a tweet occurs
(stream.on('tweet', function(tweet) {
// Log it
console.log('#' + tweet.user.screen_name + ': ' + tweet.text);
// Determine if it is a retweet and ignore
if (tweet.text.indexOf('RT') > -1) { return; }
// Set your reply
var replyString = (tweet.user.utc_offset === null) ? ' Ok' : ' Okay';
// Post your reply
T.post('statuses/update', { status: '#' + tweet.user.screen_name + replyString, in_reply_to_status_id: tweet.id_str}, function(err, data, response) {
// If an error occurs, log it
if (err) {
console.log(err);
return;
}
// Otherwise store your response and store it
tweet.botReplyId = data.id_str;
db.tweets.insert(tweet);
});
}));
// Check your stream every 10 minutes
setInterval(stream, 60000);