I have a code like this written in node.js for aws applications. I'm familiar with java and python but not with javascript.
I need to check if i have any messages left in my queue and if so i need to proccess them then delete. But as far as i understand the while loop doesn't wait for my queue processes and just run. After some time it exhausts my memory.
If i do it with for loop then no problem but i must do this with while loop so is there any way to use while loop for this?
message_count = true;
while (message_count === true)
{
queue.getQueueAttributes(params_queue, function (err, data)
{
if (err)
console.log(err, err.stack);
else
console.log(data);
if (data.Attributes.ApproximateNumberOfMessages == "0")
{
message_count = false;
}
queue.receiveMessage(function (err, data)
{
if (data)
{
message = data.Messages[0].Body
receipthandle = data.Messages[0].ReceiptHandle;
params.ReceiptHandle = receipthandle
queue.deleteMessage(params, function (err, data)
{
if (err)
console.log(err, err.stack);
else
console.log(data);
});
}
});
});
}
Here is some sample code I wrote sometime back to consume messages from queue. And when there are no messages try again after 1 minute delay.
var AWS = require('aws-sdk');
AWS.config.loadFromPath('./config.json');
AWS.config.update({region: 'us-east-1'});
var sqs = new AWS.SQS();
var sqsQueueURl = "<queueurl>";
var receiveMessageParams = {
QueueUrl : sqsQueueURl,
MaxNumberOfMessages : 10,
VisibilityTimeout : 10,
WaitTimeSeconds : 10
};
var receiveMessage = function() {
sqs.receiveMessage(receiveMessageParams, function(err, data) {
if(err){
console.log(err);
}
if (data.Messages) {
for (var i = 0; i < data.Messages.length; i++) {
var message = data.Messages[i];
var body = JSON.parse(message.Body);
// execute logic
removeFromQueue(message);
}
receiveMessage();
} else {
setTimeout(function() {
receiveMessage()
}, 60 * 1000);
}
});
};
var removeFromQueue = function(message) {
sqs.deleteMessage({
QueueUrl : sqsQueueURl,
ReceiptHandle : message.ReceiptHandle
}, function(err, data) {
err && console.log(err);
});
};
receiveMessage();
Related
I am requesting a rest server from nodejs by using nodejs request module.
I want to cancel stream if incoming data size is out of allowed limit.the purpose here is to ensure that my network is not locked.
My code example is as follows;
var http = require("http");
async function httpRequest({
host,
method,
port,
path
} = params, data) {
if (method.toUpperCase() === "GET") {
let query = "";
data = JSON.parse(data);
for (var key in data) {
if (data.hasOwnProperty(key)) {
let value = data[key];
console.log(key + " -> " + value);
query = query
.concat("&")
.concat(key)
.concat("=")
.concat(value);
}
}
if (query) {
query = "?".concat(query.substring(1));
}
path = encodeURI(path.concat(query));
console.log("path : " + path);
}
var opts = {
hostname: host,
port: port,
path: path,
method: method,
timeout: 30 * 1000,
headers: {
"Content-Type": "application/json"
}
};
return new Promise(function (resolve, reject) {
const req = http.request(opts, function (response) {
console.log("Status Code : " + response.statusCode);
if (response.statusCode < 200 || response.statusCode >= 300) {
req.end();
return reject("Fetch data failed = " + response.statusCode);
}
var str = "";
response.on("data", function (chunk) {
console.log("chunk : " + chunk);
str += chunk;
if (str.length > 256) {
req.abort();
reject(
new Error(
"The size of the incoming data is larger than the allowable limit."
)
);
}
});
response.on("end", function () {
console.log("\n Result from web service : ", str);
try {
let jsonData = JSON.parse(str);
if (jsonData.status) {
if (jsonData.status.toLowerCase === "success") {
if (!("result" in jsonData)) {
reject("Json Structure Error");
}
} else if (jsonData.status.toLowerCase === "error") {
if (!jsonData.error) {
reject("Json Structure Error");
}
}
resolve(jsonData);
} else {
reject("Json Structure Error");
}
} catch (error) {
reject("Response json error : " + error);
}
});
});
if (method.toUpperCase() !== "GET" && data) {
req.write(data);
}
//req bitti
req.on("timeout", function () {
console.log("timeout! " + opts.timeout / 1000 + " seconds expired");
req.abort();
});
req.on("error", function (err) {
console.log("Error : " + err);
if (err.code === "ECONNRESET") {
req.abort();
console.log("Timeout occurs : " + err);
reject(new Error("Timeout occurs : " + err));
} else if (err.code === "ENOTFOUND") {
req.abort();
console.log("Address cannot be reachable : " + err);
reject(new Error("Address cannot be reachable : " + err));
} else {
req.abort();
reject(new Error(err));
}
});
req.end();
});
}
let data = JSON.stringify({
username: "monetrum",
password: "123456",
name: "Loremipsumdolorsitamet,consecteturadipiscingelit" +
".Aeneaninaliquamodio,egetfac"
});
let params = {
host: "127.0.0.1",
method: "GET",
port: 3010,
path: "/login"
};
httpRequest(params, data);
So farr so good.But There is a problem.I am controlling incoming data.Size of data I allowed must not greater than 256 Bytes.But first fetch of chunk is larger than allowed size.So my size control is nonsense.Is there a way to handle it.Is it possible to limit size of chunk. Thanks in advance.
The 'readable' event
You want to use the readable event instead of the data event:
var byteCount = 0;
var chunkSize = 32;
var maxBytes = 256;
req.on('readable', function () {
var chunks = [];
var data;
while(true) {
data = this.read(chunkSize);
if (!data) { break; }
byteCount += data.byteLength;
if (byteCount > maxBytes) {
req.abort();
break;
}
chunks.push(data);
}
// do something with chunks
});
req.on('abort', function () {
// do something to handle the error
});
Since your question is very specific I made the example a little more generic so that hopefully others will be able to glean from it as well.
See https://nodejs.org/api/stream.html#stream_event_readable
However...
The Network Does't Care
However, you're going to get more data than that. TCP packet size is 64k. Over non-gigabit ethernet that gets MTU truncated to 1500 bytes (1.5k).
There's nothing that you can do to prevent the network activity from happening other than closing the connection, and you can't get less than 1.5k of data per data event unless there is less than 1.5k of data being sent (or crazy network issues happen, which you have no control over).t
P.S.
I'd recommend that you use a code editor, like VSCode. It's very difficult to read code that has mixes of tabs and spaces and different blocks at different levels. It will suggest plugins that can help you catch mistakes earlier and reformat your code so that it's easier for others (and yourself) to read it.
https://code.visualstudio.com/
https://code.visualstudio.com/docs/languages/javascript
https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode
I am trying to query a flight api with my async queue in node but it appears as my queue never "drains" (simply run the drain method). I am using this library.
It runs every request just fine but then it just stops there and the content in the drain function never executes.
let flights = [];
let q = async.queue(function (airline, callback) {
const flight_search_url = 'http://someflightapi.com/search/' + airline.code + '?date=' + date + '&from=' + originAirportCode + '&to=' + destinationAirportCode;
request(flight_search_url, function(error, response, body) {
if (error) {
return callback(error);
}
if (res.statusCode !== 200) {
return callback(res.statusCode);
}
console.log(airline.code);
flights.push(JSON.parse(body));
callback();
});
}, 10);
q.drain(function(error) {
if (error) {
res.json({
error: "There was an error while calculating flights",
destinationAirportCode: destinationAirportCode,
originAirportCode: originAirportCode,
possibleOrigins: possibleOrigins,
possibleDestinations: possibleDestinations,
flights: flights
});
} else {
res.json(flights);
}
});
q.push(airlines);
I have an array and i need process them. While task is running i need to save some information to send through ajax. I use async module, but don't do what i need.
Example:
var sendData = new Array();
async.each(mails, function(item) {
var mail = item.trim();
console.log(mail);
User.find({mail: mail},function(err, user){
if (user.length > 0) {
sendMail(mail);
var reg = new REG({mail: mail, resp:"Send Mail"});
reg.save(function(err){
if(!err){
var dat ={success: true,msg: "Seccess!"};
sendData.push(dat);
}
});
}else{
var dat ={success: false,msg: "Error!"};
sendData.push(dat);
}
});
}, function(err) {
if (err) {
console.log("ERROR");
console.log(err);
}
console.log("sendResp");
console.log(sendData);
res.send(sendData);
});
var sendData = new Array();
async.each(mails, function(item,callback) {
var mail = item.trim();
console.log(mail);
User.find({mail: mail},function(err, user){
if (user.length > 0) {
sendMail(mail);
var reg = new REG({mail: mail, resp:"Send Mail"});
reg.save(function(err){
if(!err){
var dat ={success: true,msg: "Seccess!"};
sendData.push(dat);
}
});
}else{
var dat ={success: false,msg: "Error!"};
sendData.push(dat);
}
callback();
});
}, function(err) {
if (err) {
console.log("ERROR");
console.log(err);
}
console.log("sendResp");
console.log(sendData);
res.send(sendData);
});
wish this can help u!
//Server Functions
var socketArray = [];
var socketRcon = [];
for (var i = 0; i < serversConfig.serversArray.length; i++) {
socketArray[i] = new Socket;
socketArray[i].setEncoding("utf8");
socketArray[i].setNoDelay();
socketArray[i].setTimeout(1000);
socketArray[i].connect(serversConfig.serversArray[i].port, serversConfig.serversArray[i].ip);
socketRcon[i] = serversConfig.serversArray[i].rcon;
socketArray[i].on("connect", function() {
this.write(socketRcon[i] + "\n", "utf8");
console.log("CONNECTED TO THE SERVER...");
});
socketArray[i].on("data", function(data) {
console.log(data);
});
socketArray[i].on("error", function(err) {
console.log("ERROR:" + err);
});
socketArray[i].on("close", function(err) {
console.log("CLOSED:" + err);
});
};
This is the code I have now to connect to multiple servers from a config file, and I need that each time the socket connects, I need to send a password to it. But 'socketRcon[i]' is undefined, why is that happening and how do i fix it?
Because by the time that code is run, i is equal to serversConfig.serversArray.length, meaning socketRcon[i] is undefined.
Anchor your value:
for( var i=0; i<l; i++) (function(i) {
// do stuff here
})(i);
You could also just do:
serversConfig.serversArray.forEach(function(srvconfig) {
var sock = new Socket();
sock.setEncoding("utf8");
sock.setNoDelay();
sock.setTimeout(1000);
socketArray.push(sock);
socketRcon.push(srvconfig.rcon);
sock.on("connect", function() {
this.write(srvconfig.rcon + "\n", "utf8");
console.log("CONNECTED TO THE SERVER...");
});
sock.on("data", function(data) {
console.log(data);
});
sock.on("error", function(err) {
console.log("ERROR:" + err);
});
sock.on("close", function(err) {
console.log("CLOSED:" + err);
});
sock.connect(srvconfig.port, srvconfig.ip);
});
I've run into a problem in my application's code and I would like to know the best way to handle it: I have a function that applies 5 values on callback and I would like to know the best way to use it.
Here is my function code :
var someFunc = function(callback) {
var http = require('http');
var id;
var url = 'http://somesite.com/json';
// First request to get an array of 5 elements
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var jsonResult = JSON.parse(body);
// 5 requests with a value from each of the 5 elements
for (var i=0;i<5;i++)
{
(function(idx) {
gameId = jsonResult.gameList[idx].id;
url = 'http://somesite.com' + id + '/token';
http.get(url, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
jsonRes = JSON.parse(body);
callback.apply(null, [idx, jsonRes.interestScore]);
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
})(i);
}
});
}).on('error', function(e) {
console.log("Got error: ", e);
});
};
exports.someFunc = someFunc;
When I call the function to retrieve the 5 values I do it like this :
exports.featured = function(req, res){
getSome.someFunc(function callback(result) {
var variables = {};
var variableName = result;
variables[variableName] = jsonRes.interestScore;
res.render('featured', { score0: variables[0], score1: variables[1], score2: variables[2], score3: variables[3], score4: variables[4] });
});
};
Unfortunately 'res.render' is called after the function retrieved only 1 value, so I want to know how to do it proprely, or make a proper callback.
Thanks.
The function you call is async, response end events can be happening anytime. And your code causes res.render to execute 5 times, but you only need it to execute 1 time with 5 values. You should be using a module like async which will help you to fire multiple tasks, and callback when all of them is finished.
Example:
var jsonResult = JSON.parse(body);
var arr = [];
for(var i = 0; i < 5; i++){
arr.push(jsonResult[0].interestScore);
}
async.map(arr, myAsyncFunction, function(err, results){
// results[0] => response of first index
// results[4] => response of last index
});
The first problem which I'm seeing is that you are assigning listener for the end event five times. You should do that only once. You can collect the result five times and after that call the callback. Here is an example which uses request module:
var request = require('request');
var makeRequests = function(callback) {
var result = [],
done = 0;
request('http://www.google.com', function (error, response, body) {
if (!error && response.statusCode == 200) {
// read the body here
var searchFor = [
'nodejs', // 1
'http request', // 2
'npm', // 3
'express', // 4
'javascript' // 5
];
for(var i=0; keyword = searchFor[i]; i++) {
request('https://www.google.bg/search?q=' + keyword, function (error, response, body) {
if (!error && response.statusCode == 200) {
result.push(body);
++done;
if(done == searchFor.length) {
callback(result);
}
}
});
}
}
});
}
makeRequests(function(result) {
console.log("result=" + result.length);
})