I'm creating a script that will make a request 2 times per second to a localserver of cameras network and after it gets a positive response that camera detected someone I want to log three images.
In the json config file I have the triggerURL of the server, the interval port, the dataDir where logged images should be saved and a track array which contains the url of those images and the fileName they should receive.
This is the code of the script I use after reading the JSON file:
var configGet = {
host: config.triggerURL
, port: config.interval
, method: 'GET'
};
setInterval(function () {
var request = http.request(configGet, function (response) {
var content = "";
// Handle data chunks
response.on('data', function (chunk) {
content += chunk;
});
// Once we're done streaming the response, parse it as json.
response.on('end', function () {
var data = JSON.parse(response);
if (data.track.length > 0) {
//log images
var download = function (uri, filename, callback) {
request.head(uri, function (err, res, body) {
request(uri)
.pipe(fs.createWriteStream(filename))
.on('close', callback);
});
};
for (var image in data.track) {
var path = config.dataDir + '/' + image.fileName
download(image.url, path.format(config.timestamp), function () {
console.log('done');
});
}
}
});
// Report errors
request.on('error', function (error) {
console.log("Error while calling endpoint.", error);
});
request.end();
}, 500);
});
I have the following questions:
This method produces some kind of error with the download process of the images.Can you identify it?
Is there a better way of doing this process?
Without running the code or deeper inspection; should not "data = JSON.parse(response)" rather be "data = JSON.parse(content)"? Also, if data is undefined or does not contain "track" the "if (data.track.length > 0)" will throw an error. This can be fixed with "if (data && data.track && data.track.length > 0)".
I can not think of a very much better way. I would break it up more in functions to make the code more clear though.
Related
In my Node JS server I have this route handler that sends a request to a third party API to get a username:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function (res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(JSON.stringify(data));
response.send(objectParsed);
})
if(!player) {
res.status(404).send("Not found.");
}
})
apiRequest.end();
})
This works fine to get a user that exists. However, if I put in a fake username to my /players page, that page still loads with a 200 status instead of getting a 404 response. The page loads and looks broken because it's not actually getting any data from the API.
I feel like this is a dumb question .. In my research I have found how to handle errors if it's just the route, and not if it's the route dependent on the path parameter as in /players/:player
I found a question that was similar to mine (How to throw a 404 error in express.js?) and I tried using an If statement: if (!player){res.status(404).send("Not found."); } but no dice. Am I using this if statement in the wrong place?
How can I get my Node JS server to respond with a 404 if the user from the database doesn't exist?
You have to check the result of the API call and see if you got valid data back and send the 404 there. I also added a check to make sure something was passed for the player name and send back a 400 (bad request) if there's no player specified at all:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
if (!player) {
res.status(400).send("No player specified.");
return;
}
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function(res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(data);
// test objectParsed here
if (!some condition in objectParsed) {
res.status(404).send("No data for that player name.");
} else {
response.send(objectParsed);
}
});
});
apiRequest.end();
});
Also, you don't want JSON.parse(JSON.stringify(data)) here. Your data is already a string. Just do JSON.parse(data).
FYI, if you use a small http request library such as got(), this code gets a lot simpler as it accumulates the response and parses the JSON for you in one line of code as in:
let data = await got(options).json()
I want get all files json in my directory for search into it.
$('#get-data').click(function () {
var showData = $('#show-data');
$.getJSON('/data/**all files json**', function (data) {
console.log(data);
var items = data.items.map(function (item) {
return item.key + ': ' + item.value;
});
showData.empty();
if (items.length) {
var content = '<li>' + items.join('</li><li>') + '</li>';
var list = $('<ul />').html(content);
showData.append(list);
}
});
showData.html(data);
});
Do you think its possible or i need use other method ?
Thx
You cannot make a wildcard AJAX request that will return all the possible JSON files on your server. You need to know in advance the exact endpoint that you will be sending your request to. Otherwise it would be a security vulnerability if all clients could know in advance what files are available on the server.
So one possible strategy here would be to have some /describe endpoint on your server which will return a list of all available JSON files. Then the client will first make an AJAX request to this endpoint and then for each returned endpoint in the list it will make a separate AJAX request:
$.getJSON('/data/discover', function (data) {
// suppose that data looks like this:
// ["/data/a.json", "/data/b.json", "/data/c.json"]
for (var i := 0; i < data.length; i++) {
// send an AJAX request to each individual JSON file
// available on the server as returned by the discover endpoint
$.getJSON(data[i], function (result) {
...
});
}
});
Basically you can't request for multiple files by one request.
However the scenario is perfect fit for async.parallel:
var async = require('async');
app.get('/json', function(req, res) {
var work = {
file01: async.apply(fs.readFile, __dirname + '/file01.json'),
file02: async.apply(fs.readFile, __dirname + '/file02.json')
};
async.parallel(work, function (error, results) {
if (error) {
res.status(500).send(error);
return;
}
//might need string->Object here
results['file01'] = JSON.parse(results['file01']);
results['file02'] = JSON.parse(results['file02']);
res.send(results);
});
});
Just learning Sails.js so go easy on me.
I have queried an XML service and successfully jsonified it using xml2js
var req = https.request(options, function(res) {
var xml = '';
res.on('data', function(chunk) {
xml += chunk;
});
res.on('end', function () {
var result = parseString(xml, function (err, result) {
console.log(JSON.stringify(result)); // Position 1
});
return result;
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.write(data);
var result = req.end();
console.log('Result: ' + JSON.stringify(result)); // Position 2
res.view({ message : 'hello', result : result });
The view is loading fine, and <%= message %> outputs hello. Great.
Position1 console.log is returning the stringified json object - Great.
Position 2 consile.log is returning Result: true - Not good.
I need to be able to get that json data to my view for parsing. How do I do this?
It looks like you're assuming that calling req.end() will give you the response from the https.request you started above. There are a couple of things wrong with that:
req.end() is used to finish writing to an open request, not to get a response. According to the docs, the return value is unspecified.
The https.request call is asynchronous; even if req.end() worked like you want it to, the response wouldn't have come in by the time you call it.
The solution is to put your response code (i.e. your res.view) inside the handler for the end event that you've already written. I'd also recommend refactoring your code to use different variable names for the remote request / response so that they don't collide with the req and res variables in your controller action. The whole thing would then be something like:
myAction: function (req, res) {
// Not sure how you're setting options, so just an example
var options = {url: 'http://example.com', ...}
var request = https.request(options, function(response) {
var xml = '';
response.on('data', function(chunk) {
xml += chunk;
});
response.on('end', function () {
var result = parseString(xml, function (err, result) {
return res.view({ message : 'hello', result : JSON.stringify(result)});
});
});
});
request.on('error', function(e) {
console.log('problem with request: ' + e.message);
res.serverError(e);
});
}
You might also look into using something like the Request module to simplify your external request; it would save you from having to write event handlers for data and end.
if you want to pass json to some javascript variable:
var clientJsonVar = <%- JSON.stringify(serverSideJson)%>
I am a rookie in Nodejs and asynchronous programming. I am having a problem executing a GET request inside an asynchronous function. Here I am posting the whole code. I am trying to pull a list of all Urls , add them to a list and send the list for processing to another function.
My problem is with processing them. Inturn for each url I am executing a GET request to fetch the body and to look for image elements in it. I am looking to pass the Image url to a 3rd party api as a GET param. I am unable to execute the GET request as the control doesn't seem to reach there at all.
var async = require("async"),
request = require("request"),
cheerio = require("cheerio");
async.waterfall([
function(callback) {
var url = "someSourceUrl";
var linkList = [];
request(url, function(err, resp, body) {
var $ = cheerio.load(body);
$('.list_more li').each(function() {
//Find all urls and add them to a list
$(this).find('a').each(function() {
linkList.push($(this).attr('href'));
});
});
callback(null, linkList);
});
},
//pass all the links as a list to callback
function(liksListFetched, callback) {
for (var i in liksListFetched) {
callback(null, liksListFetched[i]);
}
}],
//***********My problem is with the below code**************
function(err, curUrl) {
var cuResp = "";
console.log("Currently Processing Url : " + curUrl);
request(curUrl, function(err, resp, body) {
var $ = cheerio.load(body);
var article = $("article");
var articleImage = article.find("figure").children('img').attr('src');
var responseGrabbed = "API response : ";
//check if there is an IMG element
if (articleImage === undefined) {
console.log("No Image Found.");
articleImage = 'none';
}
else {
//if there is an img element, pass this image url to an API,
//So do a GET call by passing imageUrl to the API as a GET param
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) { //code doesn't seem to reach here
I would like to grab the response and concatenate it to the responseGrabbed var.
console.log(resp);
responseGrabbed += resp;
});
}
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
process.exit(0);
});
});
I appreciate if any one can help me understand the root cause. Thanks in advance.
request() is asynchronous, so when you're console logging the string, the string hasn't been built yet, you have to do the console log inside the callback :
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) {
responseGrabbed += resp;
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
});
Same goes for terminating the process, which should be done when all the requests have finished
I made my Request object queue up individual HTTP requests and process them one by one using process.nextTick. However, I am getting an error that I don't know how to solve:
node.js:244
callback();
^
TypeError: undefined is not a function
at process.startup.processNextTick.process._tickCallback (node.js:244:9)
I'm not sure what I am doing wrong. Here is the relevant class.
var Request = function() {
return this;
};
Request.prototype = {
queue_: []
};
Request.prototype.send = function(url, done) {
this.queue_.push(new QueueableRequest(url, done));
this.processRequest_();
}
Request.prototype.processRequest_ = function() {
if (this.queue_.length > 0) {
var request = this.queue_.shift();
var data = '';
http.get(request.url_, function(res) {
res.setEncoding('utf8');
res.on('data', function(chunk) {
data += chunk;
}).on('end', function() {
request.callback_(null, JSON.parse(data));
process.nextTick(this.processRequest_);
}).on('error', function(err) {
request.callback_(err, null);
process.nextTick(this.processRequest_);
});
});
}
}
My other question is whether this is a good method to slowing down my HTTP requests? What I am trying to do is this... I make an HTTP request for a list of threads (about 15-20), and then for each thread, I make another request to obtain its replies. Sometimes within replies, I have to make another request for the deeply nested replies. My initial solution was simply call http.get for every request, but I find that my node.js stops responding after a few requests and I have to keep restarting the server and refreshing the page. My thought was that I am perhaps sending too many requests at once, so I tried to implement this queue.
Your this inside your event handlers is incorrect, so your this.processRequest_ is undefined.
Request.prototype.processRequest_ = function() {
// Assign the outer request object to a variable so you can access it.
var self = this;
if (this.queue_.length > 0) {
var request = this.queue_.shift();
var data = '';
http.get(request.url_, function(res) {
res.setEncoding('utf8');
res.on('data', function(chunk) {
data += chunk;
}).on('end', function() {
request.callback_(null, JSON.parse(data));
process.nextTick(function(){
// Call 'processRequest_' on the correct object.
self.processRequest_()
});
}).on('error', function(err) {
request.callback_(err, null);
process.nextTick(function(){
// Call 'processRequest_' on the correct object.
self.processRequest_()
});
});
});
}
}
That said, you might consider using the request module to simplify this.
var request = require('request');
Request.prototype.processRequest_ = function() {
var self = this;
if (this.queue_.length > 0) {
var requestData = this.queue_.shift();
request(requestData.url_, function(error, response, body){
requestData.callback_(err, err ? null : JSON.parse(body));
process.nextTick(function(){
self.processRequest_();
});
});
}
};