I am a rookie in Nodejs and asynchronous programming. I am having a problem executing a GET request inside an asynchronous function. Here I am posting the whole code. I am trying to pull a list of all Urls , add them to a list and send the list for processing to another function.
My problem is with processing them. Inturn for each url I am executing a GET request to fetch the body and to look for image elements in it. I am looking to pass the Image url to a 3rd party api as a GET param. I am unable to execute the GET request as the control doesn't seem to reach there at all.
var async = require("async"),
request = require("request"),
cheerio = require("cheerio");
async.waterfall([
function(callback) {
var url = "someSourceUrl";
var linkList = [];
request(url, function(err, resp, body) {
var $ = cheerio.load(body);
$('.list_more li').each(function() {
//Find all urls and add them to a list
$(this).find('a').each(function() {
linkList.push($(this).attr('href'));
});
});
callback(null, linkList);
});
},
//pass all the links as a list to callback
function(liksListFetched, callback) {
for (var i in liksListFetched) {
callback(null, liksListFetched[i]);
}
}],
//***********My problem is with the below code**************
function(err, curUrl) {
var cuResp = "";
console.log("Currently Processing Url : " + curUrl);
request(curUrl, function(err, resp, body) {
var $ = cheerio.load(body);
var article = $("article");
var articleImage = article.find("figure").children('img').attr('src');
var responseGrabbed = "API response : ";
//check if there is an IMG element
if (articleImage === undefined) {
console.log("No Image Found.");
articleImage = 'none';
}
else {
//if there is an img element, pass this image url to an API,
//So do a GET call by passing imageUrl to the API as a GET param
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) { //code doesn't seem to reach here
I would like to grab the response and concatenate it to the responseGrabbed var.
console.log(resp);
responseGrabbed += resp;
});
}
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
process.exit(0);
});
});
I appreciate if any one can help me understand the root cause. Thanks in advance.
request() is asynchronous, so when you're console logging the string, the string hasn't been built yet, you have to do the console log inside the callback :
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) {
responseGrabbed += resp;
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
});
Same goes for terminating the process, which should be done when all the requests have finished
Related
In my Node JS server I have this route handler that sends a request to a third party API to get a username:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function (res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(JSON.stringify(data));
response.send(objectParsed);
})
if(!player) {
res.status(404).send("Not found.");
}
})
apiRequest.end();
})
This works fine to get a user that exists. However, if I put in a fake username to my /players page, that page still loads with a 200 status instead of getting a 404 response. The page loads and looks broken because it's not actually getting any data from the API.
I feel like this is a dumb question .. In my research I have found how to handle errors if it's just the route, and not if it's the route dependent on the path parameter as in /players/:player
I found a question that was similar to mine (How to throw a 404 error in express.js?) and I tried using an If statement: if (!player){res.status(404).send("Not found."); } but no dice. Am I using this if statement in the wrong place?
How can I get my Node JS server to respond with a 404 if the user from the database doesn't exist?
You have to check the result of the API call and see if you got valid data back and send the 404 there. I also added a check to make sure something was passed for the player name and send back a 400 (bad request) if there's no player specified at all:
app.get('/players/:player', apiLimiter, function(request, response) {
const player = request.params.player;
if (!player) {
res.status(400).send("No player specified.");
return;
}
const api_url = `https://api.com/shards/steam/players?filter[playerNames]=${player}`;
var options = {
method: "GET",
observe: 'body',
};
let apiRequest = https.request(api_url, options, function(res) {
let data = "";
res.on("data", chunk => {
data += chunk;
})
res.on("end", () => {
let objectParsed = JSON.parse(data);
// test objectParsed here
if (!some condition in objectParsed) {
res.status(404).send("No data for that player name.");
} else {
response.send(objectParsed);
}
});
});
apiRequest.end();
});
Also, you don't want JSON.parse(JSON.stringify(data)) here. Your data is already a string. Just do JSON.parse(data).
FYI, if you use a small http request library such as got(), this code gets a lot simpler as it accumulates the response and parses the JSON for you in one line of code as in:
let data = await got(options).json()
I have three files: user.js, influencer.js, & validate.js
In user.js, I import ./influencer (as var = influencer) & ./validate (as var = validate).
My function in user.js:
addAccount: function(){
return functions.database.ref('/acct/{accountID}/name/').onCreate(event => {
var accountID = event.params.accountID;
var name = JSON.stringify(event.data.val()).replace(/['"]+/g, '');
console.log("New Account Added ("+accountID+")");
console.log("Nickname: " +name);
influencer.getIG(name);
var data = influencer.data;
validate.validateThis(data);
});
}
With influencer.getIG(name), I am passing the name we defined above to the function getIG (inside of influencer.js). This works like a charm. The result is JSON body.
What I want to do now is take this JSON body result and pass it to the validate function (in validate.js). In influencer.js, I also added "exports.data = data;".
With that being said, I can't seem to figure out how to pass "data" to validate.js. I log it, and it returns undefined. I added a timeout before running validateThis(data) and still undefined. The validate function on its own works great; I've tested it. But clearly, I am not doing this the correct way.
This is my influencer.getIG function:
module.exports = {
getIG: function (name) {
var url = "https://www.instagram.com/"+name+"/?__a=1"
console.log(url);
request({
url: url
}, function (error, response, body) {
var data = JSON.parse(body);
console.log(data);
exports.data = data;
})
}
}
How can I pass the result of the second module to the third module in my function? What am I doing wrong?
You can try passing callback function as another parameter to getIG
Your influencer file will look like this.
module.exports = {
getIG: function (name, callback) {
var url = "https://www.instagram.com/"+name+"/?__a=1"
request({
url: url
}, callback)
}
}
And your user file will look like this
addAccount: function(){
return functions.database.ref('/acct/{accountID}/name/').onCreate(event => {
var accountID = event.params.accountID;
var name = JSON.stringify(event.data.val()).replace(/['"]+/g, '');
influencer.getIG(name, function (error, response, body) {
var data = JSON.parse(body);
validate.validateThis(data);
});
});
}
Using callback will ensure that data is retrieved before you call it.
As the two other commentors noted - you have an asynchronous function with a callback. One way around this is to define the callback inside the user.js file, and pass it to the getIG function. So you would have
user.js
<pre><code>
addAccount: function(){
return functions.database.ref('/acct/{accountID}/name/').onCreate(event => {
var accountID = event.params.accountID;
var name = JSON.stringify(event.data.val()).replace(/['"]+/g, '');
console.log("New Account Added ("+accountID+")");
console.log("Nickname: " +name);
function callback(err, res, data) {
var data = JSON.parse(body);
console.log(data);
validate.validateThis(data)
}
influencer.getIG(name, callback);
});
}
</pre></code>
then in the other file
influencer.js
module.exports = {
getIG: function (name, callback) {
var url = "https://www.instagram.com/"+name+"/?__a=1"
request({
url: url
}, callback)
}
}
This way the asynchronous function runs inside of influencer, and then calls back to the user when the result is done. Data is now in scope for the user file to utilize.
The alternative (and better) way is to use promises. In that case the user code would be along the lines of
influencer.getIg(name).then(data => //use data here in user.js//)
I want get all files json in my directory for search into it.
$('#get-data').click(function () {
var showData = $('#show-data');
$.getJSON('/data/**all files json**', function (data) {
console.log(data);
var items = data.items.map(function (item) {
return item.key + ': ' + item.value;
});
showData.empty();
if (items.length) {
var content = '<li>' + items.join('</li><li>') + '</li>';
var list = $('<ul />').html(content);
showData.append(list);
}
});
showData.html(data);
});
Do you think its possible or i need use other method ?
Thx
You cannot make a wildcard AJAX request that will return all the possible JSON files on your server. You need to know in advance the exact endpoint that you will be sending your request to. Otherwise it would be a security vulnerability if all clients could know in advance what files are available on the server.
So one possible strategy here would be to have some /describe endpoint on your server which will return a list of all available JSON files. Then the client will first make an AJAX request to this endpoint and then for each returned endpoint in the list it will make a separate AJAX request:
$.getJSON('/data/discover', function (data) {
// suppose that data looks like this:
// ["/data/a.json", "/data/b.json", "/data/c.json"]
for (var i := 0; i < data.length; i++) {
// send an AJAX request to each individual JSON file
// available on the server as returned by the discover endpoint
$.getJSON(data[i], function (result) {
...
});
}
});
Basically you can't request for multiple files by one request.
However the scenario is perfect fit for async.parallel:
var async = require('async');
app.get('/json', function(req, res) {
var work = {
file01: async.apply(fs.readFile, __dirname + '/file01.json'),
file02: async.apply(fs.readFile, __dirname + '/file02.json')
};
async.parallel(work, function (error, results) {
if (error) {
res.status(500).send(error);
return;
}
//might need string->Object here
results['file01'] = JSON.parse(results['file01']);
results['file02'] = JSON.parse(results['file02']);
res.send(results);
});
});
I'm creating a script that will make a request 2 times per second to a localserver of cameras network and after it gets a positive response that camera detected someone I want to log three images.
In the json config file I have the triggerURL of the server, the interval port, the dataDir where logged images should be saved and a track array which contains the url of those images and the fileName they should receive.
This is the code of the script I use after reading the JSON file:
var configGet = {
host: config.triggerURL
, port: config.interval
, method: 'GET'
};
setInterval(function () {
var request = http.request(configGet, function (response) {
var content = "";
// Handle data chunks
response.on('data', function (chunk) {
content += chunk;
});
// Once we're done streaming the response, parse it as json.
response.on('end', function () {
var data = JSON.parse(response);
if (data.track.length > 0) {
//log images
var download = function (uri, filename, callback) {
request.head(uri, function (err, res, body) {
request(uri)
.pipe(fs.createWriteStream(filename))
.on('close', callback);
});
};
for (var image in data.track) {
var path = config.dataDir + '/' + image.fileName
download(image.url, path.format(config.timestamp), function () {
console.log('done');
});
}
}
});
// Report errors
request.on('error', function (error) {
console.log("Error while calling endpoint.", error);
});
request.end();
}, 500);
});
I have the following questions:
This method produces some kind of error with the download process of the images.Can you identify it?
Is there a better way of doing this process?
Without running the code or deeper inspection; should not "data = JSON.parse(response)" rather be "data = JSON.parse(content)"? Also, if data is undefined or does not contain "track" the "if (data.track.length > 0)" will throw an error. This can be fixed with "if (data && data.track && data.track.length > 0)".
I can not think of a very much better way. I would break it up more in functions to make the code more clear though.
I am having issues getting node to make a database call without proceeding despite the database function has not returned a value.
Here is the basic http server code:
var http = require('http');
http.createServer(function (request, response) {
response.writeHead(200, {
'Content-Type': 'text/plain',
'Access-Control-Allow-origin': '*' // implementation of CORS
});
response.end("ok");
;
}).listen(8080,'0.0.0.0');
Using the request.on('data') function, I am able to decode JSON from requests and proceed that to make a database call:
request.on('data', function (chunk) {
var json = JSON.parse(chunk);
var id = parseInt(json["id"]);
response.end(callDatabase(id));
});
The database function goes something like this:
function callDatabase(id) {
var result;
var connection = mysql.createConnection(
{
host : '192.168.1.14',
user : 'root',
password : '',
database : 'test'
}
);
connection.connect();
var queryString = 'SELECT name FROM test WHERE id = 1';
connection.query(queryString, function(err, rows, fields) {
if (err) throw err;
for (var i in rows) {
result = rows[i].name;
}
});
connection.end();
return result;
}
}
However under testing, this proves that I am doing it wrong. I am aware that I probably want to be using the node asynchronous module, which I have tired. I have also tried using the waterfall method, as well as parallel and many other tutorials online. I feel that the request.on function should be in parallel, then the database call async, so whilst node is waiting for the response from the database server, it is free to get on with any other requests, leaving the queued time to a minimum.
Please inform me if I have miss-understood any of the concepts of node js.
You are returning result and closing the connection before the query has returned it's value from the db. Place that code inside the callback.
Fixing your code, it should look like this:
function callDatabase(id) {
var result;
var connection = mysql.createConnection(
{
host : '192.168.1.14',
user : 'root',
password : '',
database : 'test'
}
);
connection.connect();
var queryString = 'SELECT name FROM test WHERE id = 1';
connection.query(queryString, function(err, rows, fields) {
if (err) throw err;
for (var i in rows) {
result = rows[i].name;
}
connection.end();
return result;
});
}
Although, this will only solve part of the problem, since now you're still calling response.end(callDatabase(id)); before waiting for a response from the query.
In order to fix this, you need to return some kind of callback.
function callDatabase(id, callback) {
// the method code here...
connection.query(queryString, function(err, rows, fields) {
// code...
// instead of returning the result, invoke the callback!
callback(rows);
});
}
Now you can call it like this :
request.on('data', function (chunk) {
var json = JSON.parse(chunk);
var id = parseInt(json["id"]);
callDatabase(id, function(res) {
response.end(res);
});
});