We Keep Coding

Multiple HTTP async request

I try to fetch data from different HTTP sources but I wasn't able to handle the asynchronous mode even with async...
var express = require('express');
var app = express();
var https = require("https");
var timer = require("./my_modules/timer/timer.js");
var http = require('http');
var bodyParser = require('body-parser');
var async = require('async');
var request = require('request');
//These are my source from API.
//Output is a Json file
var sources = {
cnn: 'https://newsapi.org/v1/articles?source=cnn&sortBy?&apiKey=c6b3fe2e86d54cae8dcb10dc77d5c5fc',
bbc: 'https://newsapi.org/v1/articles?source=cnn&sortBy?&apiKey=c6b3fe2e86d54cae8dcb10dc77d5c5fc',
guardian: 'https://newsapi.org/v1/articles?source=cnn&sortBy?&apiKey=c6b3fe2e86d54cae8dcb10dc77d5c5fc',
othersource: "otherurls..."
};
//I want to push the JSON object in this array
var resultArray = [];
//I setup a https GET request
var getJson = function(url) {
https.get(url, (res) => {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
result = JSON.parse(body);
//push isn't working...
resultArray.push(result);
});
}).on('error', function(e) {
console.log('Got an error', e);
});
}
app.set('port', (process.env.PORT || 5000));
app.listen(
app.get('port'), () => {
console.log('We are live on port: ', app.get('port'));
getJson(sources.cnn);
});
app.use(bodyParser.urlencoded({
extended: true
}));
app.use(bodyParser.json());
app.use(function(req, res, next) {
res.setHeader('Content-Type', 'text/plain');
res.status(404).send('Page not found !');
res.status(503).send('Page not found, error 503');
});
console.log("resultArray:" + resultArray);
//resultArray = empty...
What should I do to push the result into my Array?
I couldn't find a way to set up a working callback function to push results into the Array.

Since you're already using the request package, have you tried something as simple as:
request({
url: sources.cnn,
json: true
}, function(error, response, body) {
var articles = body.articles;
// or by case, depending on what you want
// resultArray = resultArray.concat(articles);
resultArray.push({
cnn: articles
});
console.log(resultArray);
});
instead of writing your own getJson function?

Thanks Roby, your request is much clearer !
I have carefully read this really clear and helpful article : https://github.com/maxogden/art-of-node#callbacks
I think I got the logic :
//main function
function getJson(url, callback){
request({
url: url,
json: true,
callback:callback //added this
}, function(error, response, body) {
var articles = body.articles;
callback(articles);
});
}
//this callback function will be passed to the main function as the 2nd parameter
//it's possible to access "resultArray" ONLY from this function
var result = function(e){
resultArray.push(e);
console.log(resultArray);
};
//url and callback are passed as parameter
getJson(sources.cnn, result);
Thanks for the help

http.get or http.request callback only shows within shell - node.js

New to node and trying not to do any callback hell.
I have two files
routes.js
fetch.js
//routes.js
var fetchController = require("../lib/mtl_fetcher/fetcher_controller");
var express = require('express');
var router = express.Router();
router.get('/', function(req, res, next) {
res.send(fetchController.getAllTransgressor(function(results) {
return results.end();
}))
});
module.exports = router;
and
//fetch.js
var http = require('http');
var config = require('./config')
var Iconv = require('iconv').Iconv
module.exports.getAllTransgressor = function(callback) {
var req = http.get(config.urlOptions.host, function (response) {
var bufferChunk = [];
var str
if(response.statusCode == 200) {
response.on('data', function(chunk) {
bufferChunk.push(chunk);
})
response.on('end', function(callback) {
var iconv = Iconv('latin1', 'UTF-8');
str = iconv.convert(Buffer.concat(bufferChunk)).toString();
console.log(str)
});
} else {
console.log("handle this")
}
});
req.on("error", function(err) {
callback(err);
});
callback(req)
}
So the goal is to fetch and then show what has been fetch to the screen. The ressource is XML base.
Doing all of this in one block (routes.js) works, but when I try to refactor and set up some modularity my str just shows in shell stdout.
Using req.end() does not send the content back.

Firstly, you need to send inside the callback, where the result is actually available, as you can't return from an asynchronous function
router.get('/', function(req, res, next) {
fetchController.getAllTransgressor(function(error, results) {
if ( error ) {
// handle errors
} else {
res.send(results);
}
});
});
The same goes for the callback function, it has to be called when the data is available, after the request and parsing
module.exports.getAllTransgressor = function(callback) {
var req = http.get(config.urlOptions.host, function(response) {
var bufferChunk = [];
if (response.statusCode == 200) {
response.on('data', function(chunk) {
bufferChunk.push(chunk);
});
response.on('end', function() {
var iconv = Iconv('latin1', 'UTF-8');
var str = iconv.convert(Buffer.concat(bufferChunk)).toString();
callback(null, str); // here, stuff is available
});
} else {
callback('Did not return 200', err);
}
});
req.on("error", function(err) {
callback(err, null);
});
}

Returning Json with nodeJS

I try to learn nodeJS and AngularJS using Clash of clan API available here https://developer.clashofclans.com/#/
My JSON api's return is truncated and i don't know how get the full response.
there is my app.js wrinting with node and express :
var express = require('express')
, https = require('https')
, bodyParser = require('body-parser')
, request = require('request')
, app = express()
, http_port = 3000;
app.use(express.static(__dirname + '/public'));
app.use(bodyParser());
var options = {
host: 'api.clashofclans.com',
port: 443,
path: '/v1/clans/' + encodeURIComponent("#PP8JC9RQ"),
headers : {
accept : "application/json",
authorization : "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiIsImtpZCI6IjI4YTMxOGY3LTAwMDAtYTFlYi03ZmExLTJjNzQzM2M2Y2NhNSJ9.eyJpc3MiOiJzdXBlcmNlbGwiLCJhdWQiOiJzdXBlcmNlbGw6Z2FtZWFwaSIsImp0aSI6ImQ4Njg1OGFhLWQzZTUtNDNiOC05MTM1LTBjNzI1ZjI4OGFiMiIsImlhdCI6MTQ1NjMwMTE5MSwic3ViIjoiZGV2ZWxvcGVyLzk1YWM0YjdmLTY5NTQtMDE3MC01ZjAyLTcxNjY1ZDMzYjUwNyIsInNjb3BlcyI6WyJjbGFzaCJdLCJsaW1pdHMiOlt7InRpZXIiOiJkZXZlbG9wZXIvc2lsdmVyIiwidHlwZSI6InRocm90dGxpbmcifSx7ImNpZHJzIjpbIjg3LjIzMS4yMTMuMTU0Il0sInR5cGUiOiJjbGllbnQifV19.SYqeSAF-0_bM1eS2-hnovvj5j-I0SQpdr_kySIiBKw9OkrNuBzZAOAOkiH3fzdKSkcHaJfXzWdXr8JozFfAmJA"
},
method: 'GET'
};
// Route par défaut
app.get("/members", function(req, res) {
console.log("-------------------------------------\n\nroot path !! Let's get clan infos from coc api\n\n-------------------------------------");
var req = https.request(options, function(response){
response.on('data', function (chunk) {
console.log('BODY: ' + chunk);
// in console JSON is full but in my html JSON is truncated..
res.send(chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.end();
});
app.listen(http_port,function(){
console.log("Listening on " + http_port);
});
If someone can teach to a nook :)

Your code is sending the first chunk of the response immediately back to the user. You need to wait until the entire HTTP response is done. Try something like this:
app.get("/members", function(req, res) {
var req = https.request(options, function(response){
var httpResult = '';
response.on('data', function (chunk) {
console.log('BODY: ' + chunk);
httpResult += chunk;
});
response.on('end', function() {
res.send(httpResult);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.end();
});
See how I append to a var each time the data event is called and when it is done - then I send the output.

Because you use send method instead write (send non-streaming). Try this:
app.get("/members", function(req, res) {
https.request(options, function(response){
response.on('data', function (chunk) {
res.write(chunk);
});
response.on('end', function () {
res.end();
});
}).end();
});

response is a stream, you can simply pipe it to res:
app.get("/members", function(req, res) {
https.request(options, function(response) {
response.pipe(res);
});
});

node.js/express chaining multiple get commands

I have a route I that in order to get all the data needs to access the API server multiple times (according to the data that was given).
Now I need to add a third access to the server and it's getting rather unreadable.
The following code is working, but I have a feeling I'm not doing it right (promises?) - couldn't figure out what exactly is recommended in this case
The code: (stripped down to emphasise the point)
router.get('/', function(req, main_response) {
http.get(FIRST_API_COMMAND, function (res) {
var moment_respose_content = '';
res.on("data", function (chunk) {
moment_respose_content += chunk;
});
res.on('end',function(){
if (res.statusCode < 200 || res.statusCode > 299) {
main_response.send('error in getting moment');
return;
}
var response = JSON.parse(moment_respose_content );
if (response.success)
{
var data = response.data;
//doing something with the data
http.get(SECOND_API_COMMAND, function (res) {
res.on("data", function (chunk) {
comment_respose_content += chunk;
});
res.on('end',function(){
var response = JSON.parse(comment_respose_content);
if (response.success)
{
var comments = response.data;
main_response.render('the page', {data: data});
return;
}
});
}).on('error', function (e) {
console.log("Got error: " + e.message);
main_response.send('Error in getting comments');
});
return;
}
});
}).on('error', function (e) {
console.log("Got error: " + e.message);
main_response.send('Error in getting moment');
});
});

You can write a middleware for each remote action, and then use those middlewares before the get handler, so the get handler can simply access their results. (Promises can help if you need to start subsequent requests before waiting for earlier ones to finish, but that situation is rare.)
For example, using express middleware to fetch each remote data independently:
var request = require('request');
var express = require('express');
var app = express();
var router = express.Router();
/* middleware to fetch moment. will only run for requests that `router` handles. */
router.use(function(req, res, next){
var api_url = 'https://google.com/';
request.get(api_url, function(err, response, body) {
if (err) {
return next(err);
}
req.moment_response = response.headers["date"];
next();
});
});
/* middleware to fetch comment after moment has been fetched */
router.use(function(req, res, next){
var api_url = 'https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new';
request.get(api_url, function(err, response, body){
if (err) {
return next(err);
}
req.comment_response = parseInt(body);
next();
});
});
/* main get handler: expects data to already be loaded */
router.get('/', function(req, res){
res.json({
moment: req.moment_response,
comment: req.comment_response
});
});
/* error handler: will run if any middleware called next() with an argument */
router.use(function(error, req, res, next){
res.status(500);
res.send("Error: " + error.toString());
});
app.use('/endpoint', router);
app.listen(8000);
Often the remote data you want to fetch is based on some parameter of the main request. In this case you would want to use req.param() instead of App.use() to define the data-loading middleware.

Node.js passing parameters to client via express render

I'm using Node.js and I'm having issues communicating with a client.
I define Express:
var express = require("express");
var app = express();`
When I try and pass a parameter to the client upon requesting a page the variable holds no data, for example:
app.get("/", function(req, res){
res.render("index", { name: "example" });
});
On the index page, when I use the console to print the variable (name)it returns "".
More info: http://expressjs.com/api.html#app.render
Am I missing something or doing something wrong?

The variable name you sent to the render function is only available while rendering the page, after it is sent to the client, it is not accessible. You have to use it in your view on the rendering stage.
Since you are using handlebars, you can display it in your page like this, for instance:
<h1>{{ name }}</h1>
If you want to use this data in a javascript, use it inside a script tag:
<script>
var name = "{{ name }}";
console.log(name);
</script>

You are basically telling express to render your index page and providing a value for the name variable, but that doesn't necessarily make the name var available in your client side javascript. You need to edit your index template to display the name variable in the page.
The syntax varies depending on the templating engine you are using (jade, ejs, dustjs).
Another solution is to use an ajax call in your client page's javascript and use res.json on the server instead to send the data. Then you can evaluate name in the console. Ex using jquery:
index.html:
$.get( "/getvar", function( data ) {
name = data.name;
});
server.js:
app.get("/getvar", function(req, res){
res.json({ name: "example" });
});

If you want to get parameters on the clientside via javascript, you should do template like this <script>var data = data</script>, otherwise variables aren't available
If you use Jade, it will be something like this:
script(type='text/javascript').
var name = !{name}

Passing data list from node js to html
server.js
var http = require('http');
var express = require('express');
var sqlite3 = require('sqlite3').verbose();
var bodyParser = require('body-parser');
var path = require("path");
console.log('Server running at http://127.0.0.1:8081/');
var __dirname = "D:/html-files";
var app = express();
var urlencodedParser = bodyParser.urlencoded({ extended: false })
var engine = require('consolidate');
app.engine('html', engine.mustache);
app.use(express.static('./'));
app.get('/', function(req, res) {
res.render('index.html');
});
app.post('/', function (req, res) {
console.log("Got a POST request for the homepage");
res.send('Hello POST');
});
app.post('/get-user-list', urlencodedParser, function (req, res) {
let db = new sqlite3.Database('user.db', sqlite3.OPEN_READWRITE, (err) => {
if (err) {
console.error(err.message);
}
console.log('Connected to the user database.');
console.log("ID" + "\t" + "NAME" + "\t" + "EMAIL");
});
db.serialize(() => {
var dataList = "";
db.each('SELECT id, name, email FROM USER ', (err, row) => {
if (err) {
console.error(err.message);
}
if(dataList != "")
dataList = dataList + ',';
dataList = dataList + '{"id":"' + row.ID + '","name":"' + row.NAME + '","email":"' + row.EMAIL + '"}';
console.log("dataList : " + dataList);
});
db.close((err) => {
if (err) {
console.error(err.message);
}
console.log('Close the database connection.');
response = {'username':dataList};
aFunction(res, dataList);
});
});
});
var aFunction = function(res, dataList) {
console.log('return to page.');
console.log("dataList : " + dataList);
res.render(__dirname + "/list-all-users.html", response);
};
app.listen(8081, '127.0.0.1')