We Keep Coding

NodeJS returning garbage JSON

I am trying to write a simple piece of code using NodeJS to get the JSON back from the stack exchange API.
This is the API I am targetting- https://api.stackexchange.com/2.2/users?order=desc&sort=reputation&inname=donal%20rafferty&site=stackoverflow
And here is my code:
var https = require('https'); //Use NodeJS https module
function getUserDataByName(userName, callback){
var stackOverflowUserURL = 'https://api.stackexchange.com/2.2/users?order=desc&sort=reputation&inname='+encodeURIComponent(userName)+'&site=stackoverflow';
https.get(stackOverflowUserURL, function(response){
console.log("headers: ", response.headers);
if (response.statusCode == 200) {
var jsonString = '';
response.on('data', function (chunk) {
jsonString += chunk;
});
response.on('end', function () {
console.log((jsonString));
callback(JSON.stringify(jsonString));
});
}
else{
//error
console.log("Error");
}
});
}
However when I run this the data always comes back in a state of garbage like text like the following:
\"\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0004\u0000uR�n�0\f��B���ږ\u0013�2\u0010�R�m�u\\u0018\\u0004ڢ\\u001d!��Jr=�ȿ�vS\\u0004\\u0005������H����C��7ր�Q�n��\u0012\u0014{g�\\"��]����+zV\u001f����(V��%a�n|�)QU�.O�\u000e\u0012�Ѹ\u0005��\u0003\u00130a\u0006B��S�Ө�����C^��bw�I\u000bC��b�\u0017e�\u0013�q�\\"D��lO`���#^\nq\u0017|���ի�������?pFz�i�R\u000f�,[�pu�{x�\b~k��LUV��\u0012\u00194�l\u000e�ڕ\rW��\u001c���*�\u001a�9�\u001e�Q+�Q��>���o��;a'\btI�b/��
\u0007�CK̲���\u0000�jۯ����\u0003g|�\u0003�\u0002\u0000\u0000\
I'm assuming there is something wrong with my encoding/decoding but I can't figure out what to do to fix this?

You need to decode the response as it's gzipped
var https = require('https'); //Use NodeJS https module
var zlib = require("zlib");
function getUserDataByName(userName, callback){
var stackOverflowUserURL = 'https://api.stackexchange.com/2.2/users?order=desc&sort=reputation&inname='+encodeURIComponent(userName)+'&site=stackoverflow';
https.get(stackOverflowUserURL, function(response){
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function (chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
console.log((jsonString));
callback(JSON.stringify(jsonString));
});
gunzip.on('error', function (e) {
console.log(e);
});
}
else{
//error
console.log("Error");
}
});
}

node.js redirection not working to add username

I am working on a Team Treehouse project that builds a dynamic website with Node.js. The user enters in a username into the search field and it displays the user's avatar, number of badges earned and the number of JavaScript points. For some reason when I enter in the user name and click search the page just goes blank. I think there might be something wrong with the 303 redirection in my router.js file. I'm still fairly new to coding so any insight would be very helpful. Here are each of my js files.
/*****app.js file******/
var router = require('./router.js');
//Problem: We need a simple way to look at a user's badge count and JavaScript points from a web browser
//Solution: Use Node.js to perform the profile look ups and serve our templates via HTTP
//Create a web server
var http = require('http');
http.createServer(function (request, response) {
router.home(request, response);
router.user(request, response);
}).listen(3000);
console.log('Server running at http://<workspace-url>');
/*****router.js file******/
var Profile = require("./profile.js");
var renderer = require('./renderer');
var querystring = require('querystring');
var commonHeader = {'Content-Type': 'text/html'};
// Handle the HTTP route GET / and POST / i.e. Home
function home(request, response) {
//if url == "/" && GET
if (request.url === '/'){
if (request.method.toLowerCase() === "get") {
//show search
console.log(request.url);
response.writeHead(200, commonHeader);
renderer.view('header', {}, response);
renderer.view('search', {}, response);
renderer.view('footer', {}, response);
response.end();
}
else {
//if url == "/" && POST
//get the post data from body
request.on('data', function(postBody){
//extract the username
var query = querystring.parse(postBody.toString());
//redirect to /:username
response.writeHead(303, {'Location': '/' + query.username });
response.end();
});
}
}
}
// Handle the HTTP route for GET /:username i.e. /chalkers
function user(request, response) {
//if url == "/...."
var username = request.url.replace('/', '');
if(user.name.length > 0){
response.writeHead(200, commonHeader);
renderer.view('header', {}, response);
//get json from Treehouse
var studentProfile = new Profile(username);
//on "end"
studentProfile.on("end", function(profileJSON){
//show profile
//Store the values which we need
var values = {
avatarUrl: profileJSON.gravatar_url,
username: profileJSON.profile_name,
badges: profileJSON.badges.length,
javascriptPoints: profileJSON.points.JavaScript
}
//Simple response
renderer.view('profile', values, response);
renderer.view('footer', {}, response);
response.end();
});
//on "error"
studentProfile.on("error", function(error){
//show error
renderer.view('error', {errorMessage: error.message}, response);
renderer.view('search', {}, response);
renderer.view('footer', {}, response);
response.end();
});
}
}
module.exports.home = home;
module.exports.user = user;
/*****profile.js file*******/
var EventEmitter = require("events").EventEmitter;
var http = require("http");
var util = require("util");
/**
* An EventEmitter to get a Treehouse students profile.
* #param username
* #constructor
*/
function Profile(username) {
EventEmitter.call(this);
profileEmitter = this;
//Connect to the API URL (http://teamtreehouse.com/username.json)
var request = http.get("http://teamtreehouse.com/" + username + ".json", function(response) {
var body = "";
if (response.statusCode !== 200) {
request.abort();
//Status Code Error
profileEmitter.emit("error", new Error("There was an error getting the profile for " + username + ". (" + http.STATUS_CODES[response.statusCode] + ")"));
}
//Read the data
response.on('data', function (chunk) {
body += chunk;
profileEmitter.emit("data", chunk);
});
response.on('end', function () {
if(response.statusCode === 200) {
try {
//Parse the data
var profile = JSON.parse(body);
profileEmitter.emit("end", profile);
} catch (error) {
profileEmitter.emit("error", error);
}
}
}).on("error", function(error){
profileEmitter.emit("error", error);
});
});
}
util.inherits( Profile, EventEmitter );
module.exports = Profile;
/*****renderer.js file*******/
var fs = require('fs');
function mergeValues(values, content) {
//Cycle over the keys
for(var key in values) {
//Replace all the {{key}} with the value from the values object
content = content.replace('{{' + key + '}}', values[key]);
}
//return merged content
return content;
}
function view(templateName, values, response) {
//Read from the template file
var fileContents = fs.readFileSync('./views/' + templateName + '.html', {encoding: 'utf8'});
//Insert values in to the content
fileContents = mergeValues(values, fileContents);
//Write out the contents to the response
response.write(fileContents);
}
module.exports.view = view;

Treehouse changed from http to https and so this example code doesn't work any longer. The reason for that is in the profile.js file. You are making calls for an http site and it doesn't exist. You need to change the code (only in profile.js) to make it connect to the https site instead.
var http = require("http");
should be changed to
var https = require("https");
and with that all references to the variable in your profile.js code should be changed to https.
As well as the hard-coded URL start:
var request = http.get("http://teamtreehous...
should be
var request = https.get("https://teamtreehous...
That should resolve the problem. Good luck!

In order to get your code in the profile.js file to run, you need to change some instances of the "http" module to "https" but, and this is important, not all instances.
What needs to remain http is the the status code error on the profile.js page. This line of code is correct:
profileEmitter.emit("error", new Error("There was an error getting the profile for " + username + ". (" + http.STATUS_CODES[response.statusCode] + ")"));
But all other instances of the http module need to change to https. For example, these lines are correct:
var url = "https://teamtreehouse.com/" + username + ".json";
var request = https.get(url, function(response){
Remember to require both modules at the top of profile.js page
var http = require("http");
var https = require("https");

Data not saving in MongoDB using Mongoose

I am newbie in Nodejs world. I am trying to insert data in MongoDB using Mongoose. The idea is
- I will have a server running on node
- any incoming POST data will be saved in Mongo.
The problem when the below code is run no data gets saved in MongoDB and also no error is shown. Am i missing something here. Any help will be really appreciated.
I have the below code that writes data in mongoDB for an incoming http request.
var http = require('http') // http module
, fs = require('fs') // file system module
, qs = require('querystring') // querystring parser
, mongoose = require('mongoose');
mongoose.connect("mongodb://localhost/app_data_db");
var db = mongoose.connection;
var appDataSchema = new mongoose.Schema({
record_id: Number,
app_version: Number,
imei: String,
created_time: Date,
device_uid: String,
model: String
});
var appDataModel = mongoose.model("app_data_collection",appDataSchema);
var PORT=8080;
http.createServer(function(req,res){
if(req.method == "POST") {
var POST = {};
//parse query string
req.on('data', function(data) {
data = data.toString();
data = data.split('&');
for (var i = 0; i < data.length; i++) {
var _data = data[i].split("=");
POST[_data[0]] = _data[1];
}
db.once('open', function (callback) {
appDataModel.create({
record_id: POST["id"],
app_version: POST["app_version"],
imei: POST["imei"],
created_time: new Date((parseInt(POST["created_time"]) + 19800) *1000), // to set correct time zone IST
device_uid: POST["device_uid"],
model: POST["model"]
});
});
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('saved to DB:'+POST["id"]+'\n');
console.log('saved to DB:'+POST["id"]+'\n');
});
mongoose.disconnect();
}
}).listen(PORT);
To test this I am manually firing this curl call:
curl -X POST --data "id=58648148&app_version=4.8&imei=355886053224492&created_time=1417372202&device_uid=e385c8a5a4c01304&model=GT-I9082" http://localhost:8080

There are couple problems with your code:
1) you call mongoose.disconnect outside of the callback, which means that it's called before the callback is executed
2) you're creating the model inside the callback, but sending the response outside of it, so the response is sent before the model is created
3) and finally create method provides a callback when the entity is saved to the db, which you don't use it all
Here's the modified code:
mongoose.connect("mongodb://localhost/app_data_db");
db.on('open', function() {
http.createServer(function(req, res) {
if(req.method == "POST") {
var POST = {};
//parse query string
req.on('data', function(data) {
data = data.toString();
data = data.split('&');
for (var i = 0; i < data.length; i++) {
var _data = data[i].split("=");
POST[_data[0]] = _data[1];
}
appDataModel.create({
record_id: POST["id"],
app_version: POST["app_version"],
imei: POST["imei"],
created_time: new Date((parseInt(POST["created_time"]) + 19800) *1000), // to set correct time zone IST
device_uid: POST["device_uid"],
model: POST["model"]
},
function(err){
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('saved to DB:'+POST["id"]+'\n');
console.log('saved to DB:'+POST["id"]+'\n');
mongoose.disconnect();
});
});
}
}).listen(PORT);
});

I think I got this working. I have moved the mongoose connection inside. Rest all is same. Since I wanted to post the code hence answering instead of commenting. This may help others. Here is the complete code
var http = require('http') // http module
, fs = require('fs') // file system module
, qs = require('querystring') // querystring parser
, mongoose = require('mongoose');
var appDataSchema = new mongoose.Schema({
record_id: Number,
app_version: Number,
imei: String,
created_time: Date,
device_uid: String,
model: String
});
var appDataModel = mongoose.model("app_data_collection", appDataSchema);
var PORT = 8080;
http.createServer(function(req, res) {
if (req.method == "POST") {
var POST = {};
//parse query string
req.on('data', function(data) {
data = data.toString();
data = data.split('&');
for (var i = 0; i < data.length; i++) {
var _data = data[i].split("=");
POST[_data[0]] = _data[1];
}
mongoose.connect("mongodb://localhost/app_data_db");
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function() {
appDataModel.create({
record_id: POST["id"],
app_version: POST["app_version"],
imei: POST["imei"],
created_time: new Date((parseInt(POST["created_time"]) + 19800) * 1000), // to set correct time zone IST
device_uid: POST["device_uid"],
model: POST["model"]
}, function(err) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('saved to DB:' + POST["id"] + '\n');
console.log('saved to DB:' + POST["id"] + '\n');
mongoose.disconnect();
});
});
});
}
}).listen(PORT);

node.js - http with base64 [duplicate]

This question already has answers here:
How can I do Base64 encoding in Node.js?
(7 answers)
Closed 9 years ago.
I'm a node.js newbie stuck trying to implement base64 encoding. My server doesn't seem to receive/process the base64 message. Code below:
Server:
var http = require('http');
http.createServer(function (req, res) {
req.on('data',function(b) {
console.log("HEY!"); // <--- Never gets called
var content = new Buffer(b, 'base64').toString('utf8')
console.log("CLIENT SAID: "+content);
var msg = JSON.parse(content);
// do stuff and respond here...
});
}).listen(1337, '127.0.0.1');
console.log('Server running at http://127.0.0.1:1337/');
Client:
var http = require('http');
var options = {
hostname : 'localhost',
port : 1337,
method : 'POST'
};
var req = http.request(options, function(res) {
res.setEncoding('base64');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
var msg = {'name':'Fred','age':23};
var msgS = JSON.stringify(msg);
req.write(msgS,'base64');
req.end();
Any ideas what I'm doing wrong?

I came up with a fix. I noticed when using req.write(data, 'base64'); that the request never ended. I instead created a buffer that was base64 encoded, then wrote it to the request.
These exact snippets were tested localhost:
Client:
var http = require('http');
var options = {
hostname: 'localhost',
port: 1337,
method: 'POST'
};
var req = http.request(options, function (res) {
res.setEncoding('base64');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
var msg = {
'name': 'Fred',
'age': 23
};
var msgS = JSON.stringify(msg);
var buf = new Buffer(msgS, 'base64');
req.write(msgS);
req.end();
Server:
var http = require('http');
http.createServer(function (req, res) {
var content = '';
req.on('data', function (chunk) {
content += chunk;
});
req.on('end', function() {
content = content.toString('base64');
console.log(content);
//content returns {"name": "Fred","age": 23};
res.end();
});
}).listen(1337, '127.0.0.1');
console.log('Server running at http://127.0.0.1:1337/');
Aside from those things, I noticed these errors in your code.
req.on('data',function(b) {
var content = new Buffer(b, 'base64').toString('utf8')
});
Note that b in this case is actually already a buffer. You should use b.toString('base64');. Also note that b is actually only fragments of the data. You should instead collect the data of b, then listen to the end event to finally do something with the data. In your case with req.write(data, 'base64');, the end would never fire, leading to a hangup instead of the event firing.
This is how you'd collect data:
var content = '';
req.on('data', function(b) {
content += b;
});
req.on('end', function() {
//do something with content
});

Routing http requests through Node.js

I'm trying to make a cucumber test setup with Node.js that can test any website by using an iframe.
Normally the iframe is a no go because of cross script security limitations.
However if it was possible (I'm sure it is. And i trust you to come up with a solution)
to fetch the website being target for the test via the requested url when a specific url name is being requested, so that the iframe would be loaded with a copy of the test target.
Basically just a standard node.js server that fetches specific pages based on the req.url
Akin to an Address Request Router.
Here is my blatant attempt to do exactly that.
Fetching the test page via. the url works.
But i'm having a problem switching from the http server to the connection object.
Is there a way to "feed" the connection with the http server response?
PS. i also created a solution with two node.js servers.
Node 1 fetched the test target and mixing it with cucumber test page.
Node 2 hosting the cucumber test.
This solution is working. But it creates problems on websites where javascript naming conflicts occur. Which is why the iframe solution, that solves this problem by encapsulation is more appealing.
var http = require('http');
var connect = require('connect');
var port = process.env.PORT || 8788;
var server = http.createServer(function(req, webres)
{
var url = req.url;
console.log(url);
if(url == '/myWebsiteToBeTestedWithCucumberJS')
{
// Load the web site to be tested "myWebsiteToBeTestedWithCucumberJS"
// And update the references
// Finaly write the page with the webres
// The page will appear to be hosted locally
console.log('Loading myWebsiteToBeTestedWithCucumberJS');
webres.writeHead(200, {'content-type': 'text/html, level=1'});
var options =
{
host: 'www.myWebsiteToBeTestedWithCucumberJS.com,
port: 80,
path: '/'
};
var page = '';
var req = http.get(options, function(res)
{
console.log("Got response: " + res.statusCode);
res.on('data', function(chunk)
{
page = page + chunk;
});
res.on('end', function()
{
// Change relative paths to absolute (actual web location where images, javascript and stylesheets is placed)
page = page.replace(/ href="\/\//g , ' href="/');
page = page.replace(/ src="\//g , ' src="www.myWebsiteToBeTestedWithCucumberJS.com');
page = page.replace(/ data-src="\//g , ' data-src="www.myWebsiteToBeTestedWithCucumberJS.com');
page = page.replace(/ href="\//g , ' href="www.myWebsiteToBeTestedWithCucumberJS.com');
webres.write(page);
webres.end('');
});
});
}
else
{
// Load any file from localhost:8788
// This is where the cucumber.js project files are hosted
var dirserver = connect.createServer();
var browserify = require('browserify');
var cukeBundle = browserify({
mount: '/cucumber.js',
require: ['cucumber-html', './lib/cucumber', 'gherkin/lib/gherkin/lexer/en'],
ignore: ['./cucumber/cli', 'connect']
});
dirserver.use(connect.static(__dirname));
dirserver.use(cukeBundle);
dirserver.listen(port);
}
}).on('error', function(e)
{
console.log("Got error: " + e.message);
});
server.listen(port);
console.log('Accepting connections on port ' + port + '...');

Well it wasn't so difficult after all.
Being new to node.js i had to realize the possibilties of using multiple listeners.
Reading on nodejitsu's features helped me solve the problem.
Below example loads www.myWebsiteToBeTestedWithCucumberJS.com
when specifying the url as follows: http://localhost:9788/myWebsiteToBeTestedWithCucumberJS
where all other requests is handled as cucumber.js website requests.
Hope this make sense to other node.js newcucumbers.
var http = require('http');
var connect = require('connect');
var port = process.env.PORT || 9788;
var server = http.createServer(function(req, webres)
{
var url = req.url;
console.log(url);
if(url == '/myWebsiteToBeTestedWithCucumberJS')
{
loadMyWebsiteToBeTestedWithCucumberJS(req, webres);
}
else
{
loadLocal(req, webres, url);
}
}).on('error', function(e)
{
console.log("Got error: " + e.message);
});
server.listen(port);
console.log('Accepting connections on port ' + port + '...');
function loadMyWebsiteToBeTestedWithCucumberJS(req, webres)
{
console.log('Loading myWebsiteToBeTestedWithCucumberJS');
webres.writeHead(200, {'content-type': 'text/html, level=1'});
var options =
{
host: 'www.myWebsiteToBeTestedWithCucumberJS.com',
port: 80,
path: '/'
};
var page = '';
var req = http.get(options, function(res)
{
console.log("Got response: " + res.statusCode);
res.on('data', function(chunk)
{
page = page + chunk;
});
res.on('end', function()
{
page = page.replace(/ href="\/\//g , ' href="/');
page = page.replace(/ src="\//g , ' src="http://www.myWebsiteToBeTestedWithCucumberJS.com/');
page = page.replace(/ data-src="\//g , ' data-src="http://www.myWebsiteToBeTestedWithCucumberJS.com/');
page = page.replace(/ href="\//g , ' href="http://www.myWebsiteToBeTestedWithCucumberJS.com/');
webres.write(page);
webres.end('');
});
});
}
function loadLocal(req, webres, path)
{
console.log('Loading localhost');
webres.writeHead(200, {'content-type': 'text/html, level=1'});
var options =
{
host: 'localhost',
port: 9787,
path: path
};
var page = '';
var req = http.get(options, function(res)
{
console.log("Got response: " + res.statusCode);
res.on('data', function(chunk)
{
page = page + chunk;
});
res.on('end', function()
{
webres.write(page);
webres.end('');
});
});
}
// Cucumber site listening on port 9787
var dirserver = connect.createServer();
var browserify = require('browserify');
var cukeBundle = browserify(
{
mount: '/cucumber.js',
require: ['cucumber-html', './lib/cucumber', 'gherkin/lib/gherkin/lexer/en'],
ignore: ['./cucumber/cli', 'connect']
});
dirserver.use(connect.static(__dirname));
dirserver.use(cukeBundle);
dirserver.listen(9787);

var http = require('http');
// Create a server object
http.createServer(function (req, res) {
// http header
res.writeHead(200, {'Content-Type': 'text/html'});
var url = req.url;
if(url ==='/about') {
res.write(' Welcome to about us page');
res.end();
}
else if(url ==='/contact') {
res.write(' Welcome to contact us page');
res.end();
}
else {
res.write('Hello World!');
res.end();
}
}).listen(3000, function() {
// The server object listens on port 3000
console.log("server start at port 3000");
});