We Keep Coding

Using Node.js to find the value of Bitcoin on a webpage at real time

I'm trying to make a .js file that will constantly have the price of bitcoin updated (every five minutes or so). I've tried tons of different ways to web scrape but they always output with either null or nothing. Here is my latest code, any ideas?
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var url = 'https://blockchain.info/charts/';
var port = 9945;
function BTC() {
request(url, function (err, res, body) {
var $ = cheerio.load(body);
var a = $(".market-price");
var b = a.text();
console.log(b);
})
setInterval(BTC, 300000)
}
BTC();
app.listen(port);
console.log('server is running on '+port);
It successfully says what port it's running on, that's not the problem. This example (when outputting) just makes a line break every time the function happens.
UPDATE:
I changed the new code I got from Wartoshika and it stopped working, but im not sure why. Here it is:
function BTCPrice() {
request('https://blockchain.info/de/ticker', (error, response, body) => {
const data = JSON.parse(body);
var value = (parseInt(data.USD.buy, 10) + parseInt(data.USD.sell, 10)) / 2;
return value;
});
};
console.log(BTCPrice());
If I have it console.log directly from inside the function it works, but when I have it console.log the output of the function it outputs undefined. Any ideas?

I would rather use a JSON api to get the current bitcoin value instead of an HTML parser. With the JSON api you get a strait forward result set that is parsable by your browser.
Checkout Exchange Rates API
Url will look like https://blockchain.info/de/ticker
Working script:
const request = require('request');
function BTC() {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
console.log(value);
});
}
BTC();
Using the value as callback:
const request = require('request');
function BTC() {
return new Promise((resolve) => {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
resolve(value);
});
});
}
BTC().then(val => console.log(val));

As the other answer stated, you should really use an API. You should also think about what type of price you want to request. If you just want a sort of index price that aggregates prices from multiple exchanges, use something like the CoinGecko API. Also if you need real-time data you need a websocket-based API, not a REST API.
If you need prices for a particular exchange, for example you're building a trading bot for one or more exchanges, you;ll need to communicate with each exchange's websoceket API directly. For that I would recommend something like the Coygo API, a node.js package that connects you directly to each exchange's real-time data feeds. You want something that doesn't add a middleman since that would add latency to your data.

Body parser not letting localhost load NODE

var express = require('express');
var app = express();
var bodyParser = require('body-parser');
//PROBLEM LINE
**app.use(parser.json);**
///////////////
var todos = [];
var nextTodoItem = 1;
app.use(bodyParser.json);
app.get('/', function(req, res){
//console.log("ToDo Root");
res.send("ToDo Root");
});
//GET REQUEST TO GET ALL TODO ITEMS
// GET /todos
app.get('/todos', function (req, res) {
// Need to send back the array of todos
res.json(todos); //array is converted to JSON.
}
);
//GET REQUEST TO GET SOME SPECIFIC TODO
//GET todos/:id
//Express uses : (colon) to parse data.
app.get('/todos/:id', function (req, res) {
var todoID = parseInt(req.params.id, 10);
var todoObjectWithID = -1;
todos.forEach(function (todo) {
if(todo.id == todoID){
todoObjectWithID = todos[todoID - 1];
}
});
if(todoObjectWithID == -1){
res.status(404).send();
} else {
res.json(todoObjectWithID); //Send the JSON of the specific todo with id requested.
}
console.log('Asing for todo with id of ' + req.params.id);
});
//Create a POST request to create new TODO Items.
//POST /todos
app.post('/todos', function(req, res){
var body = req.body;
console.log("description");
res.json(body);
});
//Server basic start up (port and log)
app.listen(3000, function () {
console.log("Server up and running");
});
I run the server with bash (Mac OS) but I go to http://localhost:3000 nothing loads, but when I remove the app.use(bodyParser) it loads properly.
What is the problem in the body-parser?
This problem only occurs when I have that line, otherwise, the server runs up perfectly fine. I need that parser though, so what is my option?

Change that line to app.use(bodyParser.json());

NodeJS restify API caching best practice

I am very new to NodeJS and I am building my first API using restify.
I want to find out what is best practice for caching the response data - each API call must have its own cache time.
I have looked at res.cache() but that seems to be only per user request and not a global application cache.
I then looked at restify-cache but the documentation did not clearly tell me how to use it.
My application works like this:
server.js code:
var restify = require('restify');
var mysqlDB = require('./config/connection');
// REST server declaration and configuration
var server = restify.createServer({
name: 'test-api',
version: '0.0.1'
});
server.pre(restify.pre.sanitizePath());
server.use(restify.queryParser());
server.use(restify.acceptParser(server.acceptable));
server.use(restify.queryParser());
server.use(restify.bodyParser());
server.listen(9007, function() {
console.log('%s listening at %', server.name, server.url);
mysqlDB.handleDisconnect();
console.log(new Date() +': Started Cricket API on port 9007');
});
var routes = require('./routes')(server);
routes.js code:
module.exports = function(app) {
app.get('/', function(req, res, next) {
return res.send("You have reached the test API");
});
var fixtures = require('./controllers/fixtures');
app.get('/getfixtures', fixtures.getFixtures); // Get All Fixtures
};
fixtures.js code snippet:
this.getFixtures = function (req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "X-Requested-With");
console.log("Get All Fixtures");
var mysql = mysqlDB.getConnection();
var query = "SELECT * FROM fixtures WHERE fixture_date >= CURDATE() ORDER BY fixture_date, fixture_time";
mysql.query(query,function(err,rows){
if(err) {
var status = mysqlDB.getErrorStatus(err.code);
return res.status(status.code).send("Error : "+ status.Message);
} else {
var data = [];
for (i in rows){
var item = rows[i];
var output = util.formatDate(item.fixture_date);
item.fixture_date = output;
data.push(item);
};
return res.send(data);
}
});
};
Can someone please send me in the right direction? I don't know where to add the caching part?

From the library file:
server.use(cache.before); is a middleware that will be triggered to load before the request is handled, going to Redis and checking if the header_{url} key and payload_{url} exits, and at that case the value is returned.
You could put it as mentioned in this gist:
https://gist.github.com/jeffstieler/3d84fa5468c7eadb7685
var server = restify.createServer({
name: 'test-api',
version: '0.0.1'
});
server.pre(restify.pre.sanitizePath());
server.use(cache.before);
server.use(restify.queryParser());
server.use(restify.acceptParser(server.acceptable));
server.use(restify.queryParser());
server.use(restify.bodyParser());
server.on('after', cache.after);
In your code I would add the cache.before after you sanitize the path as this will be saved in Redis. also a next() should be included in every route cached.

I ended up using node-cache
It was easy to use since I come from a Java/Play Framework background - hopefully it helps someone else in future.
Example usage:
var nodeCache = require( "node-cache" );
var myCache = new nodeCache();
var cachedValue = myCache.get("alltests", true);
if (cachedValue != undefined) {
return res.send(cachedValue);
} else {
// Do work here and then:
success = myCache.set("alltests", valueHere, cacheTime);
}

return value with nested function using the request NPM package

I'm scraping the ghost blogging platform and the package I'm using to do so is request, but I'm not sure how to return a value of a nested request. I commented out the area that is causing the issue. Thanks!
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/', function(req, res){
var url = 'http://<subdomain>.ghost.io';
var articles = [];
request(url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
var post;
$('article').each(function(index) {
var self = $(this);
var article = {
header : self.find('h2.post-title').text(),
route: url + self.find('h2.post-title a').attr('href'),
content : '',
author: self.find('footer a').text(),
timestamp : self.find('time.post-date').text()
};
request(article.route, function(error, response, html) {
$ = cheerio.load(html);
post = $('section.post-content').text();
return post; //*** this is what I can't return ***//
//*** I'd like it to be the value of article.content ***//
});
console.log(post); //*** undefined ***//
articles.push(article);
});
fs.writeFile('posts.json', JSON.stringify(articles, null, 4), function(err){
console.log('Posts created.');
});
}
});
})
app.listen('8000');
console.log('Watching for changes.');
exports = module.exports = app;

So your problem boils down to having a list of URLs, and wanting to (asynchronously, because node.js) request all of them, and then know when they've all finished and then do something with the collected results.
The async module (npm install async) will let you do that like this:
var request = require('request');
var async = require('async');
var urls = ["http://google.com", "http://yahoo.com"];
async.map(urls, request, function(err, results) {
// First 100 characters of http://google.com
console.log(results[0].body.substr(0, 100));
// First 100 characters of http://yahoo.com
console.log(results[1].body.substr(0, 100));
});
So you can apply this to your problem by doing the following:
Synchronously create the entire articles list.
Use async.map on the list.
In the callback to async.map, you have a list of all the responses; you can process them synchronously.

Prime the website using Express JS

I really hope to find some answers here as i tried everything by now.
Background:
Overtime we deploy code to web server, we need to do a cache warm up, i.e. access the site and make sure it loads. First load is always the slowest since IIS require to do some manipulations with a new code and cache it.
Task:
Create a page which will a checkbox and a button. Once button is pressed, array of links sent to server. Server visits each link and provides a feedback on time it took to load each page to the user.
Solution:
I am using node JS & express JS on server side. So far i manage to POST array to the server with links, but since i have limited experience with node JS, i can not figure out server side code to work.
Here is a code i got so far (it is bits and pieces, but it gives an idea of my progress). Any help will be greatly appreciated!!!
var express = require("express");
var app = express();
var bodyParser = require('body-parser');
var parseUrlencoded = bodyParser.urlencoded({ extended: false});
var http = require("http");
function siteToPrime(url){
http.get(url, function (http_res) {
// initialize the container for our data
var data = "";
// this event fires many times, each time collecting another piece of the response
http_res.on("data", function (chunk) {
// append this chunk to our growing `data` var
data += chunk;
});
// this event fires *one* time, after all the `data` events/chunks have been gathered
http_res.on("end", function () {
// you can use res.send instead of console.log to output via express
console.log(data);
});
});
};
//Tells express where to look for static content
app.use(express.static('public'));
app.post('/', parseUrlencoded, function(request, response){
var newBlock = request.body;
console.log(Object.keys(newBlock).length);
var key = Object.keys(newBlock)[0];
console.log(newBlock[key]);
siteToPrime("www.google.com");
response.status(201);
});
app.listen(3000, function(){
console.log("Server listening on port 3000...");
});

Assuming that you have access to the array in the post route:
var express = require("express"),
request = require("request"),
app = express();
var start = new Date();
app.use(express.static('public'));
app.use(bodyParser.urlencoded({extended: false}));
function siteToPrime(req, res, urls) {
urls.forEach(function(url)) {
request(url, function(error, res, body) {
if (!error && res.statusCode == 200) {
console.log(url +' : ' + body);
console.log('Request took: ', new Date() - start, 'ms');
}
});
}
res.redirect('/');
};
app.post('/', function(req, res){
var urls = req.body.urls // Array os urls.
siteToPrime(req, res, urls);
});
app.listen(3000, function(){
console.log("Server listening on port 3000...");
});