How would you go to create a streaming API with Node? just like the Twitter streaming API.
What I want to do ultimately is get the first update from the FriendFeed api, and stream when a new one becomes available (if the id is different), and later on expose it as a web service so I can use it with WebSockets on my website :).
So far I have this:
var sys = require('sys'),
http = require('http');
var ff = http.createClient(80, 'friendfeed-api.com');
var request = ff.request('GET', '/v2/feed/igorgue?num=1',
{'host': 'friendfeed-api.com'});
request.addListener('response', function (response) {
response.setEncoding('utf8'); // this is *very* important!
response.addListener('data', function (chunk) {
var data = JSON.parse(chunk);
sys.puts(data.entries[0].body);
});
});
request.end();
Which only gets the data from FriendFeed, creating the Http server with node is easy but it can't return a stream (or I haven't yet found out how).
You would want to set up a system that keeps track of incoming requests and stores their response objects. Then when it's time to stream a new event from FriendFeed, iterate through their response objects and responses[i].write('something') out to them.
Check out LearnBoost's Socket.IO-Node, you may even just be able to use that project as your framework and not have to code it yourself.
From the Socket.IO-Node example app (for chat):
io.listen(server, {
onClientConnect: function(client){
client.send(json({ buffer: buffer }));
client.broadcast(json({ announcement: client.sessionId + ' connected' }));
},
onClientDisconnect: function(client){
client.broadcast(json({ announcement: client.sessionId + ' disconnected' }));
},
onClientMessage: function(message, client){
var msg = { message: [client.sessionId, message] };
buffer.push(msg);
if (buffer.length > 15) buffer.shift();
client.broadcast(json(msg));
}
});
Related
How I can use Swagger-generated API client source on client-site (normal browser application without NodeJs)?
In a first test I generated a javascript client for Swaggers' petstore API (https://petstore.swagger.io/v2) using editor.swagger.io
The generated code is containing a index.js which provides access to constructors for public API classes, which I try to embed and use in my web application.
The documentation describes the usage of the API like so:
var SwaggerPetstore = require('swagger_petstore');
var defaultClient = SwaggerPetstore.ApiClient.instance;
// Configure API key authorization: api_key
var api_key = defaultClient.authentications['api_key'];
api_key.apiKey = 'YOUR API KEY';
// Uncomment the following line to set a prefix for the API key, e.g. "Token" (defaults to null)
//api_key.apiKeyPrefix = 'Token';
var apiInstance = new SwaggerPetstore.PetApi();
var petId = 789; // Number | ID of pet to return
var callback = function(error, data, response) {
if (error) {
console.error(error);
} else {
console.log('API called successfully. Returned data: ' + data);
}
};
apiInstance.getPetById(petId, callback);
This works fine for NodeJs applications. But how I can use the API for conventional client-site web-apps inside the browser? For such applications the nodejs function require does not work.
From https://github.com/swagger-api/swagger-js
The example has cross origin problem, but it should work in your own project
<html>
<head>
<script src='//unpkg.com/swagger-client' type='text/javascript'></script>
<script>
var specUrl = '//petstore.swagger.io/v2/swagger.json'; // data urls are OK too 'data:application/json;base64,abc...'
SwaggerClient.http.withCredentials = true; // this activates CORS, if necessary
var swaggerClient = new SwaggerClient(specUrl)
.then(function (swaggerClient) {
return swaggerClient.apis.pet.addPet({id: 1, name: "bobby"}); // chaining promises
}, function (reason) {
console.error("failed to load the spec" + reason);
})
.then(function(addPetResult) {
console.log(addPetResult.obj);
// you may return more promises, if necessary
}, function (reason) {
console.error("failed on API call " + reason);
});
</script>
</head>
<body>
check console in browser's dev. tools
</body>
</html>
I'm trying to make a .js file that will constantly have the price of bitcoin updated (every five minutes or so). I've tried tons of different ways to web scrape but they always output with either null or nothing. Here is my latest code, any ideas?
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var url = 'https://blockchain.info/charts/';
var port = 9945;
function BTC() {
request(url, function (err, res, body) {
var $ = cheerio.load(body);
var a = $(".market-price");
var b = a.text();
console.log(b);
})
setInterval(BTC, 300000)
}
BTC();
app.listen(port);
console.log('server is running on '+port);
It successfully says what port it's running on, that's not the problem. This example (when outputting) just makes a line break every time the function happens.
UPDATE:
I changed the new code I got from Wartoshika and it stopped working, but im not sure why. Here it is:
function BTCPrice() {
request('https://blockchain.info/de/ticker', (error, response, body) => {
const data = JSON.parse(body);
var value = (parseInt(data.USD.buy, 10) + parseInt(data.USD.sell, 10)) / 2;
return value;
});
};
console.log(BTCPrice());
If I have it console.log directly from inside the function it works, but when I have it console.log the output of the function it outputs undefined. Any ideas?
I would rather use a JSON api to get the current bitcoin value instead of an HTML parser. With the JSON api you get a strait forward result set that is parsable by your browser.
Checkout Exchange Rates API
Url will look like https://blockchain.info/de/ticker
Working script:
const request = require('request');
function BTC() {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
console.log(value);
});
}
BTC();
Using the value as callback:
const request = require('request');
function BTC() {
return new Promise((resolve) => {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
resolve(value);
});
});
}
BTC().then(val => console.log(val));
As the other answer stated, you should really use an API. You should also think about what type of price you want to request. If you just want a sort of index price that aggregates prices from multiple exchanges, use something like the CoinGecko API. Also if you need real-time data you need a websocket-based API, not a REST API.
If you need prices for a particular exchange, for example you're building a trading bot for one or more exchanges, you;ll need to communicate with each exchange's websoceket API directly. For that I would recommend something like the Coygo API, a node.js package that connects you directly to each exchange's real-time data feeds. You want something that doesn't add a middleman since that would add latency to your data.
I am using service workers to intercept requests for me and provide the responses to the fetch requests by communicating with a Web worker (also created from the same parent page).
I have used message channels for direct communication between the worker and service worker. Here is a simple POC I have written:
var otherPort, parentPort;
var dummyObj;
var DummyHandler = function()
{
this.onmessage = null;
var selfRef = this;
this.callHandler = function(arg)
{
if (typeof selfRef.onmessage === "function")
{
selfRef.onmessage(arg);
}
else
{
console.error("Message Handler not set");
}
};
};
function msgFromW(evt)
{
console.log(evt.data);
dummyObj.callHandler(evt);
}
self.addEventListener("message", function(evt) {
var data = evt.data;
if(data.msg === "connect")
{
otherPort = evt.ports[1];
otherPort.onmessage = msgFromW;
parentPort = evt.ports[0];
parentPort.postMessage({"msg": "connect"});
}
});
self.addEventListener("fetch", function(event)
{
var url = event.request.url;
var urlObj = new URL(url);
if(!isToBeIntercepted(url))
{
return fetch(event.request);
}
url = decodeURI(url);
var key = processURL(url).toLowerCase();
console.log("Fetch For: " + key);
event.respondWith(new Promise(function(resolve, reject){
dummyObj = new DummyHandler();
dummyObj.onmessage = function(e)
{
if(e.data.error)
{
reject(e.data.error);
}
else
{
var content = e.data.data;
var blob = new Blob([content]);
resolve(new Response(blob));
}
};
otherPort.postMessage({"msg": "content", param: key});
}));
});
Roles of the ports:
otherPort: Communication with worker
parentPort: Communication with parent page
In the worker, I have a database say this:
var dataBase = {
"file1.txt": "This is File1",
"file2.txt": "This is File2"
};
The worker just serves the correct data according to the key sent by the service worker. In reality these will be very large files.
The problem I am facing with this is the following:
Since I am using a global dummyObj, the older dummyObj and hence the older onmessage is lost and only the latest resource is responded with the received data.
In fact, file2 gets This is File1, because the latest dummyObj is for file2.txt but the worker first sends data for file1.txt.
I tried by creating an iframe directly and all the requests inside it are intercepted:
<html>
<head></head>
<body><iframe src="tointercept/file1.txt" ></iframe><iframe src="tointercept/file2.txt"></iframe>
</body>
</html>
Here is what I get as output:
One approach could be to write all the files that could be fetched into IndexedDB in the worker before creating the iframe. Then in the Service Worker fetch those from indexed DB. But I don't want to save all the resources in IDB. So this approach is not what I want.
Does anybody know a way to accomplish what I am trying to do in some other way? Or is there a fix to what I am doing.
Please Help!
UPDATE
I have got this to work by queuing the dummyObjs in a global queue instead of having a global object. And on receiving the response from the worker in msgFromW I pop an element from the queue and call its callHandler function.
But I am not sure if this is a reliable solution. As it assumes that everything will occur in order. Is this assumption correct?
I'd recommend wrapping your message passing between the service worker and the web worker in promises, and then pass a promise that resolves with the data from the web worker to fetchEvent.respondWith().
The promise-worker library can automate this promise-wrapping for you, or you could do it by hand, using this example as a guide.
If you were using promise-worker, your code would look something like:
var promiseWorker = new PromiseWorker(/* your web worker */);
self.addEventListener('fetch', function(fetchEvent) {
if (/* some optional check to see if you want to handle this event */) {
fetchEvent.respondWith(promiseWorker.postMessage(/* file name */));
}
});
I am using IBM Bluemix to make a web service for a school project.
My project needs to request a JSON from an API, so I can use the data it provides. I use the http get method for a data set, and I am not sure if it is working properly.
When I run my code, I get the message:
Error: Protocol "https:" not supported. Expected "http:"
What is causing it and how can I solve it?
Here is my .js file:
// Hello.
//
// This is JSHint, a tool that helps to detect errors and potential
// problems in your JavaScript code.
//
// To start, simply enter some JavaScript anywhere on this page. Your
// report will appear on the right side.
//
// Additionally, you can toggle specific options in the Configure
// menu.
function main() {
return 'Hello, World!';
}
main();/*eslint-env node*/
//------------------------------------------------------------------------------
// node.js starter application for Bluemix
//------------------------------------------------------------------------------
// HTTP request - duas alternativas
var http = require('http');
var request = require('request');
// cfenv provides access to your Cloud Foundry environment
// for more info, see: https://www.npmjs.com/package/cfenv
var cfenv = require('cfenv');
//chama o express, que abre o servidor
var express = require('express');
// create a new express server
var app = express();
// serve the files out of ./public as our main files
app.use(express.static(__dirname + '/public'));
// get the app environment from Cloud Foundry
var appEnv = cfenv.getAppEnv();
// start server on the specified port and binding host
app.listen(appEnv.port, '0.0.0.0', function() {
// print a message when the server starts listening
console.log("server starting on " + appEnv.url);
});
app.get('/home1', function (req,res) {
http.get('http://developers.agenciaideias.com.br/cotacoes/json', function (res2) {
var body = '';
res2.on('data', function (chunk) {
body += chunk;
});
res2.on('end', function () {
var json = JSON.parse(body);
var CotacaoDolar = json["dolar"]["cotacao"];
var VariacaoDolar = json["dolar"]["variacao"];
var CotacaoEuro = json["euro"]["cotacao"];
var VariacaoEuro = json["euro"]["variacao"];
var Atualizacao = json["atualizacao"];
obj=req.query;
DolarUsuario=obj['dolar'];
RealUsuario=Number(obj['dolar'])*CotacaoDolar;
EuroUsuario=obj['euro'];
RealUsuario2=Number(obj['euro'])*CotacaoEuro;
Oi=1*VariacaoDolar;
Oi2=1*VariacaoEuro;
if (VariacaoDolar<0) {
recomend= "Recomenda-se, portanto, comprar dólares.";
}
else if (VariacaoDolar=0){
recomend="";
}
else {
recomend="Recomenda-se, portanto, vender dólares.";
}
if (VariacaoEuro<0) {
recomend2= "Recomenda-se, portanto, comprar euros.";
}
else if (VariacaoEuro=0){
recomend2="";
}
else {
recomend2="Recomenda-se,portanto, vender euros.";
}
res.render('cotacao_response.jade', {
'CotacaoDolar':CotacaoDolar,
'VariacaoDolar':VariacaoDolar,
'Atualizacao':Atualizacao,
'RealUsuario':RealUsuario,
'DolarUsuario':DolarUsuario,
'CotacaoEuro':CotacaoEuro,
'VariacaoEuro':VariacaoEuro,
'RealUsuario2':RealUsuario2,
'recomend':recomend,
'recomend2':recomend2,
'Oi':Oi,
'Oi2':Oi2
});
app.get('/home2', function (req,res) {
http.get('https://www.quandl.com/api/v3/datasets/BCB/432.json?api_key=d1HxqKq2esLRKDmZSHR2', function (res3) {
var body = '';
res3.on('data', function (chunk) {
body += chunk;
});
res3.on('end', function () {
var x=json.dataset.data[0][1];
console.log("My JSON is "+x); });
});
});
});
});
});
Here is a print of the error screen I get:
When you want to request an https resource, you need to use https.get, not http.get.
https://nodejs.org/api/https.html
As a side note to anyone looking for a solution from Google... make sure you are not using an http.Agent with an https request or you will get this error.
The reason for this error is that you are trying to call a HTTPS URI from a HTTP client. The ideal solution would have been for a generic module to figure out the URI protocol and take the decision to use HTTPS or HTTP internally.
The way I overcame this problem is by using the switching logic on my own.
Below is some code which did the switching for me.
var http = require('http');
var https = require('https');
// Setting http to be the default client to retrieve the URI.
var url = new URL("https://www.google.com")
var client = http; /* default client */
// You can use url.protocol as well
/*if (url.toString().indexOf("https") === 0){
client = https;
}*/
/* Enhancement : using the URL.protocol parameter
* the URL object , provides a parameter url.protocol that gives you
* the protocol value ( determined by the protocol ID before
* the ":" in the url.
* This makes it easier to determine the protocol, and to support other
* protocols like ftp , file etc)
*/
client = (url.protocol == "https:") ? https : client;
// Now the client is loaded with the correct Client to retrieve the URI.
var req = client.get(url, function(res){
// Do what you wanted to do with the response 'res'.
console.log(res);
});
Not sure why, but the issue for me happened after updating node to version 17, i was previously using version 12.
In my setup, i have node-fetch using HttpsProxyAgent as an agent in the options object.
options['agent'] = new HttpsProxyAgent(`http://${process.env.AWS_HTTP_PROXY}`)
response = await fetch(url, options)
Switching back to node 12 fixed the problem:
nvm use 12.18.3
I got this error while deploying the code.
INFO error=> TypeError [ERR_INVALID_PROTOCOL]: Protocol "https:" not supported. Expected "http:"
at new NodeError (node:internal/errors:372:5)
To fix this issue, I have updated the "https-proxy-agent" package version to "^5.0.0"
Now the error was gone and it's working for me.
I am new to node.js. How to detect client is disconnected from node.js server .
Here is my code:
var net = require('net');
var http = require('http');
var host = '192.168.1.77';
var port = 12345;//
var server = net.createServer(function (stream) {
stream.setEncoding('utf8');
stream.on('data', function (data) {
var comm = JSON.parse(data);
if (comm.action == "Join_Request" && comm.gameId =="game1") // join request getting from client
{
var reply0 = new Object();
reply0.message = "WaitRoom";
stream.write(JSON.stringify(reply0) + "\0");
}
});
stream.on('disconnect', function() {
});
stream.on('close', function () {
console.log("Close");
});
stream.on('error', function () {
console.log("Error");
});
});
server.listen(port,host);
How to know client side internet disconnection.
The best way to detect "dead sockets" is to send periodic application-level ping/keepalive messages. What that message looks like depends on the protocol you're using for communicating over the socket. Then it's just a matter of using a timer or other means of checking if you've received a "ping response" within a certain period of time after you sent the ping/keepalive message to the client.
On a semi-related note, it looks like you're using JSON messages for communication, but you're assuming a complete JSON string on every data event which is a bad assumption. Try using a delimiter (a newline is pretty common for something like this, and it makes debugging the communication more human-readable) instead.
Here is a simple example of how to achieve this:
var PING_TIMEOUT = 5000, // how long to wait for client to respond
WAIT_TIMEOUT = 5000; // duration of "silence" from client until a ping is sent
var server = net.createServer(function(stream) {
stream.setEncoding('utf8');
var buffer = '',
pingTimeout,
waitTimeout;
function send(obj) {
stream.write(JSON.stringify(obj) + '\n');
}
stream.on('data', function(data) {
// stop our timers if we've gotten any kind of data
// from the client, whether it's a ping response or
// not, we know their connection is still good.
clearTimeout(waitTimeout);
clearTimeout(pingTimeout);
buffer += data;
var idx;
// because `data` can be a chunk of any size, we could
// have multiple messages in our buffer, so we check
// for that here ...
while (~(idx = buffer.indexOf('\n'))) {
try {
var comm = JSON.parse(buffer.substring(0, idx));
// join request getting from client
if (comm.action === "Join_Request" && comm.gameId === "game1") {
send({ message: 'WaitRoom' });
}
} catch (ex) {
// some error occurred, probably from trying to parse invalid JSON
}
// update our buffer
buffer = buffer.substring(idx + 1);
}
// we wait for more data, if we don't see anything in
// WAIT_TIMEOUT milliseconds, we send a ping message
waitTimeout = setTimeout(function() {
send({ message: 'Ping' });
// we sent a ping, now we wait for a ping response
pingTimeout = setTimeout(function() {
// if we've gotten here, we are assuming the
// connection is dead because the client did not
// at least respond to our ping message
stream.destroy(); // or stream.end();
}, PING_TIMEOUT);
}, WAIT_TIMEOUT);
});
// other event handlers and logic ...
});
You could also just have one interval instead of two timers that checks a "last data received" timestamp against the current timestamp and if it exceeds some length of time and we have sent a ping message recently, then you assume the socket/connection is dead. You could also instead send more than one ping message and if after n ping messages are sent and no response is received, close the connection at that point (this is basically what OpenSSH does).
There are many ways to go about it. However you may also think about doing the same on the client side, so that you know the server didn't lose its connection either.