Using the native http.get() in Node.js, I'm trying to pipe a HTTP response to a stream that I can bind data and end events to.
I'm currently handling this for gzip data, using:
http.get(url, function(res) {
if (res.headers['content-encoding'] == 'gzip') {
res.pipe(gunzip);
gunzip.on('data', dataCallback);
gunzip.on('end', endCallback);
}
});
Gunzip is a stream and this just works. I've tried to create streams (write streams, then read streams) and pipe the response, but haven't been having much luck. Any suggestions to replicate this same deal, for non-gzipped content?
The response object from a HTTP request is an instance of readable stream. Therefore, you would collect the data with the data event, then use it when the end event fires.
var http = require('http');
var body = '';
http.get(url, function(res) {
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
// all data has been downloaded
});
});
The readable.pipe(dest) would basically do the same thing, if body in the example above were a writable stream.
Nowadays the recommended way of piping is using the pipeline function. It is supposed to protect you from memory leaks.
const { createReadStream} = require('fs');
const { pipeline } = require('stream')
const { createServer, get } = require('http')
const errorHandler = (err) => err && console.log(err.message);
const server = createServer((_, response) => {
pipeline(createReadStream(__filename), response, errorHandler)
response.writeHead(200);
}).listen(8080);
get('http://localhost:8080', (response) => {
pipeline(response, process.stdout, errorHandler);
response.on('close', () => server.close())
});
Another way of doing it that has more control would be to use async iterator
async function handler(response){
let body = ''
for await (const chunk of response) {
let text = chunk.toString()
console.log(text)
body += text
}
console.log(body.length)
server.close()
}
get('http://localhost:8080', (response) => handler(response).catch(console.warn));
Related
I'm trying to call an external API in Firebase Functions but i always get a timeout.
What can be the issue causing this?
Here is my code
exports.getCountryData = functions.https.onRequest((request, response) => {
const https = require('https');
const options = {
hostname: "api-football-v1.p.rapidapi.com/v3",
path: '/fixtures?next=5',
headers: {
"x-rapidapi-host": "api-football-v1.p.rapidapi.com/v3",
"x-rapidapi-key": "my-api-key"
}
};
var req = https.get(options, (resp) => {
let data = '';
resp.on('data', (chunk) => { data += chunk; });
resp.on('end', () => {
var result = JSON.parse(data);
console.log("Api fetched successfully");
console.log(result);
response.send({ fulfillmentText: result});
});
}).on("error", (err) => { console.log("Error: " + err.message); });
});
An event-driven function may fail to successfully complete due to errors thrown in the function code itself. Some of the reasons this might happen are as follows:
The function contains a bug and the runtime throws an exception.
The function cannot reach a service endpoint, or times out while
trying to reach the endpoint.
The function intentionally throws an exception (for example, when a
parameter fails validation).
When functions written in Node.js return a rejected promise or pass a
non-null value to a callback.
In any of the above cases, the function stops executing by default and the event is discarded. If you want to retry the function when an error occurs, you can change the default retry policy by setting the "retry on failure" property. This causes the event to be retried repeatedly for up to multiple days until the function successfully completes.
In this question, the service endpointi ‘api-football-v1.p.rapidapi.com/v3’ itself took so much time to load ( not reachable ), that was the issue. Changing the API endpoint to v3.football.api-sports.io and then calling the external API in Firebase Functions solved the issue for our user #tate_xy
It turns out using their Rapid Api url (api-football-v1.p.rapidapi.com/v3) was was resulting in a timeout. Using a direct Api url (v3.football.api-sports.io) with their domain name in it did the trick for me.
Here is my working code.
exports.getCountryData = functions.https.onRequest((request, response) => {
const https = require('https');
const options = {
hostname: "v3.football.api-sports.io",
path: '/fixtures?next=5',
headers: {
"x-rapidapi-host": "v3.football.api-sports.io",
"x-apisports-key": "my-api-key"
}
};
var req = https.get(options, (resp) => {
let data = '';
resp.on('data', (chunk) => { data += chunk; });
resp.on('end', () => {
var result = JSON.parse(data);
console.log("Api fetched successfully");
console.log(result);
response.send({ fulfillmentText: result});
});
}).on("error", (err) => { console.log("Error: " + err.message); });
});
This is the code I have written using express and node.js
const express = require("express");
const https = require("https");
const app = express();
app.get("/", function(req, res) {
// Url to my api key
const url = "https://api.spoonacular.com/recipes/random?apiKey=...&number=1";
https.get(url, function(response) {
response.on("data", function (data) {
console.log(JSON.parse(data));
// const theRecipe = JSON.parse(data);
console.log(data);
});
});
res.send("The server is up and running");
});
app.listen(3000, function () {
console.log("Server started at port 3000");
});
When I refresh my webpage on localhost, on console I get the following error:
quote
SyntaxError: Unexpected end of JSON input
at JSON.parse ()
at IncomingMessage. (C:\Users\ArunBohra\Desktop\FoodRecipes\app.js:12:33)
quote
Can anyone find what is the problem with my code.
The data event fires when a chunk of data from the response has arrived. You are trying to parse the first chunk as if it were the complete JSON text.
You need to collect the pieces from each data event but wait until the end event fires before joining them together into the complete string of JSON that you can parse.
There is an example of fetching and parsing JSON in the documentation.
You might want to look at modules like axios and node-fetch which take care of that (and the JSON parsing) for you while providing modern Promise based APIs.
If you have a new enough version of Node, you can use the native Fetch API.
If you use a package like node-fetch you can get the whole thing in one go instead of what you have now which is chunking the data
const fetch = require('node-fetch');
const url = "https://api.spoonacular.com/recipes/random?apiKey=...&number=1";
fetch(url)
.then(response => response.json())
.then(data => console.log(data));
In addition to other answers, you can do it without another package.
https.get(url, function (response) {
let result = "";
response.on("data", function (data) {
result += chunk;
});
res.on('end', () => {
// now you have the combined result
console.log(result);
});
});
INTRODUCTION
I am implementing a function for making any kind of https request to any endpoint (using the https native module). When I make a request to a specific API I get an error response in JSON format. Like this:
{
"error": {
"code": 404,
"message": "ID not found"
}
}
How can I handle this kind of errors? At a first moment, I supposed that they were handled in
request.on("error", (err) => {
reject(err);
});
HTTPs Request function code
I have comment '<---------' in the relevant parts of the code
const https = require("https");
exports.httpsRequest = function (options, body = null) {
/*
This function is useful for making requests over the HTTPs protocol
*/
return new Promise((resolve, reject) => {
const request = https.request(options, (response) => {
// Get the response content type
const contentType =
response.headers["content-type"] &&
response.headers["content-type"].split(";")[0];
// Cumulate data
let chuncks = [];
response.on("data", (chunck) => {
chuncks.push(chunck);
});
response.on("end", () => {
// Concat all received chunks
let response = Buffer.concat(chuncks);
// Some responses might be in JSON format...
if (contentType === "application/json") {
// Jsonify the response
response = JSON.parse(response);
}
// (For the future) TODO - Check and parse more content types if needed.
// Resolve the promise with the HTTPs response
resolve(response); // <--------- The JSON format error responses are resolved too!!
});
});
// Reject on request error
request.on("error", (err) => {
// <------------- At a first moment, I supposed that all error responses were handled in this part of the code
reject(err);
});
// Write the body
if (body) {
request.write(body);
}
// Close HTTPs connection.
request.end();
});
};
Question
Why the error response is not handled in request.on("error", ...) ?
Thank you. I would appreciate any help or suggestion.
You need to create a different code path for when the content type isn't what you were expecting in which you call reject() and you also need to try/catch around JSON parsing errors so you can properly catch them and reject on them too. You can solve those issues with this code:
exports.httpsRequest = function (options, body = null) {
/*
This function is useful for making requests over the HTTPs protocol
*/
return new Promise((resolve, reject) => {
const request = https.request(options, (response) => {
// Get the response content type
const contentType =
response.headers["content-type"] &&
response.headers["content-type"].split(";")[0];
// Cumulate data
let chuncks = [];
response.on("data", (chunck) => {
chuncks.push(chunck);
});
response.on("end", () => {
// Concat all received chunks
let response = Buffer.concat(chuncks);
// Some responses might be in JSON format...
if (contentType === "application/json") {
try {
// Jsonify the response
response = JSON.parse(response);
resolve(response);
return;
} catch(e) {
reject(e);
return;
}
}
reject(new Error("Not JSON content-type"))
});
});
// Reject on request error
request.on("error", (err) => {
reject(err);
});
// Write the body
if (body) {
request.write(body);
}
// Close HTTPs connection.
request.end();
});
};
FYI, libraries such as got() and others listed here, all do this work for you automatically and have a lot of other useful features. You don't really need to build this yourself.
I have some troubles while running this IBM Cloud Function:
/**
*
* main() will be run when you invoke this action
*
* #param Cloud Functions actions accept a single parameter, which must be a JSON object.
*
* #return The output of this action, which must be a JSON object.
*
*/
function main(params) {
const https = require('https');
https.get('https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log(JSON.parse(data).explanation);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}
My problem is that the first invokes of this function (at least the first 3-4) produce no output. The subsequent calls run properly and the log is correctly shown. How can I fix this unpredictable behaviour? I'd like, of course, to retrieve my data at first call of this function. Thanks.
Node.js uses an non-blocking asynchronous programming model. This main function returns before the HTTP response is available.
Returning a Promise will allow you to wait on the HTTP response.
function main(params) {
return new Promise((resolve, reject) => {
const https = require('https');
https.get('https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
const explanation = JSON.parse(data).explanation
console.log(explanation);
resolve({ explanation })
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject({ error: err.message })
});
})
}
Two additional things to check:
Make sure to append .json to your endpoint
Example: https://<ibm-domain>/api/v1/web/<username>/default/<function>.json
Make sure to select Enable as Web Action in the Endpoints sidebar menu.
Also, you should be able to return an async main function in lieu of the Promise object.
async function main(params) {
try {
// some `await` function
} catch (e) {
// catch `await` errors
}
}
module.exports = main;
I would like a response to differ depending on the finished request I recieve. I am sending a POST request and receive an xml file. The result is either a success or error. I use xml2json to convert the xml into a json object, then depending on the response I want to output json.
The problem is that I can't have a response inside a response. I also can't save the value of the callback for later usage (since its asynchronous).
I have thought about using Promises but I'm not sure. What should I do?
The order of operations should be
1) Send request
2) Get buffer response
3) Join Buffers. Process xml into JSON
4) Depending on the type of JSON entry, output either res.json('success') or res.json('error') if the xml responds with an error.
app.post('/api/submit', (req, res) => {
...
const request = https.request(options, (res) => {
let chunks = [];
res.on("data", function(chunk) {
chunks.push(chunk);
});
res.on("end", function(err) {
if (err) throw err;
let body = Buffer.concat(chunks);
xmlConverter(body, function(err, result) {
console.dir(result);
if (result.entry) {
console.log('✅ Success')
//Respond with json here --> res.json('success')
} else if (result.error) {
console.log('There was an error processing your request');
//or here if there was an error --> res.json('error')
}
});
});
});
request.end()
You can respond inside the callback. The problem is that you have two variable, both named res, so one shadows the other. You just need to change one of the res variable names so your not shadowing it. For example, you can change:
const request = https.request(options, (http_res) // <--change argument name
Then later:
if (result.entry) {
console.log('✅ Success')
http_res.json('success') // <-- use the response object from request
The problem of not being able to save the result for later is a different problem, but easy to solve. The solution though really depends one what you are trying to do. If, for example, you want to further process the data, you can set up a function to call and pass the response data in. Something like:
function process_data(response){
// use the response here
}
Then you can simply call it when you get the data:
if (result.entry) {
console.log('✅ Success')
http_res.json('success') // <-- use the response object from request
process_data(result)
Of course maybe your use case is more complicated but without more details its hard to give a specific answer.
Don't use the same name for both res, because they are different variables. And simply use the out res variable to respond the request with the value you want.
I think it would be something like this:
app.post('/
api/submit', (req, res) => {
...
const request = https.request(options, (resValue) => {
let chunks = [];
resValue.on("data", function(chunk) {
chunks.push(chunk);
});
resValue.on("end", function(err) {
if (err) throw err;
let body = Buffer.concat(chunks);
xmlConverter(body, function(err, result) {
console.dir(result);
if (result.entry) {
console.log('✅ Success')
res.json('success')
} else if (result.error) {
console.log('There was an error processing your request');
res.json('error')
}
});
});
});
request.end()
What exactly is the issue? You are perfectly able to rename the argument of the callback function supplied to https.request(options, callbackFunction) -- it is not important what this variable is named.
app.post('/api/submit', (req, res) => {
const request = https.request(options, (potato) => {
let chunks = [];
potato.on("data", function(chunk) {
chunks.push(chunk);
});
potato.on("end", function(err) {
if (err) throw err; // TODO res.status(500).json({}); ??
let body = Buffer.concat(chunks);
xmlConverter(body, function(err, result) {
console.dir(result);
if (result.entry) {
console.log('✅ Success')
res.status(200).json({});
} else if (result.error) {
console.log('There was an error processing your request');
res.status(500).json({});
}
request.end()
});
});
});
});