I am fetching data from different API with javascript's fetch API. But how can I find out how many bytes are sent on each request for analytics?
The request could be in any method.
I know that I can get the amount of bytes received with
response.headers["content-length"].
I need to find out a way to get the amount of bytes sent on the frontend (browser or mobile using React Native). Ideally, it would be the total size of the request, but just the size of the request body would be good enough.
You can get the value that will be set in the Content-Length header by reading the Request's body as text and checking the length of the returned string:
(async () => {
const formdata = new FormData();
const file = new Blob(["data".repeat(1024)])
formdata.append("key", file)
const req = new Request("/", { method: "POST", body: formdata });
// note that we .clone() the Request
// so that we can still use the original one with fetch()
console.log((await req.clone().text()).length);
fetch(req);
console.log("check the Network panel of your dev tools to see the sent header");
})();
However this only applies for requests where this header is sent, i.e not for GET and HEAD requests.
A quick solution that I used - a tiny middleware (I use Express):
const socketBytes = new Map();
app.use((req, res, next) => {
req.socketProgress = getSocketProgress(req.socket);
next();
});
/**
* return kb read delta for given socket
*/
function getSocketProgress(socket) {
const currBytesRead = socket.bytesRead;
let prevBytesRead;
if (!socketBytes.has(socket)) {
prevBytesRead = 0;
} else {
prevBytesRead = socketBytes.get(socket).prevBytesRead;
}
socketBytes.set(socket, {prevBytesRead: currBytesRead})
return (currBytesRead-prevBytesRead)/1024;
}
And then you can use req.socketProgress in your middlewares.
Related
I've never received an error like this before,
I have a file that defines functions for making API calls, currently I'm reading the endpoint base URLs from the environment variables:
/**
* Prepended to request URL indicating base URL for API and the API version
*/
const VERSION_URL = `${process.env.NEXT_PUBLIC_API_BASE_URL}/${process.env.NEXT_PUBLIC_API_VERSION}`
I tried to make a quick workaround because environment variables weren't being loaded correctly, by hardcoding the URLS incase the variable wasn't defined.
/**
* Prepended to request URL indicating base URL for API and the API version
*/
const VERSION_URL = `${process.env.NEXT_PUBLIC_API_BASE_URL || 'https://hardcodedURL.com'}/${process.env.NEXT_PUBLIC_API_VERSION || 'v1'}`
In development and production mode when running on my local machine it works fine (docker container). However, as soon as it's pushed to production, I then get the following screen:
This is the console output:
framework-bb5c596eafb42b22.js:1 TypeError: Path must be a string. Received undefined
at t (137-10e3db828dbede8a.js:46:750)
at join (137-10e3db828dbede8a.js:46:2042)
at J (898-576b101442c0ef86.js:1:8158)
at G (898-576b101442c0ef86.js:1:10741)
at oo (framework-bb5c596eafb42b22.js:1:59416)
at Wo (framework-bb5c596eafb42b22.js:1:68983)
at Ku (framework-bb5c596eafb42b22.js:1:112707)
at Li (framework-bb5c596eafb42b22.js:1:98957)
at Ni (framework-bb5c596eafb42b22.js:1:98885)
at Pi (framework-bb5c596eafb42b22.js:1:98748)
cu # framework-bb5c596eafb42b22.js:1
main-f51d4d0442564de3.js:1 TypeError: Path must be a string. Received undefined
at t (137-10e3db828dbede8a.js:46:750)
at join (137-10e3db828dbede8a.js:46:2042)
at J (898-576b101442c0ef86.js:1:8158)
at G (898-576b101442c0ef86.js:1:10741)
at oo (framework-bb5c596eafb42b22.js:1:59416)
at Wo (framework-bb5c596eafb42b22.js:1:68983)
at Ku (framework-bb5c596eafb42b22.js:1:112707)
at Li (framework-bb5c596eafb42b22.js:1:98957)
at Ni (framework-bb5c596eafb42b22.js:1:98885)
at Pi (framework-bb5c596eafb42b22.js:1:98748)
re # main-f51d4d0442564de3.js:1
main-f51d4d0442564de3.js:1 A client-side exception has occurred, see here for more info: https://nextjs.org/docs/messages/client-side-exception-occurred
re # main-f51d4d0442564de3.js:1
Viewing the source at t (137-10e3db828dbede8a.js:46:750)
I'm completely at a lost at what this means or what is happening. Why would hardcoding in a string for the path result in this client error? The lack of a readable source code is making this impossible for me to understand what's happening.
Quick googling suggests that I should upgrade some package, but the error is so vague, I'm not even sure what package is giving the issue.
This is the roughly the how the version URL path is being used
/**
* Send a get request to a given endpoint
*
* **Returns a Promise**
*/
function GET(token, data, parent, api) {
return new Promise((resolve, reject) => {
try {
let req = new XMLHttpRequest()
let endpoint = `${VERSION_URL}/${parent}/${api}` // base url with the params not included
let params = new URLSearchParams() // URLSearchParams used for adding params to url
// put data in GET request params
for (const [key, value] of Object.entries(data)) {
params.set(key, value)
}
let query_url = endpoint + "?" + params.toString() // final query url
req.open("GET", query_url, true)
req.setRequestHeader("token", token) // put token into header
req.onloadend = () => {
if (req.status === 200) {
// success, return response
resolve([req.response, req.status])
} else {
reject([req.responseText, req.status])
}
}
req.onerror = () => {
reject([req.responseText, req.status])
}
req.send()
} catch (err) {
reject(["Exception", 0])
}
})
}
From my experience, this problem can happen for multiple reasons. The most common one is because you didn't put the data accessing checker properly when data comes from an API. Sometimes this things we don't see in browser but it gives such error when you deploy.
For example:
const response = fetch("some_url");
const companyId = response.data.companyId; ❌
const companyId = response?.data?.companyId; ✔️
Every time one of my button is clicked, I trigger a call to the adviceslip API to get an advice data. I expect the payload to vary. But the exact same data is returned after each call.
I've tried to call getAdvice function as a callback but it didn't work.
Am I missing something?
'use strict'
const title = document.querySelector('.advice-title')
const desc = document.querySelector('.advice-text')
const btn = document.querySelector('.btn')
const getAdvice = async () =>{
try{
const response = await fetch('https://api.adviceslip.com/advice');
if (!response.ok){
throw new Error('Api connection problem')
}
const responseJson = await response.json();
const data = responseJson.slip;
const id = `ADVICE #${data.id}`;
const advice = `"${data.advice}"`;
title.textContent = id;
desc.textContent = advice;
}
catch (e) {
console.error(e)
}
}
btn.addEventListener('click', () => {
getAdvice()
})
There appears to be an issue with caching where multiple requests closely spaced in time from a web page will cause the same result to come back. If I add a uniquely changing value to the URL to defeat caching like this:
const response = await fetch('https://api.adviceslip.com/advice?t=' + Math.random());
Then, I get a unique response every time.
I experimented with sending various headers to disable caching instead of the t=xxx hack, but those headers trigger a CORs error and the fetch() does not succeed because custom headers like this don't pass the test for a "simple" CORS request and thus require pre-flight which the target site does not support. So, the ?t=xxx where xxx is different on each request seems to do the trick.
I found this and it worked for me. Add
const response = await fetch("https://api.adviceslip.com/advice", { cache: "no-cache" });
I finally found a useful library to parse meta data from an audio stream, here: https://github.com/ghaiklor/icecast-parser. But still, I can't get the expected response , when I send the headers as in the example below.
The first function makes the request to the radio station / channel and gets the stream:
_makeRequest() {
const request = http.request(this.getConfig('url'));
console.log("Making request to: " + this.getConfig('url'));
request.setHeader('Range', 'bytes=0-');
request.setHeader('User-Agent', 'VLC/2.2.4 LibVLC/2.2.4');
request.setHeader('Icy-MetaData', '1');
request.setHeader('Connection', 'close');
request.once('response', this._onRequestResponse.bind(this));
request.once('error', this._onRequestError.bind(this));
request.end();
return this;
}
When a request to a radio station is successfully called, this function is called:
_onRequestResponse(response) {
console.log("Received response!");
console.log("Headers:");
console.log(response.headers['content-type']);
const icyMetaInt = response.headers['icy-metaint'];
console.log("icyMetaInt= " + icyMetaInt);
if (icyMetaInt) {
const reader = new StreamReader(icyMetaInt);
reader.on('metadata', metadata => {
console.log(metadata);
this._destroyResponse(response);
this._queueNextRequest(this.getConfig('metadataInterval'));
this.emit('metadata', metadata);
});
response.pipe(reader);
this.emit('stream', reader);
} else {
this._destroyResponse(response);
this._queueNextRequest(this.getConfig('emptyInterval'));
this.emit('empty');
}
return this;
}
When I use these functions on this URL (url: 'http://streaming.radionomy.com/70-s-80-sMetal'), the reply in the console is:
audio/mpeg
icyMetaInt= undefined
I understood the most crucial header here is:
setHeader('Icy-MetaData', '1')
Still, I am not receiving the 'icyMetaInt'. The URL does seem to contain metadata when checking it with other tools.
Any ideas what is going wrong here? Thank you!
;
Requests are part of the Fetch API, and when making cross origin requests you only have access to a limited range of headers.
just posting a question as I have seen some other similar questions on here but none with a method that seemingly works for me.
I'm new to NodeJS and playing around with requesting data from an API. For my test here im just trying to pull ticker prices based on the input of a prompt from the user.
This works fine, however the object
This is the code I am using to try and make this work:
prompt.start();
prompt.get(['coin'], function (err, result) {
request({url: `https://min-api.cryptocompare.com/data/price?fsym=${result.coin}&tsyms=BTC,USD`, json:true}, function(err, res, json) {
if (err) {
throw err;
}
console.log(json);
var json = JSON.stringify(json);
var string2 = JSON.parse(json);
console.log(string2.btc_price);
console.log(json);
});
console.log('Retrieving: ' + result.coin);
});
The API request works, however it returns JSON that looks like this with my 3 console logs:
{ set_attributes: { btc_price: 1, usd_price: 15839.35 } }
undefined
{"set_attributes":{"btc_price":1,"usd_price":15839.35}} -- (Stringify'd response)
I want to be able to extract the btc_price & usd_price as variables, ive tried a few different methods and can't figure out where exactly im going wrong. Any help would be greatly appreciated!
Cheers,
J
When you attempt to extract the btc_price attribute, it's actually nested so your second console should read console.log(string2.set_attributes.btc_price);
axios has more stars on Github, more followers on Github and more forks.
Features
Make XMLHttpRequests from the browser
Make http requests from node.js
Supports the Promise API
Intercept request and response
Transform request and response data
Cancel requests
Automatic transforms for JSON data
Client side support for protecting against XSRF
Using async / await
// Make a request for a user with a given ID
var preload = null;
async function getPrice(symbol) {
preload = await axios.get('https://min-api.cryptocompare.com/data/price?fsym=${symbol}&tsyms=BTC,USD')
.then(function (response) {
preload = response.data;
})
.catch(function (error) {
console.log(error);
});
return `preload.BTC = ${preload.BTC}; preload.BTC = ${preload.BTC}`;
};
getPrice('ETH');
// return preload.BTC = 0.04689; preload.USD = 742.85
I'm trying to make a .js file that will constantly have the price of bitcoin updated (every five minutes or so). I've tried tons of different ways to web scrape but they always output with either null or nothing. Here is my latest code, any ideas?
var express = require('express');
var path = require('path');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var app = express();
var url = 'https://blockchain.info/charts/';
var port = 9945;
function BTC() {
request(url, function (err, res, body) {
var $ = cheerio.load(body);
var a = $(".market-price");
var b = a.text();
console.log(b);
})
setInterval(BTC, 300000)
}
BTC();
app.listen(port);
console.log('server is running on '+port);
It successfully says what port it's running on, that's not the problem. This example (when outputting) just makes a line break every time the function happens.
UPDATE:
I changed the new code I got from Wartoshika and it stopped working, but im not sure why. Here it is:
function BTCPrice() {
request('https://blockchain.info/de/ticker', (error, response, body) => {
const data = JSON.parse(body);
var value = (parseInt(data.USD.buy, 10) + parseInt(data.USD.sell, 10)) / 2;
return value;
});
};
console.log(BTCPrice());
If I have it console.log directly from inside the function it works, but when I have it console.log the output of the function it outputs undefined. Any ideas?
I would rather use a JSON api to get the current bitcoin value instead of an HTML parser. With the JSON api you get a strait forward result set that is parsable by your browser.
Checkout Exchange Rates API
Url will look like https://blockchain.info/de/ticker
Working script:
const request = require('request');
function BTC() {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
console.log(value);
});
}
BTC();
Using the value as callback:
const request = require('request');
function BTC() {
return new Promise((resolve) => {
// send a request to blockchain
request('https://blockchain.info/de/ticker', (error, response, body) => {
// parse the json answer and get the current bitcoin value
const data = JSON.parse(body);
value = (parseInt(data.THB.buy, 10) + parseInt(data.THB.sell, 10)) / 2;
resolve(value);
});
});
}
BTC().then(val => console.log(val));
As the other answer stated, you should really use an API. You should also think about what type of price you want to request. If you just want a sort of index price that aggregates prices from multiple exchanges, use something like the CoinGecko API. Also if you need real-time data you need a websocket-based API, not a REST API.
If you need prices for a particular exchange, for example you're building a trading bot for one or more exchanges, you;ll need to communicate with each exchange's websoceket API directly. For that I would recommend something like the Coygo API, a node.js package that connects you directly to each exchange's real-time data feeds. You want something that doesn't add a middleman since that would add latency to your data.