I am fetching data from different API with javascript's fetch API. But how can I find out how many bytes are sent on each request for analytics?
The request could be in any method.
I know that I can get the amount of bytes received with
response.headers["content-length"].
I need to find out a way to get the amount of bytes sent on the frontend (browser or mobile using React Native). Ideally, it would be the total size of the request, but just the size of the request body would be good enough.
You can get the value that will be set in the Content-Length header by reading the Request's body as text and checking the length of the returned string:
(async () => {
const formdata = new FormData();
const file = new Blob(["data".repeat(1024)])
formdata.append("key", file)
const req = new Request("/", { method: "POST", body: formdata });
// note that we .clone() the Request
// so that we can still use the original one with fetch()
console.log((await req.clone().text()).length);
fetch(req);
console.log("check the Network panel of your dev tools to see the sent header");
})();
However this only applies for requests where this header is sent, i.e not for GET and HEAD requests.
A quick solution that I used - a tiny middleware (I use Express):
const socketBytes = new Map();
app.use((req, res, next) => {
req.socketProgress = getSocketProgress(req.socket);
next();
});
/**
* return kb read delta for given socket
*/
function getSocketProgress(socket) {
const currBytesRead = socket.bytesRead;
let prevBytesRead;
if (!socketBytes.has(socket)) {
prevBytesRead = 0;
} else {
prevBytesRead = socketBytes.get(socket).prevBytesRead;
}
socketBytes.set(socket, {prevBytesRead: currBytesRead})
return (currBytesRead-prevBytesRead)/1024;
}
And then you can use req.socketProgress in your middlewares.
I have this function running in an azure function to get a sas token for a browser application to upload to azure blob storage:
var azure = require('azure-storage');
module.exports = function(context, req) {
if (req.body.container) {
// The following values can be used for permissions:
// "a" (Add), "r" (Read), "w" (Write), "d" (Delete), "l" (List)
// Concatenate multiple permissions, such as "rwa" = Read, Write, Add
context.res = generateSasToken(
context,
req.body.container,
req.body.blobName,
req.body.permissions
);
} else {
context.res = {
status: 400,
body: "Specify a value for 'container'"
};
}
context.done(null, context);
};
function generateSasToken(context, container, blobName, permissions) {
var connString = process.env.AzureWebJobsStorage;
var blobService = azure.createBlobService(connString);
// Create a SAS token that expires in an hour
// Set start time to five minutes ago to avoid clock skew.
var startDate = new Date();
startDate.setMinutes(startDate.getMinutes() - 5);
var expiryDate = new Date(startDate);
expiryDate.setMinutes(startDate.getMinutes() + 60);
permissions = azure.BlobUtilities.SharedAccessPermissions.READ +
azure.BlobUtilities.SharedAccessPermissions.WRITE +
azure.BlobUtilities.SharedAccessPermissions.DELETE +
azure.BlobUtilities.SharedAccessPermissions.LIST;
var sharedAccessPolicy = {
AccessPolicy: {
Permissions: permissions,
Start: startDate,
Expiry: expiryDate
}
};
var sasToken = blobService.generateSharedAccessSignature(
container,
blobName,
sharedAccessPolicy
);
context.log(sasToken);
return {
token: sasToken,
uri: blobService.getUrl(container, blobName, sasToken, true)
};
}
I am then calling this url in the client and I try and upload with this code:
const search = new URLSearchParams(`?${token}`);
const sig = encodeURIComponent(search.get('sig'));
const qs = `?sv=${search.get('sv')}&ss=b&srt=sco&sp=rwdlac&se=${search.get('sv')}&st=${search.get(
'st'
)}&spr=https&sig=${sig}`;
return `${url}/${containerName}/${filename}${qs}`;
Which generates a url like this:
https://mystorage.blob.core.windows.net/mycontainer/latest.png?sv=2018-03-28&ss=b&srt=sco&sp=rwdlac&se=2018-03-28&st=2019-01-30T19:11:10Z&spr=https&sig=g0sceq3EkiAQTvyaZ07C+C4SZQz9FaGTV4Zwq4HkAnc=
Which returns this error:
403 (Server failed to authenticate the request. Make sure the value of
Authorization header is formed correctly including the signature.)
If I generate the sas token from the azure portal it works, so the generated url looks like this:
https://mystorage.blob.core.windows.net/mycontainer/latest.png?sv=2018-03-28&ss=b&srt=sco&sp=rwdlac&se=2019-01-31T03:01:43Z&st=2019-01-30T19:01:43Z&spr=https&sig=ayE4gt%2FDfDzjv5DjMaD7AS%2F176Bi4Q6DWJNlnDzl%2FGc%3D
but my url looks like this:
https://mystorage.blob.core.windows.net/mycontainer/latest.png?sv=2018-03-28&ss=b&srt=sco&sp=rwdlac&se=2019-01-31T03:34:21Z&st=2019-01-30T19:34:21Z&spr=https&sig=Dx8Vm4XPnD1rn9uyzIAXZEfcdbWb0HjmOq%2BIq42Q%2FOM%3D
I have no idea what to do to get this working
Your Azure Function code is correct, and
var sasToken = blobService.generateSharedAccessSignature(
container,
blobName,
sharedAccessPolicy
);
is exactly the sasToken you need to upload blob. No need to process the token again(mishandle actually) as you have done in the 2nd code snippet.
It's expected that the sas token from the Azure portal(Account SAS) is different from the one generated in your code(Service SAS). Have a look at the doc.
To conclude,
Make sure the connection string belongs to the Storage you want to connect. You could avoid trouble and directly replace var connString = process.env.AzureWebJobsStorage; with var connString = "connectionStringGotFromPortal";
If 1 is confirmed, your Azure function code is correct and returns token as expected
{
token: sasToken,
uri: blobService.getUrl(container, blobName, sasToken, true)
};
Based on the 2nd code snippet you provide, you only need
return `${url}/${containerName}/${filename}?${token}`;
if the token is identical to what function returns.
The problem is that in your server-side code you're creating a Service SAS and then taking only signature portion of the code (sig) and creating an Account SAS on the client.
Since the parameters used to create token has now changed (in the original one, you didn't have parameters like ss, srt etc. but when you're creating your own URL, you're inserting these parameters), when you use the modified SAS URL you will get 403 error. This is happening because server again computes the signature based on the URL parameters and compare that with the signature passed in the URL. Since the two signatures won't match, you're getting the 403 error.
Since you're returning the SAS URL of the blob, there's no need for you to create the URL on the client. You can simply use the uri you're returning from your API layer on the client and use that to upload.
As Jerry Liu's answer explained your Azure function generates the correct token and already gives you the the correct uri to use which includes your blob name and token.
In your client side you can also use azure-sdk-for-js
// This is the response from your api with token and uri
const uri = response.uri;
const pipeline = StorageURL.newPipeline(new AnonymousCredential());
// Your uri already includes the full blob url with SAS signature
const blockBlobURL = BlockBlobURL.fromBlobURL(new BlobURL(uri, pipeline));
const uploadBlobResponse = await blockBlobURL.upload(
Aborter.none,
file,
file.size,
{ blobHTTPHeaders: { blobContentType: `${mime}; charset=utf-8`} }
);
I finally found a useful library to parse meta data from an audio stream, here: https://github.com/ghaiklor/icecast-parser. But still, I can't get the expected response , when I send the headers as in the example below.
The first function makes the request to the radio station / channel and gets the stream:
_makeRequest() {
const request = http.request(this.getConfig('url'));
console.log("Making request to: " + this.getConfig('url'));
request.setHeader('Range', 'bytes=0-');
request.setHeader('User-Agent', 'VLC/2.2.4 LibVLC/2.2.4');
request.setHeader('Icy-MetaData', '1');
request.setHeader('Connection', 'close');
request.once('response', this._onRequestResponse.bind(this));
request.once('error', this._onRequestError.bind(this));
request.end();
return this;
}
When a request to a radio station is successfully called, this function is called:
_onRequestResponse(response) {
console.log("Received response!");
console.log("Headers:");
console.log(response.headers['content-type']);
const icyMetaInt = response.headers['icy-metaint'];
console.log("icyMetaInt= " + icyMetaInt);
if (icyMetaInt) {
const reader = new StreamReader(icyMetaInt);
reader.on('metadata', metadata => {
console.log(metadata);
this._destroyResponse(response);
this._queueNextRequest(this.getConfig('metadataInterval'));
this.emit('metadata', metadata);
});
response.pipe(reader);
this.emit('stream', reader);
} else {
this._destroyResponse(response);
this._queueNextRequest(this.getConfig('emptyInterval'));
this.emit('empty');
}
return this;
}
When I use these functions on this URL (url: 'http://streaming.radionomy.com/70-s-80-sMetal'), the reply in the console is:
audio/mpeg
icyMetaInt= undefined
I understood the most crucial header here is:
setHeader('Icy-MetaData', '1')
Still, I am not receiving the 'icyMetaInt'. The URL does seem to contain metadata when checking it with other tools.
Any ideas what is going wrong here? Thank you!
;
Requests are part of the Fetch API, and when making cross origin requests you only have access to a limited range of headers.
I'm playing around with service workers. The following code should proxy JS files to patch the imports so that they conform to platform standards (i.e., "./", "../", "/", or "http://...").
Works great in Chromium (67.0.3396.79 on Arch Linux). And seems to work just as well in Firefox (60.0.2 (64-bit) on Arch), at least from the network tab, I can see all of the patched sources loading, but for some reason the JS modules aren't running. Can't console.log etc. or anything. Not sure how to get Firefox to bootstrap the application.
I noticed that the fetch headers are all toLowerCaseed, but I read up on that here and Mozilla also points out that header names are case-insensitive here.
I also thought maybe because the content-length was possibly changed that the file wasn't being received completely, but I didn't see any parse errors and indeed the network tab had the correct content-length changes, so I ruled that out.
const maybeAppendJS = (x) =>
x.endsWith(".js")
? x
: `${x}.js`;
const maybePatchURL = (x) =>
x.match(/(^'#.*'(.)?$)|(^"#.*"(.)?$)/)
? `"/node_modules/${maybeAppendJS(eval(x))}";`
: x;
const maybePatchImport = (x) =>
x.startsWith("import ")
? x.split(/\s/).map(maybePatchURL).join(" ")
: x;
async function maybeRewriteImportStatements(event) {
let candidate = event.request.url;
const url = maybeAppendJS(candidate);
const resp = await fetch(url);
if (!resp.headers.get("content-type").startsWith("text")) {
const text = await resp.text();
const newText = text.split(/\n/g)
.map(maybePatchImport)
.join("\n");
return new Response(newText, {headers: resp.headers});
}
if (resp.headers.get("content-type").startsWith("text/")) {
const location = `${url.substring(0, url.length - 3)}/index.js`;
return new Response(null, {status: 302, headers: {location: location}});
}
console.log("Service worker should never get here");
}
this.addEventListener('fetch', (event) => {
if (event.request.destination === "script" || event.request.referrer.endsWith(".js") || event.request.url.endsWith(".js")) {
event.respondWith(maybeRewriteImportStatements(event));
}
});
This was fixed by upgrading to the Firefox nightly (62.0a1.20180611-1).
I'm looking at the following API:
http://wiki.github.com/soundcloud/api/oembed-api
The example they give is
Call:
http://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=json
Response:
{
"html":"<object height=\"81\" ... ",
"user":"Forss",
"permalink":"http:\/\/soundcloud.com\/forss\/flickermood",
"title":"Flickermood",
"type":"rich",
"provider_url":"http:\/\/soundcloud.com",
"description":"From the Soulhack album...",
"version":1.0,
"user_permalink_url":"http:\/\/soundcloud.com\/forss",
"height":81,
"provider_name":"Soundcloud",
"width":0
}
What do I have to do to get this JSON object from just an URL?
It seems they offer a js option for the format parameter, which will return JSONP. You can retrieve JSONP like so:
function getJSONP(url, success) {
var ud = '_' + +new Date,
script = document.createElement('script'),
head = document.getElementsByTagName('head')[0]
|| document.documentElement;
window[ud] = function(data) {
head.removeChild(script);
success && success(data);
};
script.src = url.replace('callback=?', 'callback=' + ud);
head.appendChild(script);
}
getJSONP('http://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=js&callback=?', function(data){
console.log(data);
});
A standard http GET request should do it. Then you can use JSON.parse() to make it into a json object.
function Get(yourUrl){
var Httpreq = new XMLHttpRequest(); // a new request
Httpreq.open("GET",yourUrl,false);
Httpreq.send(null);
return Httpreq.responseText;
}
then
var json_obj = JSON.parse(Get(yourUrl));
console.log("this is the author name: "+json_obj.author_name);
that's basically it
In modern-day JS, you can get your JSON data by calling ES6's fetch() on your URL and then using ES7's async/await to "unpack" the Response object from the fetch to get the JSON data like so:
const getJSON = async url => {
const response = await fetch(url);
if(!response.ok) // check if response worked (no 404 errors etc...)
throw new Error(response.statusText);
const data = response.json(); // get JSON from the response
return data; // returns a promise, which resolves to this data value
}
console.log("Fetching data...");
getJSON("https://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=json").then(data => {
console.log(data);
}).catch(error => {
console.error(error);
});
The above method can be simplified down to a few lines if you ignore the exception/error handling (usually not recommended as this can lead to unwanted errors):
const getJSON = async url => {
const response = await fetch(url);
return response.json(); // get JSON from the response
}
console.log("Fetching data...");
getJSON("https://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=json")
.then(data => console.log(data));
Because the URL isn't on the same domain as your website, you need to use JSONP.
For example: (In jQuery):
$.getJSON(
'http://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=js&callback=?',
function(data) { ... }
);
This works by creating a <script> tag like this one:
<script src="http://soundcloud.com/oembed?url=http%3A//soundcloud.com/forss/flickermood&format=js&callback=someFunction" type="text/javascript"></script>
Their server then emits Javascript that calls someFunction with the data to retrieve.
`someFunction is an internal callback generated by jQuery that then calls your callback.
DickFeynman's answer is a workable solution for any circumstance in which JQuery is not a good fit, or isn't otherwise necessary. As ComFreek notes, this requires setting the CORS headers on the server-side. If it's your service, and you have a handle on the bigger question of security, then that's entirely feasible.
Here's a listing of a Flask service, setting the CORS headers, grabbing data from a database, responding with JSON, and working happily with DickFeynman's approach on the client-side:
#!/usr/bin/env python
from __future__ import unicode_literals
from flask import Flask, Response, jsonify, redirect, request, url_for
from your_model import *
import os
try:
import simplejson as json;
except ImportError:
import json
try:
from flask.ext.cors import *
except:
from flask_cors import *
app = Flask(__name__)
#app.before_request
def before_request():
try:
# Provided by an object in your_model
app.session = SessionManager.connect()
except:
print "Database connection failed."
#app.teardown_request
def shutdown_session(exception=None):
app.session.close()
# A route with a CORS header, to enable your javascript client to access
# JSON created from a database query.
#app.route('/whatever-data/', methods=['GET', 'OPTIONS'])
#cross_origin(headers=['Content-Type'])
def json_data():
whatever_list = []
results_json = None
try:
# Use SQL Alchemy to select all Whatevers, WHERE size > 0.
whatevers = app.session.query(Whatever).filter(Whatever.size > 0).all()
if whatevers and len(whatevers) > 0:
for whatever in whatevers:
# Each whatever is able to return a serialized version of itself.
# Refer to your_model.
whatever_list.append(whatever.serialize())
# Convert a list to JSON.
results_json = json.dumps(whatever_list)
except SQLAlchemyError as e:
print 'Error {0}'.format(e)
exit(0)
if len(whatevers) < 1 or not results_json:
exit(0)
else:
# Because we used json.dumps(), rather than jsonify(),
# we need to create a Flask Response object, here.
return Response(response=str(results_json), mimetype='application/json')
if __name__ == '__main__':
##NOTE Not suitable for production. As configured,
# your Flask service is in debug mode and publicly accessible.
app.run(debug=True, host='0.0.0.0', port=5001) # http://localhost:5001/
your_model contains the serialization method for your whatever, as well as the database connection manager (which could stand a little refactoring, but suffices to centralize the creation of database sessions, in bigger systems or Model/View/Control architectures). This happens to use postgreSQL, but could just as easily use any server side data store:
#!/usr/bin/env python
# Filename: your_model.py
import time
import psycopg2
import psycopg2.pool
import psycopg2.extras
from psycopg2.extensions import adapt, register_adapter, AsIs
from sqlalchemy import update
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from sqlalchemy.dialects import postgresql
from sqlalchemy import Table, Column, Integer, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
class SessionManager(object):
#staticmethod
def connect():
engine = create_engine('postgresql://id:passwd#localhost/mydatabase',
echo = True)
Session = sessionmaker(bind = engine,
autoflush = True,
expire_on_commit = False,
autocommit = False)
session = Session()
return session
#staticmethod
def declareBase():
engine = create_engine('postgresql://id:passwd#localhost/mydatabase', echo=True)
whatever_metadata = MetaData(engine, schema ='public')
Base = declarative_base(metadata=whatever_metadata)
return Base
Base = SessionManager.declareBase()
class Whatever(Base):
"""Create, supply information about, and manage the state of one or more whatever.
"""
__tablename__ = 'whatever'
id = Column(Integer, primary_key=True)
whatever_digest = Column(VARCHAR, unique=True)
best_name = Column(VARCHAR, nullable = True)
whatever_timestamp = Column(BigInteger, default = time.time())
whatever_raw = Column(Numeric(precision = 1000, scale = 0), default = 0.0)
whatever_label = Column(postgresql.VARCHAR, nullable = True)
size = Column(BigInteger, default = 0)
def __init__(self,
whatever_digest = '',
best_name = '',
whatever_timestamp = 0,
whatever_raw = 0,
whatever_label = '',
size = 0):
self.whatever_digest = whatever_digest
self.best_name = best_name
self.whatever_timestamp = whatever_timestamp
self.whatever_raw = whatever_raw
self.whatever_label = whatever_label
# Serialize one way or another, just handle appropriately in the client.
def serialize(self):
return {
'best_name' :self.best_name,
'whatever_label':self.whatever_label,
'size' :self.size,
}
In retrospect, I might have serialized the whatever objects as lists, rather than a Python dict, which might have simplified their processing in the Flask service, and I might have separated concerns better in the Flask implementation (The database call probably shouldn't be built-in the the route handler), but you can improve on this, once you have a working solution in your own development environment.
Also, I'm not suggesting people avoid JQuery. But, if JQuery's not in the picture, for one reason or another, this approach seems like a reasonable alternative.
It works, in any case.
Here's my implementation of DickFeynman's approach, in the the client:
<script type="text/javascript">
var addr = "dev.yourserver.yourorg.tld"
var port = "5001"
function Get(whateverUrl){
var Httpreq = new XMLHttpRequest(); // a new request
Httpreq.open("GET",whateverUrl,false);
Httpreq.send(null);
return Httpreq.responseText;
}
var whatever_list_obj = JSON.parse(Get("http://" + addr + ":" + port + "/whatever-data/"));
whatever_qty = whatever_list_obj.length;
for (var i = 0; i < whatever_qty; i++) {
console.log(whatever_list_obj[i].best_name);
}
</script>
I'm not going to list my console output, but I'm looking at a long list of whatever.best_name strings.
More to the point: The whatever_list_obj is available for use in my javascript namespace, for whatever I care to do with it, ...which might include generating graphics with D3.js, mapping with OpenLayers or CesiumJS, or calculating some intermediate values which have no particular need to live in my DOM.
You make a bog standard HTTP GET Request. You get a bog standard HTTP Response with an application/json content type and a JSON document as the body. You then parse this.
Since you have tagged this 'JavaScript' (I assume you mean "from a web page in a browser"), and I assume this is a third party service, you're stuck. You can't fetch data from remote URI in JavaScript unless explicit workarounds (such as JSONP) are put in place.
Oh wait, reading the documentation you linked to - JSONP is available, but you must say 'js' not 'json' and specify a callback: format=js&callback=foo
Then you can just define the callback function:
function foo(myData) {
// do stuff with myData
}
And then load the data:
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = theUrlForTheApi;
document.body.appendChild(script);