I am having an issue sending over an array of information from my client.js to the node server using an http POST request. After a couple of days I think I have narrowed it down.
My current set up is the following:
Server
After X amount of time, the server will start to ask clients to send their collected data.
The server sends out only one request at a time, waiting to receive that clients info before sending a request to the next client
Client
The client receives the request from the server
Http stuff to send POST request is set up
Array data is converted to json
1000ms is waited before sending the http post request
Server
Post request is received & that data is added to a local array
My server.js
var http = require('http')
, connect = require('connect')
, io = require('socket.io')
, fs = require('fs')
, uuid = require('node-uuid')
, _ = require('lodash');
// omitted event handers & pubnub setup
var app = connect().use(connect.static(__dirname)).use(connect.directory(__dirname));
var server = http.createServer(app);
server.listen(8888);
io = io.listen(server);
// Following handles sending request to clients
function phaseTwo() {
var count = 0;
setTimeout(function() {
pubnub.publish({
channel: 'channel' + viewers[count],
message: { message: 'measurements',
uuid: 'Server',
receiver: viewers[count] }
});
// set up server to listen for http POST
app.use(function(req, res) {
if (req.method == 'POST') {
count++;
req.on('data', function(chunk) {
// add that array into one main array
latencyData.push.apply(latencyData, chunk);
dataCollected++;
});
pubnub.publish({
channel: 'channel' + viewers[count],
message: { message: 'measurements',
uuid: 'Server',
receiver: viewers[count] }
});
}
}); // end app.use
}, msDuration);
}
client
if (message.message == 'measurements') {
console.log(' Received measurements event');
var http = new XMLHttpRequest();
var url = 'http://localhost:8888/';
http.open('POST', url, true);
http.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
// http.setRequestHeader("Content-length", latency.length);
// http.setRequestHeader("Connection", "close");
var myJsonString = JSON.stringify(latency);
setTimeout( function() {
http.send(myJsonString);
}, 1000);
}
The problem is that without that 1000ms delay on the client side, the server does not receive all the client's data. Even worse, when trying to scale up a few hundred clients, no more http posts are sent to the server after 240 clients have sent their info (there is enough ram for them)
With no delay on the client making the POST request, the server should be waiting for the current client to send over their information over before continuing. The other issue is that even with under the 240 clients, say testing 200 clients, even with no delay only about 10% of their data is received (this stuff is written to a text file at the end)
Im very new to using http post/get stuff so I think that is the issue.
Is there anything immediately wrong from anyone experiences with http or nodejs stuff?
edit: I found one mistake. Back on the server side, on the post request whatever data is gotten from the server is added right away to the local array on the server without first making sure it's an array. Im not completely sure if this would break things enough to be the source of my problem
A POST can be pretty simply handled with Express.
var express = require('express');
var bodyParser = require('body-parser');
var app = express();
app.use(express.static(__dirname + '/public'));
app.use(bodyParser());
app.post('/', function(req, res) {
console.log(req.body);
res.send('data retrieved');
});
app.listen(3000);
Make proper route in server to handle incoming requests make your app aware of such routes.
You can console.log(req) to check it in server side how you are getting the request with all the data.
Related
There is a simple web server that accepts data. Sample code below.
The idea is to track in real time how much data has entered the server and immediately inform the client about this. If you send a small amount of data, then everything works well, but if you send more than X data in size, then the on.data event on the server is triggered with a huge delay. I can see that data is transfering for 5 seconds already but on.data event is not trigerred.
on.data event seems to be triggered only when data is uploaded completely to the server, so that's why it works fine with small data (~2..20Mb), but with big data (50..200Mb) it doesnt work well.
Or maybe it is due to some kind of buffering..?
Do you have any suggestions why on.data triggered with delay and how to fix it?
const app = express();
const port = 3000;
// PUBLIC API
// upload file
app.post('/upload', function (request, response) {
request.on('data', chunk => {
// message appears with delay
console.log('upload on data', chunk.length);
// send message to the client about chunk.length
});
response.send({
message: `Got a POST request ${request.headers['content-length']}`
});
});
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`);
});
TLDR:
The delay that you are experiencing probably is the Queueing from Resource scheduling from the browser.
The Test
I did some tests with express, and then I found that it uses http to handle requests/response, so I used a raw http server listener to test this scenario, which has the same situation.
Backend code
This code, based on sample of Node transaction samples, will create a http server and give log of time on 3 situations:
When a request was received
When the first data event fires
When the end event fires
const http = require('http');
var firstByte = null;
var server = http.createServer((request, response) => {
const { headers, method, url } = request;
let body = [];
request.on('error', (err) => {
}).on('data', (chunk) => {
if (!firstByte) {
firstByte = Date.now();
console.log('received first byte at: ' + Date.now());
}
}).on('end', () => {
console.log('end receive data at: ' + Date.now());
// body = Buffer.concat(body).toString();
// At this point, we have the headers, method, url and body, and can now
// do whatever we need to in order to respond to this request.
if (url === '/') {
response.statusCode = 200;
response.setHeader('Content-Type', 'text/html');
response.write('<h1>Hello World</h1>');
}
firstByte = null;
response.end();
});
console.log('received a request at: ' + Date.now());
});
server.listen(8083);
Frontend code (snnipet from devtools)
This code will fire a upload to /upload which some array data, I filled the array before with random bytes, but then I removed and see that it did not have any affect on my timing log, so yes.. the upload content for now is just an array of 0's.
console.log('building data');
var view = new Uint32Array(new Array(5 * 1024 * 1024));
console.log('start sending at: ' + Date.now());
fetch("/upload", {
body: view,
method: "post"
}).then(async response => {
const text = await response.text();
console.log('got response: ' + text);
});
Now running the backend code and then running the frontend code I get some log.
Log capture (screenshots)
The Backend log and frontend log:
The time differences between backend and frontend:
Results
looking at the screenshoots and I get two differences between the logs:
The first, and most important, is the difference between frontend fetch start and backend request recevied, I got 1613ms which is "close" (1430ms) to Resource Scheduling in network timing tab, I think there are more things happening between the frontend fetch call and the node backend event, so I can't direct compare the times:
log.backendReceivedRequest - log.frontEndStart
1613
The second is the difference between receving data on backend, which I got
578ms, close to Request sent (585ms) in network timing tab:
log.backendReceivedAllData - log.backendReceivedFirstData
578
I also changed the frontend code to send different sizes of data and the network timing tab still matches the log
The thing that remains unknown for me is... Why does Google Chrome is queueing my fetch since I'm not running any more requests and not using the bandwidth of the server/host? I readed the conditions for Queueing but not found the reason, maybe is allocating the resources on disk, but not sure: https://developer.chrome.com/docs/devtools/network/reference/#timing-explanation
References:
https://nodejs.org/es/docs/guides/anatomy-of-an-http-transaction/
https://developer.chrome.com/docs/devtools/network/reference/#timing-explanation
I found a problem. It was in nginx config. Nginx was setup like a reverse proxy. By default proxy request buffering is enabled, so nginx grabs first whole request body and only then forwards it to nodejs, so that's why I saw delay.
https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_request_buffering
I have a node.js application which I use to interact with a REST API provided by another server. I would like to expose a web interface (html + css + javascript) using express.js in order to control the first application.
How can I let the browser talk to the server and let it make node.js actions like using http from that machine or writing to its filesystem? I tried using XMLHttpRequest, but HTTP requests are sent by my local PC instead of from my server.
The only solution I found is using XMLHttpRequest in the javascript of my web interface to invoke some middleware functions on my server, but I had some problems: when I make POST requests, I cannot read data from server. I used FormData and its append method to make the "body" of the POST request, then used body-parser in express to read that body, but it turns out to be always empty. Also tried changing the 'Content-Type' of the header.
Any suggestions? Any better solution than mine (I think it is not efficient)?
As pointed by Jonas, using node server as proxy would be the right approach.
I'm providing sample code for both frontend as well as backend app. Hope this helps you.
Frontend App Code
<html>
<head>
<script type="text/javascript">
function sendData(data) {
if (!data) {
// lets define some dummy data for testing
data = { somekey: "somevalue", anotherkey: "anothervalues" };
}
var XHR = new XMLHttpRequest();
var FD = new FormData();
// Push our data into our FormData object
for (name in data) {
FD.append(name, data[name]);
}
// Define what happens on successful data submission
XHR.addEventListener("load", function(event) {
alert("Yeah! Data sent and response loaded.");
alert(event.target.responseText);
});
// Define what happens in case of error
XHR.addEventListener("error", function(event) {
alert("Oops! Something went wrong.");
});
// Set up our request
XHR.open("POST", "http://path/to/your/nodejs/server/app");
// Send our FormData object; HTTP headers are set automatically
XHR.send(FD);
}
</script>
</head>
<body>
<button onclick="sendData()">Send Test Request to the Server</button>
</body>
</html>
Backend App code
const http = require('http');
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.get('/', (req, res) => res.send('Yeah! Server is UP! Post some data'));
app.post('/', (req, res) => {
// You'll see the posted data in req.body, simply for testing purpose return it back to the calling user
res.json(req.body || {});
});
const server = http.createServer(app);
server.listen(3000);
server.on('error', console.error);
server.on('listening', () => console.log('Listening on 3000'));
process.on('exit', (code) => console.warn('Server terminated with code=' + code));
Please note that for this backend app to run, you must have installed following npm packages: express, body-parser
So I have a node-js server and an apache server on the same machine, and one of the javascript files is sending an HTTP request to the node-js server. The node-js server receives the file, reads the data, puts it in the database, as it should, but it isn't sending back any status codes or data.
Here is the XHTMLRequest send code snippet,
// creates a new http request to be sent to the nodejs server
function createNewUser(username, password, email) {
// The url is the URL of our local nodejs server
var userCreateRequest = new XMLHttpRequest();
userCreateRequest.open( "POST", "http://<machine's IP>:8080/api/users" );
// Create json object for user data
var user = "name="+username+"&password="+password+"&email="+email;
alert(user);
// set content type for http request
userCreateRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
// Event listern for server response
// userCreateRequest.addEventListener("readystatechange", processRequest, false);
// Call process request whenever state changes
userCreateRequest.onreadystatechange = function() {
alert(this.readyState + ", " + this.status);
if (this.readyState == 4 && this.status == 200) {
var response = this.response;
alert(response.name);
}
}
// Send user data to server
userCreateRequest.send(user);
}
And here is the code for the node-js server (with express)
router.route('/users')
.post(function(req, res) { //create a new user
var user = new User();
user.name = req.body.name;
user.password = req.body.password;
user.email = req.body.email;
user.save(function(err) { //add user object to database
if(err)
res.send(err);
res.status(200).json(user);
});
});
As I said above, the code works fine in terms of putting the body of the request in the database and what-not, but the server is not sending back the 200 OK response (or I'm failing to receive it for some reason). The only times I get an alert from onreadystatechange is when it's state 2, status 0, and state 4, status 0.
Try below code snippet.
user.save(function(err, user) {
if(err)
res.send(err);
res.status(200).json(user);
});
It did end up being a CORS issue. I'm still a little iffy on exactly why, but after configuring the express/CORS package to allow requests from the IP and port of my apache server, it started working.
My understanding is that cross origin implies a different domain, where-as both of my servers are (as I understand it) on different ports on the same domain.
Either way, enabling CORS fixed the issue. Thank you to Jaromanda X for pointing it out and getting me on the right track.
I'm currently teaching myself more about server code, specifically using Node.js and Express, and I'm having a lot of trouble with receiving and parsing a JSON object sent from a POST request. I have looked at numerous other posts (linked to below) and I can't figure out for the life of me what's going wrong. Here's what I've looked at:
Javascript: Send JSON Object with AJAX
Javascript : Send JSON Object with Ajax?
How do I consume the JSON POST data in an Express application
How do I consume the JSON POST data in an Express application
Send POST data using XMLHttpRequest
Send POST data using XMLHttpRequest
How do you extract POST data in Node.js?
How do you extract POST data in Node.js?
All of these are putting me on the right track, but I'm not quite there and thus looking for help. Here's the code I'm working with:
Send POST Request
var button = document.querySelector("#button");
button.onclick = function(){
console.log("Getting data from local server");
var xhr = new XMLHttpRequest();
xhr.open("POST", "http://localhost:3000/data/test.json", true);
xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
xhr.send(JSON.stringify({"latitude": 41.2418, "longitude": -70.8898}));
};
Handle POST Request In Server
var http = require("http");
var fs = require("fs");
var express = require("express");
var app = express();
var path = require("path");
var bodyParser = require("body-parser");
var port = process.env.PORT || 3000;
//tells express where to find all the static files (HTML, CSS, etc) and load them into the browser
app.use(express.static(path.join(__dirname, '../client')));
//tells the application to use body-parser as middleware so it can handle post requests
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
//routing methods
//deal with incoming GET and POST requests to the server
app.get("/", function(req, res){
res.send("Submitted GET Request");
})
//only handles incoming POST requests to the test.json resource
app.post("/data/test.json", function(req, res){
console.info("Submitting POST Request to Server");
console.info("Request body: " + req.body);
//write the file
fs.writeFile(__dirname + "/../client/data/test.json", req.body,
function(err){
if(err){
console.error(err); //print out the error in case there is one
return res.status(500).json(err);
}
//resolve the request with the client
console.info("updated test.json");
res.send();
});
})
//tell the express object to create the server and listen on the port
app.listen(port);
console.log("Listening on localhost:" + port);
Whenever I try to print out the contents of "req.body" I get the output of "[object Object]". Any ideas?
EDIT:
My issue has been solved. I have changed
console.info("Request body: " + req.body);
To
console.info("Request body: " + JSON.stringify(req.body));
I also changed my Content-Type in my POST XMLHTTPRequest to "application/json" to help with formatting.
"[object Object]" is the default result of JavaScript's implicit toString operation, which it uses when trying to write a string representation of that object to the file.
Try writing JSON.stringify(req.data) to the file instead.
Also, on the client side – consider changing your Content-Type header to match:
xhr.setRequestHeader("Content-Type", "application/json");
If your post body is expected to be JSON, then change this line
xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
To
xhr.setRequestHeader("Content-Type", "application/json");
I'm creating node js application as a http server that communicate to tcp socket server, and the code look's like this:
var http = require('http');
var net = require('net');
var url = require('url') ;
const PORT=8080;
var client = new net.Socket();
client.connect(9000,'xxx.xxx.xxx.xxx', function(){
console.log('Connected');
});
function handleRequest(request, response){
var qo = url.parse(request.url,true).query;
if(typeof qo.q=="undefined"){
response.end("ERROR");
}else{
client.write(decodeURIComponent(qo.q));
client.on('data',function(data){
response.writeHead(200, {'Content-Type': 'text/html','Access-Control-Allow-Headers':'Content-Type,Access-Control-Allow-Headers,Access-Control-Allow-Origin,X-Powered-ByX-Powered-By','Access-Control-Allow-Origin': '*','X-Powered-By':'Poltak ganteng'});
response.end(data);
});
}
}
var server = http.createServer(handleRequest);
server.listen(PORT, function(){
console.log("Server listening on: http://localhost:%s", PORT);
});
i'm afraid that code not working properly to handle multiple request at a time. is there any way to handle multiple request and get correct response to requestor?
You want to use req/res with your socket connection, you want to send uid at each client.write end your other service responde with the same uid for conserve the relation one to one to req/res, you don't have other choice for guaranteed unicities.
The have an expensive resource as TCP client (or clients) that has to be shared among node requests, so socket-pool can help you on that.