Parsing progress live from console output - NodeJS - javascript

Link to a similar problem that has no answers, but written in C
I'm using NodeJS to parse output from ark-server-tools, which is a layer on top of SteamCMD. What I'd like to do is parse the progress of the update and assign it to a variable, which I'll return as a GET call that the client can use to check progress of the update.
I put the log results of an update into a file to run my code against, which I've put in a PasteBin for brevity.
update.js
app.get('/update', function(req, res) {
var toReturn;
var outputSoFar;
var total;
var startPos;
var endPos = 0;
//var proc = spawn('arkmanager', ['update', '--safe']);
var proc = spawn('./update-log.sh'); //for testing purposes
proc.stdout.on('data', function(data){
outputSoFar += data.toString();
//if server is already updated
if (outputSoFar.indexOf('Your server is already up to date!') !== -1) {
toReturn = 'Server is already up-to-date.';
}
//find update progress
if (outputSoFar.indexOf('progress:') !== -1) {
for(var line in outputSoFar.split('\n')){
console.log('found progress');
startPos = outputSoFar[line].indexOf('progress:', endPos) + 10; //get the value right after progress:_, which should be a number
endPos = outputSoFar[line].indexOf(' (', startPos); // find the end of this value, which is signified by space + (
console.log(outputSoFar[line].substring(startPos, endPos).trim());
updatePercent = outputSoFar[line].substring(startPos, endPos).trim(); //returned to the `checkUpdateProgress` endpoint
}
toReturn = 'Updating...';
}
});
proc.stderr.on('data', function(data){
console.log(data);
});
proc.on('close', function (code, signal) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(JSON.stringify(toReturn));
res.end();
});
}
/*
* Returns progress of an update
*/
app.get('/updateProgress', function(req, res){
console.log('updatePercent: ' + updatePercent);
res.send(JSON.stringify(updatePercent));
});
Couple questions:
1) Is this the best way to architect my RESTful API? One call for the action of updating and another for checking the progress of the update?
2) I'd love a better way to test the function, as echoing the console log returns the data in one piece, as opposed to a data stream. How do I do this?
3) I'm pretty sure the parsing function itself isn't quite right, but I'm having a hard time testing it because of #2.
If you want to take a look at the project in its entirety, here's the repo.
Thanks in advance for your help!

For one of your questions:
Is this the best way to architect my RESTful API? One call for the
action of updating and another for checking the progress of the
update?
As implemented now, I don't think your service can support concurrent requests correctly. updatePercent is a shared global variable. If i hit /update endpoint with a single client, it will start the ./update-log.sh command.
If I request /update again, it will start another update and overwrite the global updateProgress. There doesn't seem to be anything mapping an updatePercent with the correct process
Additionally, there could be serious performance issues to each request spawning a new process. Node might be able to handle hundreds or thousands of concurrent connections using a single thread, but each request is going to spawn a new process, just something to profile

Related

How to get page views on JavaScript?

How to get page views on JavaScript?
I want to get and display how many times a page is viewed, just like Stack Overflow.
How to do it by JavaScript? Thx!
This can be done in javascript. You've to use browsers local storage to store the page view count to use it. You can use either window.localStorage or window.sessionStorage.
sessionStorage will work like php session and will only be available through one browsing session, if you close the browser then sessionStorage data will be removed by browser. But, localStorage will not be removed by browser until user manually delete browser data.
I'm gonna show you two implementations:
Using localStorage: you can store a variable into localStorage which can be used to count the page view. :
var myStorage = window.localStorage, pageCount;
window.addEventListener('load', function(){
if(!myStorage.getItem("pageCount")){
myStorage.setItem('pageCount', 1);
} else {
pageCount = myStorage.getItem("pageCount");
pageCount = pageCount + 1;
myStorage.setItem('pageCount', pageCount );
}
console.log('page view count', myStorage.getItem("pageCount"));
});
Or using window.sessionStorage :
var mySession = window.sessionStorage, pageCount;
window.addEventListener('load', function(){
if(!mySession.getItem("pageCount")){
mySession.setItem('pageCount', 1);
} else {
pageCount = mySession.getItem("pageCount");
pageCount = pageCount + 1;
mySession.setItem('pageCount', pageCount );
}
console.log('page view count of current browsing session', mySession.getItem("pageCount"));
});
Using client-side JavaScript will not get the job done.
However, if you still want to use JavaScript then you should take a look at Node.js. Ideally, you would want to record the number of unique sessions, although, you can still record a page refresh as a page view.
This can be done with the following piece of code provided that you know how Node.js works:
var http = require('http'); // Require HTTP module
var pageViewCount = 0; // Declare and initialise the page view count
var server = http.createServer(function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' }); // Headers
pageViewCount++; // Increase the page view count for every successful page load request
console.log('There are currently ' + pageViewCount + ' views'); // Output in console the amount of total page views
res.end();
});
server.listen(8000);
console.log('Server is currently running');
You can find more information regarding Node.js here: https://nodejs.org/en/docs/
Keep in mind that this answer requires some knowledge of Node.js but confirms that server-side JavaScript can be used to provide a solution to this particular problem.
With nodejs (serverside javascript) and express ( a library) you could do the following:
var counter = 0;
var app = require("express");
app.get("/",function(req,res){
counter++;
res.render("main.ejs",{counter});
});
app.listen(80);
main.ejs should look like this:
The page was visited <%= counter %> times...
You might want to store that in Database.

Loading multiple files synchronously in node.js

I am learning node.js and have build a micro-app around an MVC architecture.
I have a router.js file which loads a controller based on the URI, which, in most cases, would load the views using the "fs" module. The views being the HTML elements making up the web page (basically head, body, & footer) as 3 separate files.
Here is the code for the controller:
var load_view = 'test.html';
function data(response, request, fs){
response.writeHead(200, {"Content-Type": "text/html"});
var count = 0;
var handler = function(error, content){
count++;
if(error) console.log(error);
else response.write(content);
if(count==3) response.end();
}
fs.readFile('view/elements/head.html', handler); // should load 1st
fs.readFile('view/'+load_view, handler); // should load 2nd
fs.readFile('view/elements/footer.html', handler);// should load 3rd
}
exports.data = data;
As you can see the HTML elements are supposed to load in order (head.html, then the particular view for this controller - test.html, then footer.html). But they sometimes do NOT.
They load in the "head, body, footer" order most of the time.
Sometimes they load as "head, footer, body".
Other times its "body, head, footer".
They never seem to load in any other configuration.
Please see screenshots attached.
Im am not sure what is happening here. Why are these files being loaded in any order but the one they are called??
Please note I am intentially not using a framework like Express.js for learning purposes
You cannot make assumptions about the ordering of responses for asynchronous requests. There are various factors that could affect this ordering (e.g. OS thread scheduling in the case of filesystem requests).
So the solution to this is to either call them serially by nesting callbacks or chaining Promises or by using a module like async, or bind your callbacks to include relevant contextual information so you know which file just got loaded in your callback. An example of the latter might be something like:
function data(res, req, fs) {
// Files to be loaded, in the order to be written to the response
var files = [
'view/elements/head.html',
'view/' + load_view,
'view/elements/footer.html'
];
// Start read requests, each with a callback that has an extra
// argument bound to it so it will be unshifted onto the callback's
// parameter list
for (var i = 0; i < files.length; ++i)
fs.readFile(files[i], handler.bind(null, i));
var count = 0;
function handler(index, err, content) {
// Make sure we don't send more than one response on error
if (count < 0)
return;
if (err) {
count = -1;
console.log('Error for file: ' + files[index]);
console.log(err);
res.writeHead(500);
return res.end();
}
// Reuse our `files` array by simply replacing filenames with
// their respective content
files[index] = content;
// Check if we've read all the files and write them in order if
// we are finished
if (++count === files.length) {
res.writeHead(200, { 'Content-Type': 'text/html' });
for (var i = 0; i < files.length; ++i)
res.write(files[i]);
res.end();
}
}
}
On a related note, you might look into using a templating system (that supports including partials (like headers and footers) and layouts) of some kind to avoid doing this kind of manual work.

Search Increasing URL For Text

For stekhn, here's the proper link: var location = "http://www.roblox.com/Trade/inventoryhandler.ashx?filter=0&userid=" + i + "&page=1&itemsPerPage=14";
I'm trying to create a Javascript script where I can search through a users inventory, detect if they have what I'm looking for in their inventory and output the userID if they have it.
If I type in bluesteel, I need a Javascript script which will search through http://snackyrite.com/site.ashx?userid=1 and detect if it has the text 'bluesteel' is on it - if it is, I need it to display the user id, which is 1.
You may be thinking that's easy and I can easily find the script for that - well there's a catch, my objective isn't only to get it to search userid=1, I need it to search from userid=1 up to userid=45356
If the word 'bluesteel' is found on userid=5, userid=3054 and userid=12 (these are just examples), I need it to display 5, 3054 and 12 (the ID's) on the same page where the script was ran from.
This is the script I've tried, but the userid won't increase (I'm not sure how to do that).
var location = http://snackyrite.com/site.ashx?userid=1;
if(location.indexOf("bluesteel") > -1) {
output.userid
}
I do apologize, Javascript isn't my best.
Use a loop:
for (var i = 1; i <=45356; i++) {
var loc = "http://snackyrite.com/site.ashx?userid="+i;
// get contents of location
if (contents.indexOf("bluesteel") > -1) {
console.log(i);
}
}
Since getting the contents will presumably use AJAX, the if will probably be in the callback function. See Javascript infamous Loop issue? for how to write the loop so that i will be preserved in the callback function.
This kind of web scraping can't be done in the Browser (client-side JavaScript).
I would suggest building a scraper with Node.js.
Install Node.js
Install request npm i request
Install cheerio npm i cheerio
Create a file scraper.js
Run node scraper.js
Code for scraper.js
// Import the scraping libraries
var request = require("request");
var cheerio = require("cheerio");
// Array for the user IDs which match the query
var matches = [];
// Do this for all possible users
for (var i = 1; i <= 45356; i++) {
var location = "http://snackyrite.com/site.ashx?userid="+i;
request(location, function (error, response, body) {
if (!error) {
// Load the website content
var $ = cheerio.load(body);
var bodyText = $("body").text();
// Search the website content for bluesteel
if (bodyText.indexOf("bluesteel") > -1) {
console.log("Found bluesteel in inventory of user ", i);
// Save the user ID, if bluesteel was found
matches.push(i);
}
// Something goes wrong
} else {
console.log(error.message);
}
});
console.log("All users with bluesteel in inventory: ", matches);
}
The above code seems kind of complicated, but I think this is the way it should be done. Of corse you can use any other scraping tool, library.

Why is this node.js code blocking?

I tried to play a bit with node.js and wrote following code (it doesn't make sense, but that does not matter):
var http = require("http"),
sys = require("sys");
sys.puts("Starting...");
var gRes = null;
var cnt = 0;
var srv = http.createServer(function(req, res){
res.writeHeader(200, {"Content-Type": "text/plain"});
gRes = res;
setTimeout(output,1000);
cnt = 0;
}).listen(81);
function output(){
gRes.write("Hello World!");
cnt++;
if(cnt < 10)
setTimeout(output,1000);
else
gRes.end();
}
I know that there are some bad things in it (like using gRes globally), but my question is, why this code is blocking a second request until the first completed?
if I open the url it starts writing "Hello World" 10 times. But if I open it simultaneous in a second tab, one tab waits connecting until the other tab finished writing "Hello World" ten times.
I found nothing which could explain this behaviour.
Surely it's your overwriting of the gRes and cnt variables being used by the first request that's doing it?
[EDIT actually, Chrome won't send two at once, as Shadow Wizard said, but the code as is is seriously broken because each new request will reset the counter, and outstanding requests will never get closed].
Instead of using a global, wrap your output function as a closure within the createServer callback. Then it'll have access to the local res variable at all times.
This code works for me:
var http = require("http"),
sys = require("sys");
sys.puts("Starting...");
var srv = http.createServer(function(req, res){
res.writeHeader(200, {"Content-Type": "text/plain"});
var cnt = 0;
var output = function() {
res.write("Hello World!\n");
if (++cnt < 10) {
setTimeout(output,1000);
} else {
res.end();
}
};
output();
}).listen(81);
Note however that the browser won't render anything until the connection has closed because the relevant headers that tell it to display as it's downloading aren't there. I tested the above using telnet.
I'm not familiar with node.js but do familiar with server side languages in general - when browser send request to the server, the server creates a Session for that request and any additional requests from the same browser (within the session life time) are treated as the same Session.
Probably by design, and for good reason, the requests from same session are handled sequentially, one after the other - only after the server finish handling one request it will start handling the next.

How to ping IP addresses using JavaScript

I want to run a JavaScript code to ping 4 different IP addresses and then retrieve the packet loss and latency of these ping requests and display them on the page.
How do I do this?
You can't do this from JS. What you could do is this:
client --AJAX-- yourserver --ICMP ping-- targetservers
Make an AJAX request to your server, which will then ping the target servers for you, and return the result in the AJAX result.
Possible caveats:
this tells you whether the target servers are pingable from your server, not from the user's client
so the client won't be able to test hosts its LAN
but you shouldn't let the host check hosts on the server's internal network, if any exist
some hosts may block traffic from certain hosts and not others
you need to limit the ping count per machine:
to avoid the AJAX request from timing out
some site operators can get very upset when you keep pinging their sites all the time
resources
long-running HTTP requests could run into maximum connection limit of your server, check how high it is
many users trying to ping at once might generate suspicious-looking traffic (all ICMP and nothing else)
concurrency - you may wish to pool/cache the up/down status for a few seconds at least, so that multiple clients wishing to ping the same target won't launch a flood of pings
The only method I can think of is loading e.g. an image file from the external server. When that load fails, you "know" the server isn't responding (you actually don't know, because the server could just be blocking you).
Take a look at this example code to see what I mean:
/*note that this is not an ICMP ping - but a simple HTTP request
giving you an idea what you could do . In this simple implementation it has flaws
as Piskvor correctly points out below */
function ping(extServer){
var ImageObject = new Image();
ImageObject.src = "http://"+extServer+"/URL/to-a-known-image.jpg"; //e.g. logo -- mind the caching, maybe use a dynamic querystring
if(ImageObject.height>0){
alert("Ping worked!");
} else {
alert("Ping failed :(");
}
}
I was inspired by the latest comment, so I wrote this quick piece of code.
This is a kind of "HTTP ping" which I think can be quite useful to use along with XMLHttpRequest calls(), for instance to figure out which is the fastest server to use in some case or to collect some rough statistics from the user's internet connexion speed.
This small function is just connecting to an HTTP server on an non-existing URL (that is expected to return a 404), then is measuring the time until the server is answering to the HTTP request, and is doing an average on the cumulated time and the number of iterations.
The requested URL is modified randomely at each call since I've noticed that (probably) some transparent proxies or caching mechanisms where faking results in some cases, giving extra fast answers (faster than ICMP actually which somewhat weird).
Beware to use FQDNs that fit a real HTTP server!
Results will display to a body element with id "result", for instance:
<div id="result"></div>
Function code:
function http_ping(fqdn) {
var NB_ITERATIONS = 4; // number of loop iterations
var MAX_ITERATIONS = 5; // beware: the number of simultaneous XMLHttpRequest is limited by the browser!
var TIME_PERIOD = 1000; // 1000 ms between each ping
var i = 0;
var over_flag = 0;
var time_cumul = 0;
var REQUEST_TIMEOUT = 9000;
var TIMEOUT_ERROR = 0;
document.getElementById('result').innerHTML = "HTTP ping for " + fqdn + "</br>";
var ping_loop = setInterval(function() {
// let's change non-existent URL each time to avoid possible side effect with web proxy-cache software on the line
url = "http://" + fqdn + "/a30Fkezt_77" + Math.random().toString(36).substring(7);
if (i < MAX_ITERATIONS) {
var ping = new XMLHttpRequest();
i++;
ping.seq = i;
over_flag++;
ping.date1 = Date.now();
ping.timeout = REQUEST_TIMEOUT; // it could happen that the request takes a very long time
ping.onreadystatechange = function() { // the request has returned something, let's log it (starting after the first one)
if (ping.readyState == 4 && TIMEOUT_ERROR == 0) {
over_flag--;
if (ping.seq > 1) {
delta_time = Date.now() - ping.date1;
time_cumul += delta_time;
document.getElementById('result').innerHTML += "</br>http_seq=" + (ping.seq-1) + " time=" + delta_time + " ms</br>";
}
}
}
ping.ontimeout = function() {
TIMEOUT_ERROR = 1;
}
ping.open("GET", url, true);
ping.send();
}
if ((i > NB_ITERATIONS) && (over_flag < 1)) { // all requests are passed and have returned
clearInterval(ping_loop);
var avg_time = Math.round(time_cumul / (i - 1));
document.getElementById('result').innerHTML += "</br> Average ping latency on " + (i-1) + " iterations: " + avg_time + "ms </br>";
}
if (TIMEOUT_ERROR == 1) { // timeout: data cannot be accurate
clearInterval(ping_loop);
document.getElementById('result').innerHTML += "<br/> THERE WAS A TIMEOUT ERROR <br/>";
return;
}
}, TIME_PERIOD);
}
For instance, launch with:
fp = new http_ping("www.linux.com.au");
Note that I couldn't find a simple corelation between result figures from this script and the ICMP ping on the corresponding same servers, though HTTP response time seems to be roughly-exponential from ICMP response time. This may be explained by the amount of data that is transfered through the HTTP request which can vary depending on the web server flavour and configuration, obviously the speed of the server itself and probably other reasons.
This is not very good code but I thought it could help and possibly inspire others.
The closest you're going to get to a ping in JS is using AJAX, and retrieving the readystates, status, and headers. Something like this:
url = "<whatever you want to ping>"
ping = new XMLHttpRequest();
ping.onreadystatechange = function(){
document.body.innerHTML += "</br>" + ping.readyState;
if(ping.readyState == 4){
if(ping.status == 200){
result = ping.getAllResponseHeaders();
document.body.innerHTML += "</br>" + result + "</br>";
}
}
}
ping.open("GET", url, true);
ping.send();
Of course you can also put conditions in for different http statuses, and make the output display however you want with descriptions etc, to make it look nicer. More of an http url status checker than a ping, but same idea really. You can always loop it a few times to make it feel more like a ping for you too :)
I've come up with something cause I was bored of searching hours after hours for something that everyone is saying "impossible", only thing I've found was using jQuery.
I've came up with a new simple way using Vanilla JS (nothing else than base JavaScript).
Here's my JSFiddle: https://jsfiddle.net/TheNolle/5qjpmrxg/74/
Basically, I create a variable called "start" which I give the timestamp, then I try to set an invisible image's source to my website (which isn't an image) [can be changed to any website], because it's not an image it creates an error, which I use to execute the second part of the code, at this time i create a new variable called "end" which i give the timestamp from here (which is different from "start"). Afterward, I simply make a substraction (i substract "start" from "end") which gives me the latency that it took to ping this website.
After you have the choice you can store that in a value, paste it on your webpage, paste it in the console, etc.
let pingSpan = document.getElementById('pingSpan');
// Remove all the way to ...
let run;
function start() {
run = true;
pingTest();
}
function stop() {
run = false;
setTimeout(() => {
pingSpan.innerHTML = "Stopped !";
}, 500);
}
// ... here
function pingTest() {
if (run == true) { //Remove line
let pinger = document.getElementById('pingTester');
let start = new Date().getTime();
pinger.setAttribute('src', 'https://www.google.com/');
pinger.onerror = () => {
let end = new Date().getTime();
// Change to whatever you want it to be, I've made it so it displays on the page directly, do whatever you want but keep the "end - start + 'ms'"
pingSpan.innerHTML = end - start + "ms";
}
setTimeout(() => {
pingTest();
}, 1000);
} // Remove this line too
}
body {
background: #1A1A1A;
color: white
}
img {
display: none
}
Ping:
<el id="pingSpan">Waiting</el>
<img id="pingTester">
<br> <br>
<button onclick="start()">
Start Ping Test
</button>
<button onclick="stop()">
Stop
</button>
function ping(url){
new Image().src=url
}
Above pings the given Url.
Generally used for counters / analytics.
It won't encounter failed responses to client(javascript)
I suggest using "head" to request the header only.
xhr.open('head', 'asstes/imgPlain/pixel.txt' + cacheBuster(), true);
and than ask for readystate 2 - HEADERS_RECEIVED send() has been called, and headers and status are available.
xhr.onreadystatechange = function() {
if (xhr.readyState === 2) { ...
Is it possible to ping a server from Javascript?
Should check out the above solution. Pretty slick.
Not mine, obviously, but wanted to make that clear.
You can't PING with Javascript. I created Java servlet that returns a 10x10 pixel green image if alive and a red image if dead. https://github.com/pla1/Misc/blob/master/README.md

Categories