We Keep Coding

How can I parsing JSON with Node.js in electron renderer.js file?

I'd like to parse JSON formatted dataset from a web server(edit in Electron renderer.js file)
refresh.addEventListener("click", function(){
const http = require('http');
http.get(
'http://teamparamount.cn:8080/Paramount/filesroot?username=test', (resp) =>{
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) =>{
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () =>{
// console.log(JSON.parse(data).info);
// var obj = JSON.stringify(data);
var hhh = JSON.parse(data);
var xxx = JSON.parse(data).info;
// alert(typeof obj);
// console.log(hhh.length);
// console.log(obj);
console.log(data);
console.log(hhh.status);
console.log(hhh.info);
console.log(hhh.info[1].time);
console.log(hhh.info.length);
console.log(hhh.info[408]);
// console.log(obj.info[1]);
// console.log(obj.status);
// console.log(obj.status.length);
function getJsonLth(obj){
var index = 0;
for(var i=0;i<obj.length;i++){
if (obj[i] == ':') {
index++;
}
return index;
// alert(json1.abc[i].name);
}
};
console.log(getJsonLth(xxx));
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
});
In the red circle part, the first output is JSON format dataset which server sent. The second output is the result after using JSON.parse(data).status. The third output is the result after using JSON.parse(data).info. And I think var xxx = JSON.parse(data).info xxx is an array as it's showed in the third output.
However, what I wanna do is to get the size, time, type, url these value separately in each element in the array. But, as you can see, the output of console.log(hhh.info[1].time); is undefined. Also, I wanna get this array's length, and I just use console.log(hhh.info.length) and the result is 409 and I am confused about it. This result clarify it is a string not an array. And I'd like to get these value and the length of the array at the same time. What should I do? Many thanks.

http://teamparamount.cn:8080/Paramount/filesroot?username=test returns this:
{"status":"success","info":"[{\"size\":\"10105\"...
where info property is a string, which has to be parsed separately. That's what you apparently trying to do in:
var xxx = JSON.parse(data).info;
But instead of JSON.parse(data).info you should do: JSON.parse(data.info). Then you will receive your info array into the xxx variable.

it's because the info object is a stringify object, so you need to parse it and override it, and after you will be able to read the entire data object.
var info = JSON.parse(data.info);
data.info = info;
I hope it will help you.

Node JS ignores undefined check

Im working on a NodeJs app that takes an event from FB and puts it into a local database. For every event in the first page of the api query this goes well, except for the last one.
I am getting the following error:
[December 1st 2016, 1:48:39 pm] TypeError: Cannot read property 'name'
of undefined
at IncomingMessage. (/home/node/virgo/app.js:217:32)
at IncomingMessage.emit (events.js:129:20)
at _stream_readable.js:907:16
at process._tickCallback (node.js:372:11)
Just after
console.log(resultBody);
Code:
function addFBEvent(facebookId){
console.log("getting event: " + facebookId);
var options = {
hostname: 'graph.facebook.com',
port: 443,
path: '/v2.8/'+facebookId+'?fields=name,description,start_time,end_time,place,photos{images}&access_token={INSERT API ACCESS CODE HERE}',
method: 'GET'
};
https.request(options, function(res2) {
var resultBody = "";
res2.setEncoding('utf8');
res2.on('data', function (chunk) {
resultBody = resultBody + chunk;
});
res2.on('end', function () {
dbConnection = sql.createConnection({
host : settings.dbHost,
user : settings.dbUser,
password : settings.dbPassword
});
dbConnection.connect(function(err){
if(!err) {
console.log("Database is connected ... nn");
} else {
console.log("Error connecting database ... nn");
}
});
var json = JSON.parse(resultBody);
console.log(resultBody);
if (json != undefined){
var eventName = json.name;
var eventStart = json.start_time;
var eventEnd = json.end_time;
var eventDescription = json.description;
var eventPlace = json.place.name;
var eventPoster = json.photos.data[json.photos.data.length-1].images[0].source;
var eventId = json.id;
console.log("name: " + eventName + ", start: " + eventStart + ", end: " + eventEnd + ", place: " + eventPlace + ", Poster: " + eventPoster);
//console.log("Description: " + eventDescription);
dbConnection.query('INSERT INTO SVVirgo.activities(title, description, image, start, end, price, location, facebook) VALUES ("'+eventName+'","'+eventDescription+'","'+eventPoster+'","'+eventStart+'","'+eventEnd+'",0,"'+eventPlace+'","'+eventId+'")', function (err, result){
});
}
dbConnection.end();
})
}).end();
}
See graph api explorer for the breaking event: Graph API
Event code: 1682486658666506
I've tried catching the undefined json object with no luck, i've also tried to validate the json using this solution: Stackoverflow validate JSON javascript

Instead of https.request try using request.
It will give you parsed JSON and you won't have to do it manually.
If you want to do it manually like you do, then remember to wrap var json = JSON.parse(resultBody); in a try/catch block (or use tryjson) because the JSON.parse cannot be used on unknown data outside of a try/catch - it can throw exceptions.
Another thing, don't open you database connection in your route handlers. You should open the connection once and just use it in your handlers. See this answer for more info about it.
Right now you are connecting to the database but you continue outside of the connection callback, so you run all the lines beginning from var json = JSON.parse(resultBody); before the DB connection is established.
Additionally, the error may be not because json is undefined but because json.place is undefined.
You can change this:
var eventPlace = json.place.name;
to this:
var eventPlace = json.place && json.place.name;
You must also check json.photos before you access json.photos.data and test if json.photos.data is an array before you treat it as such (you can do it with:
if (json.photos && Array.isArray(json.photos.data)) {
// ...
}
Basically, you need to make sure the values are what you want them to be before you access them. For example accessing this:
json.photos.data[json.photos.data.length-1].images[0].source
can fail when json is undefined, when json.photos is undefined, when json.photos.data is undefined, when json.photos.data is not an array, when json.photos.data.length is zero, when json.photos.data[json.photos.data.length-1] is undefined, when json.photos.data[json.photos.data.length-1].images is undefined, when json.photos.data[json.photos.data.length-1].images is not an array, when json.photos.data[json.photos.data.length-1].images is an array but it's empty, or when json.photos.data[json.photos.data.length-1].images is a non-empty array but json.photos.data[json.photos.data.length-1].images[0].source is undefined.
As you can see there are a lot of assumptions that you are doing here. When those assumptions are not met, the code will fail.

Get number of yearly GitHub user contributions

User profiles on GitHub display the number of contributions a user has made in the past year:
I'd like to display this information on my website, preferably without having to introduce any kind of backend. I've looked for it in several places. Nothing is listed in /users/(me)/. Short of scraping the page source, is there a way to retrieve this information? I don't see anything documented...

Have you seen the GithubAPI? Using it, you could run the request from your js file, without implemeting any backend.
For your purpose, I think you could use the following request https://api.github.com/users/yourUserName/events. That gives you as result, a list of events related to youUserName.
e.g
[
{
"id": "xxxxxxx",
"type": "PushEvent",
"actor": {.......},
"repo": {......},
"payload": {.......},
"public": true,
"created_at": "2016-11-17T21:33:15Z"
},
.....
]
You should go through the list and filter the type = PushEvent and created_at = lastYear.
I hope this can help you!
Update: that service is just giving the results for the last 3 months, so other possibility is to check (users/username/repos) and after that check the commits over them (repos/username/repoName/commits)

I've settled for parsing the HTML of my profile page using PHP. I begin by downloading the HTML for my profile page:
$githubProfile = file_get_contents('https://github.com/controversial');
next I find the position of the phrase contributions in the last year on the page. The whitespace is strange between contributions and in the last year so I'm using a regex to match any amount of space.
$contributionsIndex = preg_match(
'/contributions\s+in the last year/',
$githubProfile,
$matches,
PREG_OFFSET_CAPTURE
);
$index = $matches[0][1];
Finally, I iterate backwards from this point until I come across a character that is neither a number or a comma in the number. Once this condition becomes false, I know I've found the beginning of the number:
$endIndex = $index;
while (is_numeric($githubProfile[$index-2]) || $githubProfile[$index-2] == ',')
$index--;
$contributionsCount = substr($githubProfile, $index-1, $endIndex-$index);
Finally, I echo the number out (without commas)
echo(str_replace(',', '', $contributionsCount));
Finally, I use an AJAX call from my JavaScript page to get the value from the PHP script.
function get(url, callback) {
/* eslint-disable no-console */
const xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.onload = e => (callback || console.log)(e.target.responseText);
xhr.onerror = () => console.log('error');
xhr.send();
/* eslint-enable no-console */
}
// Fill in GitHub contributions count
get('contributions-count.php', (resp) => {
document.getElementById('gh-contributions-count').innerText = resp;
});
I'm sure this will be accomplishable more cleanly in the future with GitHub's GraphQL API, but that's still in preview stages.

It's also possible to parse the svg calendar data with a xml parser and then sum all data-count entries. To avoid CORS issue, you can use a proxy like urlreq living on http://urlreq.appspot.com/req :
const user = 'controversial';
fetch('https://urlreq.appspot.com/req?method=GET&url=https://github.com/users/' + user + '/contributions')
.then(function(response) {
return response.text();
})
.then(function(text) {
xmlDoc = new DOMParser().parseFromString(text, 'text/xml');
var nodes = xmlDoc.getElementsByTagName('rect');
var count = 0;
for (var i = 0; i < nodes.length; i++) {
count += parseInt(nodes[i].getAttribute('data-count'));
}
console.log('contributions count : ' + count);
})
.catch(function(error) {
console.log('Request failed', error)
});
Another example using the command line with curl & xmlstarlet can be found here

Setting up a Papa Parse progress bar with Web workers

I'm working on a CSV parsing web application, which collects data and then uses it to draw a plot graph. So far it works nicely, but unfortunately it takes some time to parse the CSV files with papaparse, even though they are only about 3MB.
So it would be nice to have some kind of progress shown, when "papa" is working. I could go for the cheap hidden div, showing "I'm working", but would prefer the use of <progress>.
Unfortunately the bar just gets updated AFTER papa has finished its work. So I tried to get into webworkers and use a worker file to calculate progress and also setting worker: true in Papa Parses configuration. Still no avail.
The used configuration (with step function) is as followed:
var papaConfig =
{
header: true,
dynamicTyping: true,
worker: true,
step: function (row) {
if (gotHeaders == false) {
for (k in row.data[0]) {
if (k != "Time" && k != "Date" && k != " Time" && k != " ") {
header.push(k);
var obj = {};
obj.label = k;
obj.data = [];
flotData.push(obj);
gotHeaders = true;
}
}
}
tempDate = row.data[0]["Date"];
tempTime = row.data[0][" Time"];
var tD = tempDate.split(".");
var tT = tempTime.split(":");
tT[0] = tT[0].replace(" ", "");
dateTime = new Date(tD[2], tD[1] - 1, tD[0], tT[0], tT[1], tT[2]);
var encoded = $.toJSON(row.data[0]);
for (j = 0; j < header.length; j++) {
var value = $.evalJSON(encoded)[header[j]]
flotData[j].data.push([dateTime, value]);
}
w.postMessage({ state: row.meta.cursor, size: size });
},
complete: Done,
}
Worker configuration on the main site:
var w = new Worker("js/workers.js");
w.onmessage = function (event) {
$("#progBar").val(event.data);
};
and the called worker is:
onmessage = function(e) {
var progress = e.data.state;
var size = e.data.size;
var newPercent = Math.round(progress / size * 100);
postMessage(newPercent);
}
The progress bar is updated, but only after the CSV file is parsed and the site is set up with data, so the worker is called, but the answer is handled after parsing. Papa Parse seems to be called in a worker, too. Or so it seems if checking the calls in the browsers debugging tools, but still the site is unresponsive, until all data shows up.
Can anyone point me to what I have done wrong, or where to adjust the code, to get a working progress bar? I guess this would also deepen my understanding of web workers.

You could use the FileReader API to read the file as text, split the string by "\n" and then count the length of the returned array. This is then your size variable for the calculation of percentage.
You can then pass the file string to Papa (you do not need to reread directly from the file) and pass the number of rows (the size variable) to your worker. (I am unfamiliar with workers and so am unsure how you do this.)
Obviously this only accurately works if there are no embedded line breaks inside the csv file (e.g. where a string is spread over several lines with line breaks) as these will count as extra rows, so you will not make it to 100%. Not a fatal error, but may look strange to the user if it always seems to finish before 100%.
Here is some sample code to give you ideas.
var size = 0;
function loadFile(){
var files = document.getElementById("file").files; //load file from file input
var file = files[0];
var reader = new FileReader();
reader.readAsText(file);
reader.onload = function(event){
var csv = event.target.result; //the string version of your csv.
var csvArray = csv.split("\n");
size = csvArray.length;
console.log(size); //returns the number of rows in your file.
Papa.parse(csv, papaConfig); //Send the csv string to Papa for parsing.
};
}

I haven't used Papa Parse with workers before, but a few things pop up after playing with it for a bit:
It does not seem to expect you to interact directly with the worker
It expects you to either want the entire final result, or the individual items
Using a web worker makes providing a JS Fiddle infeasible, but here's some HTML that demonstrates the second point:
<html>
<head>
<script src="papaparse.js"></script>
</head>
<body>
<div id="step">
</div>
<div id="result">
</div>
<script type="application/javascript">
var papaConfig = {
header: true,
worker: true,
step: function (row) {
var stepDiv = document.getElementById('step');
stepDiv.appendChild(document.createTextNode('Step received: ' + JSON.stringify(row)));
stepDiv.appendChild(document.createElement('hr'));
},
complete: function (result) {
var resultDiv = document.getElementById('result');
resultDiv.appendChild(document.createElement('hr'));
resultDiv.appendChild(document.createTextNode('Complete received: ' + JSON.stringify(result)))
resultDiv.appendChild(document.createElement('hr'));
}
};
var data = 'Column 1,Column 2,Column 3,Column 4 \n\
1-1,1-2,1-3,1-4 \n\
2-1,2-2,2-3,2-4 \n\
3-1,3-2,3-3,3-4 \n\
4,5,6,7';
Papa.parse(data, papaConfig);
</script>
</body>
</html>
If you run this locally, you'll see you get a line for each of the four rows of the CSV data, but the call to the complete callback gets undefined. Something like:
Step received: {"data":[{"Column 1":"1-1",...
Step received: {"data":[{"Column 1":"2-1",...
Step received: {"data":[{"Column 1":"3-1",...
Step received: {"data":[{"Column 1":"4","...
Complete received: undefined
However if you remove or comment out the step function, you will get a single line for all four results:
Complete received: {"data":[{"Column 1":"1-1",...
Note also that Papa Parse uses a streaming concept to support the step callback regardless of using a worker or not. This means you won't know how many items you are parsing directly, so calculating the percent complete is not possible unless you can find the length of items separately.

MongoDB / Mongoose - Adding new database entry crashes page

I have problem with adding a team to my database, I have an index page at the moment which is just showing all of the teams in the database at the moment. When I go to this localhost:3000/team which is seen below for team.jade, I enter team name and press the submit button. When I then go to the index page, there it is, my new team is there.
But when I press the submit button, I actually get an error saying:
Express
500 TypeError: Cannot read property 'teamName' of undefined
at module.exports (/home/declan/nodeapps/tournamentManager/routes/index.js:126:24)
at callbacks (/home/declan/nodeapps/tournamentManager/node_modules/express/lib/router/index.js:160:37)
at param (/home/declan/nodeapps/tournamentManager/node_modules/express/lib/router/index.js:134:11)
at pass (/home/declan/nodeapps/tournamentManager/node_modules/express/lib/router/index.js:141:5)
at Router._dispatch (/home/declan/nodeapps/tournamentManager/node_modules/express/lib/router/index.js:169:5)
at Object.router (/home/declan/nodeapps/tournamentManager/node_modules/express/lib/router/index.js:32:10)
at next (/home/declan/nodeapps/tournamentManager/node_modules/express/node_modules/connect/lib/proto.js:190:15)
at Object.handle (/home/declan/nodeapps/tournamentManager/app.js:34:5)
at next (/home/declan/nodeapps/tournamentManager/node_modules/express/node_modules/connect/lib/proto.js:190:15)
at Object.static (/home/declan/nodeapps/tournamentManager/node_modules/express/node_modules/connect/lib/middleware/static.js:55:61)
I'm not sure why I get this, line 126 of index.js is actually this line from below:
var name = teamForm.teamName;
This is one of the first lines creating the new database entry, so if it couldn't read that property, it isn't adding it to the database. That was my thinking, but when I reload the index page, there is my new database entry, so why do you think it isn't working?
index.js # routes
/**
* Add a new Team to database
*/
app.post('/team', function(req, res) {
util.log('Serving request for url[GET] ' + req.route.path);
var teamForm = req.body.teamForm;
var name = teamForm.teamName;
var newTeam = new Team();
newTeam.name = name;
newTeam.save(function(err, savedTeam){
var message = '';
var retStatus = '';
if(!err){
util.log('Successfully created team with Name : ' + name);
message = 'Successfully created new team : ' + name;
retStatus = 'success';
} else {
util.log('Error while creating team : ' + name + ' error : ' + util.inspect(err));
if(err.code === 11000){
message = 'Team already exists';
}
retStatus = 'failure';
}
res.json({
'retStatus' : retStatus,
'message' : message
});
});
});
team.jade # views (html template, you can see the needed id's etc)
extends index
block content
div.row-fluid
div.span9
h2 New Team
div.well.sidebar-nav
div.teamList
form.form-horizontal(method="post", id="team-form")
div.control-group
label.control-label(for="teamName") Team Name :
div.controls
input.input-small(type="text", id="teamName")
div.control-group
div.controls
button#teamConfirm.btn.btn-primary.btn-mini(href='#') To DB
br
p This page is used to demonstrate how an 8 team single elimination tournament would be represented in the final design of my project. The pseudo code conjured up in my initial hand-in document was originally used here to produce the same result. The pseudo code helped create this set of seeded brackets, so that the matches correspond to the seeds of the tournament. For the 8 team bracket, I worked back through from the final, where seeds 1 and 2 should face each other, until I was left at round 1 of the brackets. This was used to give myself a 'new' set of seeds which were in the correct order to just be placed into these brackets.
div.span3
div.well.sidebar-nav
h4 To-Do List:
p - Enter list of teams manually from this page.
p - Do a test to show a bracket with random seed.
p - Show rankings after tournament
team.js # js
var newTeam = function(){
$('#teamConfirm').click(function(){
newTeam.teamForm();
});
};
newTeam.teamForm = function(){
var teamForm = {
teamName : $('#teamName').val()
};
// Basic validation
$.post('/team', {'teamForm' : teamForm}, function(response) {
console.log(response);
});
};
newTeam();
index.js # js
var Main = {};
Main.loadScript = function(url){
var footer = document.getElementById('footer');
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = url;
footer.appendChild(script);
}
$(document).ready(function(response){
Main.loadScript('js/login.js');
Main.loadScript('js/signup.js');
Main.loadScript('js/team.js');
});

I assume that this is because req.body is not being parsed to JavaScript object ( it remains as JSON string ). First check whether you are using
app.use(express.bodyParser());
before any route. If it is not that, then try using .ajax instead of .post with contentType set:
$.ajax({
// some other settings
contentType : "application/json"
});
It might be that content types are messed up and Express is not parsing JSON string received from the client. The other thing you might try is simply doing ( on the server side ):
app.post('/team', function(req, res) {
var body = JSON.parse( req.body );
// other code
}
Perhaps wrapping the additional line with try{}catch{} block.