I have spawned a python child_process in node.js that returns JSON data. In my python file I used json.dumps() before passing it back to node.
Whenever the list "results" in the JSON contains more than 62 entries, my node.js server dies when trying to parse it. Otherwise it works all fine?
Even when the node app dies, the data gets still sent back correctly to the client.
The error message is only thrown when calling JSON.parse(), but I cannot leave it out here, because the received data from python is in a buffered format:
<Buffer 7b 0d 0a 20 20 22 70 61 72 61 6d 65 74 65 72 73 22 3a 20 5b 0d 0a 20 20 20 20 7b 0d 0a 20 20 20 20 20 20 22 70 72 6f 70 65 72 74 79 22 3a 20 22 42 65 ... 9136 more bytes>
<Buffer 0d 0a>
Has anyone an idea what the problem could be?
The thrown error message is:
undefined:2
SyntaxError: Unexpected end of JSON input
at JSON.parse (<anonymous>)
at Socket.<anonymous> (D:\Projects\data-farming-framework-api\app.js:52:23)
at Socket.emit (events.js:310:20)
at addChunk (_stream_readable.js:286:12)
at readableAddChunk (_stream_readable.js:268:9)
at Socket.Readable.push (_stream_readable.js:209:10)
at Pipe.onStreamRead (internal/stream_base_commons.js:186:23)
My node.js code:
app.post('/api/ps/executeExperiments', async (req, res, next) => {
let path = req.body.path;
let experiments = JSON.stringify(req.body.data);
let python = await spawn('python', ['./plantsim.py', 'execute_experiments()', path, experiments]);
python.stdout.on('data', data => {
if (data.toString() == 'ERROR') {
callback(new Error('ERROR'))
}
let result = JSON.parse(data);
res.send(result);
});
});
The returned JSON has following format:
{
parameters: [
{ property: 'Bearbeitungszeit', object: 'Einzelstation' },
{ property: 'Bearbeitungszeit', object: 'Einzelstation2' }
],
results: [
{ id: 1, input: 252, output: 249, values: [Array] },
{ id: 2, input: 198, output: 195, values: [Array] },
{ id: 3, input: 165, output: 162, values: [Array] },
{ id: 4, input: 204, output: 201, values: [Array] },
{ id: 5, input: 169, output: 166, values: [Array] },
{ id: 6, input: 239, output: 236, values: [Array] },
{ id: 7, input: 225, output: 222, values: [Array] },
{ id: 8, input: 312, output: 309, values: [Array] },
{ id: 9, input: 324, output: 321, values: [Array] },
{ id: 10, input: 248, output: 245, values: [Array] },
{ id: 11, input: 199, output: 196, values: [Array] },
{ id: 12, input: 186, output: 183, values: [Array] },
{ id: 13, input: 161, output: 158, values: [Array] },
{ id: 14, input: 149, output: 146, values: [Array] },
{ id: 15, input: 175, output: 172, values: [Array] },
{ id: 16, input: 201, output: 198, values: [Array] },
{ id: 17, input: 242, output: 239, values: [Array] },
{ id: 18, input: 162, output: 159, values: [Array] },
{ id: 19, input: 184, output: 181, values: [Array] },
{ id: 20, input: 157, output: 154, values: [Array] },
{ id: 21, input: 160, output: 157, values: [Array] },
{ id: 22, input: 290, output: 287, values: [Array] },
{ id: 23, input: 358, output: 355, values: [Array] },
{ id: 24, input: 153, output: 150, values: [Array] },
{ id: 25, input: 155, output: 152, values: [Array] },
{ id: 26, input: 195, output: 192, values: [Array] },
{ id: 27, input: 174, output: 171, values: [Array] },
{ id: 28, input: 269, output: 266, values: [Array] },
{ id: 29, input: 341, output: 338, values: [Array] },
{ id: 30, input: 263, output: 260, values: [Array] },
{ id: 31, input: 156, output: 153, values: [Array] },
{ id: 32, input: 167, output: 164, values: [Array] },
{ id: 33, input: 295, output: 292, values: [Array] },
{ id: 34, input: 230, output: 227, values: [Array] },
{ id: 35, input: 189, output: 186, values: [Array] },
{ id: 36, input: 172, output: 169, values: [Array] },
{ id: 37, input: 254, output: 251, values: [Array] },
{ id: 38, input: 182, output: 179, values: [Array] },
{ id: 39, input: 206, output: 203, values: [Array] },
{ id: 40, input: 232, output: 229, values: [Array] },
{ id: 41, input: 259, output: 256, values: [Array] },
{ id: 42, input: 159, output: 156, values: [Array] },
{ id: 43, input: 305, output: 302, values: [Array] },
{ id: 44, input: 151, output: 148, values: [Array] },
{ id: 45, input: 181, output: 178, values: [Array] },
{ id: 46, input: 302, output: 299, values: [Array] },
{ id: 47, input: 187, output: 184, values: [Array] },
{ id: 48, input: 245, output: 242, values: [Array] },
{ id: 49, input: 149, output: 146, values: [Array] },
{ id: 50, input: 279, output: 276, values: [Array] },
{ id: 51, input: 215, output: 212, values: [Array] },
{ id: 52, input: 170, output: 167, values: [Array] },
{ id: 53, input: 288, output: 285, values: [Array] },
{ id: 54, input: 329, output: 326, values: [Array] },
{ id: 55, input: 283, output: 280, values: [Array] },
{ id: 56, input: 193, output: 190, values: [Array] },
{ id: 57, input: 177, output: 174, values: [Array] },
{ id: 58, input: 348, output: 345, values: [Array] },
{ id: 59, input: 265, output: 262, values: [Array] },
{ id: 60, input: 219, output: 216, values: [Array] },
{ id: 61, input: 352, output: 349, values: [Array] },
{ id: 62, input: 151, output: 148, values: [Array] }
]
}
Values is also a list, however i don't know why it is not displayed in the terminal correctly. The data is still there as it gets passed to the client correctly.
data events deliver chunks of data on arbitrary boundaries. As long as you are getting the entire JSON in one chunk (something you do not control), your code probably just happens to work. But, as soon as the data gets large enough of takes long enough to generate that it gets sent in more than one chunk, then your code dies trying to parse an individual chunk of data. This is why it is size dependent. At some size of data, it will start arriving in more than one chunk. And, you can't parse each chunk as JSON separately (thus your JSON.parse() error.
Instead, if you're going to parse it into JSON, you need to accumulate all the chunks and then when it's done, parse the entire thing. JSON is not a format that can easily be incrementally parsed (there is JSON stream code that can do it, but it's a lot of work).
app.post('/api/ps/executeExperiments', async (req, res, next) => {
let path = req.body.path;
let experiments = JSON.stringify(req.body.data);
let python = spawn('python', ['./plantsim.py', 'execute_experiments()', path, experiments]);
let allData = [];
// collect from all the data events here
python.stdout.on('data', data => {
allData.push(data.toString());
}).on('error', e => {
console.log(e);
res.sendStatus(500);
}).on('close', () => {
// we have all data now
let result = allData.join(allData, "");
if (result.startsWith('ERROR')) {
console.log(result);
res.sendStatus(500);
} else {
res.send(allData);
}
});
});
In your existing code what is callback()? You don't show that and it has me confused so I removed it.
You also need to send an error response to the incoming POST request if you get an error.
And, there's no point to using await on the return value from spawn(). It doesn't return a promise so await doesn't do anything useful.
Also, it appears there is no need to even parse the JSON. If you're going to just parse it and then send it right off with res.send(), all res.send() is going to do is turn it back into JSON again. So, perhaps all you really need to do it to just stream python.stdout as your response?
The issue is not with the long file, the issue is with the way you tried to collect data. python.stdout.on is the stream of data. Data comes in the chunk. So you need to collect all data then return to response. You can simplify, using basic util method as given below.
const spawn = require("child_process").spawn;
function run(path, experiments) {
let command = spawn("python", [
"./plantsim.py",
"execute_experiments()",
path,
experiments,
]);
return new Promise((resolve) => {
var result = "";
command.stdout.on("data", function (data) {
result += data.toString();
});
command.on("close", function (code) {
resolve(result);
});
});
}
app.post("/api/ps/executeExperiments", async (req, res, next) => {
let path = req.body.path;
let experiments = JSON.stringify(req.body.data);
const data = await run(path, experiments);
res.send(result);
});
If you are just returning file[no parse or transform] you can just pipe the data to the response.
Sample:
const { spawn } = require("child_process");
const express = require("express");
const app = express();
app.get("/json", (req, res) => {
let command = spawn("cat", [
__dirname + "/test.json",
]);
command.stdout.pipe(res)
command.on("close", function (data) {
console.log("done writing");
});
});
app.listen(3000)
Related
I've created a couple arrays. arrLocations holds a series of 2-digit country names, arrResults counts how many times each occurs and holds the country name with it's count as see here:
[
FR: 40, US: 1511, AU: 82,
CN: 151, IE: 170, SG: 108,
GB: 66, KR: 52, JP: 137,
IN: 45, BR: 68, SE: 39,
ZA: 19, NL: 19, BH: 19,
LU: 3, CA: 41, DE: 79,
ID: 1, HK: 1
]
My intention is to sort this array by ascending occurrence, I want this:
[
ID: 1, HK: 1, LU: 3,
BH: 19, NL: 19, ZA: 19,
....and so on........
]
However, no matter what way I implement sort, it does absolutely nothing. I even did a test sort on a dumby array and was able to sort it fine, but this array I cannot sort.
Here is what I have so far:
var geoip = require('geoip-lite')
const fs = require('fs')
const { Console } = require('console')
let data = fs.readFileSync('./ip_new.txt')
let iplocation = ""
let arrLocations = []
let arrResults = []
let arrIP = data.toString().split("\r\n")
for(let item of arrIP){
iplocation = geoip.lookup(item)
arrLocations.push(iplocation.country)
}
for(let item of arrLocations){
if (item in arrResults){
arrResults[item]++
}
else{
arrResults[item]=1
}
}
arrResults.sort()
console.log(arrResults)
OK, so first of all I'm assuming this
[
FR: 40, US: 1511, AU: 82,
CN: 151, IE: 170, SG: 108,
GB: 66, KR: 52, JP: 137,
IN: 45, BR: 68, SE: 39,
ZA: 19, NL: 19, BH: 19,
LU: 3, CA: 41, DE: 79,
ID: 1, HK: 1
]
Is actually this
[
{ FR: 40 }, { US: 1511 }, { AU: 82 },
{ CN: 151 }, { IE: 170 }, { SG: 108 },
{ GB: 66 }, { KR: 52 }, { JP: 137 },
{ IN: 45 }, { BR: 68 }, { SE: 39 },
{ ZA: 19 }, { NL: 19 }, { BH: 19 },
{ LU: 3 }, { CA: 41 }, { DE: 79 },
{ ID: 1 }, { HK: 1 }
]
If this is true, then sort method will not work on it's own. The following will do though:
const arrResults = [
{ FR: 40 }, { US: 1511 }, { AU: 82 },
{ CN: 151 }, { IE: 170 }, { SG: 108 },
{ GB: 66 }, { KR: 52 }, { JP: 137 },
{ IN: 45 }, { BR: 68 }, { SE: 39 },
{ ZA: 19 }, { NL: 19 }, { BH: 19 },
{ LU: 3 }, { CA: 41 }, { DE: 79 },
{ ID: 1 }, { HK: 1 }
];
arrResults.sort((a, b) => {
const countryCodeA = Object.keys(a)[0];
const countryCodeB = Object.keys(b)[0];
return a[countryCodeA] - b[countryCodeB];
});
The data I'm looking to process is a form of web-based game data communicated through JSON. I'm trying to do statistical analysis on correlations between let's say attributes api_ship_id and api_lv. I was following a Tutorial (Mostly in Japanese) on how to export some JSON files with a chrome plugin. On the other hand, I have a chromium-based web browser that supports similar JSON export but has a slightly different structure. The code this tutorial provided as was
copy(JSON.stringify(Object.entries(temp1.model.ship._map).map(([, v]) => v._o).filter(v => v.api_locked), ['api_ship_id', 'api_lv', 'api_kyouka', 'api_exp']))
but my chromium-based browser have trouble parsing this JSON when going filter.
Uncaught TypeError: Cannot read property 'api_locked' of undefined.
tl;dr: The 4 catagories I want are
['api_ship_id', 'api_lv', 'api_kyouka', 'api_exp']
JSON:
{"1": {
"api_id": 1,
"api_sortno": 1337,
"api_ship_id": 237,
"api_lv": 70,
"api_exp": [
274115,
885,
91
],
"api_nowhp": 30,
"api_maxhp": 30,
"api_soku": 10,
"api_leng": 1,
"api_slot": [
-1,
-1,
-1,
-1,
-1
],
"api_onslot": [
0,
0,
0,
0,
0
],
"api_slot_ex": 0,
"api_kyouka": [
37,
51,
34,
36,
0,
0,
0
],
"api_backs": 4,
"api_fuel": 15,
"api_bull": 20,
"api_slotnum": 3,
"api_ndock_time": 0,
"api_ndock_item": [
0,
0
],
"api_srate": 4,
"api_cond": 49,
"api_karyoku": [
49,
49
],
"api_raisou": [
79,
79
],
"api_taiku": [
49,
49
],
"api_soukou": [
49,
49
],
"api_kaihi": [
76,
89
],
"api_taisen": [
48,
59
],
"api_sakuteki": [
29,
39
],
"api_lucky": [
12,
59
],
"api_locked": 1,
"api_locked_equip": 0
},
"2": {
"api_id": 2,
"api_sortno": 1350,
"api_ship_id": 250,
"api_lv": 29,
"api_exp": [
40635,
2865,
1
],
"api_nowhp": 31,
"api_maxhp": 31,
"api_soku": 10,
"api_leng": 1,
"api_slot": [
1921,
356,
-1,
-1,
-1
],
"api_onslot": [
0,
0,
0,
0,
0
],
"api_slot_ex": 0,
"api_kyouka": [
22,
37,
15,
19,
0,
0,
0
],
"api_backs": 4,
"api_fuel": 15,
"api_bull": 20,
"api_slotnum": 3,
"api_ndock_time": 0,
"api_ndock_item": [
0,
0
],
"api_srate": 2,
"api_cond": 73,
"api_karyoku": [
36,
49
],
"api_raisou": [
65,
79
],
"api_taiku": [
38,
49
],
"api_soukou": [
33,
49
],
"api_kaihi": [
58,
89
],
"api_taisen": [
34,
59
],
"api_sakuteki": [
17,
39
],
"api_lucky": [
12,
59
],
"api_locked": 1,
"api_locked_equip": 0
},
"3": {
"api_id": 3,
"api_sortno": 1302,
"api_ship_id": 202,
"api_lv": 30,
"api_exp": [
43597,
2903,
3
],
"api_nowhp": 30,
"api_maxhp": 30,
"api_soku": 10,
"api_leng": 1,
"api_slot": [
2131,
5074,
-1,
-1,
-1
],
"api_onslot": [
0,
0,
0,
0,
0
],
"api_slot_ex": 0,
"api_kyouka": [
18,
27,
16,
17,
0,
0,
0
],
"api_backs": 4,
"api_fuel": 15,
"api_bull": 20,
"api_slotnum": 3,
"api_ndock_time": 0,
"api_ndock_item": [
0,
0
],
"api_srate": 2,
"api_cond": 61,
"api_karyoku": [
32,
49
],
"api_raisou": [
62,
79
],
"api_taiku": [
32,
49
],
"api_soukou": [
30,
49
],
"api_kaihi": [
60,
89
],
"api_taisen": [
34,
59
],
"api_sakuteki": [
16,
39
],
"api_lucky": [
12,
49
],
"api_locked": 1,
"api_locked_equip": 0
}}
What I've tried
copy(JSON.stringify(Object.entries(temp2).map(([, v]) => v._o), ['api_ship_id', 'api_lv', 'api_kyouka', 'api_exp']))
Result: undefined
Ideal results:
{"api_ship_id": 237,"api_lv": 70,"api_kyouka": [37,51,34,36,0,0,0],"api_exp": [40635,2865,1],}
{"api_ship_id": 238,"api_lv": 68,"api_kyouka": [30,0,4,6,0,0,0],"api_exp": [565,285,1],}
I have a JSON array containing several objects. I would like to return objects containing a certain value. For example, I would like to return
[
service_wog: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 76,
service: 'WOG',
slug: 'wog'
},
service_gojoy: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 77,
service: 'GoJoy',
slug: 'gojoy'
}
]
How do I return the object that contains 'gojoy' in slug?
I tried the following way:
let u = Object.values(a);
u.filter(i => i.slug === 'gojoy');
It doesn't seem to be working... Did I misunderstand how the filter() works?
No, it seems that you're using filter correctly.
However, what are you using as an input:
[
service_wog: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 76,
service: 'WOG',
slug: 'wog'
},
service_gojoy: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 77,
service: 'GoJoy',
slug: 'gojoy'
}
]
it's not a valid array, but an object.
So instead of [ and ] - { and } should be used:
{
service_wog: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 76,
service: 'WOG',
slug: 'wog'
},
service_gojoy: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 77,
service: 'GoJoy',
slug: 'gojoy'
}
}
So eventually:
const a = {
service_wog: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 76,
service: 'WOG',
slug: 'wog'
},
service_gojoy: {
count: 48,
popular: false,
code: 33,
price: 20,
id: 77,
service: 'GoJoy',
slug: 'gojoy'
}
}
let u = Object.values(a);
console.log(u.filter(i => i.slug === 'gojoy'));
I'll preface this question by stating that I am very new to Javascript and programming as a whole as I'm a current first year student. I am attempting to create an app which makes use of data given to me by an API. However, this API only returns the data as one whole String with different subsections.
This is how the return looks when logged:
User {
id: 'redacted',
username: 'Despacito II',
platform: 'PC',
url: 'https://fortnitetracker.com/profile/pc/Despacito II',
stats:
{ solo:
Mode {
score: 101032,
kd: 0.35,
matches: 916,
kills: 324,
kills_per_match: 0.35,
score_per_match: 110.3,
wins: 1,
top_3: 1,
top_5: 2,
top_6: 4,
top_12: 8,
top_25: 231 },
duo:
Mode {
score: 29650,
kd: 0.47,
matches: 198,
kills: 92,
kills_per_match: 0.46,
score_per_match: 149.75,
wins: 2,
top_3: 2,
top_5: 23,
top_6: 27,
top_12: 104,
top_25: 158 },
squad:
Mode {
score: 166404,
kd: 0.42,
matches: 795,
kills: 323,
kills_per_match: 0.41,
score_per_match: 209.31,
wins: 31,
top_3: 120,
top_5: 151,
top_6: 488,
top_12: 790,
top_25: 1580 },
current_solo:
Mode {
score: 16660,
kd: 0.62,
matches: 128,
kills: 80,
kills_per_match: 0.62,
score_per_match: 130.16,
wins: 0,
top_3: 0,
top_5: 0,
top_6: 0,
top_12: 0,
top_25: 34 },
current_duo:
Mode {
score: 11907,
kd: 0.78,
matches: 60,
kills: 45,
kills_per_match: 0.75,
score_per_match: 198.45,
wins: 2,
top_3: 2,
top_5: 12,
top_6: 16,
top_12: 55,
top_25: 87 },
current_squad:
Mode {
score: 24822,
kd: 0.56,
matches: 130,
kills: 70,
kills_per_match: 0.54,
score_per_match: 190.94,
wins: 4,
top_3: 19,
top_5: 23,
top_6: 69,
top_12: 115,
top_25: 230 },
lifetime:
[ [Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object] ] } }
I was wondering how I could take this return and retrieve variables from it. For example, have a new variable called 'kdSolo' and have it equal 0.35 as per the APIs return?
I'd also love to know if there is a specific term for this and what it is for future reference and research. Sorry again for the noobie question.
if you want to walk the current object tree then you could simply do:
var kdSolo = User.stats.solo.Mode.kd
// kdSolo will now be 0.35
The reason I asked about the JSON is because what you have above is not a valid JSON object.
I'm in need to use d3-timeseries graph given on link:
http://mcaule.github.io/d3-timeseries/
I'm having some JSON data which I will use to plot on this graph. I'm trying to make this work on d3 plunker.
Being new to D3 and plunker,I'm not sure if I'm doing the code at right places or not as nothing is coming up. Please guide me.
Code I'm trying to use on d3-plunker:
<!DOCTYPE html>
<svg width="960" height="500"></svg>
<script src="https://d3js.org/d3.v4.min.js"></script>
<script>
var data : [{date:new Date('2013-01-01'),n:120,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-02'),n:121,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-03'),n:122,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-04'),n:123,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-05'),n:124,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-06'),n:125,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-07'),n:126,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-08'),n:127,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-09'),n:128,n3:124,ci_up:130,ci_down:118},
{date:new Date('2013-01-10'),n:129,n3:124,ci_up:130,ci_down:118}]
var chart = d3.timeseries()
.addSerie(data.slice(0,60),{x:'date',y:'n'},{interpolate:'linear',color:"#a6cee3",label:"value"})
.addSerie(data.slice(50),
{x:'date',y:'n3',ci_up:'ci_up',ci_down:'ci_down'},
{interpolate:'monotone',dashed:true,color:"#a6cee3",label:"prediction"})
.width(900)
</script>
First of all, you have to reference d3-timeseries:
<script src="https://mcaule.github.io/d3-timeseries/src/d3_timeseries.js"></script>
After that, at the end of your code, you have to call it:
chart("body")
Finally, have in mind that this is not a valid JavaScript:
var data : [];
It should be var data = [] instead.
Here is your working code (click "run code snippet"):
var data = [{
date: new Date('2013-01-01'),
n: 120,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-02'),
n: 121,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-03'),
n: 122,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-04'),
n: 123,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-05'),
n: 124,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-06'),
n: 125,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-07'),
n: 126,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-08'),
n: 127,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-09'),
n: 128,
n3: 124,
ci_up: 130,
ci_down: 118
}, {
date: new Date('2013-01-10'),
n: 129,
n3: 124,
ci_up: 130,
ci_down: 118
}]
var chart = d3.timeseries()
.addSerie(data.slice(0, 60), {
x: 'date',
y: 'n'
}, {
interpolate: 'linear',
color: "#a6cee3",
label: "value"
})
.addSerie(data.slice(50), {
x: 'date',
y: 'n3',
ci_up: 'ci_up',
ci_down: 'ci_down'
}, {
interpolate: 'monotone',
dashed: true,
color: "#a6cee3",
label: "prediction"
})
.width(900)
chart("body")
.axis line, .axis path {
fill: none;
stroke: black;
}
<script src="https://d3js.org/d3.v3.min.js"></script>
<script src="https://mcaule.github.io/d3-timeseries/src/d3_timeseries.js"></script>