We Keep Coding

How do I receive multiple JSON objects and use each one seperately?

I am trying to receive JSON objects via a websocket server. And I receive one JSON object every second that I want to be able to use each one separately as it gives a status on whether someone is active or not. However, whenever I try and store all the JSON's into a data structure only the final JSON I receive will be stored the others get deleted.
So my question is: is there a way to store JSON into a data structure whilst receiving it?
This is the JSON I receive looks something like this:
{"studentNum":"21127985","active":false,"currentScore":0}
here is the code for receiving the json from the server:
const WebSocket = require('ws');
// const serverAddress = "ws://127.0.0.1:5000";
const serverAddress = ' *SERVER NAME*';
const ws = new WebSocket(serverAddress, {
headers: {
"user-agent": "Mozilla"
}
});
ws.on('open', function () {
});
ws.on('message', function (msg) {
fs = require('fs')
console.log(msg.toString())
var obj = JSON.parse(msg);
/* convert buff to string
const json = JSON.parse(file.toString())
json.push(msg);*/
var writerStream = fs.createWriteStream('output.json')
writerStream.write(msg, 'UTF-8')
writerStream.end();
writerStream.on('finish', function () {
});
writerStream.on('error', function (err) {
console.log(err.stack);
});
});

you can put fs outside of your WS message callback
const WebSocket = require('ws');
fs = require('fs');
var writerStream = fs.createWriteStream('output.json')
const serverAddress = "ws://127.0.0.1:5000";
const serverAddress = ' *SERVER NAME*';
const ws = new WebSocket(serverAddress, {
headers: {
"user-agent": "Mozilla"
}
});
ws.on('open', function () {
});
ws.on('message', function (msg) {
writerStream.write(msg, 'UTF-8')
writerStream.on('error', function (err) {
console.log(err.stack);
});
});
Or alternatively you can write file in chunks i.e. store 10 messages in an array and write it using fs.appendFile function.

I got my JSON, now I need it to download inot an specific directory

I have an API that periodically makes a request to RESTAPI and stores the data into my database, now I want to add a new feature for certain data: download some data by a request to my API, then, my API makes another request to the RESTAPI, but I dont want this data to store in my database I want it to download as JSON or CSV.
I managed to create the request, I coded the request to the RESTAPI and managed to get the JSON into a variable with all the data, Im stuck there, How do I make this data to get downloaded into a direcory?
Im using javascript nodeJS with bent ,getJSON ,mssql ,https.
the code of the function:
async function descargarDatosOpendata(anioIni, anioFin, Indicativo){
try{
while(anioIni == anioFin || anioIni < anioFin){
console.log("first");
var http = require("https");
var options = {
"method": "GET",
"hostname": "opendata.aemet.es",
"path": "/opendata/api/valores/climatologicos/mensualesanuales/datos/anioini/"+ anioIni +"/aniofin/"+ anioIni +"/estacion/"+ Indicativo +"",
"headers": {
"cache-control": "no-cache",
"api_key": "MYKEY"
}
};
console.log("second");
var req = http.request(options, function (res) {
console.log("tercera");
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", async function () {
console.log("endChunk");
var body = Buffer.concat(chunks);
console.log(body);
var bodyString = body.toString();
bodyJSON = JSON.parse(bodyString);
console.log(bodyJSON);
console.log(bodyJSON.datos);
if (bodyJSON.estado == 200 && bodyJSON.descripcion == "exito") {
let obj = await getJSON(bodyJSON.datos);
console.log(obj)
}
});
});
anioIni++;
req.end();
}
}
catch (err) {
console.log(err);
}
}
obj log is the data in json format: [{data}]

If this code is running in Node, you should use the Node fs module. The appendFile method will create a new file or append to it if it already exists. Learn more about fs
Example code
var fs = require('fs');
fs.appendFile('mynewfile1.txt', 'Hello content!', function (err) {
if (err) throw err;
console.log('Saved!');
});

Node.js nested function scoping question -> fetching data stream piping to a file

This works and pipes downloaded data into a file:
const fs = require('fs')
await fetch(downloadURL).then(res => {
const dest = fs.createWriteStream('/tmp/output.xlsx');
res.body.pipe(dest);
});
This also works:
const buffer = await page.evaluate(({downloadURL}) =>
{
return fetch(downloadURL, {
method: 'GET'
}).then(r => r.text());
}, {downloadURL});
But to read a binary stream inside of page.evaluate(), I need to replace r => r.text() with the res.body.pipe from the first code snipped. When I do that:
const fs = require('fs')
const buff = await page.evaluate(({ downloadURL, fs }) => {
return fetch(downloadURL, fs, {
method: 'GET'
}).then(res => {
const dest = fs.createWriteStream('/tmp/output.xlsx');
res.body.pipe(dest);
});
}, { downloadURL, fs });
The error I get is TypeError: fs.createWriteStream is not a function
I don't think it has anything to do with "fs" per se; my bet is that somehow the "fs" is out of scope for this function structure.
How do I fix this last snippet so the data read is piped to a file?
My gut tells me it's some syntactical fix someone more skilled than I am can do...
thx

Node.js error in reading file

I am writing code in node.js where i want to read from a file and then export it to a web api. The problem is that I get an error on the code when i am using let.
The error appears to be in my function "render_html my views.js file:
"use strict";
const fs = require('fs');
const model = require('./model');
exports.car = (request, response, params) => {
if (Object.keys(params).length === 0) {
render_JSON(response, model.cars())
}else{
render_JSON(response, model.cars(parseInt(params.number)))
}
};
function render_html(response, file) {
fs.readFile(file, (err, data) => {
if (err) {
console.error(err)
} else {
response.write(data);
response.end();
}
});
}
function render_JSON(response, object) {
const responseJSON = JSON.stringify(object);
response.write(responseJSON);
response.end()
}
I also have problem in "function setHeaders" in router.js file:
"use strict";
const views = require('./views');
const url = require('url');
const routes = [
{
url: ['/api/cars'],
view: views.car,
methods: ['GET'],
headers: {'Content-Type': 'application/json; charset=UTF-8'} // application/json as per RFC4627
}];
function setHeaders(response, headers = {'Content-Type': 'text/plain'}, code = 200) {
response.writeHeader(code, headers);
}
// Filters trailing slash in the url
// for example allowing /api/cars and /api/cars/ to be treated equally by removing trailing slash in the second case
function filterURL(requestURL) {
if (requestURL.pathname.endsWith('/')) {
requestURL.pathname = requestURL.pathname.replace(/\/$/, '');
}
}
exports.route = (request, response) => {
for (let r of routes) {
const requestURL = url.parse(request.url, true);
// url matched and correct method
//if requestURL.pathname
filterURL(requestURL);
if (r.url.includes(requestURL.pathname) && r.methods.includes(request.method)) {
if (r.headers) {
setHeaders(response, r.headers);
} else {
setHeaders(response)
}
r.view(request, response, requestURL.query);
return;
}// if unsupported HTTP method
else if (r.url.includes(requestURL.pathname) && !r.methods.includes(request.method)) {
setHeaders(response, undefined, 405);
response.end();
return;
}
}
// if route not found respond with 404
setHeaders(response, undefined, 404);
response.end('404 Not Found!')
};
Somebody knows what the problem could be?
thanks.

about your problem in "render_html" function I think the problem is you are missing the encoding of the file, as fs doc says if you dont set a encoding the result will be a buffer. You can easy fix it using:
fs.readFile(file, 'utf8', callback)
(Assuming that you are using utf8 as encoding)
And I think your problem in "router.js" file is you should use "writeHead" instead "writeHeader" you can check it in http doc.
I hope it solves your issue, greetings.

how to require from URL in Node.js

Is there a standard way to require a Node module located at some URL (not on the local filesystem)?
Something like:
require('http://example.com/nodejsmodules/myModule.js');
Currently, I am simply fetching the file into a temporary file, and requiring that.

You can fetch module using http.get method and execute it in the sandbox using vm module methods runInThisContext and runInNewContext.
Example
var http = require('http')
, vm = require('vm')
, concat = require('concat-stream'); // this is just a helper to receive the
// http payload in a single callback
// see https://www.npmjs.com/package/concat-stream
http.get({
host: 'example.com',
port: 80,
path: '/hello.js'
},
function(res) {
res.setEncoding('utf8');
res.pipe(concat({ encoding: 'string' }, function(remoteSrc) {
vm.runInThisContext(remoteSrc, 'remote_modules/hello.js');
}));
});
IMO, execution of the remote code inside server application runtime may be reasonable in the case without alternatives. And only if you trust to the remote service and the network between.

Install the module first :
npm install require-from-url
And then put in your file :
var requireFromUrl = require('require-from-url/sync');
requireFromUrl("http://example.com/nodejsmodules/myModule.js");

0 dependency version (node 6+ required, you can simply change it back to ES5)
const http = require('http'), vm = require('vm');
['http://example.com/nodejsmodules/myModule.js'].forEach(url => {
http.get(url, res => {
if (res.statusCode === 200 && /\/javascript/.test(res.headers['content-type'])) {
let rawData = '';
res.setEncoding('utf8');
res.on('data', chunk => { rawData += chunk; });
res.on('end', () => { vm.runInThisContext(rawData, url); });
}
});
});
It is still the asynchronous version, if sync load is the case, a sync http request module for example should be required

If you want something more like require, you can do this:
var http = require('http')
, vm = require('vm')
, concat = require('concat-stream')
, async = require('async');
function http_require(url, callback) {
http.get(url, function(res) {
// console.log('fetching: ' + url)
res.setEncoding('utf8');
res.pipe(concat({encoding: 'string'}, function(data) {
callback(null, vm.runInThisContext(data));
}));
})
}
urls = [
'http://example.com/nodejsmodules/myModule1.js',
'http://example.com/nodejsmodules/myModule2.js',
'http://example.com/nodejsmodules/myModule3.js',
]
async.map(urls, http_require, function(err, results) {
// `results` is an array of values returned by `runInThisContext`
// the rest of your program logic
});

You could overwrite the default require handler for .js files:
require.extensions['.js'] = function (module, filename) {
// ...
}
You might want to checkout better-require as it does pretty much this for many file formats. (I wrote it)

const localeSrc = 'https://www.trip.com/m/i18n/100012631/zh-HK.js';
const http = require('http');
const vm = require('vm');
const concat = require('concat-stream');
http.get(
localeSrc,
res => {
res.setEncoding('utf8');
res.pipe(
concat({ encoding: 'string' }, remoteSrc => {
let context = {};
const script = new vm.Script(remoteSrc);
script.runInNewContext(context);
console.log(context);
}),
);
},
err => {
console.log('err', err);
},
);