I am trying to receive JSON objects via a websocket server. And I receive one JSON object every second that I want to be able to use each one separately as it gives a status on whether someone is active or not. However, whenever I try and store all the JSON's into a data structure only the final JSON I receive will be stored the others get deleted.
So my question is: is there a way to store JSON into a data structure whilst receiving it?
This is the JSON I receive looks something like this:
{"studentNum":"21127985","active":false,"currentScore":0}
here is the code for receiving the json from the server:
const WebSocket = require('ws');
// const serverAddress = "ws://127.0.0.1:5000";
const serverAddress = ' *SERVER NAME*';
const ws = new WebSocket(serverAddress, {
headers: {
"user-agent": "Mozilla"
}
});
ws.on('open', function () {
});
ws.on('message', function (msg) {
fs = require('fs')
console.log(msg.toString())
var obj = JSON.parse(msg);
/* convert buff to string
const json = JSON.parse(file.toString())
json.push(msg);*/
var writerStream = fs.createWriteStream('output.json')
writerStream.write(msg, 'UTF-8')
writerStream.end();
writerStream.on('finish', function () {
});
writerStream.on('error', function (err) {
console.log(err.stack);
});
});
you can put fs outside of your WS message callback
const WebSocket = require('ws');
fs = require('fs');
var writerStream = fs.createWriteStream('output.json')
const serverAddress = "ws://127.0.0.1:5000";
const serverAddress = ' *SERVER NAME*';
const ws = new WebSocket(serverAddress, {
headers: {
"user-agent": "Mozilla"
}
});
ws.on('open', function () {
});
ws.on('message', function (msg) {
writerStream.write(msg, 'UTF-8')
writerStream.on('error', function (err) {
console.log(err.stack);
});
});
Or alternatively you can write file in chunks i.e. store 10 messages in an array and write it using fs.appendFile function.
Related
I have an API that periodically makes a request to RESTAPI and stores the data into my database, now I want to add a new feature for certain data: download some data by a request to my API, then, my API makes another request to the RESTAPI, but I dont want this data to store in my database I want it to download as JSON or CSV.
I managed to create the request, I coded the request to the RESTAPI and managed to get the JSON into a variable with all the data, Im stuck there, How do I make this data to get downloaded into a direcory?
Im using javascript nodeJS with bent ,getJSON ,mssql ,https.
the code of the function:
async function descargarDatosOpendata(anioIni, anioFin, Indicativo){
try{
while(anioIni == anioFin || anioIni < anioFin){
console.log("first");
var http = require("https");
var options = {
"method": "GET",
"hostname": "opendata.aemet.es",
"path": "/opendata/api/valores/climatologicos/mensualesanuales/datos/anioini/"+ anioIni +"/aniofin/"+ anioIni +"/estacion/"+ Indicativo +"",
"headers": {
"cache-control": "no-cache",
"api_key": "MYKEY"
}
};
console.log("second");
var req = http.request(options, function (res) {
console.log("tercera");
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", async function () {
console.log("endChunk");
var body = Buffer.concat(chunks);
console.log(body);
var bodyString = body.toString();
bodyJSON = JSON.parse(bodyString);
console.log(bodyJSON);
console.log(bodyJSON.datos);
if (bodyJSON.estado == 200 && bodyJSON.descripcion == "exito") {
let obj = await getJSON(bodyJSON.datos);
console.log(obj)
}
});
});
anioIni++;
req.end();
}
}
catch (err) {
console.log(err);
}
}
obj log is the data in json format: [{data}]
If this code is running in Node, you should use the Node fs module. The appendFile method will create a new file or append to it if it already exists. Learn more about fs
Example code
var fs = require('fs');
fs.appendFile('mynewfile1.txt', 'Hello content!', function (err) {
if (err) throw err;
console.log('Saved!');
});
My Node.js server receives a stream of data from an external API.
I serve my client after receiving the data completely. Like this,
async function getFile(req, res) {
const { id } = req.body;
const file = await get(process.env.FILE_API_URL + id);
res.send(file);
}
But instead of waiting to receive the whole stream, I would like to stream it to the client as soon as I have some data. Kind of like this,
function getFile(req, res) {
const { id } = req.body;
const stream = get(process.env.FILE_API_URL + id);
stream.on('data', (data) > {
res.write(data);
});
stream.on('end', res.end);
}
How can I implement this?
I'm working on learning Express JS and practicing security. I have moved some coupon code logic to the server, to make it more secure. That means that I send a coupon code to the server, it checks whether it is a valid code, and then returns a JSON object with a message and a win indicator (1 for the win, 0 for loss).
My issue is that I cannot figure out how to get both the message and the win indicator on the client from the response. It's always undefined
This is the client:
window.addEventListener("load", () => {
const nameInput = document.getElementById("name");
const couponCodeInput = document.getElementById("coupon-code");
const button = document.getElementById("button");
const nowinElem = document.getElementById("no-win");
const winElem = document.getElementById("you-won");
button.addEventListener("click", async e => {
e.preventDefault();
winElem.style.display = "none"; //new click of button:
nowinElem.style.display = "none"; // hide previous messages
let resp = await fetch('winner', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: nameInput.value, code: couponCodeInput.value })
});
if (resp.status !== 200) console.log('Winner report failed');
var res = resp.body;
if(res.win == "1"){
winElem.style.display = "block";
winElem.innerText = res.msg;
} else {
nowinElem.style.display = "block";
nowinElem.innerText = res.msg;
}
})
})
And this is the server:
const express = require('express')
const fs = require('fs')
const app = express()
const html = fs.readFileSync('coupon-client.html');
const js = fs.readFileSync('coupon-client.js');
const winnerCodes = ["123", "secret", "abc321"];
app.get('/', (req, res) => {
res.writeHead(200, {"Content-Type": "text/html; charset=utf-8"});
res.end(html);
})
app.get('/coupon-client.js', (req, res) => {
res.writeHead(200, {"Content-Type": "application/javascript"});
res.end(js);
})
app.post('/winner', express.json(), (req, res) => {//use built-in JSON middle-ware
let jsonObj = req.body //JSON already parsed: { "name": "my name" }
if(winnerCodes.includes(jsonObj.code)){
console.log(`Congratulations to ${jsonObj.name}!`);
console.log("We'll send you a diploma.");
res.json({msg: 'Congratulations - and thanks!', win: '1'});
} else {
console.log(`Condolences to ${jsonObj.name}!`);
console.log("We'll send you nothing.");
res.json({msg: 'Sorry, you did not win - but thanks for playing!', win: '0'});
}
})
app.listen(8080, () => console.log('Listening...'))
As I mentioned above, I cannot get the JSON data object to give me the win and message variables. I have tried to do "JSON.parse(resp.body)" but then it gives me an error saying "unexpected character at....", which I read means that the body is already parsed, so I'm just taking the body now.
Can anyone give me a hint or help me out?
Reading a Response object works differently from how you currently are using it.
Use the json() method on the response object. It will return a promise which parses the response from JSON into usable data.
About the Response object and its usage.
Currently you're accessing the body property on the Response object, which it inherited from the Body mixin.
The body read-only property of the Body mixin is a simple getter used to expose a ReadableStream of the body contents. MDN
So you're accessing a ReadableStream object and try to read the win property from it, which doesn't exist on the stream.
To help with this the Body mixin has methods to convert the stream into usable data. Examples are Body.json() and Body.text(). These methods read the stream and convert the body into an object, array, string or number when it has a JSON structure or into a single string, which is useful when you're sending HTML or just raw text.
Both methods return Promises in which you have to wait for the result to be ready for usage.
let resp = await fetch('winner', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
name: nameInput.value,
code: couponCodeInput.value
})
});
if (resp.status !== 200) console.log('Winner report failed');
// Decode body from JSON.
let { win, msg } = await resp.json();
if (win === "1") {
winElem.style.display = "block";
winElem.innerText = msg;
} else {
nowinElem.style.display = "block";
nowinElem.value = msg;
}
I'm trying to use Node.js from here for my IAP Receipt Validation, but it always returns error on a server's log: "The data in the receipt-data property was malformed."
Can someone help me to properly send base64 string to Node.js and to decode it there as same base64 string for receipt validation? I have zero experience with javascript and been trying to make this simple code work for two days now, to no avail.
Here is my Swift code:
let receiptData = try Data(contentsOf: appStoreReceiptURL, options: .alwaysMapped)
let receiptString = receiptData.base64EncodedString(options: [])
var request = URLRequest(url: URL(string: "https://us-central1-calendizer-6a849.cloudfunctions.net/receiptValidation")!)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
// attach receipt data to a request
request.httpBody = receiptString.data(using: .utf8)
print("httpBody: \(request.httpBody?.base64EncodedString())")
Logs.log("✉️ Receipt Validation -> sending request ...")
let task = URLSession.shared.dataTask(with: request) { data, response, error in
...
}
Here is my Node.js code (Firebase Cloud Functions):
const functions = require('firebase-functions');
var iap = require('in-app-purchase');
exports.receiptValidation = functions.https.onRequest((request, response) => {
var receipt_try1 = request.body
var receipt_try2 = request.body.toString('base64');
var receipt_try3 = JSON.stringify(receipt);
var receipt_try4 = new Buffer(request.body.toString(), 'base64')
console.log("receipt: " + receipt_try1)
iap.config({
applePassword: 'my shared key',
test: true
});
iap.setup(function (error) {
if (error) {
console.log("Setup error:" + error) // Failed to validate
}
iap.validate(iap.APPLE, receipt_try1, function (error, appleResponse) {
if (error)
{
console.log("Validation error:" + error) // Failed to validate
}
if (iap.isValidated(appleResponse)) {
console.log("successful validation" + appleResponse)
response.send(appleResponse)
}
});
});
});
did it using json:
let receiptData = try Data(contentsOf: appStoreReceiptURL, options: .alwaysMapped)
let receiptBase64String = receiptData.base64EncodedString(options: [])
//.replacingOccurrences(of: "+", with: "%2B")
// prepare json data
let json: [String: Any] = ["receipt_string": receiptBase64String]
let jsonData = try? JSONSerialization.data(withJSONObject: json, options: .prettyPrinted)
// create post request
var request = URLRequest(url: URL(string: "https://us-central1-calendizer-6a849.cloudfunctions.net/receiptValidation")!)
//request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
request.httpBody = jsonData
and in Node.js simply:
var receipt = request.body.receipt_string
Background
I am doing some experimentation with Node.js and would like to read a JSON object, either from a text file or a .js file (which is better??) into memory so that I can access that object quickly from code. I realize that there are things like Mongo, Alfred, etc out there, but that is not what I need right now.
Question
How do I read a JSON object out of a text or js file and into server memory using JavaScript/Node?
Sync:
var fs = require('fs');
var obj = JSON.parse(fs.readFileSync('file', 'utf8'));
Async:
var fs = require('fs');
var obj;
fs.readFile('file', 'utf8', function (err, data) {
if (err) throw err;
obj = JSON.parse(data);
});
The easiest way I have found to do this is to just use require and the path to your JSON file.
For example, suppose you have the following JSON file.
test.json
{
"firstName": "Joe",
"lastName": "Smith"
}
You can then easily load this in your node.js application using require
var config = require('./test.json');
console.log(config.firstName + ' ' + config.lastName);
Asynchronous is there for a reason! Throws stone at #mihai
Otherwise, here is the code he used with the asynchronous version:
// Declare variables
var fs = require('fs'),
obj
// Read the file and send to the callback
fs.readFile('path/to/file', handleFile)
// Write the callback function
function handleFile(err, data) {
if (err) throw err
obj = JSON.parse(data)
// You can now play with your datas
}
At least in Node v8.9.1, you can just do
var json_data = require('/path/to/local/file.json');
and access all the elements of the JSON object.
Answer for 2022, using ES6 module syntax and async/await
In modern JavaScript, this can be done as a one-liner, without the need to install additional packages:
import { readFile } from 'fs/promises';
let data = JSON.parse(await readFile("filename.json", "utf8"));
Add a try/catch block to handle exceptions as needed.
In Node 8 you can use the built-in util.promisify() to asynchronously read a file like this
const {promisify} = require('util')
const fs = require('fs')
const readFileAsync = promisify(fs.readFile)
readFileAsync(`${__dirname}/my.json`, {encoding: 'utf8'})
.then(contents => {
const obj = JSON.parse(contents)
console.log(obj)
})
.catch(error => {
throw error
})
Using fs-extra package is quite simple:
Sync:
const fs = require('fs-extra')
const packageObj = fs.readJsonSync('./package.json')
console.log(packageObj.version)
Async:
const fs = require('fs-extra')
const packageObj = await fs.readJson('./package.json')
console.log(packageObj.version)
using node-fs-extra (async await)
const readJsonFile = async () => {
const myJsonObject = await fs.readJson('./my_json_file.json');
console.log(myJsonObject);
}
readJsonFile() // prints your json object
https://nodejs.org/dist/latest-v6.x/docs/api/fs.html#fs_fs_readfile_file_options_callback
var fs = require('fs');
fs.readFile('/etc/passwd', (err, data) => {
if (err) throw err;
console.log(data);
});
// options
fs.readFile('/etc/passwd', 'utf8', callback);
https://nodejs.org/dist/latest-v6.x/docs/api/fs.html#fs_fs_readfilesync_file_options
You can find all usage of Node.js at the File System docs!
hope this help for you!
function parseIt(){
return new Promise(function(res){
try{
var fs = require('fs');
const dirPath = 'K:\\merge-xml-junit\\xml-results\\master.json';
fs.readFile(dirPath,'utf8',function(err,data){
if(err) throw err;
res(data);
})}
catch(err){
res(err);
}
});
}
async function test(){
jsonData = await parseIt();
var parsedJSON = JSON.parse(jsonData);
var testSuite = parsedJSON['testsuites']['testsuite'];
console.log(testSuite);
}
test();
Answer for 2022, using v8 Import assertions
import jsObject from "./test.json" assert { type: "json" };
console.log(jsObject)
Dynamic import
const jsObject = await import("./test.json", {assert: { type: "json" }});
console.log(jsObject);
Read more at:
v8 Import assertions
So many answers, and no one ever made a benchmark to compare sync vs async vs require. I described the difference in use cases of reading json in memory via require, readFileSync and readFile here.
If you are looking for a complete solution for Async loading a JSON file from Relative Path with Error Handling
// Global variables
// Request path module for relative path
const path = require('path')
// Request File System Module
var fs = require('fs');
// GET request for the /list_user page.
router.get('/listUsers', function (req, res) {
console.log("Got a GET request for list of users");
// Create a relative path URL
let reqPath = path.join(__dirname, '../mock/users.json');
//Read JSON from relative path of this file
fs.readFile(reqPath , 'utf8', function (err, data) {
//Handle Error
if(!err) {
//Handle Success
console.log("Success"+data);
// Parse Data to JSON OR
var jsonObj = JSON.parse(data)
//Send back as Response
res.end( data );
}else {
//Handle Error
res.end("Error: "+err )
}
});
})
Directory Structure: