Read value at index in a huge json array - javascript

I need to get the value at a given index in a json array containing millions of entries.
The json file looks a bit like this:
[
{"street":"123 Here street", "city":"Here city"},
{"street":"456 There street", "city":"There city"}
]
I can get away with it using stream-json with the following code:
const { chain } = require('stream-chain');
const { parser } = require('stream-json');
const { streamArray } = require('stream-json/streamers/StreamArray');
const fs = require('fs');
const chosenIndex = 8
const pipeline = chain([
fs.createReadStream(filepath),
parser(),
streamArray()
]);
pipeline.on('data', data => {
if (data.key == chosenIndex) {
console.log(data.value);
}
});
pipeline.on('end', () => { console.log('All Done'); });
I'm just afraid this is not the fastest way to do this.
Thanx in advance

Related

how to convert to json from excel in js i have prublem with spilt lines

i have excel file That I uploaded to react i'm use XLSX library
this is my excel
my orginal excel
and i try to do Something like that
item[{
CJ Order Number:14,
Your Order Number:100101,
SKU:{
CJNSXZHL:1,
CJNSX:1},
},
{
CJ Order Number:15,
Your Order Number:100102,
SKU:{
CJNSXZN:1,// 1 is quentity
CJNS:1},
},
{
CJ Order Number:16,
Your Order Number:100103,
SKU:{
CJNSX:1,
CJNS:1},
},
{
CJ Order Number:17,
Your Order Number:100103,
SKU:{
CJNSTX:2
},
}]
but i got Something like this
[{CJ Order Number: '14',
Your Order Number: '100101',
SKU: 'CJNSXZHL',
CJ Quantity: '1'}
{SKU: 'CJNSXL',
CJ Quantity: '1'}]
my code is
const [items, setItems] = useState([]);
const readExcel = (file) => {
const promise = new Promise((resolve, reject) => {
const fileReader = new FileReader();
fileReader.readAsArrayBuffer(file);
fileReader.onload = (e) => {
const bufferArray = e.target.result;
const wb = XLSX.read(bufferArray, { type: "buffer" });
const wsname = wb.SheetNames[0];
const ws = wb.Sheets[wsname];
const data = XLSX.utils.sheet_to_json(ws);
resolve(data);
};
fileReader.onerror = (error) => {
reject(error);
};
});
promise.then((d) => {
setItems(d);
});
}
I compare products that want to buy
Do I have them in my inventory
It could be that a person has ordered 2 products and I only have one order number so I want to know how I pass the Excel standard
That every line that is double he will assign it to one person
And not to be treated as another separate line
That it's a mistake
I solved the problem using the following code I hope it will help more people who have had difficulty with it
I continue the code above
const [items, setItems] = useState([]);
setItems(d);
let i = 0
for ( i ; i <= items.length - 1; i++) {
try {
if (items[i].Orders) {
arr.push({
"OrderNumber":items[i].Orders,
"Address":items[i].Address,
"SKU": [items[i].SKU],
"Quantity":[parseInt(items[i].Quantity)],
"Province":items[i].Province,
"Product_Name":[items[i].Product_Name],
"Name":items[i].Name,
"City":items[i].City,
"ZipCode":items[i].ZipCode,
"Phone":(items[i].Phone),
});
} else {
arr[arr.length - 1].SKU.push(items[i].SKU);
arr[arr.length - 1].Quantity.push(parseInt(items[i].Quantity));
arr[arr.length - 1].Product_Name.push(items[i].Product_Name);
}
} catch (err) {
console.log("err",err);
}
}

How to wait data from useLazyQuery

I'm trying to run two sequencitally graphQL requests, the first one give me data that I need into the second one parameters. And I don't know how to wait to the first.
My program is the following one:
I have the declaration of my GraphQL requests:
const [
addConfigurableProductToCart,
{ error: errorAddingSimpleProduct, loading: isAddSimpleLoading }
] = useMutation(ADD_CONFIGURABLE_MUTATION);
const [getDataParentSku, { error, loading, data }] = useLazyQuery(
GET_PARENT_SKU
);
And the main workflow are in this function.
const handleAddProductsToCart = useCallback(
async csvProducts => {
let tempSkuErrorList = [];
for (let i = 0; i < csvProducts.length; i++) {
const orParentSku = getDataVariable(csvProducts[i][0]);
const variables = {
cartId,
quantity: parseInt(csvProducts[i][1], 10),
sku: csvProducts[i][0],
parentSku: orParentSku.then(res => {
return res.products.items[0].orParentSku;
})
};
try {
await addConfigurableProductToCart({
variables
});
} catch {
tempSkuErrorList.push(csvProducts[i][0]);
}
}
},
[
addConfigurableProductToCart,
cartId,
]
);
getDataVariable() is the function who call the first query (useLazyQuery()). And its content is:
const getDataVariable = useCallback(
async sku => {
getDataParentSku({
variables: { sku: sku }
});
return await data;
},
[getDataParentSku, data]
);
The error that I have been finding all the time is that when I need the data, is undefined.
Another option was the idea of using this library https://www.npmjs.com/package/graphql-lodash, in order to merge the query into one, but is outdated.
Thanks a lot for your help.

Problem with getting one item from json file through get method in Node js

Basically I can get the whole list from json file but I cannot figure out how to get concrete item. When I use postman, path: http://localhost:3000/book/2 for example it returns bookcode does not exist
app.get('/book/list', (req, res) => {
const books = getBookData()
res.send(books)
})
app.get("/book/:code", (req, res) => {
const code = req.params.code
const existsBook = getBookData();
const filterBook = existsBook.filter(book => book.code !== code)
if (existsBook.length === filterBook.length) {
return res.status(400).send({error: true, msg: 'bookcode does not exist'})
}
})
const getBookData = () => {
const jsonData = fs.readFileSync('books.json')
return JSON.parse(jsonData.toString())
}
here is json file
[{"code":2,"name":"Name of the wind","author":"Patrrick Rothfuss"},{"code":3,"name":"Wise Man's Fear","author":"Patrrick Rothfuss"},{"code":4,"name":"The Doors of Stone","author":"Patrrick Rothfuss"}]
This may be because req.query.code is by default a string, not number. This tripped me up once.
A reliable solution would be to use parseInt.
const code = parseInt(req.params.code)
Not working? You could also try a traditional Number ()
const code = Number(req.params.code)

A JSON file created with NodeJS holds the wrong data

I wrote the following code to create a formatted JSON file that holds all names of files (testfile1, testfile2, testfile3) that exist in a directory called "uploads".
const fs = require("fs");
let toJson = {files: []};
let file = new Object();
function createJSON(){
fs.readdir("./uploads", (err, files) => {
//add each files name to obj
files.forEach((item, index) => {
file.name = item;
console.log(index, item);
toJson.files.push(file);
});
var saveJson = JSON.stringify(toJson, null, 4);
fs.writeFile('./files.json', saveJson, 'utf8', (err) => {
if(err) throw err;
console.log(err);
});
});
}
The output from console.log(item, index) is as expected:
0 testfile1
1 testfile2
2 testfile3
However, the JSON file that is being created holds the following names:
{
"files": [
{
"name": "testfile3"
},
{
"name": "testfile3"
},
{
"name": "testfile3"
}
]
}
instead of the intended
{
"files": [
{
"name": "testfile1"
},
{
"name": "testfile2"
},
{
"name": "testfile3"
}
]
}
Your problem lies in your memory referencing. In the following code, you are setting the name of the file and pushing the file to an array. However, you are not pushing the item to the array, you are pushing the reference of the item to the array. This means that on subsequent iterations, you are changing the attributes of the elements that are in the array as well.
files.forEach((item, index) => {
file.name = item;
console.log(index, item);
toJson.files.push(file);
});
See an example below of these memory issues.
const array = [];
const item = {};
item.name = 5;
array.push(item);
item.name = 4;
console.log(array);
const copy = Object.assign({}, item);
copy.name = 3;
array.push(copy);
item.name = 6;
console.log(array);
To fix this, you need to create a copy of the object at each step. There should be enough information in this post to assist you with this: What is the most efficient way to deep clone an object in JavaScript?
An example of fixed code would be the following:
files.forEach((item, index) => {
const f = Object.assign({}, file);
f.name = item;
console.log(index, item);
toJson.files.push(f);
});
You are not changing the file object inside your forEach loop. You are only changing its name property. That's why you end up with all the same file names
Just move the line file = new Object() or file = {} to the beginning of the forEach loop
You should be calling synchronously. You are also overwriting the same object. You can greatly simplify your code.
const fs = require("fs");
function createJSON(){
const files = fs.readdirSync("./uploads").map((file, index) => {
console.log(index, file)
return {
name: file
}
});
const output = JSON.stringify({
files
}, undefined, 4)
fs.writeFileSync('./files.json', output)
}

NodeJS script to modify a JSON file

I need to write a NodeJS script for the following task:
I have a temp.json file with content like:
{
"name": "foo",
"id": "1.2.15"
}
When we run the script, I want the temp.json files content changed. Specifically, I want the number after the 2nd decimal in id to be incremented as follows:
{
"name": "foo",
"id": "1.2.16"
}
I don't know JavaScript and would appreciate any help.
Thanks!
"use strict";
const fs = require('fs');
const data = JSON.parse(fs.readFileSync("file.json"));
const nums = data.id.split('.');
++nums[2];
data.id = nums.join('.');
fs.writeFileSync("file.json", JSON.stringify(data, null, 4));
And if you want to do it without breaking the async nature of Node, you can do it with the async functions as well:
const fs = require('fs');
fs.readFile('temp.json', 'utf8', (e, data) => {
const obj = JSON.parse(data);
const idParts = obj.id.split('.').map((el) => parseInt(el, 10))
idParts[2] = idParts[2] + 1;
obj.id = idParts.join('.');
fs.writeFile('out.json', JSON.stringify(obj), (err) => {
console.log(err || 'complete');
});
});

Categories