I iterate over the repository and parse JSON files there. I populate batch array with the data in order to use it later for batch insert using pg.
This is how I populate batch array
const batch = [];
dir.readFiles('../resources/data/', { match: /\.json$/ }, (err, content, next) => {
if (err) throw err;
const jsn = JSON.parse(content);
const generatorName = jsn.name;
const generatorParts = jsn.name_parts;
Object.entries(generatorParts).forEach(([key, value]) => {
const pr_sp = value.precede_space;
const ordr = value.order;
const nm_prts = value.parts;
nm_prts.forEach(item => {
batch.push([generatorName, key, item, ordr, pr_sp]);
});
});
next();
});
console.log(batch);
But console.log shows that the batch is empty. What is the problem?
Related
I am using Algolia for search purposes and we got a huge pile of records. We want to delete some records and have decided to delete records that are older than X date.
First I was using this
const records = [];
const deleteRecordsBeforeDateAlgolia = (date) => {
let client;
*****
const index = client.initIndex('function_message');
//get the records before the given date
try {
index.search('',{
filters: `time_stamp < ${date}`
}).then(({hits}) => {
if(hits.length > 0) {
for (const hit of hits) {
index.deleteObject(hit.objectID);
records.push(hit.objectID);
}
}
if(hits.length === 0) {
console.log(`Deleted ${records.length} records`);
} else {
deleteRecordsBeforeDateAlgolia(date);
}
});
} catch (err) {
console.error(err);
}
};
but I realized this isnt that optimized + will be very slow when deleting on prod. Can you tell me how I can get huge amounts of data with a filter (timestamp in this case) and then delete all of them?
EDIT
const records = [];
const deleteRecordsBeforeDateAlgolia = (date) => {
let client;
//creds stuff
const index = client.initIndex('function_message');
//get the records before the given date
try {
const search = index.browseObjects({
filters: `time_stamp < ${date}`
}).then(res => {
//IT SHOWS RESPONSE IS UNDEFINED
res.forEach(record => {
records.push(record);
});
console.log(`found ${records.length} records`);
});
} catch (err) {
console.error(err);
}
};
browseObjects takes a batch callback function that's called on every batch of hits where you can specify what to do with the batch. The optional parameter list can be found here
Something like this should work
const records = [];
const deleteFromIndex = (idArray,index) => {
index.deleteObjects(idArray).then(({ objectIDs }) => {
console.log(objectIDs);
});
}
const deleteRecordsBeforeDateAlgolia = (date) => {
let client;
client = algoliasearch('algoliaApp', 'algoliaKey');
const index = client.initIndex('function_message');
try {
index.browseObjects({
filters: `time_stamp<${date}`,
query: '',
batch: (batch) => {
records.push(...batch); //push each batch into records array
}
})
.then(() => {
const idArray = records.map(({objectID}) => objectID) //get an array of objectIDs
deleteFromIndex(idArray, index)
});
} catch (err) {
console.error(err);
}
};
deleteRecordsBeforeDateAlgolia('some date')
I have a cloud function receiving a json string in a pubsub topic.
The goal is to extracts some data into a new json string.
Next parse it as JSONL.
And finally stream it to Google Cloud Storage.
I notice that sometimes the files seem to contain data and sometimes they do not.
The pubsub is working fine and data is coming into this cloud function just fine.
I tried adding some async awaits where I seem it might fit but I am afraid it has do to with the bufferstream. Both topics on where I have trouble getting my head around.
What could be the issue?
const stream = require('stream');
const { Storage } = require('#google-cloud/storage');
// Initiate the source
const bufferStream = new stream.PassThrough();
// Creates a client
const storage = new Storage();
// save stream to bucket
const toBucket = (message, filename) => {
// Write your buffer
bufferStream.end(Buffer.from(message));
const myBucket = storage.bucket(process.env.BUCKET);
const file = myBucket.file(filename);
// Pipe the 'bufferStream' into a 'file.createWriteStream' method.
bufferStream.pipe(file.createWriteStream({
validation: 'md5',
}))
.on('error', (err) => { console.error(err); })
.on('finish', () => {
// The file upload is complete.
console.log(`${filename} is uploaded`);
});
};
// extract correct fields
const extract = (entry) => ({
id: entry.id,
status: entry.status,
date_created: entry.date_created,
discount_total: entry.discount_total,
discount_tax: entry.discount_tax,
shipping_total: entry.shipping_total,
shipping_tax: entry.shipping_tax,
total: entry.total,
total_tax: entry.total_tax,
customer_id: entry.customer_id,
payment_method: entry.payment_method,
payment_method_title: entry.payment_method_title,
transaction_id: entry.transaction_id,
date_completed: entry.date_completed,
billing_city: entry.billing.city,
billing_state: entry.billing.state,
billing_postcode: entry.billing.postcode,
coupon_lines_id: entry.coupon_lines.id,
coupon_lines_code: entry.coupon_lines.code,
coupon_lines_discount: entry.coupon_lines.discount,
coupon_lines_discount_tax: entry.coupon_lines.discount_tax,
});
// format json to jsonl
const format = async (message) => {
let jsonl;
try {
// extract only the necessary
const jsonMessage = await JSON.parse(message);
const rows = await jsonMessage.map((row) => {
const extractedRow = extract(row);
return `${JSON.stringify(extractedRow)}\n`;
});
// join all lines as one string with no join symbol
jsonl = rows.join('');
console.log(jsonl);
} catch (e) {
console.error('jsonl conversion failed');
}
return jsonl;
};
exports.jsonToBq = async (event, context) => {
const message = Buffer.from(event.data, 'base64').toString();
const { filename } = event.attributes;
console.log(filename);
const jsonl = await format(message, filename);
toBucket(jsonl, filename);
};
it's fixed by moving the bufferstream const into the tobucket function.
var emp = db.collection('BookedTicketData').get().then((snapshot) => {
snapshot.docs.forEach((doc) => {
data = doc.data();
bseat = data.AllSeat
// console.log(bseat)
allseat.concat(bseat)
})
console.log(allseat)
return allseat;
}).then((alls) => {
console.log(alls)
})
I have done this code to get the array from the doucumnets of firebase and it is coming seperatly i want to combine all the array in single array and print the array in console.log(alls)
1-> [4,46,324,346,345,234,3446,36]
2-> [324,6,3,44,6,2,6,35,2,7,23]
alls -> [4,46,324,346,345,234,3446,36,3244,6,3,44,6,2,6,35,2,7,23]
If I correctly understand your question, the following should do the trick:
var emp = db
.collection('BookedTicketData')
.get()
.then((snapshot) => {
let allseat = [];
snapshot.docs.forEach((doc) => {
data = doc.data();
bseat = data.AllSeat;
// console.log(bseat)
allseat = allseat.concat(bseat);
});
console.log(allseat);
return allseat;
})
So I'm triyng to update some ids from a categories tree using TreeModelJS.
after editing I would like to dump the tree to a file in JSON format.
but when outputing other keys from TreeModel gets outputed as well.
How could I output edited tree as JSON (model only)?
I managed to replace other keys values with null and so far I got this:
const axios = require('axios')
const TreeModel = require('tree-model')
const fs = require('fs')
const url = 'https://my-api-uri-for-categories'
const dumpPath = `${process.cwd()}/data/test/categories.json`
const getCategories = async () => {
try {
const response = await axios.get(url)
return response.data.categories
} catch (error) {
console.log('Error reading categories', error)
}
}
const dumpJsonTofile = data => {
try {
console.log('Dumping to file')
console.log(data)
fs.writeFileSync(
dumpPath,
JSON.stringify(data, (k, v) => {
if (k === 'parent' || k === 'config' || k === 'children') return null
else return v
}),
'utf8'
) // write it back
} catch (error) {
console.log('Error dumping categories', error)
}
}
const scraping = async category => {
try {
const response = await axios.get(category.url)
const document = response.data
const json = document.match(/{"searchTerm"(.*);/g)[0]
const data = JSON.parse(json.replace(';', ''))
return data
} catch (error) {
console.log(`Error while scraping category: ${category.name}`, error)
}
}
async function run() {
const categories = await getCategories()
const categoriesTree = new TreeModel({
childrenPropertyName: 'items',
})
const root = categoriesTree.parse({ id: 0, origin: {}, items: categories })
root.walk(async node => {
const category = node.model
console.log(`scraping category: ${category.name}...`)
if (!category.url) return console.log(`skipping (root?)...`)
const data = await scraping(category)
category.id = data.categoryId
})
dumpJsonTofile(root)
}
run()
but that still outputs a Node object like this:
{
"config":null,
"model":{},
"children":null
}
I need to output all the tree showing only the model key value for each item
Try JSON.stringify(root.model).
I am struggling to find a way to write data to a CSV in Node.js.
There are several CSV plugins available however they only 'write' to stdout.
Ideally I want to write on a row-by-row basis using a loop.
You can use fs (https://nodejs.org/api/fs.html#fs_fs_writefile_file_data_options_callback):
var dataToWrite;
var fs = require('fs');
fs.writeFile('form-tracking/formList.csv', dataToWrite, 'utf8', function (err) {
if (err) {
console.log('Some error occured - file either not saved or corrupted file saved.');
} else{
console.log('It\'s saved!');
}
});
The docs for node-csv-parser (npm install csv) specifically state that it can be used with streams (see fromStream, toStream). So it's not hard-coded to use stdout.
Several other CSV parsers also come up when you npm search csv -- you might want to look at them too.
Here is a simple example using csv-stringify to write a dataset that fits in memory to a csv file using fs.writeFile.
import stringify from 'csv-stringify';
import fs from 'fs';
let data = [];
let columns = {
id: 'id',
name: 'Name'
};
for (var i = 0; i < 10; i++) {
data.push([i, 'Name ' + i]);
}
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err;
fs.writeFile('my.csv', output, (err) => {
if (err) throw err;
console.log('my.csv saved.');
});
});
If you want to use a loop as you say you can do something like this with Node fs:
let fs = require("fs")
let writeStream = fs.createWriteStream('/path/filename.csv')
someArrayOfObjects.forEach((someObject, index) => {
let newLine = []
newLine.push(someObject.stringPropertyOne)
newLine.push(someObject.stringPropertyTwo)
....
writeStream.write(newLine.join(',')+ '\n', () => {
// a line was written to stream
})
})
writeStream.end()
writeStream.on('finish', () => {
console.log('finish write stream, moving along')
}).on('error', (err) => {
console.log(err)
})
In case you don't wanna use any library besides fs, you can do it manually.
let fileString = ""
let separator = ","
let fileType = "csv"
let file = `fileExample.${fileType}`
Object.keys(jsonObject[0]).forEach(value=>fileString += `${value}${separator}`)
fileString = fileString.slice(0, -1)
fileString += "\n"
jsonObject.forEach(transaction=>{
Object.values(transaction).forEach(value=>fileString += `${value}${separator}`)
fileString = fileString.slice(0, -1)
fileString += "\n"
})
fs.writeFileSync(file, fileString, 'utf8')
For those who prefer fast-csv:
const { writeToPath } = require('#fast-csv/format');
const path = `${__dirname}/people.csv`;
const data = [{ name: 'Stevie', id: 10 }, { name: 'Ray', id: 20 }];
const options = { headers: true, quoteColumns: true };
writeToPath(path, data, options)
.on('error', err => console.error(err))
.on('finish', () => console.log('Done writing.'));
**In case you don't wanna use any library besides fs, you can do it manually. More over you can filter the data as you want to write to CSV file
**
router.get('/apiname', (req, res) => {
const data = arrayOfObject; // you will get from somewhere
/*
// Modify old data (New Key Names)
let modifiedData = data.map(({ oldKey1: newKey1, oldKey2: newKey2, ...rest }) => ({ newKey1, newKey2, ...rest }));
*/
const path = './test'
writeToFile(path, data, (result) => {
// get the result from callback and process
console.log(result) // success or error
});
});
writeToFile = (path, data, callback) => {
fs.writeFile(path, JSON.stringify(data, null, 2), (err) => { // JSON.stringify(data, null, 2) help you to write the data line by line
if (!err) {
callback('success');
// successfull
}
else {
callback('error');
// some error (catch this error)
}
});
}
this is the code that worked for me in nest js
import { Parser } from "json2csv";
const csv = require('csvtojson');
const csvFilePath = process.cwd() + '/' + file.path;
let csv data = await csv().fromFile(csvFilePath); /// read data from csv into an array of json
/// * from here how to write data into csv *
data.push({
label: value,
.......
})
}
const fields = [
'field1','field2', ...
]
const parser = new Parser({ fields, header:false }); /// if dont want header else remove header: false
const csv = parser.parse(data);
appendFileSync('./filename.csv',`${csv}\n`); // remove /n if you dont want new line at the end