calculate list of the sums of each column from csv - javascript

I'm trying to calculate sums of each columns of csv. I'm able to read a csv in js using readfile method. I also was able to loop through it and parsed data into array of objects. Now I just to figure out a way to add up all the column elements, that's where I'm struggling. My csv object is in array of object format which looks like this.
[
{ item: '18', count: '180' },
{ item: '19', count: '163' },
{ item: '20', count: '175' },
{ item: '', count: undefined }
]
CSV input is like this:
item,count
18,180
19,163
20,175
I want to add 18 + 19 + 20 and final answer should look like this [57,518].
Here's I've done so far, I just need help to make this better and column wise adding logic in JS, please help.
const fs = require('fs')
let result = []
var dataArray = []
fs.readFile(filename, 'utf8', function (err, data) {
dataArray = data.split(/\r?\n/);
// console.log("dataArray", dataArray)
var headers = dataArray[0].split(",");
for (var i = 1; i < dataArray.length; i++) {
var obj = {};
console.log("dataArray", dataArray)
var currentline = dataArray[i].split(",");
for (var j = 0; j < headers.length; j++) {
obj[headers[j]] = currentline[j];
}
result.push(obj);
}
})

You can iterate through each row of your csv and sum up values of items and count using array#reduce and array#forEach.
const fs = require('fs').promises;
const fileName = 'data.csv'
const calculateSum = async () => {
const data = await fs.readFile(fileName, 'utf-8');
const dataArray = data.split(/\r?\n/);
const header = dataArray[0].split(',')
const result = dataArray.slice(1).reduce((sum, arr) => {
arr.split(',').forEach((v, i) => {
sum[i] = (sum[i] || 0) + Number(v.trim());
})
return sum;
}, []);
console.log(result);
}

Generic function
let dataArray = [
{ item: '18', count: '180' },
{ item: '19', count: '163' },
{ item: '20', count: '175' },
{ item: '', count: undefined }
]
const sums = dataArray.reduce((sum, tableRow) => {
Object.keys(tableRow).forEach((obj) => {
if (Number(tableRow[obj])) sum[obj] = (sum[obj] || 0) + Number(tableRow[obj]);
})
return sum;
}, []);
console.log(sums) // [ item: 57, count: 518 ]

Related

Object transformation assistance

I have a list of single key value pairs, where the key is a 2 part string that describes where the value should be plotted on a table. This is the first time I've asked a questions on SO so please go easy on me.
let tiles = [
{ 'A~baz': 'x' },
{ 'A~buzz': 'o' },
{ 'A~fam': '' },
{ 'B~baz': 'x' },
{ 'B~buzz': '' },
{ 'B~fam': '' },
{ 'C~baz': 'x' },
{ 'C~buzz': 'x' },
{ 'C~fam': 'x' }
]
I want to convert it into the below format.
[
{ _id: 'A', baz: 'x', buzz: 'o', fam: '' },
{ _id: 'B', baz: 'x', buzz: '', fam: '' },
{ _id: 'C', baz: 'x', buzz: 'x', fam: 'x' }
]
Note I will need to perform this operation on hundreds of thousands of key value pairs.
What I have done so far, this works, but I was hoping there could be places I can make improvements.
let tiles = [
{ 'C~fam': "x" },
{ 'B~buzz': "" },
{ 'B~fam': "" },
{ 'A~fam': "" },
{ 'A~buzz': "o" },
{ 'B~baz': "x" },
{ 'A~baz': "x" },
{ 'C~baz': "x" },
{ 'C~buzz': "x" },
];
// I thought it would help to sort the array
tiles.sort((a, b) => Object.keys(a)[0].localeCompare(Object.keys(b)[0]));
let obj = {};
tiles.forEach((kvp) => { //kvp = key value pair
let [row,col] = Object.keys(kvp)[0].split('~') //destruct by '~'
let val = Object.values(kvp)[0];
obj[row] = obj[row] ?? {}
obj[row][col] = val;
})
let keys = Object.keys(obj);
let values = Object.values(obj)
let output = [];
for (let i = 0, len = keys.length; i < len; i++) {
output.push(Object.assign({_id : `${keys[i]}`}, values[i]));
}
You condemned your algorithm's complexity to O(nlog(n)) by sorting the array. You can solve this problem without the need to sort it. Since we must iterate through all of the array, the best complexity possible would be O(n). Assuming the input format will always remain the same, try this:
function changeFormat (arr){
const hash = {}
arr.forEach(element => {
const key = Object.keys(element)[0];
const _id = key[0];
if (hash[_id] === undefined)
hash[_id] ={_id, baz:'', buzz:'', fam:''};
const type = key.slice(2);
hash[_id][type] = element[key];
});
return Object.values(hash);
}
Here you've got single loop solution
let res = {};
let arrayRes = [];
tiles.forEach(function(tile) {
let tileKey = Object.keys(tile)[0];
let tileKeySplitted = tileKey.split('~');
let column = tileKeySplitted[0];
let key = tileKeySplitted[1];
if (res[column] == null) {
res[column] = {'_id': column};
arrayRes.push(res[column]);
}
res[column][key] = tile[tileKey];
});
console.log(arrayRes);
You can put this code in a function and reuse, this code will work even if the props change from baz, buzz, fam.
let requiredFormat = tiles.reduce((acc, tile) => {
let keys = Object.keys(tile);
let firstKey = keys[0];
let firstKeyArray = firstKey.split("~");
let id = firstKeyArray[0];
let propName = firstKeyArray[1];
let objWithId = acc.find(obj => obj._id === id);
if(objWithId) {
let accumulatorWithoutCurrentObject = acc.filter(obj => obj._id !== id);
let upadtedObjWithId = {...objWithId, [propName]: tile[firstKey]};
let updatedAcc = [
...accumulatorWithoutCurrentObject,
upadtedObjWithId
];
return updatedAcc;
}
let updatedAcc = [
...acc,
{_id: id, [propName]: tile[firstKey]}
];
return updatedAcc;
}, []);

How to invert the structure of nested array of objects in Javascript?

I currently have an array that has the following structure:
data = [
{
time: 100,
info: [{
name: "thing1",
count: 3
}, {
name: "thing2",
count: 2
}, {
}]
},
{
time: 1000,
info: [{
name: "thing1",
count: 7
}, {
name: "thing2",
count: 0
}, {
}]
}
];
But I would like to restructure the array to get something like this:
data = [
{
name: "thing1",
info: [{
time: 100,
count: 3
}, {
time: 1000,
count: 7
}, {
}]
},
{
name: "thing2",
info: [{
time: 100,
count: 2
}, {
time: 1000,
count: 0
}, {
}]
}
];
So basically the key would have to be switched from time to name, but the question is how. From other posts I have gathered that using the map function might work, but since other posts had examples to and from different structures I am still not sure how to use this.
There are a number of ways to achieve this however, the key idea will be to perform a nested looping of both data items and their (nested) info items. Doing that allows your algorithm to "visit" and "map" each piece of input data, to a corresponding value in the resulting array.
One way to express that would be to use nested calls to Array#reduce() to first obtaining a mapping of:
name -> {time,count}
That resulting mapping would then be passed to a call to Object.values() to transform the values of that mapping to the required array.
The inner workings of this mapping process are summarized in the documentation below:
const data=[{time:100,info:[{name:"thing1",count:3},{name:"thing2",count:2},{}]},{time:1e3,info:[{name:"thing1",count:7},{name:"thing2",count:0},{}]}];
const result =
/* Obtain array of values from outerMap reduce result */
Object.values(
/* Iterate array of data items by reduce to obtain mapping of
info.name to { time, count} value type */
data.reduce((outerMap, item) =>
/* Iterate inner info array of current item to compound
mapping of info.name to { time, count} value types */
item.info.reduce((innerMap, infoItem) => {
if(!infoItem.name) {
return innerMap
}
/* Fetch or insert new { name, info } value for result
array */
const nameInfo = innerMap[ infoItem.name ] || {
name : infoItem.name, info : []
};
/* Add { time, count } value to info array of current
{ name, info } item */
nameInfo.info.push({ count : infoItem.count, time : item.time })
/* Compound updated nameInfo into outer mapping */
return { ...innerMap, [ infoItem.name] : nameInfo }
}, outerMap),
{})
)
console.log(result)
Hope that helps!
The approach I would take would be to use an intermediate mapping object and then create the new array from that.
const data = [{time: 100, info: [{name: "thing1", count: 3}, {name: "thing2", count: 2}, {}]}, {time: 1e3, info: [{name: "thing1", count: 7}, {name: "thing2", count: 0}, {}]} ];
const infoByName = {};
// first loop through and add entries based on the name
// in the info list of each data entry. If any info entry
// is empty ignore it
data.forEach(entry => {
if (entry.info) {
entry.info.forEach(info => {
if (info.name !== undefined) {
if (!infoByName[info.name]) {
infoByName[info.name] = [];
}
infoByName[info.name].push({
time: entry.time,
count: info.count
});
}
});
}
});
// Now build the resulting list, where name is entry
// identifier
const keys = Object.keys(infoByName);
const newData = keys.map(key => {
return {
name: key,
info: infoByName[key]
};
})
// newData is the resulting list
console.log(newData);
Well, the other guy posted a much more elegant solution, but I ground this one out, so I figured may as well post it. :)
var data = [
{
time: 100,
info: [{
name: "thing1",
count: 3
}, {
name: "thing2",
count: 2
}, {
}]
},
{
time: 1000,
info: [{
name: "thing1",
count: 7
}, {
name: "thing2",
count: 0
}, {
}]
}
];
var newArr = [];
const objInArray = (o, a) => {
for (var i=0; i < a.length; i += 1) {
if (a[i].name === o)
return true;
}
return false;
}
const getIndex = (o, a) => {
for (var i=0; i < a.length; i += 1) {
if (a[i].name === o) {
return i;
}
}
return false;
}
const getInfoObj = (t, c) => {
let tmpObj = {};
tmpObj.count = c;
tmpObj.time = t;
return tmpObj;
}
for (var i=0; i < data.length; i += 1) {
let t = data[i].time;
for (var p in data[i].info) {
if ("name" in data[i].info[p]) {
if (objInArray(data[i].info[p].name, newArr)) {
let idx = getIndex(data[i].info[p].name, newArr);
let newInfoObj = getInfoObj(t, data[i].info[p].count);
newArr[idx].info.push(newInfoObj);
} else {
let newObj = {};
newObj.name = data[i].info[p].name;
let newInfo = [];
let newInfoObj = getInfoObj(t, data[i].info[p].count);
newInfo.push(newInfoObj);
newObj.info = newInfo;
newArr.push(newObj);
}}
}
}
console.log(newArr);
try to use Object.keys() to get the key

How to convert an unorganized array into an grouped array by id

I'm trying to create an array that contains objects with an id and amount, grouped by id. The ids needs to be unique. So if there is 2 objects with same id, the amount will be added.
I can do it with nested for-loops, but I find this solution inelegant and huge. Is there a more efficient or cleaner way of doing it?
var bigArray = [];
// big Array has is the source, it has all the objects
// let's give it 4 sample objects
var object1 = {
id: 1,
amount: 50
}
var object2 = {
id: 2,
amount: 50
}
var object3 = {
id: 1,
amount: 150
}
var object4 = {
id: 2,
amount:100
}
bigArray.push(object1,object2,object3,object4);
// organizedArray is the array that has unique ids with added sum. this is what I'm trying to get
var organizedArray = [];
organizedArray.push(object1);
for(var i = 1; i < bigArray.length; i++ ) {
// a boolean to keep track whether the object was added
var added = false;
for (var j = 0; j < organizedArray.length; j++){
if (organizedArray[j].id === bigArray[i].id) {
organizedArray[j].amount += bigArray[i].amount;
added = true;
}
}
if (!added){
// it has object with new id, push it to the array
organizedArray.push(bigArray[i]);
}
}
console.log(organizedArray);
You can definitly make it cleaner and shorter by using reduce, not sure about efficiency though, i would say a traditional for loop is more efficient :
var bigArray = [];
var object1 = {id: 1, amount: 50}
var object2 = {id: 2, amount: 50}
var object3 = {id: 1, amount: 150}
var object4 = {id: 2, amount: 100}
bigArray.push(object1, object2, object3, object4);
var organizedArray = bigArray.reduce((acc, curr) => {
// check if the object is in the accumulator
const ndx = acc.findIndex(e => e.id === curr.id);
if(ndx > -1) // add the amount if it exists
acc[ndx].amount += curr.amount;
else // push the object to the array if doesn't
acc.push(curr);
return acc;
}, []);
console.log(organizedArray)
Rather than an organized array, how about a single object whose keys are the ids and values are the sums.
var bigArray = [
{ id: 1, amount: 50 },
{ id: 2, amount: 50 },
{ id: 1, amount: 150 },
{ id: 2, amount: 100 }
];
let total = {}
bigArray.forEach(obj => {
total[obj.id] = (total[obj.id] || 0) + obj.amount;
});
console.log(total);
If you really need to convert this to an array of objects then you can map the keys to objects of your choosing like this:
var bigArray = [
{ id: 1, amount: 50 },
{ id: 2, amount: 50 },
{ id: 1, amount: 150 },
{ id: 2, amount: 100 }
];
let total = {}
bigArray.forEach(obj => {
total[obj.id] = (total[obj.id] || 0) + obj.amount;
});
console.log(total);
// If you need the organized array:
let organizedArray = Object.keys(total).map(key => ({ id: key, amount: total[key] }));
console.log(organizedArray);
function getUniqueSums(array) {
const uniqueElements = [];
const arrayLength = array.length;
for(let index = 0; index < arrayLength; index++) {
const element = array[index];
const id = element.id;
const uniqueElement = findElementByPropertyValue(uniqueElements, 'id', id);
if (uniqueElement !== null) {
uniqueElement.amount += element.amount;
continue;
}
uniqueElements.push(element);
}
return uniqueElements;
}
function findElementByPropertyValue(array, property, expectedValue) {
const arrayLength = array.length;
for(let index = 0; index < arrayLength; index++) {
const element = array[index];
const value = element[property];
if (value !== expectedValue) {
continue;
}
return element;
}
return null;
}
This is an untested code. You will be able to understand the logic. Logic is almost same yours. But, perhaps a more readable code.

How to fix csv to json converter module?

I can't figure out how to match the title and genre correctly based on what I have in my module.
The csv_json module has an exception where it doesn't match each of the properties accordingly and that is when the title has "The" in it.
//csv file
movieId,title,genre
1,"American President, The (1995)",Comedy|Drama|Romance
2,"Creation, The creator(xxxx)",Comedy|Drama|Romance
3,"Destruction, The destroyer(xxxxx)",Comedy|Drama|Romance
//csv_json module
const readline = require('readline');
const fs = require('fs');
function readCsv(pathToFile) {
return new Promise((resolve, reject) => {
const csvReader = readline.createInterface({
input: fs.createReadStream(pathToFile)
});
let headers;
const rows = [];
let tempRows = [];
csvReader
.on('line', row => {
if (!headers) {
headers = row.split(','); // header name breed age
} else {
rows.push(row.split(','));
}
})
.on('close', () => {
// then iterate through all of the "rows", matching them to the "headers"
for (var i = 0; i < rows.length; i++) {
var obj = {};
var currentline = rows[i];
for (var j = 0; j < headers.length; j++) {
obj[headers[j]] = currentline[j]; //Kitty Siamese 14
}
tempRows.push(obj);
}
resolve(JSON.stringify(tempRows));
});
// This would be in place of the "return" statement you had before
});
}
module.exports = readCsv;
//js file
const readCsv = require('./csvjson.js');
readCsv('movieTest.csv').then((data) => {
console.log(data)
let movieJson = JSON.parse(data);
console.log(movieJson)
/*data output:
[{"movieId":"1","title":"\"American President","genre":" The (1995)\""},{"movieId":"2","title":"\"Creation","genre":" The creator(xxxx)\""},{"movieId":"3","title":"\"Destruction","genre":" The destroyer(xxxxx)\""}]
*/
/*movieJson output:
[ { movieId: '1',
title: '"American President',
genre: ' The (1995)"' },
{ movieId: '2',
title: '"Creation',
genre: ' The creator(xxxx)"' },
{ movieId: '3',
title: '"Destruction',
genre: ' The destroyer(xxxxx)"' } ]
*/
});
I expect the output to match:
[ { movieId: '1',
title: "American President, The (1995)",
genre:'Comedy|Drama|Romance' },
{ movieId: '2',
title: "The creator(xxxx) Creation",
genre: ' Comedy|Drama|Romance' },
{ movieId: '3',
title: "Destruction The destroyer(xxx)",
genre: ' Comedy|Drama|Romance' } ]
This is probably since you're splitting each row on every occurrence of a comma.
const row = '1,"American President, The (1995)",Comedy|Drama|Romance'
row.split(',')
// returns ["1", ""American President", " The (1995)"", "Comedy|Drama|Romance"]
Try replacing every comma that is not followed by a whitespace with some unique string that wouldn't occur anywhere else in the CSV file, and then split on that:
row.replace(/\,(\S)/g, '&unique;$1').split('&unique;')
// returns ["1", ""American President, The (1995)"", "Comedy|Drama|Romance"]
Hope this helps! :)

how to count duplicate values object to be a value of object

how to count the value of object in new object values
lets say that i have json like this :
let data = [{
no: 3,
name: 'drink'
},
{
no: 90,
name: 'eat'
},
{
no: 20,
name: 'swim'
}
];
if i have the user pick no in arrays : [3,3,3,3,3,3,3,3,3,3,3,90,20,20,20,20]
so the output should be an array
[
{
num: 3,
total: 11
},
{
num: 90,
total: 1
},
{
num:20,
total: 4
}
];
I would like to know how to do this with a for/of loop
Here is the code I've attempted:
let obj = [];
for (i of arr){
for (j of data){
let innerObj={};
innerObj.num = i
obj.push(innerObj)
}
}
const data = [{"no":3,"name":"drink"},{"no":90,"name":"eat"},{"no":20,"name":"swim"}];
const arr = [3,3,3,3,3,3,3,3,3,3,3,20,20,20,20,80,80];
const lookup = {};
// Loop over the duplicate array and create an
// object that contains the totals
for (let el of arr) {
// If the key doesn't exist set it to zero,
// otherwise add 1 to it
lookup[el] = (lookup[el] || 0) + 1;
}
const out = [];
// Then loop over the data updating the objects
// with the totals found in the lookup object
for (let obj of data) {
lookup[obj.no] && out.push({
no: obj.no,
total: lookup[obj.no]
});
}
document.querySelector('#lookup').textContent = JSON.stringify(lookup, null, 2);
document.querySelector('#out').textContent = JSON.stringify(out, null, 2);
<h3>Lookup output</h3>
<pre id="lookup"></pre>
<h3>Main output</h3>
<pre id="out"></pre>
Perhaps something like this? You can map the existing data array and attach filtered array counts to each array object.
let data = [
{
no: 3,
name: 'drink'
},
{
no:90,
name: 'eat'
},
{
no:20,
name: 'swim'
}
]
const test = [3,3,3,3,3,3,3,3,3,3,3,90,20,20,20,20]
const result = data.map((item) => {
return {
num: item.no,
total: test.filter(i => i === item.no).length // filters number array and then checks length
}
})
You can check next approach using a single for/of loop. But first I have to create a Set with valid ids, so I can discard noise data from the test array:
const data = [
{no: 3, name: 'drink'},
{no: 90, name: 'eat'},
{no: 20, name: 'swim'}
];
const userArr = [3,3,3,3,3,3,3,3,7,7,9,9,3,3,3,90,20,20,20,20];
let ids = new Set(data.map(x => x.no));
let newArr = [];
for (i of userArr)
{
let found = newArr.findIndex(x => x.num === i)
if (found >= 0)
newArr[found].total += 1;
else
ids.has(i) && newArr.push({num: i, total: 1});
}
console.log(newArr);

Categories