We Keep Coding

Optimize javascript loop on CSV data

I am plotting a graph using d3.js by loading an external .CSV file.
The code i have so far works fine with a small amount of data but when i load a larger file with thousands of lines then it kills the page.
The data has a usage column which is a value for every 30 mins throughout the day, which will go on over several months.
See Plunker example.
var avgClientArr = [];
var dateArr = [];
var dateGroupArr = [];
function csvParseClient() {
d3.xhr('client.csv').get(function(err, response) {
var dirtyCSV = response.responseText;
var initialClientKeys = /TYPE,DATE,START TIME,END TIME,USAGE,UNITS,NOTES/i;
var newClientKeys = "TYPE,x,startTime,endTime,y,UNITS,NOTES";
var csvDataClient = dirtyCSV.replace(initialClientKeys, newClientKeys);
var validData = csvDataClient.substr(csvDataClient.indexOf(newClientKeys));
var csvData = d3.csv.parse(validData);
csvData.customForEach(function(val, i) {
// filter data
//var keep = ['x', 'startTime', 'endTime', 'UNITS', 'y'];
//for (var key in val[i]) {
// if (keep.indexOf(key) === -1) {
// delete val[i][key];
// }
//}
// parse data
var date = val.x;
var usage = val.y;
var startTime = val.startTime;
var endTime = val.endTime;
var x = new Date(date);
var y = parseFloat(usage);
dateArr.push({
"date": x,
"usage": y
})
dateGroupArr = groupBy(dateArr, 'date');
})
console.log(dateGroupArr);
var objDates = objectValues(dateGroupArr);
objDates.customForEach(function(f) {
var avg = f.reduce(function(a, b) {
return a + b.usage;
}, 0) / f.length;
var date = f.reduce(function(a, b) {
return new Date(b.date);
}, 0);
avgClientArr.push({
"x": date,
"y": avg
})
})
//console.log("avgClientArr", avgClientArr);
document.getElementById('arrayDiv').innerHTML = '<pre>' + JSON.stringify(avgClientArr, null, 4) + '</pre>';
})
}
function groupBy(arr, key) {
var reducer = (grouped, item) => {
var group_value = item[key]
if (!grouped[group_value]) {
grouped[group_value] = []
}
grouped[group_value].push(item)
return grouped
}
return arr.reduce(reducer, {})
}
function objectValues(object) {
var values = []
for (var property in object) {
if (object.hasOwnProperty(property)) {
values.push(object[property])
}
}
return values
}
function foreach(fn) {
var arr = this;
var len = arr.length;
for (var i = 0; i < len; ++i) {
fn(arr[i], i);
}
}
Object.defineProperty(Array.prototype, 'customForEach', {
enumerable: false,
value: foreach
});
var t0 = performance.now();
csvParseClient();
var t1 = performance.now();
console.log("Call csvParseClient() " + (t1 - t0) + " milliseconds.");
What i need to happen
I need the average value of usage for the whole day returned as y and the date for that day returned as x for each day.
The slow process i have
Start the loop from a specified line in the CSV file as there is unwanted data on the first few lines.
Group unique date and store each usage value for that date in an object.
Average the usage values for each date.
Output an array of objects with property x being the date and y being the average usage value.
If you can give me any help on how to make this run faster that would be great!

I solved this by using the d3 nest() and rollup() functions, its simple and really fast.
d3.nest()
.key(function(d) {
return d.x;
})
.rollup(function(d) {
var avg = d3.mean(d, function(g) {return g.y; });
return avg;
}).entries(dateArr);

Array of Arrays to Json

I am looping to convert the array of array to object, I am splitting using comma"," but if my values have some comma it is also getting split. Eg: If my value has a date May 2, 2017 it is actually a single value, but it is splitting as May 2 as one value and 2017 as another value so, my whole is object getting changed.need help
I am reading from an excel sheet. My Output is array of array. I need to convert that to a json object.
var xls = require('excel');
describe('TransactionDetails-Excel', function () {
it('External Type G Tranasaction Details validation', function (done) {
xls('transactions.xlsx', function(err, data) {
if(err){
throw err;
}
else{
console.log(data);
console.log(JSON.stringify(convertToJSON(data)));
done();
}
});
function convertToJSON(array) {
var first = array[0].join();
var headers = first.split(',');
var jsonData = [];
for ( var iterator_i = 1, length = array.length; iterator_i < length; iterator_i++ ){
var myRow = array[iterator_i].join();
var row = myRow.split(',');
var data = {};
for ( var x = 0; x < row.length; x++ ){
data[headers[x]] = row[x];
}
jsonData.push(data);
}
return jsonData;
}
});
});
Excel sheet: transactions.xlsx
click here to view screenshot of excel sheet
output: data
[ [ 'Transaction Id', 'Paid ', 'Time Created' ],
[ '0YC0',
'Joe\'s Generic Business \nperson#gmail.com',
'Mar 2, 2012 14:28:36 PST' ],
[ '8uyhr', 'mail#gmail.com', 'May 2, 2015 15:20:20 PST' ],
[ '786YUH', 'mailtome#gmail.com', 'May 18, 2017 13:20:30 PST' ] ]
Output : //this is what i get
[{"Transaction Id":"0YC0","Paid ":"Joe's Generic Business \nperson#gmail.com","Time Created":"Mar 2","undefined":" 2012 14:28:36 PST"},
{"Transaction Id":"8uyhr","Paid ":"mail#gmail.com","Time Created":"May 2","undefined":" 2015 15:20:20 PST"},
{"Transaction Id":"786YUH","Paid ":"mailtome#gmail.com","Time Created":"May 18","undefined":" 2017 13:20:30 PST"}]
But final object should be:
[{"Transaction Id":"0YC0","Paid ":"Joe's Generic Business \nperson#gmail.com","Time Created":"Mar 2, 2012 14:28:36 PST"},
{"Transaction Id":"8uyhr","Paid ":"mail#gmail.com","Time Created":"May 2, 2015 15:20:20 PST"},
{"Transaction Id":"786YUH","Paid ":"mailtome#gmail.com","Time Created":"May 18, 2017 13:20:30 PST"}]
Thanks in Advance.

use this function insted
function convertToJSON(array) {
var objArray = [];
for (var i = 1; i < array.length; i++) {
objArray[i - 1] = {};
for (var k = 0; k < array[0].length && k < array[i].length; k++) {
var key = array[0][k];
objArray[i - 1][key] = array[i][k]
}
}
return objArray;
}

You can do it easily by splitting out the labels and the data, and doing the following:
var labels = data[0]
var output = data.slice(1).map(item => item.reduce((obj, val, index) => {
obj[labels[index]] = val
return obj
}, {}))
This works by maping each item in the array to a new array. With each item undergoing a reduce that converts the data values into an object with the correct keys.
Fiddle here

Join with some custom value and then split
Replace this with
var myRow = array[iterator_i].join(';');
var row = myRow.split(';');

you can try this one also.....
let keys = array.shift();
let arrayofObjects = array.map(subArray => _.object(keys,subArray));
console.log(arrayofObjects);

Iterating over an object in a Google Apps script and printing to Google Sheets

I'm having trouble printing more than one row to my google sheet with this loop.
The first row appends fine, but I want the function to append all objects from the data var.
The data object is properly pulling from Firebase when I verify with a Logger.
var firebaseUrl = "https://test.firebaseio.com/alerts";
var secret = "sssssssssssssssssssss";
var base = FirebaseApp.getDatabaseByUrl(firebaseUrl, secret);
var data = base.getData();
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("feed");
var selection = sheet.getActiveRange();
var range = sheet.getActiveRange();
var values = range.getValues();
var columns = selection.getNumColumns();
var rows = selection.getNumRows();
var num = 2;
function writeToSheets() {
for(var i in data) {
var values = [
[ data[i].id, data[i].two, data[i].three, data[i].four ]
];
var keys = Object.keys(values[0]);
var sheetRow = [];
var entryKeys;
for (j in keys) {
sheetRow = [];
entryKeys = Object.keys(values[keys[j]])
for (k in entryKeys) {
sheetRow.push(values[keys[j]][entryKeys[k]]);
}
sheet.appendRow(sheetRow);
}
}
}

I've just tried this code (assuming that I guessed the data structure correctly):
function myFunction() {
var data = [
{'id': 1, 'two': 'test2', 'three': 'test3', 'four': 'test4'},
{'id': 2, 'two': 'test2-2', 'three': 'test3-2', 'four': 'test4-2'}
]
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("feed");
var selection = sheet.getActiveRange();
var range = sheet.getActiveRange();
var values = range.getValues();
var columns = selection.getNumColumns();
var rows = selection.getNumRows();
var num = 2;
function writeToSheets() {
for(var i in data) {
var values = [
[ data[i].id, data[i].two, data[i].three, data[i].four ]
];
var keys = Object.keys(values[0]);
var sheetRow = [];
var entryKeys;
for (j in keys) {
sheetRow = [];
entryKeys = Object.keys(values[keys[j]])
for (k in entryKeys) {
sheetRow.push(values[keys[j]][entryKeys[k]]);
}
sheet.appendRow(sheetRow);
}
}
}
writeToSheets();
}
When I run it, it fails after printing the first line with an error TypeError: Expected argument of type object, but instead had type undefined. (line 26, file "Code").
And it is easy to see what exactly happens if you run it in debug mode:
You have values array with one element (line 18)
The var keys = Object.keys(values[0]); becomes [0,1,2,3] (we have 4 values inside the first element of values array)
Then, having j from 0 to 3 we get entryKeys = Object.keys(values[keys[j])
When j = 0, values[keys[j]] = values[0] - we get the first element from values
When j = 1, values[keys[j]] = values[1] - here we fail, because there is only 1 element in values
I am not really sure what you are trying to do here with all these keys, but if you just want to print the data, it can be done simpler:
function writeToSheets() {
for(var i in data) {
var item = data[i];
sheetRow = [];
for (key in item) {
sheetRow.push(item[key]);
}
sheet.appendRow(sheetRow);
}
}

Remove Matching Objects from Array While Adding them to Another Array

I am trying to build a 3-tiered function:
First, an array lists available workshops (array is called 'workshops').
Second, another array lists workshops that a user has selected (this array is called 'selectedWorkshops').
Third, I have a final array called 'registeredWorkshops'.
When my function is run, I want objects within 'selectedWorkshops' to be added to 'registeredWorkshops', then I want to delete any objects within 'selectedWorkshops' from both 'selectedWorkshops' and any matching elements from 'workshops'. So, where those objects used to exist in both 'selectedworkshops' and 'workshops', now they only exist in 'registeredWorkshops'.
Here's what I've got so far:
addRemoveWorkshops = function(){
var numberOfWorkshops = selectedWorkshops.length;
for(var i = 0; i < numberOfWorkshops; i++ ){
registeredWorkshops.push(selectedWorkshops[i]);
for(var j = 0, arrayLength = workshops.length; j < arrayLength; j++) {
var searchTerm = selectedWorkshops[i].WorkshopId;
if (workshops[j].WorkshopId === searchTerm) {
workshops = workshops.slice(j);
}
}
selectedWorkshops = selectedWorkshops.slice(i);
}
};
addRemoveWorkshops();
However, the function doesn't appear to work properly. It doesn't seem to be deleting the correct workshops, and it only seems to add one of the selectedWorkshops to registeredWorkshops. What am I doing wrong?
Here's a codepen demonstration: http://codepen.io/trueScript/pen/GgVWMx

If it's not possible to add other properties to the objects (as per my other answer) then I'd tackle it like this:
function registration(workshops, selected, registered) {
// add the selected workshops to registered
selected.forEach(function(workshop) {
registered.push(workshop);
});
// remove them from the other lists
registered.forEach(function(workshop) {
removeWorkshop(selected, workshop);
removeWorkshop(workshops, workshop);
});
}
function removeWorkshop(list, workshop) {
var index = list.indexOf(workshop);
if(index >= 0) {
list.splice(index, 1);
}
}
The function expects each of the arrays to be passed in as arguments and it will modify them in place. Things always become clearer and easier to test if you move your loops out into functions before nesting them.
There should be no reason not to use the indexOf method here, as it saves you having to write an extra loop. However, if for some reason you needed to use the WorkshopId property to locate the item within the list, you could create another helper method to do this for you.
function findWorkshop(list, workshop) {
for(var i = 0; i < list.length; i++) {
if(list[i].WorkshopId === workshop.WorkshopID) {
return i;
}
}
return -1;
}
Then you just amend the removeWorkshop function to reflect that.
function removeWorkshop(list, workshop) {
var index = findWorkshop(list, workshop);
list.splice(index, 1);
}

I think it would be easier to slightly rethink your data structure. If you go for the imperative solution above, you run this risk of ending up with duplicate values in more than one list.
Would it not be easier to add registered and selected properties to your workshop objects?
var workshops = [
{
name: 'apples',
WorkshopId: '19',
registered: true,
selected: false
},
{
name: 'oranges',
WorkshopId: '3b',
selected: true,
registered: false
},
// ...
];
Then if you need to be able to get a list of all the registered workshops, you can create it using a filter.
// helper function for filtering based
// on a given property
function property(name) {
return function(object) {
return object[name];
}
}
var registered = workshops.filter(property('registered'));
var selected = workshops.filter(property('selected'));
To select a workshop, all you need to do is change the select property to true:
workshops[3].selected = true;
You could then write the original function to register all workshops that were selected like this:
function registration(workshops) {
workshops.forEach(function(workshop) {
if(workshop.selected) {
workshop.registered = true;
workshop.selected = false;
}
});
}

A while loop + a for one :
var workshops = [{
name: 'apples',
WorkshopId: '19'
}, {
name: 'oranges',
WorkshopId: '3b'
}, {
name: 'pears',
WorkshopId: 'x6'
}, {
name: 'pineapples',
WorkshopId: '55'
}, {
name: 'watermelons',
WorkshopId: '8v'
}];
var selectedWorkshops = [{
name: 'oranges',
WorkshopId: '3b'
}, {
name: 'watermelons',
WorkshopId: '8v'
}, {
name: 'pears',
WorkshopId: 'x6'
}];
var registeredWorkshops = [];
var numberOfWorkshops;
addRemoveWorkshops = function () {
numberOfWorkshops = selectedWorkshops.length;
// A single while statment is enough and lighter
while (selectedWorkshops.length) {
var removedWorkshop;
numberOfWorkshops = registeredWorkshops.push(selectedWorkshops[0]);
for (var i = 0; i < workshops.length; i++)
if (workshops[i].WorkshopId == selectedWorkshops[0].WorkshopId) {
workshops.splice(i, 1);
break;
}
selectedWorkshops.splice(0, 1);
}
};
addRemoveWorkshops();
// Better for viewing the content (in firefox I have just "Object") :
console.log("workshops : ");
for (var i = 0; i < workshops.length; i++)
console.log('- ' + workshops[i].name);
console.log("selectedWorkshops : ");
for (var i = 0; i < selectedWorkshops.length; i++)
console.log('- ' + selectedWorkshops[i].name);
console.log("registeredWorkshops : ");
for (var i = 0; i < registeredWorkshops.length; i++)
console.log('- ' + registeredWorkshops[i].name);

addRemoveWorkshops = function(){
var numberOfWorkshops = selectedWorkshops.length;
for(var i = 0; i < numberOfWorkshops; i++ ){
registeredWorkshops.push(selectedWorkshops[i]);
for(var j = 0, arrayLength = workshops.length; j < arrayLength; j++) {
var searchTerm = selectedWorkshops[i].WorkshopId;
if (workshops[j].WorkshopId === searchTerm) {
workshops = workshops.splice(j,1);
}
}
selectedWorkshops = selectedWorkshops.splice(i,1);
}
};

Find string in array of objects- javascript or jquery

I've got a JSON response that looks like this:
{
"COLUMNS":["SETTING_NAME","SETTING_VALUE","COLOR"],
"DATA": [
["setting_1",100.0,"yellow"],
["setting_2",150.0,"red"],
["setting_3",30.0,"green"],
["setting_4",11.0,"blue"]
]
}
How do I find the 'color' for the setting 'setting_4'? Acceptable solutions would either be am easy way to access the data, or a function to transform this into an exploded key/value array like
[
setting_1_value: '100',
setting_1_color: 'yellow',
setting_2_value: "150"
...
]

You can use this code to put the data into the type of data structure that you asked for:
var response = {"COLUMNS":["SETTING_NAME","SETTING_VALUE","COLOR"],
"DATA":[["setting_1",100.0,"yellow"],["setting_2",150.0,"red"],
["setting_3",30.0,"green"],["setting_4",11.0,"blue"]]};
var data = response.DATA;
var columns = response.COLUMNS;
var hash = {}, item, name, i;
var cols = {};
// remember order of columns
for (i = 0; i < columns.length; i++) {
cols[columns[i]] = i;
}
// fetch data from correct column
for (i = 0; i < data.length; i++) {
item = data[i];
name = item[cols["SETTING_NAME"]];
hash[name + "_value"] = item[cols["SETTING_VALUE"]];
hash[name + "_color"] = item[cols["COLOR"]];
}
hash.num = data.length;
As you requested, this gives you a data structure like this so you can directly read any value you want:
{
"setting_1_value":100,
"setting_1_color":"yellow",
"setting_2_value":150,
"setting_2_color":"red",
"setting_3_value":30,
"setting_3_color":"green",
"setting_4_value":11,
"setting_4_color":"blue",
"num":4
}
jsFiddle here: http://jsfiddle.net/jfriend00/HZmYN/ that generated this result.
Personally, I would rather use this code to parse it into this type of data structure:
var response = {"COLUMNS":["SETTING_NAME","SETTING_VALUE","COLOR"],
"DATA":[["setting_1",100.0,"yellow"],["setting_2",150.0,"red"],
["setting_3",30.0,"green"],["setting_4",11.0,"blue"]]};
var data = response.DATA;
var columns = response.COLUMNS;
var newData = [], item, obj, i, num, match;
var cols = {};
// remember order of columns
for (i = 0; i < columns.length; i++) {
cols[columns[i]] = i;
}
for (i = 0; i < data.length; i++) {
item = data[i];
obj = {};
obj.value = item[cols["SETTING_VALUE"]];
obj.color = item[cols["COLOR"]];
obj.name = item[cols["SETTING_NAME"]];
match = obj.name.match(/\d+$/);
if (match && match.length > 0) {
obj.settingNumber = parseInt(match[0], 10);
}
newData.push(obj);
}
// now sort the array by the number in the name setting
newData.sort(function(a, b) {
return(a.settingNumber- b.settingNumber);
});
And generates this data structure:
[
{"value":100,"color":"yellow","name":"setting_1","settingNumber":1},
{"value":150,"color":"red","name":"setting_2","settingNumber":2},
{"value":30,"color":"green","name":"setting_3","settingNumber":3},
{"value":11,"color":"blue","name":"setting_4","settingNumber":4}
]
Illustrated in this jsFiddle: http://jsfiddle.net/jfriend00/A23Jd/.
The reason I prefer this structure, is you can more easily access the "n" settings as an array of objects:
newData[0].color
newData[0].value
newData[0].name
newData[1].color
....
And, it's easier to iterate through the various settings

Using $.grep will allow you to access the data without mapping them before:
var json={"COLUMNS":["SETTING_NAME","SETTING_VALUE","COLOR"],
"DATA":[["setting_1",100.0,"yellow"],
["setting_2",150.0,"red"],
["setting_3",30.0,"green"],
["setting_4",11.0,"blue"]]}
alert($.grep(json.DATA, function(item){return(item[0]=='setting_4');})[0][2])//returns 'blue'
//or using the labels provided by COLUMNS:
alert($.grep(json.DATA,
function(a){return(a[0]=='setting_4');})[0][$.inArray('COLOR',json.COLUMNS)])

You can do this with a simple for loop:
var obj = {"COLUMNS":["SETTING_NAME","SETTING_VALUE","COLOR"],
"DATA":[["setting_1",100.0,"yellow"],["setting_2",150.0,"red"],
["setting_3",30.0,"green"],["setting_4",11.0,"blue"]]};
for(var i = 0; i < obj.DATA.length; i++)
{
var row = obj.DATA[i]
if (row[0] == 'setting_4')
{
console.log(row[2]);
break;
}
}
Prints:
blue

You could simply reduce that list of DATA:
DATA.reduce(function (value, item) { if (item[0] === "setting_4") return item[2] })
You could wrap that whole thing into a function for easier use, passing in the "setting_4" part. E.g.
var getColour = function (id) {
return DATA.reduce(function (value, item) {
if (item[0] === id) return item[2]
})
}
UPDATE: you could zip the two lists together, perhaps that would make access easier?
obj['DATA'].map(function (row) {
return obj['COLUMNS'].reduce(function (memo, columnName, index) {
memo[columnName] = row[index]
return memo
}, {})
})
This will return something like the following:
[{
COLOR: "yellow",
SETTING_NAME: "setting_1",
SETTING_VALUE: 100
}]

A generic algorithm for translating the dataset into a more-easily-addressed structure.
var json = {
"COLUMNS": [
"SETTING_NAME",
"SETTING_VALUE",
"COLOR"],
"DATA": [
["setting_1",100.0,"yellow"],
["setting_2",150.0,"red"],
["setting_3",30.0,"green"],
["setting_4",11.0,"blue"]
]
};
function translateJSON(json) {
var oHash = {};
var data = json['DATA'];
var cols = json['COLUMNS'];
for(var i = 0, imax = data.length; i < imax; i++) {
var row = data[i]; // shorthand
for(var j = 1, jmax = cols.length; j < jmax; j++) {
var c = cols[j]; // shorthand
oHash[(row[0] + '_' + c.replace(/[^_]+_/, '')).toLowerCase()] = row[j];
}
}
return oHash;
}
var h = translateJSON(json);
console.log(h['setting_4_color']);
Edit: updated the code. translateJSON will transform the JSON into the data structure you described, for easier property access. If you anticipate needing to access more than one property from the same JSON payload, it will be much more efficient to do a one-time transform before data access than to use something like $.grep, and much less terse than doing the column-name cross-referencing by hand.
That said, I don't think the target data structure you asked for is necessarily the best one. Assuming you can't change the structure of the JSON payload, it would still probably be better to transform that into something like:
data = {
'setting_1': { 'value': 100.0, 'color': 'yellow' },
'setting_2': { 'value': 150.0, 'color': 'red' }
...
};