We Keep Coding

Optimize javascript loop on CSV data

I am plotting a graph using d3.js by loading an external .CSV file.
The code i have so far works fine with a small amount of data but when i load a larger file with thousands of lines then it kills the page.
The data has a usage column which is a value for every 30 mins throughout the day, which will go on over several months.
See Plunker example.
var avgClientArr = [];
var dateArr = [];
var dateGroupArr = [];
function csvParseClient() {
d3.xhr('client.csv').get(function(err, response) {
var dirtyCSV = response.responseText;
var initialClientKeys = /TYPE,DATE,START TIME,END TIME,USAGE,UNITS,NOTES/i;
var newClientKeys = "TYPE,x,startTime,endTime,y,UNITS,NOTES";
var csvDataClient = dirtyCSV.replace(initialClientKeys, newClientKeys);
var validData = csvDataClient.substr(csvDataClient.indexOf(newClientKeys));
var csvData = d3.csv.parse(validData);
csvData.customForEach(function(val, i) {
// filter data
//var keep = ['x', 'startTime', 'endTime', 'UNITS', 'y'];
//for (var key in val[i]) {
// if (keep.indexOf(key) === -1) {
// delete val[i][key];
// }
//}
// parse data
var date = val.x;
var usage = val.y;
var startTime = val.startTime;
var endTime = val.endTime;
var x = new Date(date);
var y = parseFloat(usage);
dateArr.push({
"date": x,
"usage": y
})
dateGroupArr = groupBy(dateArr, 'date');
})
console.log(dateGroupArr);
var objDates = objectValues(dateGroupArr);
objDates.customForEach(function(f) {
var avg = f.reduce(function(a, b) {
return a + b.usage;
}, 0) / f.length;
var date = f.reduce(function(a, b) {
return new Date(b.date);
}, 0);
avgClientArr.push({
"x": date,
"y": avg
})
})
//console.log("avgClientArr", avgClientArr);
document.getElementById('arrayDiv').innerHTML = '<pre>' + JSON.stringify(avgClientArr, null, 4) + '</pre>';
})
}
function groupBy(arr, key) {
var reducer = (grouped, item) => {
var group_value = item[key]
if (!grouped[group_value]) {
grouped[group_value] = []
}
grouped[group_value].push(item)
return grouped
}
return arr.reduce(reducer, {})
}
function objectValues(object) {
var values = []
for (var property in object) {
if (object.hasOwnProperty(property)) {
values.push(object[property])
}
}
return values
}
function foreach(fn) {
var arr = this;
var len = arr.length;
for (var i = 0; i < len; ++i) {
fn(arr[i], i);
}
}
Object.defineProperty(Array.prototype, 'customForEach', {
enumerable: false,
value: foreach
});
var t0 = performance.now();
csvParseClient();
var t1 = performance.now();
console.log("Call csvParseClient() " + (t1 - t0) + " milliseconds.");
What i need to happen
I need the average value of usage for the whole day returned as y and the date for that day returned as x for each day.
The slow process i have
Start the loop from a specified line in the CSV file as there is unwanted data on the first few lines.
Group unique date and store each usage value for that date in an object.
Average the usage values for each date.
Output an array of objects with property x being the date and y being the average usage value.
If you can give me any help on how to make this run faster that would be great!

I solved this by using the d3 nest() and rollup() functions, its simple and really fast.
d3.nest()
.key(function(d) {
return d.x;
})
.rollup(function(d) {
var avg = d3.mean(d, function(g) {return g.y; });
return avg;
}).entries(dateArr);

How to restructure my JSON object

I implemented an aggregation function but the only problem I have now is that I lost my key: value format e.g [{name:"Apples",val:8},{name:"Banana",val: 9}].
function agrregate(a){
var targetObj = {};
var result;
var b = JSON.parse(JSON.stringify(a));
var trees= b.length;
if(!trees){
trees = 0
}
for (var i = 0; i < trees; i++) {
if (!targetObj.hasOwnProperty(b[i].key)) {
targetObj[b[i].key] = 0;
}
targetObj[b[i].key] += b[i].val;
}
result = JSON.stringify(targetObj);
return result;
}
This is the result i get when agrregate function completes.
{"Apple":8,"Banana":9}
Instead of
{name:"Apple", val:8}, {name:"Banana", val:9}

Use a reducer to aggregate. You don't need to do stuff with JSON stringify/parse.
To get back to an array of objects, you use map and Object.keys
var test = [{name:"Apples",val:5},{name:"Banana",val: 9},{name:"Apples",val:3}]
var aggregate = function(arr) {
return arr.reduce(function(result, obj) { // Create one object (result)
result[obj.name] = (result[obj.name] || 0) + obj.val; // Add a new key/or increase
return result // Return the object
}, {});
};
var wrap = function(obj) {
return Object.keys(obj) // Create an array of keys
.map(function(key) {
return { // Specify the format
name: key,
val: obj[key]
};
});
};
console.log(aggregate(test));
console.log(wrap(aggregate(test)));

How to make a function that contains a promise return a value instead of a promise?

Say, I have a function F1 that will be called in many other function. F1 is meant to return a value VAL that will be used in F2. A promise is needed to retrieve that needed data that will help calculate VAL. Having F1 as a promise would cause a lot of confusion in F2, for F1 is often called inside IF statements and FOR loops. Let me illustrate this scenario:
function F1(param1, param2) {
var VAL = 0;
promise(param1, param2).then(function(data) {
for (var i = 0; i < data.length; i++) {
// Do some calculation here
}
});
return VAL;
}
function F2(x1, x2) {
var myArray = [],
someValue = 0;
if ([conditional expression]) {
someValue = F1(x1, x2);
call_some_function();
myArray.push({
val: someValue,
...
});
}
var x = someValue + y;
myArray.push({
id: x,
...
});
return myArray;
}
How do I make sure that F1 returns VAL (integer) so I can use it as a synchronous function?
Thanks in advance for your help.
EDIT:
Here is how the code looks like:
function myFunc(x, y) {
return init()
.then(function() {
return getData(x, y).then(function(data) {
if (data.length == 0) return [];
var id, name,
firstPass = true,
headIn = 0,
headOut = 0,
currentHead = 0,
payWtIn = 0,
payWtOut = 0,
expectedAdg = 0,
weight = 0,
results = [];
for (var i = 0; i < data.length; i++) {
if (firstPass) {
id = data[i].id();
name = data[i].name();
headIn = data[i].headIn();
headOut = data[i].headOut();
expectedAdg = data[i].expectedAdg();
firstPass = false;
}
if (id != data[i].id()) {
buildFunc();
reset();
}
headIn += data[i].headIn();
headOut += data[i].headOut();
payWtIn += data[i].payWtIn();
payWtOut += data[i].payWtOut();
}
buildFunc();
return results;
function buildFunc() {
currentHead = headIn - headOut;
var headDays = getHeadDays({ locationId: locationId, groupId: groupId, callDate: null });
var totalWeight = headIn != 0
? ((((headDays * expectedAdg) + payWtIn) / headIn) * currentHead) + payWtOut
: 0;
results.push({
id: id,
name: name,
headIn: headIn,
headOut: headOut,
headDays: headDays,
currentHd: currentHead,
totalWt: totalWeight
});
}
function reset() {
id = data[i].id();
name = data[i].name();
headIn = data[i].headIn();
headOut = data[i].headOut();
expectedAdg = data[i].expectedAdg();
payWtIn = 0;
payWtOut = 0;
weight = 0;
}
});
});
}
function getHeadDays(params) {
var VAL = 0;
promise(params.a, params.b).then(function(data) {
for (var i = 0; i < data.length; i++) {
// Make calculation to determine VAL here
}
});
return VAL;
}
The init function loads needed entities in the cache (I'm working with BreezeJs) for querying. The getData function gets raw data that are sorted by id from database, and those data are used to determine the results array. As the data are looped through, as long as the id of each record is the same, headIn, headOut, payWtIn and payWtOut are incremented by the record fields, and when the previous and current id are different, we can calculate totalWeight and push a new record to the results array with the buildFunc function. Inside that buildFunc function, we retrieve the headDays in order to calculate totalWeight. The getHeadDays function will be called in many other functions. Please, let me know if you have any questions. Thanks in advance for your help.

You can't.
If you need to return a promise, then that is because the value won't be available until some event happens, and the function will (or at least may) return before then. That's the point of promises.

Create an array [n,[v,..,z]] from a list of key-value pairs

I have this input sample:
var c1 = "s_A_3";
var c2 = "s_B_10";
var c3 = "s_B_9";
var c4 = "s_C_18";
var c5 = "s_C_19";
var c6 = "s_C_20";
Which can easily be concatenated to:
var keypairs = ["A_3","B_10","B_9","C_18","C_19","C_20"];
And I want to convert this to a multidimensional array like this:
var groupArray = [["A",[3]],["B",[10,9]],["C",[18,19,20]]];
It's like a kind of card-sorting. How can I achieve this?

Maybe something like this:
function makeGroups(arr) {
var result = [], prev;
for(var i = 0; i < arr.length; i++) {
var x = arr[i].split("_");
if (prev !== x[0]) {
prev = x[0];
result.push([prev, []]);
}
result[result.length - 1][1].push(x[1]); // or .push(parseInt(x[1], 10))
}
return result;
}
var keypairs = ["A_3","B_10","B_9","C_18","C_19","C_20"];
console.log(makeGroups(keypairs));
// [["A",["3"]],["B",["10","9"]],["C",["18","19","20"]]]
Demonstration
The above method assumes the groups will be contiguous (e.g. all B_ elements appear together). In case your input may be out of order, you can tweak this algorithm to still group all elements together regardless of where they appear in the input:
function makeGroups(arr) {
var result = [], keys = {};
for(var i = 0; i < arr.length; i++) {
var x = arr[i].split("_");
if (!(x[0] in keys)) {
keys[x[0]] = [];
result.push([x[0], keys[x[0]]]);
}
keys[x[0]].push(x[1]); // or .push(parseInt(x[1], 10))
}
return result;
}
var keypairs = ["A_3","B_10","C_18","C_19","C_20","B_9"];
console.log(makeGroups(keypairs));
// [["A",["3"]],["B",["10","9"]],["C",["18","19","20"]]]
Demonstration

When you need to mention "key value pairs" in a JS program, it's usually most appropriate to use... key value pairs =D.
function solution(input) {
var kvp = {},
result = [];
input.forEach(function (el) {
var cut = el.split("_"),
alpha = cut[0],
numeric = cut[1],
elsWithSameAlpha = kvp[alpha] = kvp[alpha] || [];
elsWithSameAlpha.push(numeric);
});
Object.keys(kvp).forEach(function (key) {
result.push([key, kvp[key]]);
});
return result;
}

Return variable from method in a loop

Something like this
var joined = function(arr) {
var res = [];
for (var i in arr) {
var u = DB.getUser(arr[i].user_id, function(user) {
return user;
});
arr[i].user = u;
res = arr[i];
}
return res;
}
I need to get user variable from DB.getUser scope.

Is just inserted some comments into you code to help understand the async flow:
var joined = function(arr) {
// Timestamp: 0
var res = [];
for (var i in arr) {
// Timestamp: 1
var u = DB.getUser(arr[i].user_id, function(user) {
// Timestamp 4 ... length of arr
// user contains what you are looking for
// but this is not return to u, because we passed that a long time ago
return user;
});
// u is null or undefined, because DB.getUser returns nothing
// is a async function, you need wait for the callback
arr[i].user = u;
// Timestamp: 2 return useless arr
res = arr[i];
}
// Timestamp: 3 again, return a useless array
return res;
}
Edit:
You need to this before you pass everything to the template, e.g.:
var joined = function(arr, doneCallback) {
var res = []
var count = arr.length;
for (var i in arr) {
DB.getUser(arr[i].user_id, function(user) {
count--;
res.push(user);
if (count == 0) {
doneCallback(res);
}
})
}
}
joined(somedata, function(mydata) {
render(template, mydata)
});
Take a look at some flow control libraries. (My favorite async)

var getJoinedAndDoSomeThingWithThem = function(ids) {
var joined = [];
var i = 0;
var getAUser = function () {
DB.getUser(ids[i].user_id, function(user) {
joined.push(user);
i++;
if (i == ids.length -1) {
doSomeThingWithTheResult(joined);
return;
}
getAUser();
});
}
getAUser();
}