Optimize javascript loop on CSV data - javascript

I am plotting a graph using d3.js by loading an external .CSV file.
The code i have so far works fine with a small amount of data but when i load a larger file with thousands of lines then it kills the page.
The data has a usage column which is a value for every 30 mins throughout the day, which will go on over several months.
See Plunker example.
var avgClientArr = [];
var dateArr = [];
var dateGroupArr = [];
function csvParseClient() {
d3.xhr('client.csv').get(function(err, response) {
var dirtyCSV = response.responseText;
var initialClientKeys = /TYPE,DATE,START TIME,END TIME,USAGE,UNITS,NOTES/i;
var newClientKeys = "TYPE,x,startTime,endTime,y,UNITS,NOTES";
var csvDataClient = dirtyCSV.replace(initialClientKeys, newClientKeys);
var validData = csvDataClient.substr(csvDataClient.indexOf(newClientKeys));
var csvData = d3.csv.parse(validData);
csvData.customForEach(function(val, i) {
// filter data
//var keep = ['x', 'startTime', 'endTime', 'UNITS', 'y'];
//for (var key in val[i]) {
// if (keep.indexOf(key) === -1) {
// delete val[i][key];
// }
//}
// parse data
var date = val.x;
var usage = val.y;
var startTime = val.startTime;
var endTime = val.endTime;
var x = new Date(date);
var y = parseFloat(usage);
dateArr.push({
"date": x,
"usage": y
})
dateGroupArr = groupBy(dateArr, 'date');
})
console.log(dateGroupArr);
var objDates = objectValues(dateGroupArr);
objDates.customForEach(function(f) {
var avg = f.reduce(function(a, b) {
return a + b.usage;
}, 0) / f.length;
var date = f.reduce(function(a, b) {
return new Date(b.date);
}, 0);
avgClientArr.push({
"x": date,
"y": avg
})
})
//console.log("avgClientArr", avgClientArr);
document.getElementById('arrayDiv').innerHTML = '<pre>' + JSON.stringify(avgClientArr, null, 4) + '</pre>';
})
}
function groupBy(arr, key) {
var reducer = (grouped, item) => {
var group_value = item[key]
if (!grouped[group_value]) {
grouped[group_value] = []
}
grouped[group_value].push(item)
return grouped
}
return arr.reduce(reducer, {})
}
function objectValues(object) {
var values = []
for (var property in object) {
if (object.hasOwnProperty(property)) {
values.push(object[property])
}
}
return values
}
function foreach(fn) {
var arr = this;
var len = arr.length;
for (var i = 0; i < len; ++i) {
fn(arr[i], i);
}
}
Object.defineProperty(Array.prototype, 'customForEach', {
enumerable: false,
value: foreach
});
var t0 = performance.now();
csvParseClient();
var t1 = performance.now();
console.log("Call csvParseClient() " + (t1 - t0) + " milliseconds.");
What i need to happen
I need the average value of usage for the whole day returned as y and the date for that day returned as x for each day.
The slow process i have
Start the loop from a specified line in the CSV file as there is unwanted data on the first few lines.
Group unique date and store each usage value for that date in an object.
Average the usage values for each date.
Output an array of objects with property x being the date and y being the average usage value.
If you can give me any help on how to make this run faster that would be great!

I solved this by using the d3 nest() and rollup() functions, its simple and really fast.
d3.nest()
.key(function(d) {
return d.x;
})
.rollup(function(d) {
var avg = d3.mean(d, function(g) {return g.y; });
return avg;
}).entries(dateArr);

Related

Javascript: making functions at runtime

update
solution works in foreach loop but not in for loop
function x(number){
return number - 10;
}
var i = 0
var runtimefunctions = {};
var allLevels = {"1":"State","2":"Educational_Services","3":"Principal_Networks","4":"Schools"}
for (var key in allLevels) {
runtimefunctions[i] = function() { return x(i); };
i++;
};
console.log(runtimefunctions[1]()); // -6
console.log(runtimefunctions[2]()); // -6
console.log(runtimefunctions[3]()); // -6
tried hard to make functions but it's first time to create such thing so cant understand the proper way...
I have a function..
function x(number){
return number - 10;
}
runtimefunctions = {};
now I have a loop to run
[1,2,3].forEach(function(y){
//here I want to create a function.. which will make a function x(y) -- like this
runtimefunctions[x] = new Function("return function x_" + levelIterator + "(levelIterator){ console.log(levelIterator); x(" + y + ") }")();
});
so basically..want to make functions like this.
runtimefunctions= {
"1": x(1),
"2": x(2),
and so on
}
Is this what you need?
function x(number){
return number - 10;
}
var runtimefunctions = {};
[1,2,3].forEach(function(y){
runtimefunctions[y] = function() { return x(y); };
});
console.log(runtimefunctions[1]()); // -9
console.log(runtimefunctions[2]()); // -8
console.log(runtimefunctions[3]()); // -7
To satisfy your next (for-in) requirement, you need to closure the index variable with additional function call:
var runtimefunctions = {}, i = 0;
var allLevels = {"1":"State","2":"Educational_Services","3":"Principal_Networks","4":"Schools"}
for (var key in allLevels) {
runtimefunctions[i] = function(index){ return function() { return x(index); } }(i++);
};
It is much easier.
For example:
const createFunctionWith = (x) => {
return (param) => console.log(x, param)
}
let a = [1,2,3].map(x => createFunctionWith(x));
console.log(a[1]("bebe")); // 2, "bebe"
https://jsfiddle.net/muLxoxLd/
You could do something like this
// Found in your code
var x = (a) => {
console.log(a)
};
var runtimefunctions = {};
[1, 2, 3].forEach(function(y) {
//Create a function with a parameter named "levelIterator"
runtimefunctions[y] = Function("levelIterator", "{ console.log(levelIterator); x(" + y + ") }");
});
runtimefunctions[1]('test')

MongoDB mapReduce array key values

i'm fullstack web developer. I'm using asyncawait and mapReduce and i want sum all client events by conditions in DB.
My code:
var mapReduceLoyalty = async(function (clients, dtTill, eventTypes, periodLoyalty) {
// Init mapReduce object
var mapReduce = {};
mapReduce.scope = {
// this array of ObjectIds of clients
clients: clients,
// this array of ObjectIds of eventTypes (4 elements)
eventTypes: eventTypes
};
mapReduce.jsMode = true;
// i want sum count of events by condition of each client in array
mapReduce.map = function () {
for (var i = 0, len = clients.length; i < len; i++) {
var key = clients[i];
var value = {
count: (eventTypes.indexOf(this._eventType)) ? 1 : 0
};
emit(key, value);
}
};
mapReduce.reduce = function (key, values) {
var reduceObject = {};
reduceObject.client = key;
reduceObject.count = 0;
values.forEach(function (value) {
reduceObject.count += value
});
return reduceObject;
};
mapReduce.query = {};
mapReduce.keeptemp = false;
// where dtEnd less then dtTill date
mapReduce.query['dtEnd'] = {
"$lt": dtTill
};
var result = await(Event.mapReduce(mapReduce).exec());
return result;
This code doing nothing . Only stops at the line
var result = await(Event.mapReduce(mapReduce).exec());
I'm newbie in map Reduce. What i'm doing wrong ? May be asyncawait ?

How to make a function that contains a promise return a value instead of a promise?

Say, I have a function F1 that will be called in many other function. F1 is meant to return a value VAL that will be used in F2. A promise is needed to retrieve that needed data that will help calculate VAL. Having F1 as a promise would cause a lot of confusion in F2, for F1 is often called inside IF statements and FOR loops. Let me illustrate this scenario:
function F1(param1, param2) {
var VAL = 0;
promise(param1, param2).then(function(data) {
for (var i = 0; i < data.length; i++) {
// Do some calculation here
}
});
return VAL;
}
function F2(x1, x2) {
var myArray = [],
someValue = 0;
if ([conditional expression]) {
someValue = F1(x1, x2);
call_some_function();
myArray.push({
val: someValue,
...
});
}
var x = someValue + y;
myArray.push({
id: x,
...
});
return myArray;
}
How do I make sure that F1 returns VAL (integer) so I can use it as a synchronous function?
Thanks in advance for your help.
EDIT:
Here is how the code looks like:
function myFunc(x, y) {
return init()
.then(function() {
return getData(x, y).then(function(data) {
if (data.length == 0) return [];
var id, name,
firstPass = true,
headIn = 0,
headOut = 0,
currentHead = 0,
payWtIn = 0,
payWtOut = 0,
expectedAdg = 0,
weight = 0,
results = [];
for (var i = 0; i < data.length; i++) {
if (firstPass) {
id = data[i].id();
name = data[i].name();
headIn = data[i].headIn();
headOut = data[i].headOut();
expectedAdg = data[i].expectedAdg();
firstPass = false;
}
if (id != data[i].id()) {
buildFunc();
reset();
}
headIn += data[i].headIn();
headOut += data[i].headOut();
payWtIn += data[i].payWtIn();
payWtOut += data[i].payWtOut();
}
buildFunc();
return results;
function buildFunc() {
currentHead = headIn - headOut;
var headDays = getHeadDays({ locationId: locationId, groupId: groupId, callDate: null });
var totalWeight = headIn != 0
? ((((headDays * expectedAdg) + payWtIn) / headIn) * currentHead) + payWtOut
: 0;
results.push({
id: id,
name: name,
headIn: headIn,
headOut: headOut,
headDays: headDays,
currentHd: currentHead,
totalWt: totalWeight
});
}
function reset() {
id = data[i].id();
name = data[i].name();
headIn = data[i].headIn();
headOut = data[i].headOut();
expectedAdg = data[i].expectedAdg();
payWtIn = 0;
payWtOut = 0;
weight = 0;
}
});
});
}
function getHeadDays(params) {
var VAL = 0;
promise(params.a, params.b).then(function(data) {
for (var i = 0; i < data.length; i++) {
// Make calculation to determine VAL here
}
});
return VAL;
}
The init function loads needed entities in the cache (I'm working with BreezeJs) for querying. The getData function gets raw data that are sorted by id from database, and those data are used to determine the results array. As the data are looped through, as long as the id of each record is the same, headIn, headOut, payWtIn and payWtOut are incremented by the record fields, and when the previous and current id are different, we can calculate totalWeight and push a new record to the results array with the buildFunc function. Inside that buildFunc function, we retrieve the headDays in order to calculate totalWeight. The getHeadDays function will be called in many other functions. Please, let me know if you have any questions. Thanks in advance for your help.
You can't.
If you need to return a promise, then that is because the value won't be available until some event happens, and the function will (or at least may) return before then. That's the point of promises.

Counting array elements with specific date in javascript

I have an array of Date() objects in javascript and I want to count the number of events on each day.
Here is an example:
What I have is:
Array [ Date 2014-12-04T10:30:20.000Z, Date 2014-12-05T11:04:58.056Z, Date 2014-12-05T11:04:58.056Z, Date 2014-12-05T11:04:58.056Z ]
What I want is:
Array [{date: '2014-12-04', counts: 1}, {date: '2014-12-05', counts: 3}]
Thanks a lot!
Max
Basic answer:
var arr = [], // fill it with array with your data
results = {}, rarr = [], i, date;
for (i=0; i<arr.length; i++) {
// get the date
date = [arr[i].getFullYear(),arr[i].getMonth(),arr[i].getDate()].join("-");
results[date] = results[date] || 0;
results[date]++;
}
// you can always convert it into an array of objects, if you must
for (i in results) {
if (results.hasOwnProperty(i)) {
rarr.push({date:i,counts:results[i]});
}
}
These can be made much easier with lodash functions, and Array.forEach() in ES5
You much better off having a simple object with the keys as the date and the value as the count. I've added a simple pad function that prefixes a zero where the number is a single digit as per your output requirements.
function pad(n) {
return n.toString().length == 1 ? '0' + n : n;
}
function getCount(arr) {
var obj = {};
for (var i = 0, l = arr.length; i < l; i++) {
var thisDate = arr[i];
var day = pad(thisDate.getDate());
var month = pad(thisDate.getMonth() + 1);
var year = thisDate.getFullYear();
var key = [year, day, month].join('-');
obj[key] = obj[key] || 0;
obj[key]++;
}
return obj;
}
getCount(arr); // Object { 2014-04-12: 1, 2014-05-12: 3 }
DEMO
I came across the same issue and found this solution which uses Map()
`
calc = (obj) => {
const orders = []
const dates_map = new Map()
//iterate through all the objects inside the orders array
orders.forEach(order => {
// format and get the date
const date = new Date(order.created_at).toLocaleDateString('en-GB')
//check if the date key exists in the Map() and save it in a temp
const temp = dates_map.get(date) || false
// if it does not exist
if (temp) {
// clone the object
const previous = {...temp}
// increase counter
previous.count += 1
dates_map.set(date, previous)
}else{
//create new object to avoid overwriting
const result = {}
result.count = 1
dates_map.set(date, result)
}
})
console.log(dates_map)
}
And this is the output
Output: Map(3) {
'08/05/2021' => { count: 2 },
'09/05/2021' => { count: 1 },
'11/05/2021' => { count: 2,}
}
`

JS calculate mean of same elements in 2d array

I've got a 'table' of two columns represented as an array. The first column are numbers from 1 to 20 and they are labels, the second column are the corresponding values (seconds):
my_array = [ [ 3,4,5,3,4,5,2 ],[ 12,14,16,11,12,10,20 ] ];
I need the mean (average) for each label:
my_mean_array = [ [ 2,3,4,5 ],[ 20/1, (12+11)/2, (14+12)/2, (16+10)/2 ] ];
// edit: The mean should be a float - the notion above is just for clarification.
// Also the number 'labels' should remain as numbers/integers.
My try:
var a = my_array[0];
var b = my_array[1];
m = [];
n = [];
for( var i = 0; a.length; i++){
m[ a[i] ] += b[i]; // accumulate the values in the corresponding place
n[ a[i] ] += 1; // count the occurences
}
var o = [];
var p = [];
o = m / n;
p.push(n);
p.push(o);
How about this (native JS, will not break on older browsers):
function arrayMean(ary) {
var index = {}, i, label, value, result = [[],[]];
for (i = 0; i < ary[0].length; i++) {
label = ary[0][i];
value = ary[1][i];
if (!(label in index)) {
index[label] = {sum: 0, occur: 0};
}
index[label].sum += value;
index[label].occur++;
}
for (i in index) {
if (index.hasOwnProperty(i)) {
result[0].push(parseInt(i, 10));
result[1].push(index[i].occur > 0 ? index[i].sum / index[i].occur : 0);
}
}
return result;
}
FWIW, if you want fancy I've created a few other ways to do it. They depend on external libraries and are very probably an order of magnitude slower than a native solution. But they are nicer to look at.
It could look like this, with underscore.js:
function arrayMeanUnderscore(ary) {
return _.chain(ary[0])
.zip(ary[1])
.groupBy(function (item) { return item[0]; })
.reduce(function(memo, items) {
var values = _.pluck(items, 1),
toSum = function (a, b) { return a + b; };
memo[0].push(items[0][0]);
memo[1].push(_(values).reduce(toSum) / values.length);
return memo;
}, [[], []])
.value();
}
// --------------------------------------------
arrayMeanUnderscore([[3,4,5,3,4,5,2], [12,14,16,11,12,10,20]]);
// -> [[2,3,4,5], [20,11.5,13,13]]
or like this, with the truly great linq.js (I've used v2.2):
function arrayMeanLinq(ary) {
return Enumerable.From(ary[0])
.Zip(ary[1], "[$, $$]")
.GroupBy("$[0]")
.Aggregate([[],[]], function (result, item) {
result[0].push(item.Key());
result[1].push(item.Average("$[1]"));
return result;
});
}
// --------------------------------------------
arrayMeanLinq([[3,4,5,3,4,5,2], [12,14,16,11,12,10,20]]);
// -> [[3,4,5,2], [11.5,13,13,20]]
As suspected, the "fancy" implementations are an order of magnitude slower than a native implementation: jsperf comparison.
var temp = {};
my_array[0].map(function(label, i) {
if (! temp[label])
{
temp[label] = [];
}
temp[label].push(my_array[1][i]);
});
var result = [ [], [] ];
for (var label in temp) {
result[0].push(label);
result[1].push(
temp[label].reduce(function(p, v) { return p + v }) / temp[label].length
);
}
This function do not sort the resulted array like in your result example. If you need sorting, just say me and i will add it.
function getMeanArray(my_array)
{
m = {}; //id={count,value}
for( var i = 0; i<my_array[0].length; i++){
if (m[my_array[0][i]]===undefined)
{
m[my_array[0][i]]={count:0, value:0};
}
m[ my_array[0][i] ].value += my_array[1][i]; // accumulate the values in the corresponding place
m[ my_array[0][i] ].count++; // count the occurences
}
var my_mean_array=[[],[]];
for (var id in m)
{
my_mean_array[0].push(id);
my_mean_array[1].push(m[id].count!=0?m[id].value/m[id].count:0);
}
return my_mean_array;
}

Categories