I am working on a Dimple/D3 chart that plots missing days' data as 0.
date fruit count
2013-12-08 12:12 apples 2
2013-12-08 12:12 oranges 5
2013-12-09 16:37 apples 1
<- oranges inserted on 12/09 as 0
2013-12-10 11:05 apples 6
2013-12-10 11:05 oranges 2
2013-12-10 20:21 oranges 1
I was able to get nrabinowitz's excellent answer to work, nearly.
My data's timestamp format is YYYY-MM-DD HH-MM, and the hashing + D3.extent time interval in days results in 0-points every day at midnight, even if there is data present from later in the same day.
An almost-solution I found was to use .setHours(0,0,0,0) to discard the hours/minutes, so that all data would appear to be from midnight:
...
var dateHash = data.reduce(function(agg, d) {
agg[d.date.setHours(0,0,0,0)] = true;
return agg;
}, {});
...
This works as expected when there is just 1 entry per day everyday, BUT on days when there are multiple entries the values are added together. So in the data above on 12/10: apples: 6 , oranges: 3.
Ideally (in my mind) I would separate the plotting data from the datehash, and on the hash discard hours/minutes. This would compare the midnight-datehash with the D3 days interval, fill in 0s at midnight on days with missing data, and then plot the real points with hours/minutes intact.
I have tried data2 = data.slice() followed by setHours, but the graph still gets the midnight points:
...
// doesn't work, original data gets converted
var data2 = data.slice();
var dateHash = data2.reduce(function(agg, d) {
agg[d.date.setHours(0,0,0,0)] = true;
return agg;
}, {});
...
Props to nrabinowitz, here is the adapted code:
// get the min/max dates
var extent = d3.extent(data, function(d) { return d.date; }),
// hash the existing days for easy lookup
dateHash = data.reduce(function(agg, d) {
agg[d.date] = true;
// arrr this almost works except if multiple entries per day
// agg[d.date.setHours(0,0,0,0)] = true;
return agg;
}, {}),
headers = ["date", "fruit", "count"];
// make even intervals
d3.time.days(extent[0], extent[1])
// drop the existing ones
.filter(function(date) {
return !dateHash[date];
})
// fruit list grabbed from user input
.forEach(function(date) {
fruitlist.forEach(function(fruits) {
var emptyRow = { date: date };
headers.forEach(function(header) {
if(header === headers[0]) {
emptyRow[header] = fruits;}
else if(header === headers[1]) {
emptyRow[header] = 0;};
// and push them into the array
data.push(emptyRow);
});
// re-sort the data
data.sort(function(a, b) { return d3.ascending(a.date, b.date); });
(I'm not concerned with 0-points in the hour-scale, just the dailies. If the time.interval is changed from days to hours I suspect the hash and D3 will handle it fine.)
How can I separate the datehash from the data? Is that what I should be trying to do?
I can't think of a smooth way to do this but I've written some custom code which works with your example and can hopefully work with your real case.
var svg = dimple.newSvg("#chartContainer", 600, 400),
data = [
{ date : '2013-12-08 12:12', fruit : 'apples', count : 2 },
{ date : '2013-12-08 12:12', fruit : 'oranges', count : 5 },
{ date : '2013-12-09 16:37', fruit : 'apples', count : 1 },
{ date : '2013-12-10 11:05', fruit : 'apples', count : 6 },
{ date : '2013-12-10 11:05', fruit : 'oranges', count : 2 },
{ date : '2013-12-10 20:21', fruit : 'oranges', count : 1 }
],
lastDate = {},
filledData = [],
dayLength = 86400000,
formatter = d3.time.format("%Y-%m-%d %H:%M");
// The logic below requires the data to be ordered by date
data.sort(function(a, b) {
return formatter.parse(a.date) - formatter.parse(b.date);
});
// Iterate the data to find and fill gaps
data.forEach(function (d) {
// Work from midday (this could easily be changed to midnight)
var noon = formatter.parse(d.date).setHours(12, 0, 0, 0);
// If the series value is not in the dictionary add it
if (lastDate[d.fruit] === undefined) {
lastDate[d.fruit] = formatter.parse(data[0].date).setHours(12, 0, 0, 0);
}
// Calculate the days since the last occurance of the series value and fill
// with a line for each missing day
for (var i = 1; i <= (noon - lastDate[d.fruit]) / dayLength - 1; i++) {
filledData.push({
date : formatter(new Date(lastDate[d.fruit] + (i * dayLength))),
fruit : d.fruit,
count : 0 });
}
// update the dictionary of last dates
lastDate[d.fruit] = noon;
// push to a new data array
filledData.push(d);
}, this);
// Configure a dimple line chart to display the data
var chart = new dimple.chart(svg, filledData),
x = chart.addTimeAxis("x", "date", "%Y-%m-%d %H:%M", "%Y-%m-%d"),
y = chart.addMeasureAxis("y", "count"),
s = chart.addSeries("fruit", dimple.plot.line);
s.lineMarkers = true;
chart.draw();
You can see this working in a fiddle here:
http://jsfiddle.net/LsvLJ/
Related
I use an ajax call to get data on grades based on a 12-month date range. Then I calculate the ratio of passes and fails by looping through a for loop of the dates and a nested for loop of the data. If the date and the data's date match, perform calculations for that date.
The problem is that I loop through a list of 12 with a nested for loop of up to thousands. The algorithm therefore seems to have an efficiency of x^2. How can I improve the efficiency of this algorithm? Can it be improved to 2x?
dataSet = [];
dateList = [
01 - 2023,
02 - 2023,
//...
12 - 2023,
];
$.ajax({
url: '...',
type: 'GET',
global: false,
success: function(data) {
// Iterate through each date
for (date of dateList) {
var passes = 0;
var fails = 0;
var passFailRatio = 0;
// Iterate through each data result
for (result of data['results']) {
//If data's grade date matches with date increment pass or fail
if (result.gradeDate == date) {
if (result.pass == true) {
passes++;
} else if (result.fail == true) {
fails++;
}
}
}
if ((passes + fails) == 0) {
passFailRatio = 0;
} else {
passFailRatio = (100 * (passes / (passes + fails))).toFixed(0);
}
// Push a dictionary to a list based on the date
dataSet.push({
'date': date,
'passFailRatio': passFailRatio
});
}
}
});
I was thinking about creating the list of dates while iterating through the data results. If the data result has a unique grade date, then add that date to the list and increment passes and fails for that date.
The algorithm as written has complexity O(n*m), with n being the length of dateList and m being the length of data[results].
You can improve this by sorting both lists, followed by a standard merge.
The sort/merge complexity is:
O(n*log(n)) to sort the dates
O(m*log(m)) to sort the results
O(n+m) to merge
Merging the two sorted lists is, in pseudo-code (you'll have to convert to JavaScript):
dateIdx = 0
rsltIds = 0
passes = 0
fails = 0
while (dateIdx < length(datelist) and rsltIdx < length(results))
{
if (results[rsltIdx].gradeDate < datelist[dateIdx])
{
// skip this result because there's no matching date
++rsltIdx;
} else if (results[rsltIdx].gradeDate == dateList[dateIdx])
{
if (results[rsltIdx].pass)
++passes;
else
++fails;
++rsltIdx;
} else
{
// result date is greater. Compute ratio for this date.
if (passes + fails) == 0
passFailRatio = 0;
else
passFailRatio = (100 * (passes / (passes + fails))).toFixed(0);
// Push a dictionary to a list based on the date
dataSet.push({
'date': datelist[dateIdx],
'passFailRatio': passFailRatio
});
passes = 0;
fails = 0;
// and go to the next date
++dateIdx;
}
}
// Output '0' entries for any remaining dates
while (dateIdx < length(dateList))
{
dataSet.push({
'date': datelist[dateIdx],
'passFailRatio': 0
});
}
// Don't care about remaining results because there is no
// corresponding dateList entry for them.
I have a sales data for a couple of years in an array:
var= ['Jan-2019',325678], ['feb-2019', 456789], ['Mar-2019',-12890],.....['Dec-2021', 987460]
1 -want to calculate the net total amount of profit/losses over the entire period.
2- Average of the (changes) in profit/losses over the entire period - track the total change in profits from month to month and find the average(total/number of months)
3 - greatest increase in profit (month and amount)over the whole period
4 - greatest decrease3in profit (month and amount)over the whole period
Tried to solve number 1 using :
`
const profitMonths = data.filter(el => el[1] > 0);
console.log("Total:", profitMonths.map(el => el[1]).reduce((a, b) => a + b));
console.log;
`
but the sum I am getting is different from what excel and calculator is giving me.
I will appreciate some help here
Not sure what is the format of your original data records for each of the months. I assumed that your data format is like below. But you could get the sum of each months growth or loss (earnings) like this and also get what you were trying as well (profit months total sales):
const data = [
['Jan-2019', 325678],
['Feb-2019', 456789],
['Mar-2019', -12890],
];
const earningsArray = data.map((el) => el[1]);
const profitMonths = data.filter((el) => el[1] > 0);
const salesOnProfitMonths = profitMonths
.map((el) => el[1])
.reduce((accVal, curVal) => accVal + curVal, 0);
const avgOfProfitAndLoss =
earningsArray.reduce((accVal, curVal) => accVal + curVal, 0) / data.length; // get the average of all total and losses
const maxMonth = {
monthName: '',
profit: 0,
};
const minMonth = {
monthName: '',
profit: 0,
};
data.forEach((month) => {
if (month[1] > maxMonth.profit) {
maxMonth.monthName = month[0];
maxMonth.profit = month[1];
}
if (month[1] < minMonth.profit) {
minMonth.monthName = month[0];
minMonth.profit = month[1];
}
return { maxMonth, minMonth };
});
console.log('Total sale of profit months: ', salesOnProfitMonths);
console.log('Total average : ', avgOfProfitAndLoss);
console.log('The month with max profit is : ', maxMonth);
console.log('The month with min profit is : ', minMonth);
Using .reduce() you can actually build an object to the returned based on all of the data from your original array.
const data = [['Jan-2019', 325678], ['feb-2019', 456789], ['Mar-2019',-12890], ['Dec-2021', 987460]]
let result = data.reduce((a, b, i) => {
let d = (i > 1) ? a : {total: a[1], average: a[1], sumChange: 0, lastMonth: a[1], increase: a, decrease: a},
change = b[1] - d.lastMonth
d.total += b[1]
d.sumChange += change
d.lastMonth = b[1]
d.average = d.sumChange / i
d.increase = (d.increase[1] > change) ? d.increase : [b[0], change]
d.decrease = (d.decrease[1] < change) ? d.decrease : [b[0], change]
return d
})
console.log(result) // Return the full object
console.log(result.total) // Only return one value, the total
Based on the array/input you provided, this should provide a net total, average profit/loss, highest increase from the previous month, and highest decrease from the previous month.
EDIT
I had to make a few adjustments after getting some clarification. But this again should return a single object that holds values for everything requested by OP. (the sumChange and lastMonth values are only there to help with the .reduce() function month over month)
NOTES
Just for clarity as OP claimed they were not getting the right values, here is a breakdown based on the provided data:
Date
Sales
Change
Jan-2019
$325,678
N/A
Feb-2019
$456,789
$131,111
Mar-2019
-$12,890
-$469,679
Dec-2021
$987,460
$1,000,350
Based on this input, calculated manually:
The "Average of the (changes) in profit/losses over the entire period" is $220,594 (($131,111 + $469,679 + $1,000,350) / 3).
The "greatest increase in profit (month and amount)over the whole period" would be Dec-2021 with a $1,000,350 increase.
And the "greatest decrease in profit (month and amount)over the whole period" would be Mar-2019 with -$469,679.
This is exactly what my .reduce() does produce, so I'm not sure what actual input or output OP is getting (or how they are applying this to their code/data).
I have bunch of electricity meter readings which have irregular dates. See below :
ReadingDate Meter
19/01/2021 5270
06/03/2021 5915
11/05/2021 6792
08/07/2021 7367
9/9/2021 8095
8/11/2021 8849
02/12/2021 9065
17/01/2022 9950
Now I'd like to transform this into monthly readings, using just this data, to end up with a table like this
Month Usage
2021-01 452
2021-02 393
2021-03 416
2021-04 399
2021-05 341
2021-06 297
2021-07 347
2021-08 358
2021-09 369
2021-10 389
2021-11 295
2021-12 586
2022-01 308
Now, I have a working solution, but I'm sure there's a more beautiful concise way of doing it.
What I do is to create an intermediate array that has one line for each date between first and last meter readings.
Each item in the array has 3 values :
the date
the average value for that date (calculated by counting the days between meter readings and dividing that by change in the meter.
the corresponding month
The last step then is to loop over this intermediate array and sum the values for each different month.
Here's the working code (its taken from Google Apps Script so please ignore the spreadsheet specific stuff:
var DailyAveragesArray = [['Date','Usage','Month']];
var monthlyObject = {};
var monthlyArray = [['Month','Usage']];
function calculateAverageDailyFigures() {
// give indices for the useful columns, 0 numbered
var ReadingDateColumn = 0;
var MeterReading = 1;
// Read into an array
var MeterReadingData = ss.getDataRange().getValues() // Get array of values
const sortedReadings = MeterReadingData.slice(1).sort((a, b) => a[0] - b[0]);
// from https://flaviocopes.com/how-to-sort-array-by-date-javascript/
// First calculate the number of days and average daily figure for each row
// Note we don't do this for the last row
for(i=0; i < sortedReadings.length - 1 ; i++){
var NumberOfDays = (sortedReadings[i+1][0] - sortedReadings[i][0])/(1000*3600*24);
sortedReadings[i].push(NumberOfDays);
var MeterDifference = sortedReadings[i+1][1] - sortedReadings[i][1];
var AverageDailyFigure = MeterDifference/NumberOfDays;
sortedReadings[i].push(AverageDailyFigure);
}
BuildDailyArray(sortedReadings);
}
function BuildDailyArray(sortedReadings){
// For each row in sorted , loop from the date to the next date-1 and create columns date and Usage
for(i=0; i<sortedReadings.length -1 ;i++){
for (var d = sortedReadings[i][0]; d < sortedReadings[i+1][0]; d.setDate(d.getDate() + 1)) {
var newDate = new Date(d);
var month = newDate.getFullYear() + '-' + ('0' + (newDate.getMonth() + 1)).slice(-2);
DailyAveragesArray.push([newDate,sortedReadings[i][3],month]);
// Check if the month is in the object and add value, otherwise create object an add value
if(month in monthlyObject){
monthlyObject[month] = monthlyObject[month] + sortedReadings[i][3];
} else {
Logger.log('Didnt find month so create it');
monthlyObject[month] = sortedReadings[i][3];
}
}
}
Logger.log(DailyAveragesArray.length);
Logger.log(monthlyObject);
var DailyUsageData = ss.getRange('D1:F'+DailyAveragesArray.length);
DailyUsageData.setValues(DailyAveragesArray);
BuildMonthlyArray();
}
function BuildMonthlyArray(){
const keys = Object.keys(monthlyObject);
Logger.log(keys);
keys.forEach((key, index) => {
monthlyArray.push([key,Math.round(monthlyObject[key])]);
});
var MonthlyUsageData = ss.getRange('H1:I'+monthlyArray.length);
MonthlyUsageData.setValues(monthlyArray);
}
So, my question is, how would I do this nicer, more beautifully, not so verbose ?
I'm not sure what the correct term is for what I want to do. I don't think it's resampling .
I'd appreciate any comments.
Thanks / Colm
Here is my shot on this.
The way i'm doing it:
Initializing all days and its value
Grouping by month
Calculating the average per month
Explanation a bit more precise
initDateFromString
The method initDateFromString takes a dates with the format DD/MM/YYYY and return the associated js date object
initAllDates
The method initAllDates will split the data into day and add the average value of the difference for each day
for example, for the first two readings, it will result to an array of dates looking like :
date
value
19/01/2021
14.02
20/01/2021
14.02
....
....
05/03/2021
14.02
06/03/2021
14.02
The value 14.02 comme from the following calcul :
(newReadingMeter - oldReadingMeter)/nbDaysBetweenDates
Which in this example is (5915 - 5270)/46 = 14.02
joinToMonth
The joinToMonth method will then group the days into month with all the days value summed !
const data = [{
ReadingDate: '19/01/2021',
Meter: 5270
},
{
ReadingDate: '06/03/2021',
Meter: 5915
},
{
ReadingDate: '11/05/2021',
Meter: 6792
},
{
ReadingDate: '08/07/2021',
Meter: 7367
},
{
ReadingDate: '9/9/2021',
Meter: 8095
},
{
ReadingDate: '8/11/2021',
Meter: 8849
},
{
ReadingDate: '02/12/2021',
Meter: 9065
},
{
ReadingDate: '17/01/2022',
Meter: 9950
}
]
function initDateFromString(dateString){
let dateParts = dateString.split("/");
return new Date(+dateParts[2], dateParts[1] - 1, +dateParts[0]);
}
function initAllDates(data){
let dates = []
let currentValue = data.shift()
const currentDate = initDateFromString(currentValue.ReadingDate)
data.forEach(metric => {
const date = initDateFromString(metric.ReadingDate)
const newDates = []
while(currentDate < date){
newDates.push({date: new Date(currentDate)})
currentDate.setDate(currentDate.getDate() + 1)
}
dates = dates.concat(newDates.map(x => {
return {Usage: (metric.Meter - currentValue.Meter) / newDates.length, date: x.date}}
))
currentDate.setDate(date.getDate())
currentValue = metric
})
return dates
}
function joinToMonth(dates){
return dates.reduce((months, day) => {
const month = day.date.getMonth()
const year = day.date.getFullYear()
const existingObject = months.find(x => x.month === month && x.year === year)
if (existingObject) {
existingObject.total += day.Usage
} else {
months.push({
month: day.date.getMonth(),
year: day.date.getFullYear(),
total: day.Usage,
})
}
return months;
}, []);
}
const dates = initAllDates(data)
const joinedData = joinToMonth(dates)
console.log(joinedData)
I'm using LocalStorage to save an array of Dates and Costs.
When I'm writing localStorage.getItem("todos"); into the console, the format will be like this:
"[{"due":"28/10/2017","task":"80"},{"due":"06/10/2017","task":"15"}]"
Where due is the Date, and TASK is the AMOUNT.
I managed to get the TOTAL of AMOUNTS by:
total: {
type: String,
value: () => {
var values = localStorage.getItem("todos");
if (values === undefined || values === null) {
return "0";
}
var data = JSON.parse(values);
var sum = 0;
data.forEach(function(ele){ sum+=Number(ele.task)}); return sum;
}
}
Now I'm trying to get the TOTAL of last 6 MONTHS.
I have no idea on how to approach this.
How should I be able to do this?
During your iteration you need to add a check to make sure the sum is only including values where the due date is within your range. If you can use a library like moment, this would greatly simplify your logic.
const data = [
{ due: '28/10/2017', task: 80 },
{ due: '06/10/2017', task: 15 },
{ due: '10/05/2000', task: 3000 }
];
const sixMonthsAgo = moment().subtract(6, 'months');
const total = data.reduce((acc, item) => {
const dueDate = moment(item.due, 'DD/MM/YYYY');
return acc + (dueDate.isAfter(sixMonthsAgo) ? item.task : 0);
}, 0);
console.log('total should equal 95: ', total);
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.19.1/moment.min.js"></script>
Here is a solution for your issue :
make a test in the forEach loop :
I've put 4 dates : 2 under 6 months and 2 older
The result is 80+15 = 95
// After JSON.parse
var todos=[{"due":"28/10/2017","task":"80"},{"due":"06/10/2017","task":"15"},{"due":"06/04/2017","task":"15"},{"due":"06/02/2017","task":"15"}];
var sum = 0;
var minDate = new Date();
var month = minDate.getMonth()+1-6; // get month minus 6 months
var year = minDate.getFullYear(); // get year
if(month < 1){ // if month is under January then change year
month+=6;
year-= 1;
}
minDate.setMonth(month); // Replace our min date with our - 6 m
minDate.setYear(year); // set year in case we have changed
todos.forEach(function(ele){
var arr = ele.due.split("/"); // split french string date into d,m,y
if(arr.length==3){
var dueDate = new Date(arr[2],arr[1],arr[0]); // get the task date
if(dueDate>minDate){ // if task is not to old then
sum+=parseInt(ele.task); // sum it
}
}
});
console.log(sum);
I have an array with the following values (example):
[
1491408000000,
1491494400000,
1491753600000,
1493222400000,
1493308800000,
1493568000000
]
Where the index is a date time. The date time will always be at 12:00:00 on a date.
In this example, the first 3 dates are consecutive cross weekend (weekend is holiday so count as leave), then another group of 3 dates cross weekend and month.
Now, what I am trying to do is find sequential dates (cross week and month) and put them into an array as follows:
[
1491408000000,
1491494400000,
1491753600000
],
[
1493222400000,
1493308800000,
1493568000000
]
I have tried the following code to get the sequential dates but this cannot cross week and month, how to modify the code to get above result? Any help would be much appreciated!
var timeValue = new Date(dateReview).getTime();
valueCon.push(timeValue);
var k = 0;
sortedValue[k] = [];
valueCon.sort( function ( a, b ){
return +a > +b ? 1 : +a == +b ? 0: -1;
})
.forEach( function( v , i ){
var a = v,b = valueCon[i+1]||0;
sortedValue[k].push( +a );
if ( (+b - +a) > 86400000) {
sortedValue[++k] = []
}
return 1;
});
sortedValue.sort( function ( a,b ){
return a.length > b.length ? -1: 1;
});
This requires help from a function to test if two dates are in the same week. The following goes over the set of time values provided in an array and puts the first value into an array within the array. For each subsequent value, it tests if it's in the same week as the first value in each array within the outer array.
If it's in the same week as the first value in any existing array, it's pushed into that array. Otherwise, it's put in a new array and pushed into the outer array.
There may be a neater way to implement the algorithm, but I'll leave that for others.
Due to time zone differences, they are adjusted to the host time zone based on the original time values representing noon in the source time zone.
// Given 2 dates, return true if they are in the same week (Mon to Sun).
// Otherwise, return false
function sameWeek(a, b){
var e = new Date(+a);
// Week starts at 00:00:00.000 on Monday on or before date
var s = new Date(e.setDate(e.getDate() - ((e.getDay()||7) -1)));
s.setHours(0,0,0,0);
// Week ends at 23:59:59.999 the following Sunday
e.setDate(e.getDate() + 6);
e.setHours(23,59,59,999);
// Test b and return value
return b >= s && b <= e;
}
// Given time value for UTC-0400, adjust to same date and time
// in local time zone and return a date
function adjust(n) {
var d = new Date(n);
d.setMinutes(d.getMinutes() - 240 + d.getTimezoneOffset());
return d;
}
var result = [1491408000000,1491494400000,1491753600000,1493222400000,1493308800000,1493568000000
].reduce(function(acc, n) {
var d = adjust(n);
var used;
if (acc.length != 0) {
used = acc.some(function(arr) {
if (sameWeek(adjust(arr[0]), d)) {
arr.push(n);
return true;
}
});
}
if (!used || acc.length == 0) {
acc.push([n]);
}
return acc;
},[]);
// Result array
console.log(result);
// Printed as date strings adjusted to same host local time
result.forEach(arr => {
arr.forEach(n => console.log(adjust(n).toString()))
console.log('\n');
});
Manipulation of timestamps is a pain. JavaScript has a built-in Date type, as you know, and I would suggest you use it. Date#getUTCDay returns the day of the week as an integer (for reference, 4 is Friday, or the day before a weekend), while Date#setUTCDate and Date#getUTCDate together allow you to adjust the date in day increments (and have it overflow/underflow to the next/previous month). Thus, to determine whether a timestamp b follows "sequentially" (excluding weekends) after a, you can use:
function sequential (a, b) {
a = new Date(a)
return a.setUTCDate(a.getUTCDate() + (a.getUTCDay() === 4 ? 3 : 1)) === b
}
Grouping is just an exercise after that; the code above contains all of the real logic behind this solution.
Example Snippet
var dates = [
1491408000000,
1491494400000,
1491753600000,
1493222400000,
1493308800000,
1493568000000
]
function sequential (a, b) {
a = new Date(a)
return a.setUTCDate(a.getUTCDate() + (a.getUTCDay() === 4 ? 3 : 1)) === b
}
function groupSequential(dates) {
if (dates.length < 2) return [dates.slice()]
dates.sort(function(a, b) { return a - b })
var result = [], group
for (var i = 0; i < dates.length; i++) {
sequential(dates[i - 1], dates[i]) || result.push(group = [])
group.push(dates[i])
}
return result
}
console.log(groupSequential(dates))