We Keep Coding

Convert the date format from MM/DD/YYYY to YYYY-MM-DDT00:00:00.000Z in a list of dictionaries

I want to add my own data in a sample HTML chart. The javascript codes of the sample data is:
var data = [];
var visits = 10;
for (var i = 1; i < 50000; i++) {
visits += Math.round((Math.random() < 0.5 ? 1 : -1) * Math.random() * 10);
data.push({
date: new Date(2018, 0, i),
value: visits
});
}
chart.data = data
I ran the output of the sample data with console.log(data) and found the sample data is the following list of dictionaries:
[{"date":"2018-11-10T05:00:00.000Z","value":-459},
{"date":"2018-11-11T05:00:00.000Z","value":-459},
{"date":"2018-11-12T05:00:00.000Z","value":-464},
...
{"date":"2020-11-22T05:00:00.000Z","value":-493}]
I used pandas to create a list of dictionaries as my own data. I changed my own data to match the format of the above sample data list, the following list is my own data:
rnow =
[{'date': '01/02/2020', 'value': 13},
{'date': '01/03/2020', 'value': 2},
{'date': '01/06/2020', 'value': 5},
...
{'date': '01/07/2020', 'value': 6}]
I realized the date format is different from the sample data. I used the following code to change the date format to yyyy-mm-dd.
dfrnow['date'] = pd.to_datetime(dfrnow.date)
rnow = dfrnow.to_dict('records')
Now my data is shown as:
rnow =
[{'date': Timestamp('2020-01-02 00:00:00'), 'value': 13},
{'date': Timestamp('2020-01-03 00:00:00'), 'value': 2},
{'date': Timestamp('2020-01-06 00:00:00'), 'value': 5},
...
{'date': Timestamp('2020-01-07 00:00:00'), 'value': 6}]
When I replace the sample data with my own data, the chart doesn't show up any data. Is that because it has this "Timestamp" thing shown in my list and how should I remove it? Or are there other better methods I can use to convert my dates?
(The following codes are all the js codes of this template chart. I listed them here just in case I missed anything.)
var chart = am4core.create("chartdiv", am4charts.XYChart);
chart.paddingRight = 20;
var data = [];
var visits = 10;
for (var i = 1; i < 50000; i++) {
visits += Math.round((Math.random() < 0.5 ? 1 : -1) * Math.random() * 10);
data.push({
date: new Date(2018, 0, i),
value: visits
});
}
chart.data = {{rnow}};
var dateAxis = chart.xAxes.push(new am4charts.DateAxis());
dateAxis.renderer.grid.template.location = 0;
dateAxis.minZoomCount = 5;
dateAxis.groupData = true;
dateAxis.groupCount = 500;
var valueAxis = chart.yAxes.push(new am4charts.ValueAxis());
var series = chart.series.push(new am4charts.LineSeries());
series.dataFields.dateX = "date";
series.dataFields.valueY = "value";
series.tooltipText = "{valueY}";
series.tooltip.pointerOrientation = "vertical";
series.tooltip.background.fillOpacity = 0.5;
chart.cursor = new am4charts.XYCursor();
chart.cursor.xAxis = dateAxis;
var scrollbarX = new am4core.Scrollbar();
scrollbarX.marginBottom = 20;
chart.scrollbarX = scrollbarX;
var selector = new am4plugins_rangeSelector.DateAxisRangeSelector();
selector.container = document.getElementById("selectordiv");
selector.axis = dateAxis;

I solved it by:
rnow = dfrnow.to_dict('records')
def new_date(date):
return datetime.strptime(date, '%m/%d/%Y').strftime('%Y-%m-%dT00:00:00Z')
for i in range (len(rnow)):
rnow[i]['date'] = new_date(rnow[i]['date'])

How to group values by week in plotly.js

Seen below is a time series bar graph with a range selector in plotly.js.
In it, I am trying to figure out how to group the values by week, but cannot seem to accomplish this. Is there a setting in plotly.js to group these by week when changing the time range selection? I cannot seem to figure out if it is possible.
Here are the main documentation pages they offer, of which I tried as many settings as I thought pertained to accomplishing this, but could not figure it out.
https://plot.ly/javascript/time-series/
https://plot.ly/javascript/bar-charts/
var days = (function(start,count){
var days = [];
var MSday = 1000 * 60 * 60 * 24;
for(var i = 0; i < count; i++){
days.push(new Date(+start + i*MSday));
}
return days;
})(new Date(2018,0,1),100);
function vals(){
var vals = [];
for(var i = 0; i < 100; i++){
vals.push((Math.random() * 2 * i) | 0);
}
return vals;
}
var selectorOptions = {
buttons: [{
step: 'month',
stepmode: 'backward',
count: 1,
label: '1m'
}, {
step: 'month',
stepmode: 'backward',
count: 6,
label: '6m'
}, {
step: 'year',
stepmode: 'todate',
count: 1,
label: 'YTD'
}, {
step: 'year',
stepmode: 'backward',
count: 1,
label: '1y'
}, {
step: 'all',
}],
};
var trace1 = {
x: days,
y: vals(),
type: 'bar',
name: 'Trace 1'
};
var trace2 = {
x: days,
y: vals(),
type: 'bar',
name: 'Trace 2'
};
var data = [trace1, trace2];
var layout = {
title: 'Bar Demo',
barmode: 'group',
xaxis: {
rangeselector: selectorOptions
}
};
Plotly.newPlot('myDiv', data, layout);
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<div id="myDiv"><!-- Plotly chart will be drawn inside this DIV --></div>
How can I make the 6 month selection group by week instead of by day on the graph?

Apparently this isn't built in. If it is, or becomes built in at some point, please indicate that here in a comment or another answer.
The only option I was able to determine as viable was to hook into the relayout event using .on('plotly_relayout', function () {, taking the arguments from the range selector buttons (which seem limited, only a from and to date, if there is a better way to determine the origination please also let me know and I will update here), and then roughly based on that to bin the dates by week and adjust the x and y values in the plot.
This is just a basic implementation as proof of concept. Using it in production would require refactoring this code to work with the existing data structures with regards to design and page implementation.
There is a lot going on here. Basically, it will iterate through the set of dates to create sunday bins which will hold the weekly data (note that it still lacks a display update to show it is a week from the start date). Once it has the bins it sums the dates in each bin range. Then it replaces the data set using restyle. If the range selected is not 6m then it will use the a slice of the backup data because plotly modifies arrays in place, and as a result it will overwrite the data if there is no backup copy in addition with a single copy every time the backup is used.
See below for a working demo.
function sum(array){
return array.reduce(function(sum,curr){
return sum + curr;
},0);
};
Date.MSday = 1000 * 60 * 60 * 24;
Date.prototype.floor = function(){
return new Date(this.getFullYear(),this.getMonth(),this.getDate());
}
Date.prototype.addDays = function(days){
var time = +this - +this.floor();
var addedDays = new Date(+this.floor() + Date.MSday*days);
return new Date(+addedDays + time);
}
function weeksFromDates(datesArray, valsArray){
var lastDay = datesArray[datesArray.length -1];
var firstDay = datesArray[0];
var dayOfWeek = firstDay.getDay();
var firstSunday = firstDay.addDays(-dayOfWeek);
var sundays = [];
var currentSunday = firstSunday;
while(currentSunday < lastDay){
sundays.push(currentSunday);
currentSunday = currentSunday.addDays(7);
}
currentSunday = currentSunday.addDays(7);
sundays.push(currentSunday);
var valSets = [];
var n = 0;
for(var i = 1; i < sundays.length; i++){
var last = sundays[i-1];
var next = sundays[i];
var theseVals = [];
for(; n < datesArray.length && last <= datesArray[n] && next > datesArray[n]; n++){
theseVals.push(valsArray[n]);
}
valSets.push(sum(theseVals));
}
sundays.pop();
return {x: sundays, y: valSets};
}
var MSday = 1000 * 60 * 60 * 24;
var days = (function(start,count){
var days = [];
for(var i = 0; i < count; i++){
days.push(new Date(+start + i*MSday));
}
return days;
})(new Date(2018,0,1),100);
function vals(){
var vals = [];
for(var i = 0; i < 100; i++){
vals.push((Math.random() * 2 * i) | 0);
}
return vals;
}
var selectorOptions = {
buttons: [{
step: 'month',
stepmode: 'backward',
count: 1,
label: '1m'
}, {
step: 'month',
stepmode: 'backward',
count: 6,
label: '6m'
}, {
step: 'year',
stepmode: 'todate',
count: 1,
label: 'YTD'
}, {
step: 'year',
stepmode: 'backward',
count: 1,
label: '1y'
}, {
step: 'all',
}],
};
var trace1 = {
x: days,
y: vals(),
type: 'bar',
name: 'Trace 1',
orientation: 'v'
};
var trace2 = {
x: days,
y: vals(),
type: 'bar',
name: 'Trace 2',
orientation: 'v'
};
var data = [trace1, trace2];
var dataBackup = $.extend(true,{},data);
var layout = {
title: 'Bar Demo',
barmode: 'group',
xaxis: {
rangeselector: selectorOptions
}
};
Plotly.newPlot('myDiv', data, layout);
$('#myDiv').on('plotly_relayout', function () {
var lower = new Date(arguments[1]['xaxis.range[0]']);
var upper = new Date(arguments[1]['xaxis.range[1]']);
var dayRange = (+upper - +lower) / MSday;
if( dayRange < 190 && dayRange > 170 ){
//6m
for(var n = 0; n < data.length; n++){
var weekly = weeksFromDates(dataBackup[n].x,dataBackup[n].y);
Plotly.restyle('myDiv',{x:[weekly.x],y: [weekly.y]},n);
}
}else{
for(var n = 0; n < data.length; n++){
Plotly.restyle('myDiv',{x:[dataBackup[n].x.slice()],y: [dataBackup[n].y.slice()]},n);
}
}
});
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<div id="myDiv"><!-- Plotly chart will be drawn inside this DIV --></div>

Blimey! There is a much simpler option...
use 7 days:
step: 'day',
stepmode: 'backward',
count: 7,
label: '1w'

Dynamically created object not populated as expected

I am trying to extract data from one object, restructure and create a new one.
Simplified example for the source object
var res = [{
DateTime: '00:00',
Number: 1,
WeekDay: 1
},
{
DateTime: '00:00',
Number: 4,
WeekDay: 1
},
{
DateTime: '00:00',
Number: 1,
WeekDay: 2
},
{
DateTime: '00:30',
Number: 1,
WeekDay: 2
}]
From here I want to create a new object where "number" is summed by
1. WeekDay
2. Half hour interval
var intervals = ['00:00', '00:30']
var weekdays = [1, 2]
var target = []
var intervalObj = [];
for (i = 0; i < intervals.length; i++) {
intervalObj.push({
interval: intervals[i],
number: 0
})
}
for (i = 0; i < weekdays.length; i++) {
var day = i + 1;
target.push({
day: day,
data: intervalObj,
})
}
And then populate the new object like this:
for(var row in res) {
var dt = res[row].DateTime;
var wd = res[row].WeekDay;
var wdidx = weekdays.indexOf(wd)
var dtidx = intervals.indexOf(dt)
var num = res[row].Number;
target[wdidx].data[dtidx].number += num;
}
This does not work when creating the target object like above. The summed results gets repeated for the same interval over all week days.
However, when object is statically:
var target = [{
day: 1,
data: [{
interval: '00:00',
number: 0
},
{
interval: '00:30',
number: 0
}]
},
{
day: 2,
data: [{
interval: '00:00',
number: 0
},
{
interval: '00:30',
number: 0
}]
}]
It works as expected. I cannot figure out why.
Here is a fiddle example:
https://jsfiddle.net/oceansmoving/wkfL9e3o/

You are using the same array reference intervalObj for data in each instance of loop. Need to create new array for each instance
Change
var intervalObj = [];
for (i = 0; i < weekdays.length; i++) {
var day = i + 1;
target.push({
day: day,
data: intervalObj,
})
}
To
//var intervalObj = [];
for (i = 0; i < weekdays.length; i++) {
var day = i + 1;
target.push({
day: day,
data: [],
})
}

send multiple requests in parallel

I have to make a bridge (script) between two databases (Mongo and Oracle).
I run three find queries on my MONGO database in three different collections.
Collection 1 = [{
name: 'Toto',
from: 'Momo',
note: 5,
cat: 'noCategori'
}]
Collection 2 = [{
city: 'london',
country: 'UK'
}]
...etc
I retrieve lists of documents from each collection I make a treatment on this list (Ex: average).
RowToSend = {
name: 'Toto',
note: 17, // average all document
city: 'london'
}
I establish a connection with Oracle and I persist this row.
I would like to run queries to mongo in parallel. at the end of the three requests and treatments. I compose my recording and I persist it on my database.
I have with async but it does not work. With await too. the function (Script) ends before the request is processed
function RequestOne(dateStart, dateEnd, RowToInsert) =
CollectionOne.find({
AQS_REF_TIME_EVENT_MSR: {
$gte: startInterv,
$lte: endInerv
},
})
.exec((err, arrAqsOneHourMoy) => {
if (err)
return err;
if (arrAqsOneHourMoy) {
AttributeOneAverge = 0;
AttributeTwoAverage = 0;
for (i = 0; i < arrAqsOneHourMoy.length; i++) {
AttributeOneAverge = (arrAqsOneHourMoy[i].AttributeOne + AttributeOneAverge);
AttributeTwoAverge = (arrAqsOneHourMoy[i].AttributeTwo + AttributeTwoAverge);
}
RowToInsert.AttributeOne = ((AttributeOneAverge) / (i + 1));
RowToInsert.AttributeOne = (AttributeTwoAverge) / (i + 1);
}
})
}
Function TWO .. same logic
Globale function :
function GenericFunction(time, sensorID, TEST_oracle_save) {
var d = new Date(2018, 00, 03, 11, 00, 00, 000);
var x = d.toISOString();
var d2 = new Date(2018, 00, 03, 11, 59, 59, 999);
var y = d2.toISOString();
RequestOne(x, y, TEST_oracle_save);
RequestTwo(x, y, TEST_oracle_save);
//Connexion to ORACLE DATABASE
//Connexion etablished
// Persist TEST_oracle_save on field Oracle
}

function RequestOne(dateStart, dateEnd, RowToInsert) =
CollectionOne.find({
AQS_REF_TIME_EVENT_MSR: {
$gte: startInterv,
$lte: endInerv
},
})
.exec((err, arrAqsOneHourMoy) => {
if (err)
return err;
if (arrAqsOneHourMoy) {
AttributeOneAverge = 0;
AttributeTwoAverage = 0;
for (i = 0; i < arrAqsOneHourMoy.length; i++) {
AttributeOneAverge = (arrAqsOneHourMoy[i].AttributeOne + AttributeOneAverge);
AttributeTwoAverge = (arrAqsOneHourMoy[i].AttributeTwo + AttributeTwoAverge);
}
RowToInsert.AttributeOne = ((AttributeOneAverge) / (i + 1));
RowToInsert.AttributeOne = (AttributeTwoAverge) / (i + 1);
}
})
}
Function TWO .. same logic
Globale function :
function GenericFunction(time, sensorID, TEST_oracle_save) {
var d = new Date(2018, 00, 03, 11, 00, 00, 000);
var x = d.toISOString();
var d2 = new Date(2018, 00, 03, 11, 59, 59, 999);
var y = d2.toISOString();
RequestOne(x, y, TEST_oracle_save);
RequestTwo(x, y, TEST_oracle_save);
//Connexion to ORACLE DATABASE
//Connexion etablished
// Persist TEST_oracle_save on field Oracle
}

Set custom X-Axis values in Flot

My custom X-axis values are not displayed in flot js.
Code for drawing the line chart as below :
var length = 0;
var length1 = 0;
var dataXY = [];
var data = [];
var dataX = [];
length = allVenues.length;
for (var i = 0; i < length; i++) {
length1 = allVenues[i].length;
for (var j = 0; j < length1; j++) {
dataX.push([j, returnDate(allVenues[i][j].date)]);
dataXY.push([returnTimestamp(allVenues[i][j].date), allVenues[i][j].price, "<b>X</b> : " + returnDate(allVenues[i][j].date) + " | " + " <b>Y</b>: " + allVenues[i][j].price]);
}
}
var result = {'label': 'Demo Graph', 'data': dataXY};
data = [result];
var options = {
lines: {
show: true
},
points: {
show: true
},
xaxis: {
ticks: dataX
},
grid: {
hoverable: true,
clickable: true
},
tooltip: {
show: true,
content: "%s | X: %x | Y: %y"
}
};
function returnTimestamp(val) {
var dateTime = new Date(val);
return moment(dateTime).unix();
}
function returnDate(val) {
var dateTime = new Date(val);
return moment(dateTime).format("YYYY-MM-DD hh:mm:ss A");
}
$.plot("#placeholder", data, options);
dataXY array values are:
{"label":"Demo Graph","data":[[1455776629,12],[1455801889,30],[1455962948,45]]}
dataX array values are:
[[0, "2016-02-18 11:53:49 AM"], [1, "2016-02-18 06:54:49 PM"], [2, "2016-02-20 03:39:08 PM"]]
Now i want to set this "dataX" array as X axis values (ticks) on the chart.
This values display below in each point of line graph with X-Axis.
Here dataX and dataXY from allVenues Json Array.
My graph is woking fine except the X-Axis data. You can see in the image below.

Your dataX array for the ticks must have the same x values (timestamps) as your real data array:
[[1455776629, "2016-02-18 11:53:49 AM"], [1455801889, "2016-02-18 06:54:49 PM"], [1455962948, "2016-02-20 03:39:08 PM"]]
PS: I would also suggest to put a linebreak (<br>) between date and time in the labels.