We Keep Coding

How to accumulate category counted fixed-size array in `$group` stage?

Let books collection be,
db.books.insertMany([
{ "name": "foo", "category": 0, publishedAt: ISODate("2008-09-14T00:00:00Z") },
{ "name": "bar", "category": 1, publishedAt: ISODate("1945-08-17T00:00:00Z") },
{ "name": "baz", "category": 1, publishedAt: ISODate("2002-03-01T00:00:00Z") },
{ "name": "qux", "category": 2, publishedAt: ISODate("2002-01-21T00:00:00Z") },
{ "name": "quux", "category": 4, publishedAt: ISODate("2018-04-18T00:00:00Z") },
])
I want to calculate total amount of books published between 2000-2010 inclusive for each year and also count of published categories. Let category be defined as an enum with 5 variants represented with integer in MongoDB schema e.g Fiction, Fantasy, Classic, Horror, Comic.
I achieved other requirements with this aggregation pipeline.
db.books.aggregate([
{
$match: {
publishedAt: {
$gte: ISODate("2000-01-01T00:00:00Z"),
$lt: ISODate("2011-01-01T00:00:00Z"),
},
},
},
{
$group: {
_id: {
$year: "$publishedAt",
},
totalCount: {
$count: {},
},
},
},
{
$sort: {
_id: 1,
},
},
]);
With following output,
[
{
_id: 2002,
totalCount: 2,
},
{
_id: 2008,
totalCount: 1,
},
]
But I also want a field that represents number of categories in an array. For example,
[
{
_id: 2002,
totalCount: 2,
categoryCount: [0, 1, 1, 0, 0],
},
{
_id: 2008,
totalCount: 1,
categoryCount: [1, 0, 0, 0, 0],
},
]
Array's length needs to be 5 since category is defined with 5 variants. In the example, the year 2002 has total of 2 books, which totalCount represents and has 1 book in category 1 which is why categoryCount[1] is 1. Likewise 1 book in category 2.

Using $accumulate
db.collection.aggregate([
{
$match: {
publishedAt: {
$gte: ISODate("2000-01-01T00:00:00Z"), $lt: ISODate("2011-01-01T00:00:00Z")
}
}
},
{
$group: {
_id: {
$year: "$publishedAt"
},
totalCount: {
$count: {}
},
categoryCount: {
$accumulator: {
init: function(){
return [0,0,0,0,0] //start with array with all entries as 0
},
accumulate: function(state, category) {
state[category] = state[category] + 1; //increment the value at index specified by the category
return state;
},
accumulateArgs: ["$category"],
merge: function(state1, state2) {
for (i = 0; i < state.length; i++) //incase the merge is needed add the values at each indexes
{
state[i] = state1[i] + state2[i];
}
return state;
},
lang: "js"
}
}
},
},
{
$sort: {
_id: 1
}
}
]);

You can achieve results like that without accumulator, using two $group stages: first by year and category, and then by year only, and then apply some MongoDB functions to transform the result to the desired format
The resulting query is long and looks quite complicated, duh. But works on your data example:
db.collection.aggregate([
{
$match: {
publishedAt: {
$gte: ISODate("2000-01-01T00:00:00Z"),
$lt: ISODate("2011-01-01T00:00:00Z")
}
}
},
{
$group: {
_id: {
year: {
$year: "$publishedAt"
},
category: "$category"
},
totalCount: {
$count: {}
}
}
},
{
$group: {
"_id": "$_id.year",
"totalCount": {
"$sum": "$totalCount"
},
"categoryCount": {
"$push": {
"k": {
"$toString": "$_id.category"
},
"v": "$totalCount"
}
}
}
},
{
"$addFields": {
"categoryCount": {
"$arrayToObject": "$categoryCount"
}
}
},
{
"$addFields": {
"categoryCount": {
"$mergeObjects": [
{
"0": 0,
"1": 0,
"2": 0,
"3": 0,
"4": 0,
"5": 0
},
"$categoryCount"
]
}
}
},
{
"$addFields": {
"categoryCount": {
"$objectToArray": "$categoryCount"
}
}
},
{
"$addFields": {
"categoryCount": {
"$map": {
"input": "$categoryCount",
"as": "x",
"in": {
"$mergeObjects": [
"$$x",
{
"k": {
"$toInt": "$$x.k"
}
}
]
}
}
}
}
},
{
"$addFields": {
"categoryCount": {
"$sortArray": {
"input": "$categoryCount",
"sortBy": {
"$k": 1
}
}
}
}
},
{
"$addFields": {
"categoryCount": "$categoryCount.v"
}
},
{
$sort: {
_id: 1
}
}
])
MongoDB playground
Step-by-step explanation:
$match - your initial filter
$group - pass both year and category into _id to preserve the count for each category
$group - group by year only, collect a "categoryCount" as a list of objects for each category that appeared in this year
$addFields - combine the list into a single document, keys are categories, and values are their counts. Notice, that keys can only be a strings, so we must cast them
$addFields - "densify" object to fill missing categories with zeros
$addFields - convert object back to the array, so we can extract values only
$addFields - cast categories back to numbers for correct sorting, if you have more than 10 of them
$addFields - sort by categories to ensure order (actually I'm not sure if this step is really needed)
$addFields - extract the count for each category into a flat list
Try to add these stages one by one to your query to see how it actually works.
In fact, my suggestion is to use aggregation as an end-to-end transformation, but rather stop at stage 3 or 4, and finish the transformation with your programming language, if you can. Good luck

Modifying the values inside an object

I am trying to alter object returned from an api to have a similar structure to another data set in my app and push them into an Array. So that I can add this data to my original data. But I am getting a strange result from my reduce function.
The data is coming in like this:
{
count: 4,
datapoints: [
{
Date: "2021-05-22",
score: 0,
},
{
Date: "2021-05-23",
score: 0,
},
{
Date: "2021-05-24",
score: 0,
},
{
Date: "2021-05-25",
score: 114,
},
],
};
I have a reduce function that looks like this:
const riskScores = await api.PCC.riskAssessment(userID, startdate, endDate);
const riskScoresFormatted = riskScores.datapoints.reduce((result, data) => {
const scores = result["riskAssessment"] || [];
scores.push({
value: data.score,
unit: "none",
recordedDate: data.Date,
method: "none",
});
result["riskAssessment"] = scores;
return result;
});
When I console out riskScoresFormatted Im getting something that looks like what I want but its coming out with the newly formatted objects nested inside the first object iterated over. like this:
{ Date: "2021-05-22"
riskAssessment: [{value: 0, unit: "none", recorded...}{...}] <- the data I need
score: 0
}
So I see clearly Im doing something wrong. I want to have all the newly formatted objects pushed inside an array with the key of "riskAssessment". Can anyone spot what Im doing wrong?

Provide the empty array [] as the initial data for the array at the end of reduce function.
const riskScoresFormatted = riskScores.datapoints.reduce((result, data) => {
const scores = result["riskAssessment"] || [];
scores.push({
value: data.score,
unit: "none",
recordedDate: data.Date,
method: "none",
});
result["riskAssessment"] = scores;
return result;
},[]);
Output
[
{
"value": 0,
"unit": "none",
"recordedDate": "2021-05-22",
"method": "none"
},
{
"value": 0,
"unit": "none",
"recordedDate": "2021-05-23",
"method": "none"
},
{
"value": 0,
"unit": "none",
"recordedDate": "2021-05-24",
"method": "none"
},
{
"value": 114,
"unit": "none",
"recordedDate": "2021-05-25",
"method": "none"
}
]

From what I gather, you want riskAssessment to be a new array with your new objects.
You're setting scores to be the previous value of riskAssessment. It will inherit the previous state, and you're just adding to it. Try to initialize it as an empty array directly, and then push your values to it.
const riskScoresFormatted = riskScores.datapoints.reduce((result, data) => {
const scores = [];
scores.push({
value: data.score,
unit: "none",
recordedDate: data.Date,
method: "none",
});
result["riskAssessment"] = scores;
return result;
});

How to calculate average from nested values in an array using javascript?

I am trying to calculate the average duration for each stage. So in the array below - I should be able to get the average duration for 'test1', which would be 2.
jobs = [
{
"build_id": 1,
"stage_executions": [
{
"name": "test1"
"duration": 1,
},
{
"name": "test2"
"duration": 16408,
},
{
"name": "test3"
"duration": 16408,
},
]
},
{
"build_id": 2,
"stage_executions": [
{
"name": "test1"
"duration": 3,
},
{
"name": "test2"
"duration": 11408,
},
{
"name": "test3"
"duration": 2408,
},
]
}
]
My failed attempt:
avgDuration: function(jobs) {
let durationSum = 0
for (let item = 0; item < this.jobs.length; item++) {
for (let i = 0; i < this.jobs[item].stage.length; item++) {
durationSum += stage.duration
}
durationAverage = durationSum/this.jobs[item].stage.length
}
return durationAverage
What am I doing wrong? I'm not sure how to accomplish this since the duration is spread out between each job.
UPDATE:
This is return a single average for all stages rateher than per stage
<template>
<div class="stages">
<h3>
Average Duration
</h3>
<table>
<tbody>
<tr v-for="item in durations">
<td>
<b>{{ item.average}} {{ item.count }}</b>
// this returns only 1 average and 177 count instead of 10
<br />
</td>
</tr>
</tbody>
</table>
</div>
</template>
<script>
import { calculateDuration } from "../../helpers/time.js";
import { liveDuration } from "../../helpers/time.js";
import moment from "moment";
export default {
name: "Stages",
data() {
return {
jobs: [],
durations: []
};
},
methods: {
avgDuration: function(jobs) {
var averageByName = {}; // looks like { 'name': { average: 111, count: 0 }}
for (var job of jobs) {
for(var stage of job.stage_execution) {
if (averageByName[stage.name] == null) { // we need a new object
averageByName[stage.name] = { average: 0, count: 0 };
}
// just name it so its easier to read
var averageObj = averageByName[stage.name];
// update count
averageObj.count += 1;
// Cumulative moving average
averageObj.average = averageObj.average + ( (stage.duration - averageObj.average) / averageObj.count );
console.log(averageObj.count)
}
}
return averageByName
},
},
created() {
this.JobExecEndpoint =
process.env.VUE_APP_TEST_URL +
"/api/v2/jobs/?limit=10";
fetch(this.JobExecEndpoint)
.then(response => response.json())
.then(body => {
for (let i = 0; i < body.length; i++) {
this.jobs.push({
name: body[i].job.name,
job: body[i].job,
stage_execution: body[i].stage_executions,
});
}
})
.then(() => {
this.$emit("loading", true);
})
.then(() => {
this.durations = this.avgDuration(this.jobs);
})
.catch(err => {
console.log("Error Fetching:", this.JobExecEndpoint, err);
return { failure: this.JobExecEndpoint, reason: err };
});
}
};
</script>

We can do this pretty simply and without overflow from having too many numbers by using a Cumulative moving average and a few loops.
Here is a line the relevant Wikipedia page on Moving Averages and the most relvant formula below.
I will not go into much detail with the above as there are a lot of documents describing this sort of thing. I will however say that the main reason to this over adding all the values together is that there is a far lower chance of overflow and that is why I am using it for this example.
Here is my solution with comments made in code.
var jobs = [ { "build_id": 1, "stage_executions": [ { "name": "test1", "duration": 1, }, { "name": "test2", "duration": 16408, }, { "name": "test3", "duration": 16408, }, ] }, { "build_id": 2, "stage_executions": [ { "name": "test1", "duration": 3, }, { "name": "test2", "duration": 11408, }, { "name": "test3", "duration": 2408, }, ] } ];
var averageByName = {}; // looks like { 'name': { average: 111, count: 0 }}
for (var job of jobs) {
for(var stage of job.stage_executions) {
if (averageByName[stage.name] == null) { // we need a new object
averageByName[stage.name] = { average: 0, count: 0 };
}
// just name it so its easier to read
var averageObj = averageByName[stage.name];
// update count
averageObj.count += 1;
// Cumulative moving average
averageObj.average = averageObj.average + ( (stage.duration - averageObj.average) / averageObj.count );
}
}
// print the averages
for(var name in averageByName) {
console.log(name, averageByName[name].average);
}
Let me know if you have any questions or if anything is unclear.

You could collect the values in an object for each index and map later only the averages.
var jobs = [{ build_id: 1, stage_executions: [{ name: "test1", duration: 1 }, { name: "test2", duration: 16408 }, { name: "test3", duration: 16408 }] }, { build_id: 2, stage_executions: [{ name: "test1", duration: 3 }, { name: "test2", duration: 11408 }, { name: "test3", duration: 2408 }] }],
averages = jobs
.reduce((r, { stage_executions }) => {
stage_executions.forEach(({ duration }, i) => {
r[i] = r[i] || { sum: 0, count: 0 };
r[i].sum += duration;
r[i].avg = r[i].sum / ++r[i].count;
});
return r;
}, []);
console.log(averages.map(({ avg }) => avg));
console.log(averages);
.as-console-wrapper { max-height: 100% !important; top: 0; }

I've used Array.prototype.flatMap to flatten the jobs array into an array of {name:string,duration:number} object. Also, to make more solution a bit more dynamic the function takes in a field argument which returns the average for that specific field.
const jobs = [
{
"build_id": 1,
"stage_executions": [
{
"name": "test1",
"duration": 1,
},
{
"name": "test2",
"duration": 16408,
},
{
"name": "test3",
"duration": 16408,
},
]
},
{
"build_id": 2,
"stage_executions": [
{
"name": "test1",
"duration": 3,
},
{
"name": "test2",
"duration": 11408,
},
{
"name": "test3",
"duration": 2408,
},
]
}
];
const caller = function(jobs, field) {
const filtered = jobs
.flatMap((item) => item.stage_executions)
.filter(item => {
return item.name === field;
})
const total = filtered.reduce((prev, curr) => {
return prev + curr.duration;
}, 0)
return total / filtered.length;
}
console.log(caller(jobs, 'test1'))
console.log(caller(jobs, 'test2'))
console.log(caller(jobs, 'test3'))
In case you get the error flatMap is not a function. You can add this code snippet in your polyfill or at the top of your js file.
Array.prototype.flatMap = function(lambda) {
return Array.prototype.concat.apply([], this.map(lambda));
};
PS: for demostration, I obtained the flatMap implementation from here

How to Convert an Array of Objects into an Object of Associative Arrays in Javascript

I am receiving the following structure from a system. I am attempting to bend it into the form needed for a particular graph utilizing chartjs. Given the JSON data structure … an array of objects in an object:
{
"chart": [
{
"date": "2018-10-29",
"done": 3,
"todo": 10
},
{
"date": "2018-10-30",
"done": 4,
"todo": 7
},
{
"date": "2018-10-31",
"done": 5,
"todo": 12
}
]
}
I need the desired JSON data structure ... an object of arrays (in one array, in one object)
{
"chart": [{
"date": [
"2018-10-29",
"2018-10-29",
"2018-10-31"
],
"done": [
3,
4,
5
],
"todo": [
10,
7,
12
]
}]
}
I have attempted to use the .map function but I don't seem to have the correct map-fu.

You could take an object and get all keys with ther values in single array.
var data = { chart: [{ date: "2018-10-29", done: 3, todo: 10 }, { date: "2018-10-30", done: 4, todo: 7 }, { date: "2018-10-31", done: 5, todo: 12 }] },
result = { chart: data.chart.reduce((r, o) => {
Object.entries(o).forEach(([k, v]) => (r[k] = r[k] || []).push(v));
return r;
}, {})
};
console.log(result);
.as-console-wrapper { max-height: 100% !important; top: 0; }

What about using reduce ?
const output = input.reduce((acc, curr) => ({
date: acc.date.concat(curr.date),
done: acc.done.concat(curr.done),
todo: acc.todo.concat(curr.todo),
}), { date: [], done: [], todo: [] });
const chartData = {
chart: [output],
};
Reference for reduce is here : https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/Array/reduce

Here's a very explicit solution. There may be some slicker Javascript solutions; certainly you can do multiple .map calls, but that makes it less efficient.
// Variables
var dates = [];
var doneValues = [];
var todoValues = [];
// Loop through the original data once, collect the data.
originalJSON.forEach(function(data) {
dates.push(data["date"]);
doneValues .push(data["done"]);
todoValues .push(data["todo"]);
});
// Put it all together.
return {"chart": [{"date": dates, "done": doneValues , "todo": todoValues}]};
Modify it to suit your needs.

How to group an array of objects in JavaScript on specific fields using Lodash?

EDIT:
I believe this is a different problem than a simple _.groupBy because I am trying to group by all keys and returning an hashmap not an array.
I have an array of the following objects:
items = [
{ created_at: "01/01/2016", name: "bob", age: 21, height: 60 },
{ created_at: "01/02/2016", age: 22, height: 70 },
{ created_at: "01/03/2016", name: "alice", age: 23 }
]
And I am trying to transform it into this format:
{
"name": [
{
"value": "bob",
"created_at": "01/01/2016"
},
{
"value": "alice",
"created_at": "01/03/2016"
}
],
"age": [
{
"value": 21,
"created_at": "01/01/2016"
},
{
"value": 22,
"created_at": "01/02/2016"
},
{
"value": 23,
"created_at": "01/03/2016"
}
],
"height": [
{
"value": 60,
"created_at": "01/01/2016"
},
{
"value": 70,
"created_at": "01/02/2016"
}
]
}
My requirement is I am ignoring the created_at field, but grouping everything else.
One solution is:
var items = [
{ created_at: "01/01/2016", name: "bob", age: 21, height: 60 },
{ created_at: "01/02/2016", age: 22, height: 70 },
{ created_at: "01/03/2016", name: "alice", age: 23 }
]
var store = {}
for(var i=0; i < items.length; i++) {
var item = items[i]
for(key in item) {
if(key === "created_at") {
continue
}
value = item[key]
if(!store[key]) {
store[key] = []
}
store[key].push({
value: value,
created_at: item.created_at
})
}
}
$('pre').text(JSON.stringify(store, null, 2))
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<body><pre></pre></body>
I was wondering if there was some cool way to do it in lodash ... maybe using .map() or .groupBy() or something? I don't quite get how to use them properly and am trying to learn new, more elegant ways to write the code.

You asked specifically about lodash, and so this answer is constructed using lodash.
Note I'm using the _.chain method - this makes it convenient to chain multiple operations rather than running them each separately.
One way (I'm sure there are many others) that gets it pretty close (not perfect) is this:
items = _.chain(items)
.map(function(n) {
var created = _.get(n, 'created_at');
var arr = [];
_.each(n, function(n, k) {
if (k != 'created_at') {
arr.push({
'field': k,
'value': n,
'created_at': created
});
}
});
return arr;
})
.flatten()
.groupBy('field')
.value();
Which results in a collection that looks like this:
{ "name":
[
{ "field": "name", "value": "bob", "created_at": "01/01/2016" },
{ "field": "name", "value": "alice", "created_at": "01/03/2016" }
],
"age": [
....
Here's a Fiddle for playing around with this code.

Using reduce might be the shortest way.
var baseKey = 'created_at';
var result = items.reduce(function(col, item) {
Object.keys(item).forEach(function(key) {
if (key === baseKey || !item[key]) return;
var o = { value: item[key] };
o[baseKey] = item[baseKey];
(col[key] = col[key] || []).push(o);
});
return col;
}, {});
JSFiddle Demo: https://jsfiddle.net/pLcv1am2/6/