Can I make Mongo map reduce count multiple values of an object? - javascript

I have a collection that I am trying to map reduce by id and date to produce a graph for sales of a product in store vs online. A new object is created for each transaction, so I would like to reduce them to a total count for a given day. An object looks something like this:
object
{
"ProductID": 1
"Purchase Method": In Store
"Date": 2018-01-16
"Count": 5
}
What I am trying to achieve as output is to have in store and online purchases combined into 1 object with a key being the id and the date and then the value being the counts of each method as shown below:
ProductID: 1
Date: 2018-01-16
[
{Name: "In store", Count: 3}
{Name: "Online", Count: 2}
]
My current method was to map the objects by Id, date, and Purchase Method so the reduce would get the total count for that id on that date using that method, but this leads to having two entries for an id and date, 1 for in store and 1 for online. This is the current state of my functions:
var mapDailySales = function() {
var sale = this;
/*Converts timestamp to just date */
var pad = function pad(n, width, z) {
z = z || '0';
n = n + '';
return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n;
};
var d = sale.Date;
var date = d.getFullYear() + "-" + pad(d.getMonth() + 1, 2, 0) + "-" + pad(d.getDate(), 2, 0);
emit({ProductId: sale.ProductID, Date:date, Method: sale.PurchaseMethod},
{Name: sale.PurchaseMethod, Count: 1})
};
var reduceDailySales = function(key, value) {
var res = {Name: 0, Count: 0};
value.forEach(function(value){
res.Name = value.Name;
res.Count += value.Count;
});
return res;
};
Current Output looks something like this:
{
"_id" : {
"ProductId" : 1,
"Date" : "2018-01-16",
"Method" : "Online"
},
"value" : {
"Name" : "Online",
"Count" : 3
}
}
Is there a way to achieve my desired output without map reducing again on the current output?

You can use aggregation pipeline to get the results instead of mapReduce, $group by ProductID and Date, with $project you can map counts to an array
added $out to write the results to new collection, removing it will return a cursor
db.prod.aggregate([
{$group : {
_id : {ProductID : "$ProductID", Date : "$Date"},
onlineCount : {$sum : {$cond : [{$eq : ["$PurchaseMethod", "Online"]}, "$Count" , 0]}},
storeCount : {$sum : {$cond : [{$eq : ["$PurchaseMethod", "In Store"]}, "$Count" , 0]}}
}
},
{$project : {
_id : 0,
ProductID : "$_id.ProductID",
Date : "$_id.Date",
counts : [{Name: "In Store", Count: "$storeCount"},{Name : "Online", Count: "$onlineCount"}]
}},
{$out : "count_stats"}
]).pretty()
collection
> db.prod.find()
{ "_id" : ObjectId("5a98ce4a62f54862fc7cd1f5"), "ProductID" : 1, "PurchaseMethod" : "In Store", "Date" : "2018-01-16", "Count" : 5 }
{ "_id" : ObjectId("5a98ce4a62f54862fc7cd1f6"), "ProductID" : 1, "PurchaseMethod" : "Online", "Date" : "2018-01-16", "Count" : 2 }
>
result
> db.count_stats.find()
{ "_id" : ObjectId("5a98d3366a5f43b12a39b4ac"), "ProductID" : 1, "Date" : "2018-01-16", "counts" : [ { "Name" : "In Store", "Count" : 5 }, { "Name" : "Online", "Count" : 2 } ] }
>
if you want to use mapReduce, you can use finalize to reduce or transform the result further
db.prod.mapReduce(
<map>,
<reduce>,
{
out: <collection>,
finalize: <function>
}
)

Related

Calculus with objects in javascript

I have an object that contains 2 fields: day_active and day_inactive. (the object is in the javascript snippet below)
And what I want to obtain is another object that is based on this formula:
count(day_active (on date x))-count(day_inactive (on date x)
{
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 2.0 // (5.0 - 3.0) - how many groupC were on day_active(2019-09-19) minus how many groupC were on day_inactive(2019-09-19)
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : -2.0 // (3.0 - 5.0)
},
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : -2.0 // (8.0 - 10.0)
},
{
"date" : "2019-10-08",
"type" : "groupW",
"count" : 7.0 // (7.0 - 0.0)
}
}
I tried this code but is not taking all the cases and the result is incomplete:
let items = {
"day_inactive" : [
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 3.0
},
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : 10.0
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : 5.0
},
{
"date" : "2019-10-07",
"type" : "groupW",
"count" : 9.0
},
{
"date" : "2019-10-05",
"type" : "groupW",
"count" : 3.0
},
],
"day_active" : [
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : 8.0
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : 3.0
},
{
"date" : "2019-10-08",
"type" : "groupW",
"count" : 7.0
},
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 5.0
}
]
}
let auxObj = {}
for (let i = 0; i < items.day_active.length; i++) {
for (let j = 0; j < items.day_inactive.length; j++) {
if (items.day_active[i].date == items.day_inactive[j].date && items.day_active[i].type == items.day_inactive[j].type) {
// console.log("yes")
auxObj.date = items.day_active[i].date
auxObj.type = items.day_active[i].type
auxObj.count = items.day_active[i].count - items.day_inactive[j].count
}
}
}
console.log(auxObj)
How can I solve this in a simple way? Thank you for your time!
Follow along the comments for explanation...
// let's create an empty object
let output = {};
// and start copying active days...
for (const obj of items.day_active) {
// the following `key` is just for grouping purposes...
const key = `${obj.date}-${obj.type}`;
output[key] = { ...obj };
}
// Now let's look at inactive days...
for (const obj of items.day_inactive) {
// the following `key` is just for grouping purposes...
const key = `${obj.date}-${obj.type}`;
// is this the first time we're looking at this `date-type`? let's add it with 0 count
if (!output[key]) {
output[key] = { ...obj, count: 0 };
}
// and subtract it from active days count
output[key].count -= obj.count;
}
// let's remove the `key` we created earlier...
output = Object.values(output);
// here's the output
console.log(output);
From the sample input given, this is the result we get:
[ { date: '2019-10-11', type: 'groupW', count: -2 },
{ date: '2019-10-19', type: 'groupW', count: 3 },
{ date: '2019-10-08', type: 'groupW', count: 7 },
{ date: '2019-10-19', type: 'groupC', count: 5 },
{ date: '2019-09-19', type: 'groupC', count: -3 },
{ date: '2019-09-19', type: 'groupW', count: -5 },
{ date: '2019-10-07', type: 'groupW', count: -9 },
{ date: '2019-10-05', type: 'groupW', count: -3 } ]
I think this one will be more efficient, i named the object after me cause...idn
let marios = {};
items.day_active.forEach(d => marios[d.date+'_'+d.type] = d.count || 0);
items.day_inactive.forEach(d => marios[d.date+'_'+d.type] = marios[d.date+'_'+d.type] ? marios[d.date+'_'+d.type] - (d.count || 0) : (d.count || 0));
console.log(marios);
The logic behind it is that we create an object and we create a property for each date present in the data sets, starting with the first terms of the formla, ending with the subtraction of the second part, and by defaulting to 0 for each appropriate case.
In the end you can iterate the properties of the object, split the string on '_' to read each propertie's date and group and create an array of the results ( if you have trouble with this let me know )

Mongoose percentage of total

I have a model that looks like this :
Model :
createdAt: {
type: String,
default: Moment(new Date()).format('YYYY-MM-DD')
},
loginTrack: [
{
user_id: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Users',
}
}
With some data :
[
{
_id: ...,
createdAt : '2018-03-22',
loginTrack: [
{user_id : 1,...}
{user_id : 1, ...},
{user_id : 2, ...}
]
},
{
_id: ...,
createdAt : '2018-03-23',
loginTrack : [
{user_id : 4, ...},
{user_id : 1, ...}
]
},
{
_id : ...,
createdAt: '2018-03-24',
loginTrack : [
{user_id : 2, ...}
]
]
I'd like to have the percentage of total unique new sessions per day, that mean count the number of sessions for each previous day, is it possible with mongodb ?
With an output like this
[{
date : '2018-03-22',
newSessionsAvg : 2 (unique sessions only : maybe it's 100 % ?)
},
{
date : '2018-03-23',
newSessionAvg: 100
},
{
date : '2018-03-24',
newSessionAvg : 25 (1/ (2+2) * 100)
}]
Is it possible using an aggregation/project/group ?
This is what I tried :
AnalyticsModel.aggregate([
{
"$project" : {
users: {$size: "$loginTrack"},
"createdAt" : 1,
"_id": 0
}},
{
"$group": {
"_id": "$createdAt",
"count": { "$sum": 1 }
}
}
The output looks like this :
[{"_id":"2018-03-22","count":3},{"_id":"2018-03-21","count":2}]
Thanks
Maybe just create a occurence map at first:
User.find({}, function(err, users) {
const occurences = {};
for(const {createdAt} of users){
occurences[createdAt] = (occurences[createdAt] || 0) +1;
}
Then you can sort that data after the date and build up the results:
const timeline = Object.entries(occurences);
timeline.sort((a,b) => a[0].localeCompare(b[0]));
const result = [];
let previous = 0;
for(const [date, total] of timeline){
result.push({ date, avg: (total / (total + previous) || 0) * 100 });
previous = total;
}

How to access node in the firebase databse

My aim is to have each user select a total of 6 players. when each player is selected, the player id is sent to a node on the database called 'total'using the push () method. The code is written below
var ref = firebase.database().ref().child('players');
var ref3 = firebase.database().ref().child('users').child(uid).child('total');
$scope.players = $firebaseArray(ref);
console.log ($scope.players);
$scope.history = [];
$scope.buy = function(player) {
//remove if already added locally
var index = $scope.history.indexOf(player);
if(index>=0){
$scope.history.splice(index,1);
return;
}
//remove if already added on firebase
//max 6 allowed
if($scope.history.length>=6){
alert('max 6 allowed');
return;
}
var selected = $scope.history.reduce(function(a,b){
a[b.position] = (a[b.position] || 0) + 1;
return a;
}, {}) || {};
if(!selected[player.position] || selected[player.position]<2){
$scope.history.push(player);
ref3.push(player.id);
}else{
alert('You can add only two players per position');
}
};
$scope.getTotal = function(){
return $scope.history.reduce(function(tot, p){
tot = tot - p.price;
return tot;
}, $scope.total);
};
this is how the database is structured :
{
"players" : [ {
"R" : 0,
"Team" : "Industry",
"Y" : 0,
"assists" : 0,
"goals" : 0,
"id" : 1,
"name" : "Yasin 'YB' Amusan",
"position" : "forward",
"price" : 8000000
}, {
"R" : 0,
"Team" : "Industry",
"Y" : 0,
"assists" : 0,
"goals" : 0,
"id" : 2,
"name" : "Hassan 'Hasi' Akinyera",
"position" : "defender",
"price" : 5000000
}],
"users" : {
"l3J1TVsFNLMi0Oxg6hz4AJpacE53" : {
"email" : "awoniyideji#yahoo.com",
"fullname" : "Djflex11",
"teamname" : "deji awoniyi",
"total" : {
"-Kpl19z_25IEhiCEFrui" : 1,
"-Kpl1ARqT-v_btJ7OAq2" : 2,
"-Kpl1AsdodYWVPWWd5xA" : 2,
"-Kpl1iN7a7PLU-tnkKc4" : 1,
"-Kpl1j5CtN6c_mnmWLP-" : 1,
"-Kpl1k0BNCP5NNFV5uX6" : 1
},
"userName" : "awoniyideji#yahoo.com",
"week" : "no",
}
}
}
My ISSUE
The aim is that a player cannot be selected twice by the same user. MY code currently prevents a player from being selected twice locally, but the same player id can be pushed to the firebase database. My issue is how to basically check the total node so that the selected player if already "pushed" into database, will be removed instead of inserted into the database. I know that 'indexOf' is to be avoided with firebase.
This should work:
if(ref3.toJSON().hasOwnProperty(playerId)){
ref3.child(playerId).remove();
}
References:
Check for key in JSON
Remove from DB

Underscore sortBy algorithm

I need your help for a little algorithme for my app :
i have an object like this :
var obj = { "response" : [
"candidate" : {
"id":"1",
"price" : 10,
"distance" : 20
},
"candidate" : {
"id":"2"
"price" : 14,
"distance" : 2
},
"candidate" : {
"id":"3",
"price" : 200,
"distance" : 1
}
] }
Which i sort by price like this :
var sortPrice = _(obj.response).sortBy(function(p){
return p.candidate.price
})
It works fine and sort the object (ids) : 1,2,3
Now if candidate has the same price but different distance, i should show first candidate with the same price and the lowest distance :
var obj = { "response" : [
"candidate" : {
"id":"1",
"price" : 10,
"distance" : 20
},
"candidate" : {
"id":"2"
"price" : 10,
"distance" : 2
},
"candidate" : {
"id":"3",
"price" : 200,
"distance" : 1
}
] }
var sorted = _(obj.response).chain().sortBy(function (p) {
return parseInt(p.candidate.price) ;
}).sortBy(function(d){
return parseInt(d.candidate.distance)
}).value();
But it sort me the lowest distance first (ids) : 3(with distance 1), 2(with distance 2), 1(with distance 20) than 2,1,3
Do you have any suggestion?
Thank you.
In pure js you can use sort() like this.
var obj = {
"response": [{
"candidate": {
"id": "1",
"price": 8,
"distance": 20
}
}, {
"candidate": {
"id": "2",
"price": 8,
"distance": 2
}
}, {
"candidate": {
"id": "3",
"price": 200,
"distance": 1
}
}]
}
obj.response.sort(function(a, b) {
return a.candidate.price - b.candidate.price || a.candidate.distance - b.candidate.distance;
})
console.log(obj.response)
Lodash is a fork of underscore that allows you to sort by several properties of the object.
Using it, a solution could be:
_(obj.response).map(_.partial(_.get, _, 'candidate')).sortBy(['price', 'distance']).value();
Here's the fiddle in case you want to play with it.

JavaScript SUM and GROUP BY of JSON data

This is my first attempt at doing JavaScript with some JSON data objects and need some advice on the proper way to attain my goal.
Some server-side code actually generates a JSON formatted string that I have to work with and assign it to a string:
var dataString='$DATASTRING$';
But the end-result I have to work with after the server substitutes its data (without the \r\n, of course):
var dataString='[
{ "category" : "Search Engines", "hits" : 5, "bytes" : 50189 },
{ "category" : "Content Server", "hits" : 1, "bytes" : 17308 },
{ "category" : "Content Server", "hits" : 1, "bytes" : 47412 },
{ "category" : "Search Engines", "hits" : 1, "bytes" : 7601 },
{ "category" : "Business", "hits" : 1, "bytes" : 2847 },
{ "category" : "Content Server", "hits" : 1, "bytes" : 24210 },
{ "category" : "Internet Services", "hits" : 1, "bytes" : 3690 },
{ "category" : "Search Engines", "hits" : 6, "bytes" : 613036 },
{ "category" : "Search Engines", "hits" : 1, "bytes" : 2858 }
]';
And then I can change it to an object to work with.
var dataObject=eval("("+dataString+")");
This allows me to access the data individual rows of data, but I need to sum, group by, and order the values.
I need to the equivalent of an SQL statement like this:
SELECT category, sum(hits), sum(bytes)
FROM dataObject
GROUP BY category
ORDER BY sum(bytes) DESC
My desired output would be an object like this that I can further process:
var aggregatedObject='[
{ "category" : "Search Engines", "hits" : 13, "bytes" : 673684 },
{ "category" : "Content Server", "hits" : 3, "bytes" : 88930 },
{ "category" : "Internet Services", "hits" : 1, "bytes" : 3690 },
{ "category" : "Business", "hits" : 1, "bytes" : 2847 }
]';
...but i don't know where to start.
I could loop through all the category values and find the unique categories first, then loop again and sum the hits and bytes, then again to sort, but it seems there has got to be an easier way.
prototype.js (1.7) is already included on the client page, but I could add Underscore, jQuery, or some other small library if I had to.
I just don't know what would be best, easiest, smallest with the least amount of code to process the query.
Any suggestions?
You can use the native functions .reduce() to aggregrate the data, and then .sort() to sort by bytes.
var result = dataObject.reduce(function(res, obj) {
if (!(obj.category in res))
res.__array.push(res[obj.category] = obj);
else {
res[obj.category].hits += obj.hits;
res[obj.category].bytes += obj.bytes;
}
return res;
}, {__array:[]}).__array
.sort(function(a,b) { return b.bytes - a.bytes; });
If you're supporting older implementations, you'll need to use a shim for .reduce().
If you go the LINQ.js route, you can do it like this:
var aggregatedObject = Enumerable.From(dataArray)
.GroupBy("$.category", null,
function (key, g) {
return {
category: key,
hits: g.Sum("$.hits"),
bytes: g.Sum("$.bytes")
}
})
.ToArray();
Working demo with Stack Snippets:
var dataArray = [
{ category: "Search Engines", hits: 5, bytes: 50189 },
{ category: "Content Server", hits: 1, bytes: 17308 },
{ category: "Content Server", hits: 1, bytes: 47412 },
{ category: "Search Engines", hits: 1, bytes: 7601 },
{ category: "Business", hits: 1, bytes: 2847 },
{ category: "Content Server", hits: 1, bytes: 24210 },
{ category: "Internet ", hits: 1, bytes: 3690 },
{ category: "Search Engines", hits: 6, bytes: 613036 },
{ category: "Search Engines", hits: 1, bytes: 2858 }
];
var aggregatedObject = Enumerable.From(dataArray)
.GroupBy("$.category", null,
function (key, g) {
return {
category: key,
hits: g.Sum("$.hits"),
bytes: g.Sum("$.bytes")
}
})
.ToArray();
console.log(aggregatedObject);
<script src="//cdnjs.cloudflare.com/ajax/libs/linq.js/2.2.0.2/linq.min.js"></script>
Also, you can find more info on linqjs group by with a sum
Given the dataString above, the below code seems to work. It goes through each object; if the category exists in the groupedObjects array, its hits and bytes are added to the existing object. Otherwise, it is considered new and added to the groupedObjects array.
This solution makes use of underscore.js and jQuery
Here's a jsfiddle demo: http://jsfiddle.net/R3p4c/2/
var objects = $.parseJSON(dataString);
var categories = new Array();
var groupedObjects = new Array();
var i = 0;
_.each(objects,function(obj){
var existingObj;
if($.inArray(obj.category,categories) >= 0) {
existingObj = _.find(objects,function(o){return o.category === obj.category; });
existingObj.hits += obj.hits;
existingObj.bytes += obj.bytes;
} else {
groupedObjects[i] = obj;
categories[i] = obj.category;
i++;
}
});
groupedObjects = _.sortBy(groupedObjects,function(obj){ return obj.bytes; }).reverse();
var obj = [{Poz:'F1',Cap:10},{Poz:'F1',Cap:5},{Poz:'F1',Cap:5},{Poz:'F2',Cap:20},{Poz:'F1',Cap:5},{Poz:'F1',Cap:15},{Poz:'F2',Cap:5},{Poz:'F3',Cap:5},{Poz:'F4',Cap:5},{Poz:'F1',Cap:5}];
Array.prototype.sumUnic = function(name, sumName){
var returnArr = [];
var obj = this;
for(var x = 0; x<obj.length; x++){
if((function(source){
if(returnArr.length == 0){
return true;
}else{
for(var y = 0; y<returnArr.length; y++){
var isThere = [];
if(returnArr[y][name] == source[name]){
returnArr[y][sumName] = parseInt(returnArr[y][sumName]) + parseInt(source[sumName]);
return false;
}else{
isThere.push(source);
}
}
if(isThere.length>0)returnArr.push(source);
return false;
}
})(obj[x])){
returnArr.push(obj[x]);
}
}
return returnArr;
}
obj.sumUnic('Poz','Cap');
// return "[{"Poz":"F1","Cap":45},{"Poz":"F2","Cap":25},{"Poz":"F3","Cap":5},{"Poz":"F4","Cap":5}]"
Hi here is one solution written by me Visit: aggregate_groupby_js on npm
or in aggregate_groupby_js on github
The javascript library for using aggregate functions on array of objects. Basic functions like SUM, MIN, MAX, AVG, DISTINCT_COUNT for entire javascript objects
Example:
var arr = [{`"shape"`:`"square"`,`"color"`:`"red"`,`"used"`:1,`"instances"`:1},
{`"shape"`:`"square"`,`"color"`:`"red"`,`"used"`:2,`"instances"`:1},
{`"shape"`:`"circle"`,`"color"`:`"blue"`,`"used"`:0,`"instances"`:0},
{`"shape"`:`"square"`,`"color"`:`"blue"`,`"used"`:4,`"instances"`:4},
{`"shape"`:`"circle"`,`"color"`:`"red"`,"`used"`:1,`"instances"`:1},
{`"shape"`:`"circle"`,`"color"`:`"red"`,`"used"`:1,`"instances"`:0},
{`"shape"`:`"square"`,`"color"`:`"blue"`,`"used"`:4,`"instances"`:5},
{`"shape"`:`"square"`,`"color"`:`"red"`,`"used"`:2,`"instances"`:1}];
// Specify columns
var columns =[`"used"`, `"instances"`];
// Initialize object
var gb = new GroupBy(arr,columns);
// or
var gb = new GroupBy(arr,[`"used"`, `"instances"`]);
// Call the aggregate functions
gb.sum();
gb.min();
gb.max();
gb.avg();
gb.distinctCount();

Categories