I am attempting to modify a collection via the mongo shell to trim all strings, then, if the field is a string but is numeric, change its data type to a number.
Here is the code I am using:
function isNumeric(num) {
return !isNaN(parseFloat(num)) && isFinite(num);
}
var bulk = db.yale.initializeOrderedBulkOp();
var counter = 0;
db.yale.find().forEach((doc) => {
let updoc = {
"$set" : {}
};
// get document fields but ignore immutable id field
let fields = Object.keys(doc);
let badField = fields.indexOf("_id");
if (badField > -1) {
fields.splice(badField, 1);
}
fields.forEach((field) => {
if (typeof doc[field] === "string") {
updoc["$set"][field] = doc[field].trim();
}
});
fields.forEach((field) => {
if (typeof doc[field] === "string") {
if (isNumeric(doc[field])) {
updoc["$set"][field] = parseFloat(doc[field]);
} else {
if (doc[field].length === 0) {
updoc["$set"][field] = null;
}
}
}
});
bulk.find({
"_id" : doc._id
}).update(updoc);
counter++;
if (counter % 1000 === 0) {
bulk.execute();
bulk = db.yale.initializeOrderedBulkOp();
}
});
(Note 1: the dataset is the Yale Bright Star catalog from the Vizier service, hence the name).
(Note 2: I prefer a isNumeric() function to !isNan() because I honestly think it looks better, personal preference)
Before the code is run, the first couple of results from db.yale.find() looks like this:
{ "_id" : ObjectId("5d945e8baea250effd9dd0dc"), "HR" : " 1", "Name" : " ", "HD" : " 3", "SAO" : " 36042", "ADS" : " 46", "VarID" : " ", "RAJ2000" : "00 05 09.9", "DEJ2000" : "+45 13 45", "Vmag" : " 6.70", "BV" : " 0.07", "SpType" : " A1Vn ", "NoteFlag" : "" }
{ "_id" : ObjectId("5d945e8baea250effd9dd0dd"), "HR" : " 11", "Name" : " ", "HD" : " 315", "SAO" : 128595, "ADS" : " ", "VarID" : "Var? ", "RAJ2000" : "00 07 44.1", "DEJ2000" : "-02 32 56", "Vmag" : " 6.43", "BV" : -0.14, "SpType" : " B8IIIpSi ", "NoteFlag" : "*" }
After the run, the first lines look like this:
{ "_id" : ObjectId("5d945e8baea250effd9dd0dc"), "HR" : 1, "Name" : "", "HD" : 3, "SAO" : 36042, "ADS" : 46, "VarID" : "", "RAJ2000" : "00 05 09.9", "DEJ2000" : "+45 13 45", "Vmag" : 6.7, "BV" : 0.07, "SpType" : "A1Vn", "NoteFlag" : null }
{ "_id" : ObjectId("5d945e8baea250effd9dd0dd"), "HR" : 11, "Name" : "", "HD" : 315, "SAO" : 128595, "ADS" : "", "VarID" : "Var?", "RAJ2000" : "00 07 44.1", "DEJ2000" : "-02 32 56", "Vmag" : 6.43, "BV" : -0.14, "SpType" : "B8IIIpSi", "NoteFlag" : "*" }
Notice that in the first Document, the "ADS" field has been converted to a number, but in the second Document, the "ADS" field is still an empty string, empty though it should have been converted to null.
I do not want to remove the field from the collection entirely. For the sake of the service that will be using it, the value of null means something.
Can anyone see an issue with the javascript that is causing the field not to convert to null? It's probably just a dumb error I am overlooking.
Jason
Related
I have an object that contains 2 fields: day_active and day_inactive. (the object is in the javascript snippet below)
And what I want to obtain is another object that is based on this formula:
count(day_active (on date x))-count(day_inactive (on date x)
{
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 2.0 // (5.0 - 3.0) - how many groupC were on day_active(2019-09-19) minus how many groupC were on day_inactive(2019-09-19)
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : -2.0 // (3.0 - 5.0)
},
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : -2.0 // (8.0 - 10.0)
},
{
"date" : "2019-10-08",
"type" : "groupW",
"count" : 7.0 // (7.0 - 0.0)
}
}
I tried this code but is not taking all the cases and the result is incomplete:
let items = {
"day_inactive" : [
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 3.0
},
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : 10.0
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : 5.0
},
{
"date" : "2019-10-07",
"type" : "groupW",
"count" : 9.0
},
{
"date" : "2019-10-05",
"type" : "groupW",
"count" : 3.0
},
],
"day_active" : [
{
"date" : "2019-09-11",
"type" : "groupW",
"count" : 8.0
},
{
"date" : "2019-09-19",
"type" : "groupW",
"count" : 3.0
},
{
"date" : "2019-10-08",
"type" : "groupW",
"count" : 7.0
},
{
"date" : "2019-09-19",
"type" : "groupC",
"count" : 5.0
}
]
}
let auxObj = {}
for (let i = 0; i < items.day_active.length; i++) {
for (let j = 0; j < items.day_inactive.length; j++) {
if (items.day_active[i].date == items.day_inactive[j].date && items.day_active[i].type == items.day_inactive[j].type) {
// console.log("yes")
auxObj.date = items.day_active[i].date
auxObj.type = items.day_active[i].type
auxObj.count = items.day_active[i].count - items.day_inactive[j].count
}
}
}
console.log(auxObj)
How can I solve this in a simple way? Thank you for your time!
Follow along the comments for explanation...
// let's create an empty object
let output = {};
// and start copying active days...
for (const obj of items.day_active) {
// the following `key` is just for grouping purposes...
const key = `${obj.date}-${obj.type}`;
output[key] = { ...obj };
}
// Now let's look at inactive days...
for (const obj of items.day_inactive) {
// the following `key` is just for grouping purposes...
const key = `${obj.date}-${obj.type}`;
// is this the first time we're looking at this `date-type`? let's add it with 0 count
if (!output[key]) {
output[key] = { ...obj, count: 0 };
}
// and subtract it from active days count
output[key].count -= obj.count;
}
// let's remove the `key` we created earlier...
output = Object.values(output);
// here's the output
console.log(output);
From the sample input given, this is the result we get:
[ { date: '2019-10-11', type: 'groupW', count: -2 },
{ date: '2019-10-19', type: 'groupW', count: 3 },
{ date: '2019-10-08', type: 'groupW', count: 7 },
{ date: '2019-10-19', type: 'groupC', count: 5 },
{ date: '2019-09-19', type: 'groupC', count: -3 },
{ date: '2019-09-19', type: 'groupW', count: -5 },
{ date: '2019-10-07', type: 'groupW', count: -9 },
{ date: '2019-10-05', type: 'groupW', count: -3 } ]
I think this one will be more efficient, i named the object after me cause...idn
let marios = {};
items.day_active.forEach(d => marios[d.date+'_'+d.type] = d.count || 0);
items.day_inactive.forEach(d => marios[d.date+'_'+d.type] = marios[d.date+'_'+d.type] ? marios[d.date+'_'+d.type] - (d.count || 0) : (d.count || 0));
console.log(marios);
The logic behind it is that we create an object and we create a property for each date present in the data sets, starting with the first terms of the formla, ending with the subtraction of the second part, and by defaulting to 0 for each appropriate case.
In the end you can iterate the properties of the object, split the string on '_' to read each propertie's date and group and create an array of the results ( if you have trouble with this let me know )
I am having a problem on firebase functions. What I trying to do is when Items's child gets updated, then I want to get the value of Count and do further calculation, But what I am facing is that the firebase log console always shows an erreor "TypeError: Cannot read property 'val' of undefined".
JSON structure
"VTEST" : {
"A" : {
"Count" : 5,
"Items" : {
"item1" : "apple"
},
"NUMBER" : 5
},
"B" : {
"Count" : 8,
"Items" : {
"item1" : "orange;"
},
"NUMBER" : 3
},
"C" : {
"Count" : 10,
"Items" : {
"item1" : "grape"
},
"NUMBER" : 7
},
"D" : {
"Count" : 12,
"Items" : {
"item1" : "grava"
},
"NUMBER" : 10
},
"E" : {
"Count" : 15,
"Items" : {
"item1" : "fish"
},
"NUMBER" : 12
},
"F" : {
"Count" : 18,
"Items" : {
"item1" : "chicken;"
},
"NUMBER" : 8
}
}
My code:
exports.ItemCount = functions.database.ref('/VTEST/{ID}/Items').onUpdate((updateRef, context) => {
var childCount = updateRef.after.numChildren();
var newReference = updateRef.after.ref.parent.child('/Count');
var Count = newReference.val();
Count = Count + childCount;
return updateRef.ref.update({Count})
})
What I expect is the Count's value will be update, but it always show error : "TypeError: Cannot read property 'val' of undefined"
Can anyone tell me what am I doing wrong here, I don't get it.
The problem comes from the fact that a Reference does not have a val() method. You need to use the once() method to get the value of the corresponding database location.
The following adapted code should work:
exports.ItemCount = functions.database
.ref('/VTEST/{ID}/Items')
.onUpdate((updateRef, context) => {
var childCount = updateRef.after.numChildren();
var newReference = updateRef.after.ref.parent.child('/Count');
return newReference.once('value').then(dataSnapshot => {
var Count = dataSnapshot.val();
Count = Count + childCount;
return newReference.parent.update({ Count: Count });
});
});
However, depending on your exact requirements, you may decide to use a Transaction, see https://firebase.google.com/docs/database/web/read-and-write#save_data_as_transactions
I have a collection that I am trying to map reduce by id and date to produce a graph for sales of a product in store vs online. A new object is created for each transaction, so I would like to reduce them to a total count for a given day. An object looks something like this:
object
{
"ProductID": 1
"Purchase Method": In Store
"Date": 2018-01-16
"Count": 5
}
What I am trying to achieve as output is to have in store and online purchases combined into 1 object with a key being the id and the date and then the value being the counts of each method as shown below:
ProductID: 1
Date: 2018-01-16
[
{Name: "In store", Count: 3}
{Name: "Online", Count: 2}
]
My current method was to map the objects by Id, date, and Purchase Method so the reduce would get the total count for that id on that date using that method, but this leads to having two entries for an id and date, 1 for in store and 1 for online. This is the current state of my functions:
var mapDailySales = function() {
var sale = this;
/*Converts timestamp to just date */
var pad = function pad(n, width, z) {
z = z || '0';
n = n + '';
return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n;
};
var d = sale.Date;
var date = d.getFullYear() + "-" + pad(d.getMonth() + 1, 2, 0) + "-" + pad(d.getDate(), 2, 0);
emit({ProductId: sale.ProductID, Date:date, Method: sale.PurchaseMethod},
{Name: sale.PurchaseMethod, Count: 1})
};
var reduceDailySales = function(key, value) {
var res = {Name: 0, Count: 0};
value.forEach(function(value){
res.Name = value.Name;
res.Count += value.Count;
});
return res;
};
Current Output looks something like this:
{
"_id" : {
"ProductId" : 1,
"Date" : "2018-01-16",
"Method" : "Online"
},
"value" : {
"Name" : "Online",
"Count" : 3
}
}
Is there a way to achieve my desired output without map reducing again on the current output?
You can use aggregation pipeline to get the results instead of mapReduce, $group by ProductID and Date, with $project you can map counts to an array
added $out to write the results to new collection, removing it will return a cursor
db.prod.aggregate([
{$group : {
_id : {ProductID : "$ProductID", Date : "$Date"},
onlineCount : {$sum : {$cond : [{$eq : ["$PurchaseMethod", "Online"]}, "$Count" , 0]}},
storeCount : {$sum : {$cond : [{$eq : ["$PurchaseMethod", "In Store"]}, "$Count" , 0]}}
}
},
{$project : {
_id : 0,
ProductID : "$_id.ProductID",
Date : "$_id.Date",
counts : [{Name: "In Store", Count: "$storeCount"},{Name : "Online", Count: "$onlineCount"}]
}},
{$out : "count_stats"}
]).pretty()
collection
> db.prod.find()
{ "_id" : ObjectId("5a98ce4a62f54862fc7cd1f5"), "ProductID" : 1, "PurchaseMethod" : "In Store", "Date" : "2018-01-16", "Count" : 5 }
{ "_id" : ObjectId("5a98ce4a62f54862fc7cd1f6"), "ProductID" : 1, "PurchaseMethod" : "Online", "Date" : "2018-01-16", "Count" : 2 }
>
result
> db.count_stats.find()
{ "_id" : ObjectId("5a98d3366a5f43b12a39b4ac"), "ProductID" : 1, "Date" : "2018-01-16", "counts" : [ { "Name" : "In Store", "Count" : 5 }, { "Name" : "Online", "Count" : 2 } ] }
>
if you want to use mapReduce, you can use finalize to reduce or transform the result further
db.prod.mapReduce(
<map>,
<reduce>,
{
out: <collection>,
finalize: <function>
}
)
My aim is to have each user select a total of 6 players. when each player is selected, the player id is sent to a node on the database called 'total'using the push () method. The code is written below
var ref = firebase.database().ref().child('players');
var ref3 = firebase.database().ref().child('users').child(uid).child('total');
$scope.players = $firebaseArray(ref);
console.log ($scope.players);
$scope.history = [];
$scope.buy = function(player) {
//remove if already added locally
var index = $scope.history.indexOf(player);
if(index>=0){
$scope.history.splice(index,1);
return;
}
//remove if already added on firebase
//max 6 allowed
if($scope.history.length>=6){
alert('max 6 allowed');
return;
}
var selected = $scope.history.reduce(function(a,b){
a[b.position] = (a[b.position] || 0) + 1;
return a;
}, {}) || {};
if(!selected[player.position] || selected[player.position]<2){
$scope.history.push(player);
ref3.push(player.id);
}else{
alert('You can add only two players per position');
}
};
$scope.getTotal = function(){
return $scope.history.reduce(function(tot, p){
tot = tot - p.price;
return tot;
}, $scope.total);
};
this is how the database is structured :
{
"players" : [ {
"R" : 0,
"Team" : "Industry",
"Y" : 0,
"assists" : 0,
"goals" : 0,
"id" : 1,
"name" : "Yasin 'YB' Amusan",
"position" : "forward",
"price" : 8000000
}, {
"R" : 0,
"Team" : "Industry",
"Y" : 0,
"assists" : 0,
"goals" : 0,
"id" : 2,
"name" : "Hassan 'Hasi' Akinyera",
"position" : "defender",
"price" : 5000000
}],
"users" : {
"l3J1TVsFNLMi0Oxg6hz4AJpacE53" : {
"email" : "awoniyideji#yahoo.com",
"fullname" : "Djflex11",
"teamname" : "deji awoniyi",
"total" : {
"-Kpl19z_25IEhiCEFrui" : 1,
"-Kpl1ARqT-v_btJ7OAq2" : 2,
"-Kpl1AsdodYWVPWWd5xA" : 2,
"-Kpl1iN7a7PLU-tnkKc4" : 1,
"-Kpl1j5CtN6c_mnmWLP-" : 1,
"-Kpl1k0BNCP5NNFV5uX6" : 1
},
"userName" : "awoniyideji#yahoo.com",
"week" : "no",
}
}
}
My ISSUE
The aim is that a player cannot be selected twice by the same user. MY code currently prevents a player from being selected twice locally, but the same player id can be pushed to the firebase database. My issue is how to basically check the total node so that the selected player if already "pushed" into database, will be removed instead of inserted into the database. I know that 'indexOf' is to be avoided with firebase.
This should work:
if(ref3.toJSON().hasOwnProperty(playerId)){
ref3.child(playerId).remove();
}
References:
Check for key in JSON
Remove from DB
I am a newbie to javascript..have gone thru multiple scope related blogs and several answers on stackoverflow..but not sure why this code doesnt work..
function checkPhoneEmail(element, index, array) {
var _contact = {};
var _phone_empty = true;
var _email_empty = true;
var _phones_to_store = [];
var _emails_to_store = [];
var _prev_phone_number;
var _phone;
var i;
//function to check if this phone
// should be included
function checkMobilePhone(ph_element) {
var _match;
_match = ph_element.type.match(/mobile/i);
if (_match && ph_element.value.length >= 10 && !(ph_element.value.match(/^800/) || ph_element.value.match(/^1800/) || ph_element.value.match(/^1-800/))) {
return true;
};
return false;
};
if (!_.isEmpty(element.phoneNumbers)) {
for (i = 0; i < element.phoneNumbers.length; i++) {
console.log('prev num: ' + _prev_phone_number);
console.log('curr num: ' + element.phoneNumbers[i].value)
if (!_.isEqual(_prev_phone_number, element.phoneNumbers[i].value)) {
if (checkMobilePhone(element.phoneNumbers[i])) {
_phone = {
id: element.phoneNumbers[i].id,
value: element.phoneNumbers[i].value
};
_phones_to_store.push(_phone);
console.log('phone to store: ' + element.phoneNumbers[i].value)
};
};
_prev_phone_number = element.phoneNumbers[i].value;
console.log('prev1 num: ' + _prev_phone_number);
};
_phone_empty = false;
};
if (!_.isEmpty(element.emails)) {
};
};
why is the _prev_phone_number not being set ? I see it at prev1 num..but when you look for the next element its set back to undefined...my understanding is for doesnt create a new scope ? is this incorrect ?
I am trying to remove duplicates from phone contacts array (from cordova contacts) and doing some very basic checks to eliminate all numbers except a valid us mobile # for a mobile app..using above logic if a contact has multiple entries in contacts for same phone number i am seeing duplicates..Tried above logic with foreach and also _.uniq ... but same result..
Any help is appreciated.
sample Data:
{
"id" : "1916",
"rawId" : "1911",
"displayName" : "John Doe",
"name" : {
"familyName" : "Doe",
"formatted" : "John Doe",
"givenName" : "John"
},
"nickname" : null,
"phoneNumbers" : [{
"type" : "mobile",
"value" : "+1 999 666 9175",
"id" : "11994",
"pref" : false
}, {
"type" : "mobile",
"value" : "+1 999 666 9175",
"id" : "12001",
"pref" : false
}
],
"emails" : null,
"addresses" : null,
"ims" : null,
"organizations" : null,
"birthday" : null,
"note" : "",
"photos" : null,
"categories" : null,
"urls" : null
}
Instead of using the variable _prev_phone_number, why don't you just access element.phoneNumbers[i-1].value? No need to introduce a new variable for it.
If that doesn't work, then something is clobbering your object and we will need to look deeper.
A for loop does not create its own scope. The issue is that you are trying to log to the console an undefined value.
for (i = 0; i < element.phoneNumbers.length; i++) {
//_prev_phone_number is undefined here.
console.log('prev num: ' + _prev_phone_number);
console.log('curr num: ' + element.phoneNumbers[i].value)
if (!_.isEqual(_prev_phone_number, element.phoneNumbers[i].value)) {
if (checkMobilePhone(element.phoneNumbers[i])) {
_phone = {
id: element.phoneNumbers[i].id,
value: element.phoneNumbers[i].value
};
_phones_to_store.push(_phone);
console.log('phone to store: ' + element.phoneNumbers[i].value)
};
};
//This is the first time you have defined _prev_phone_number
_prev_phone_number = element.phoneNumbers[i].value;
console.log('prev1 num: ' + _prev_phone_number);
};
You should probably check to see if _prev_phone_number has been defined before logging it to the console.
if(_prev_phone_number){
console.log('prev num: ' + _prev_phone_number);
}
This would ensure that you are not logging an undefined value.