I have a records of 15K in my collection with min 40 fields, I created a table which is generated from records. In this table i have various fileds as shown in image(from Excel sheet).
/client/main.js
Template.GetTable.helpers({
'getTotalData':function(key1,key2){
console.log("-------inside totalData()---------");
const projects1 = Template.instance().distinct1.get();
var filter1= _.map(projects1, col_8 => {
return {col_8};
});
q={};
p={};
console.log(filter1);
console.log(JSON.stringify(q));
//var queryData=test.find(q).fetch();
Meteor.call('getCountData',"all",function(err,count){
if(err)
console.log("Failed to call meteor..!");
else{
console.log(count);
return count;
}
});
},
});
/server/main.js
Meteor.methods({
'getCountData':function(type){
return test.find({"col_17" : " Query->:#####->x","col_8": { $regex: /^CQI?/i}}).count();
},
});
I was just testing for testing and i know how to get the count from DB.
My problem is after all the rendering and the helpers are called the UI will load without any count data. But when i checked in debugger i got the right counts printed using "console.log()" and the UI is not updated.
How can i resolve this issue? or is there any efficient way to solve this?
The UI problem is that you're doing a Meteor calling inside a helper and returning the result of the call to itself, not the helper.
Here's an example of what you're doing and what you SHOULD be doing.
SHOULD NOT:
Template.GetTable.helpers({
'getTotalData':function(key1,key2){
Meteor.call('getCountData',"all",function(err,count){
if(err)
console.log("Failed to call meteor..!");
else {
return count; // Being returned to this function, not helper fuction
}
});
},
});
SHOULD:
var recVar = new ReactiveVar(false);
Template.GetTable.onCreated({
Meteor.call('getCountData',"all",function(err,count){
if(err)
console.log("Failed to call meteor..!");
else {
recVar.set(count);
}
});
});
Template.GetTable.helpers({
'getTotalData':function(key1,key2){
return recVar.get();
},
});
Related
I want to make a counter of how many time the server js file has started, for my website using mongodb driver for angular js.
I want to save a varible named counter which has a value of 0 and then increment that value each time that the server is running. my code is below. as you can see my code doesn't acutally update the field in the db. just the varible.
beside that... well.. the whole code I wrote seems like bad practise. I basically have a document with {id:<>,count:0} and I am looping through all the count fields which are greater the -1 (i.e. integers) although I have only got just 1 count field.
isn't there any simple way to persist/get this 1 value from the db?
How can I update the field inside the db itself using something like $inc, in the easiest way possible?
Thanks
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
if (err) {
console.log(err);
}
else {
console.log("Connected correctly to DB.");
var dbusers =db.collection('users');
var cursor =dbusers.find( { "count": { $gt: -1 } } );
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
doc.count=doc.count+1;
}
}
);
}
db.close();
});
Try this:
MongoClient.connect(url, function(err, db) {
if (err) {
console.log(err);
return db.close();
}
console.log("Connected correctly to DB.");
// update a record in the collection
db.users.update(
// find record with name "MyServer"
{ name: "MyServer" },
// increment it's property called "ran" by 1
{ $inc: { ran: 1 } }
);
return db.close();
});
This should be enough to get you started. It sounds like you're trying to do something like:
get me all the objects in the collection that have a property 'count' greater than -1
increase it's value by 1
save it to the collection.
The step you're missing is step 3. Doing it your way you'd have to do a bulk update. The example I gave you is updating a single record.
here is the documentation for increment. And here is the documentation for bulk updates.
I currently have a collection in Mongodb say "Collection1".
I have the following array of objects that need to be into inserted into MongoDB. I am using Mongoose API. For now, I am iterating through the array and inserting each of them into mongo.
This is ok for now, but will be a problem when the data is too big.
I need a way of inserting the data in bulk into MongoDB without repetition.
I am not sure how to do this. I could not find a bulk option in Mongoose.
My code below
myData = [Obj1,Obj2,Obj3.......]
myData.forEach(function(ele){
//console.log(ele)
saveToMongo(ele);
});
function saveToMongo(obj){
(new Collection1(obj)).save(function (err, response) {
if (err) {
// console.log('Error while inserting: ' + obj.name + " " +err);
} else {
// console.log('Data successfully inserted');
}
});
return Collection1(obj);
}
You might want to use the insertMany() method here if you're using the latest Mongoose version 4.4.X and greater, which essentially uses Model.collection.insertMany() under the hood and the driver might handle parallelizing >= 1000 docs for you.
myData = [Obj1, Obj2, Obj3.......];
Collection1.insertMany(myData, function(error, docs) {});
or using Promises for better error handling
Collection1.insertMany(myData)
.then(function(docs) {
// do something with docs
})
.catch(function(err) {
// error handling here
});
It works by creating a bunch of documents, calls .validate() on them in parallel, and then calls the underlying driver's insertMany() on the result of toObject({ virtuals: false }); of each doc.
Although insertMany() doesn't trigger pre-save hooks, it has better performance because it only makes 1 round-trip to the server rather than 1 for each document.
For Mongoose versions ~3.8.8, ~3.8.22, 4.x which support MongoDB Server >=2.6.x, you could use the Bulk API as follows
var bulk = Collection1.collection.initializeOrderedBulkOp(),
counter = 0;
myData.forEach(function(doc) {
bulk.insert(doc);
counter++;
if (counter % 500 == 0) {
bulk.execute(function(err, r) {
// do something with the result
bulk = Collection1.collection.initializeOrderedBulkOp();
counter = 0;
});
}
});
// Catch any docs in the queue under or over the 500's
if (counter > 0) {
bulk.execute(function(err,result) {
// do something with the result here
});
}
you can pass an array of objects to mongoose model create function
var Collection1 = mongoose.model('Collection1');
Collection1.create(myData,function(err){
if(err) ...
});
I am using Sails v0.11 and am developing an standalone importer script in order to import data to mongoDB and - that is now the not-working part - build the associations between the models.
For this process I introduced temporary helper properties in the models in order to find the associated records and replace them by in real MongoDB _ids.
The script starts Sails in order to be able use its features (waterline, etc.):
var app = Sails();
app.load({
hooks: { grunt: false },
log: { level: 'warn' }
}, function sailsReady(err){
processUsers() finds all users and their _ids and iterates over them to invoke a second function addOrgsToOneUser()
var processUsers = function() {
// Iterate through all users in order to retrieve their _ids and
app.models['user'].native(function(err, collection) {
collection.find({}, projectionOrgInUser).toArray(function (err, users) {
Async.eachSeries(users, function (user, next){
// prepare userInOrgs
whereUserInOrg = { orgId: { $in: userInOrgs } };
//This is invoking
addOrgsToOneUser(user, whereUserInOrg);
next();
}, function afterwards (err) {
if (err) {
console.error('Import failed, error details:\n',err);
return process.exit(1);
}
console.log("done");
return process.exit(0); // This returns too early, not executing the addOrgsToOneUser
});
});
});
};
addOrgsToOneUser() finds all orgs belonging to THIS user and updates then the orgs array property of THIS user
var addOrgsToOneUser = function(user, whereUserInOrg) {
var projectionUserInOrg = "...";
// Find all orgs that this user is associated to and store it in inOrgs
app.models['org'].native(function(err, collection) {
collection.find(whereUserInOrg, projectionUserInOrg).toArray(function (err, orgs) {
// prepare inOrgs which is needed for updating
//update user to have an updated orgs array based on inOrgs.
app.models['user'].update({'id' : user._id.toString()}, {'orgs': inOrgs}).exec(function afterwards(err, updated){
console.log('Updated user ' + user._id.toString() + ' to be in their orgs');
});
});
});
}
Problem:
Process.exit(0) is called before the query/update of saddOrgsToOneUser() has completed. It behaves as expected if saddOrgsToOneUser() contains just a console.log for instance, but queries are triggered ansynchronously of course.
In case I comment out Process.exit(0), the script never stops, but the queries are executed as intented.
As the script will have further nested queries, I need a better approach to this as manually kill this script ...
How is nesting queries and iterating over their results done properly?
Thank you very much,
Manuel
addOrgsToOneUser is asynchronous. next() needs to be called after everything is done inside addOrgsToOneUser. The way I would do it is to pass in a callback (next) and call it when everything is done. So the call is
addOrgsToOneUser(user, whereUserInOrg, next);
and the addOrgsToOneUser will have an extra argument:
var addOrgsToOneUser = function(user, whereUserInOrg, callback) {
var projectionUserInOrg = "...";
// Find all orgs that this user is associated to and store it in inOrgs
app.models['org'].native(function(err, collection) {
collection.find(whereUserInOrg, projectionUserInOrg).toArray(function (err, orgs) {
// prepare inOrgs which is needed for updating
//update user to have an updated orgs array based on inOrgs.
app.models['user'].update({'id' : user._id.toString()}, {'orgs': inOrgs}).exec(function afterwards(err, updated){
console.log('Updated user ' + user._id.toString() + ' to be in their orgs');
callback(); // your original next() is called here
});
});
});
}
Ive written and basic Node app (my first) to insert many csv rows into mongo (items array in the code below). Once all items have been inserted the db connection should be closed and the program exited.
The issue ive been working with is figuring out when to close the db connection once all inserts have returned a result. Ive gotten it working by counting all of the insert result callbacks but to me this feels clunky. I know one improvement I could make is to batch the inserts via an array to the insert function but ill still need to have my code be aware of when all inserts have completed (assuming it would be bad to insert 100k items in one query). Is there and better way (my code feels hacky) to do this?
Hack part...
function (err, result) {
queryCompletedCount++;
if (err) console.log(err);
//Not sure about doing it this way
//Close db once all queries have returned a result
if (queryCompletedCount === items.length) {
db.close();
console.log("Finish inserting data: " + new Date());
}
}
Full insert code
MongoClient.connect(dbConnectionURL, function (err, db) {
if (err) {
console.log("Error connecting to DB: " + err);
} else {
var productCollection = db.collection('products');
console.log("Connected to DB");
console.log("Start inserting data: " + new Date());
var queryCompletedCount = 0;
for (var i = 0; i < items.length; i++) {
productCollection.insert([{
manufacturerCode: null,
name: items[i].name,
description: null
}], function (err, result) {
queryCompletedCount++;
if (err) console.log(err);
//Not sure about doing it this way
//Close db once all queries have returned a result
if (queryCompletedCount === items.length) {
db.close();
console.log("Finish inserting data: " + new Date());
}
});
}
}
});
What do you think about realizing this issue with async module like this:
async = require('async')
async.eachSeries(items, function (item, next) {
productCollection.insert(productCollection.insert(
[{
manufacturerCode: null,
name: item.name,
description: null
}], function (err, result) {
if (err) {
return next(err);
}
next();
})
)
}, function () {
// this will be called after all insertion completed
db.close();
console.log("Finish inserting data: " + new Date());
});
What you need here is MongoDB's Write Concern, configured in the strictest way.
There are two levels of Write Concern. The first is the write mode, in which case the query returns only if the result is written to the configured number of mongo instances. In your case I suppose there is a single instance, but for future you may configure it as "w": "majority". The second level is the Journal concern, where by setting "j": 1 your query will return only when the data is written into the journal.
So in your case you best Write Concern configuration might be {"w": "majority", "j": 1}. Just add it as the last argument of your insert statement.
I'm encountering few problems when dealing with mongoose.
I wrote the following snippet of code:
if (!usr.settings) usr.settings = {};
async.forEach(Object.keys(params), function (item, nextitem){
usr.settings[item] = params[item];
nextitem();
}, function (err) {
if (err) return callback(err);
usr.save(function(err) {
if (err) return callback(err);
return callback();
});
});
When I first use it, it works just fine, it creates all the items in the document perfectly, but when I use it the 2nd time (lets say I want to update those items) it doesn't change their values nor gives me any error.
The data stays just the same.
I tried to debug it.
if (!usr.settings) usr.settings = {};
async.forEach(Object.keys(params), function (item, nextitem){
usr.settings[item] = params[item];
nextitem();
}, function (err) {
if (err) return callback(err);
usr.save(function(err) {
console.log(usr); <------------------- [At this point it shows the updated data but for some reason it doesnt save it to the db]
if (err) return callback(err);
return callback();
});
});
Any idea why it could happen?
Thanks.
After messing with it, found the solution.
lets say you are using the following schema:
user_schema = {
settings = Object
}
you won't be able to set setting., therefore the following code:
usr.settings.something = 123;
usr.save(function(err) {
if (err) return callback(err);
return callback();
});
will do nothing and still will not result in an error.
If you want to fix the issue, apply this fix to the schema:
user_schema = {
settings = {
something : Number
something_else: Object
}
}
now you will be able to set something with a value :)
I wish it would helpful to someone