Collections Missing with Fawn Transactions - javascript
In index.js, i create my database correctly, and i add a genre collection in the db, and is added fine.
However, when i add my rental collection, it isn't added or viewed in mongodb compass
My code for rental.js:
const mongoose = require('mongoose')
const joi = require('joi')
const rentalSchema = new mongoose.Schema({
customer: {
type: new mongoose.Schema({
name: {
type: String,
required: true,
minlength: 2,
maxlength: 255
},
phone: {
type: String,
required: true,
minlength: 2,
maxlength: 255
},
isGold: {
type: Boolean,
default: false,
required: false
},
}),
movie: {
type: new mongoose.Schema({
title: {
type: String,
required: true,
trim: true,
minlength: 2,
maxlength: 500
},
dailyRentalRate: {
type: Number,
min: 2,
required: true,
max: 255
}
}),
},
dateOut: {
type: Date,
required: true,
default: Date.now
},
dateReturned: {
type: Date
},
rentalFee: {
type: Number,
min: 0
}
}
})
const Rental = mongoose.model('Rental', rentalSchema)
function validate(obj) {
const schema = {
customerId: joi.string().required(),
movieId: joi.string().required()
}
return joi.validate(obj, schema)
}
exports.Rental = Rental
exports.validate = validate
My index.js Code (Where i initialise the database):
const mongoose = require('mongoose')
const movies = require('./routes/movies')
const rentals = require('./routes/rentals')
mongoose.connect('mongodb://localhost/vidly', { useNewUrlParser: true})
.then(() => console.log('Connected to mongodb..'))
.catch(() => console.error('Error connecting...'))
This is unusual, as i do the same thing for genre, but it is added and viewed in mongodb compass!
[The image of mongo db compass is here: ]
Here is my rentals.js file, that uses rental.js for models:
const express = require('express')
const router = express.Router()
const {Customer} = require('../models/customer')
const Fawn = require('fawn')
const mongoose = require('mongoose')
const {Movie} = require('../models/movie')
const {Rental, validate} = require('../models/rental')
Fawn.init(mongoose)
router.get('/rentals', async (req, res) => {
const rentals = await Rental.find().sort('-dateOut')
res.send (rentals)
})
router.post('/rentals', async (req, res) => {
const {error} = validate(req.body)
if (error) return res.status(400).send('Error')
// Makes sure the customerId/customer sends us is valid
const customer = await Customer.findById(req.body.customerId)
if (!customer) return res.status(404).send('Invalid customerId')
const movie = await Movie.findById(req.body.movieId)
if (!movie) return res.status(404).send('Invalid movieId')
let rental = new Rental({
customer: {
_id: customer._id,
name: customer.name,
phone: customer.phone
},
movie: {
_id: movie._id,
title: movie.title,
dailyRentalRate: movie.dailyRentalRate
}
})
// This is for our success scenario
try {
// All args in here treated all together as unit
new Fawn.Task()
// First arg is collection we work with, and second is obj we wanna save
.save('rentals', rental)
// Update movies collection Second Arg is movie that should be updated Third is we increment the numInstock prop, and decrement by 1
.update('movies', { _id: movie._id}, {
$inc: { numberInStock: -1}
})
.run()
res.send(rental)
}
catch(ex) {
// 500 means Internal server error
res.status(500).send('Something failed.')
}
})
module.exports = router
Here is mongodb compass, and the collections seen
Using Fawn
The issue is one of usage with the Fawn library and comes from some misconceptions about the naming of mongoose models and how these interact with the library itself. As such the best way to demonstrate is with a minimal example of working code:
const { Schema } = mongoose = require('mongoose');
const Fawn = require('fawn');
const uri = 'mongodb://localhost:27017/fawndemo';
const opts = { useNewUrlParser: true };
// sensible defaults
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
// schema defs
const oneSchema = new Schema({
name: String
});
const twoSchema = new Schema({
counter: Number
});
// don't even need vars since we access model by name
mongoose.model('One', oneSchema);
mongoose.model('Two', twoSchema);
// log helper
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri, opts);
// init fawm
Fawn.init(mongoose);
// Clean models
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.deleteMany())
)
// run test
let task = Fawn.Task();
let results = await task
.save('One', { name: 'Bill' })
.save('Two', { counter: 0 })
.update('Two', { }, { "$inc": { "counter": 1 } })
.run({ useMongoose: true });
log(results);
// List objects in models
for ( [k,m] of Object.entries(conn.models) ) {
let result = await m.find();
log(result);
}
} catch(e) {
console.error(e)
} finally {
mongoose.disconnect()
}
})()
Note how the mongoose models are registered here:
mongoose.model('One', oneSchema);
mongoose.model('Two', twoSchema);
That first argument is the registered name which mongoose uses for the model in it's internal logic. From the perspective of mongoose itself, once you have registered the model name with the schema as above, you can actually call an instance of the model as follows:
const One = mongoose.model('One');
Typically people export the result of the initial registration and then just use the returned value which is a reference to mongoose's own internal storage of the model details and attached schema. But the line of code is equivalent to that same thing as long as the registration code has already been run.
A typical exports considering this can therefore be used as:
require('./models/one');
require('./models/two');
let results = await mongoose.model('One').find();
So you might not see that often in other code examples, but that is really to show what is actually happening from the perspective of the Fawn library with later code.
With that knowledge you can consider the following code in the listing:
let task = Fawn.Task();
let results = await task
.save('One', { name: 'Bill' })
.save('Two', { counter: 0 })
.update('Two', { }, { "$inc": { "counter": 1 } })
.run({ useMongoose: true });
Here the methods of update() and save() familiar to mongoose and MongoDB users actually have a different first argument specific to their implementation on the Fawn.Task() result. That first argument is the "registered model name" for mongoose, which is what we just explained with the previous example.
What the Fawn library is actually doing is calling similar code to:
mongoose.model('One').save({ name: 'Bill' })
Well actually it's doing something a lot more complicated than that as is evidenced in the output of the example listing. It's actually doing a lot of other things related to two phase commits and writing temporary entries in another collection and eventually moving those over to the target collections. But when it does actually go to the collections for the registered models, then that is basically how it is doing it.
So the core issue in the code in the question is that you are not using the names that were actually registered to the mongoose models, and a few other things are missing from the documentation steps.
You're also not awaiting asynchronous functions correctly, and the try..catch within the question code is not doing anything with calls in this context. The listing here however demonstrates how to do that correctly using async/await.
You can alternately just use the native Promise.then(...).catch(...) aproach if your NodeJS version does not have async/await support, but there really is little other change than doing that and of course removing the try..catch since promises in that form will ignore it. Which is why you catch() instead.
NOTE - With some brief testing there appear to be a number of things which are supported mongoose/mongodb features which are not actually implemented and supported on this library's methods. Notably "upserts" was a prime example of a useful and common thing which the "two phase commit" system implemented here does not appear to support at all.
This partly seems an oversight in the code of the library where certain "options" to the methods are actually being ignored or stripped completely. This is a concern for getting the most out of MongoDB features.
Transactions
The whole usage of this library though at least seems suspicious to me that you picked it up because you "thought" this was "Transactions". Put plainly the two phase commit is NOT a transaction. Furthermore the implementation of any attempt at such control and rollback etc seem very loose at best.
If you have a modern MongoDB 4.0 server or above, and where you actually configured it to be named as a "replica set" ( which you can also do for a single member, where a common misconception is you need more than one ) then there is support for real transactions, and they are very easy to implement:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/trandemo';
const opts = { useNewUrlParser: true };
// sensible defaults
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
// schema defs
const orderSchema = new Schema({
name: String
});
const orderItemsSchema = new Schema({
order: { type: Schema.Types.ObjectId, ref: 'Order' },
itemName: String,
price: Number
});
const Order = mongoose.model('Order', orderSchema);
const OrderItems = mongoose.model('OrderItems', orderItemsSchema);
// log helper
const log = data => console.log(JSON.stringify(data, undefined, 2));
// main
(async function() {
try {
const conn = await mongoose.connect(uri, opts);
// clean models
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.deleteMany())
)
let session = await conn.startSession();
session.startTransaction();
// Collections must exist in transactions
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.createCollection())
);
let [order] = await Order.create([{ name: 'Bill' }], { session });
let items = await OrderItems.insertMany(
[
{ order: order._id, itemName: 'Cheese', price: 1 },
{ order: order._id, itemName: 'Bread', price: 2 },
{ order: order._id, itemName: 'Milk', price: 3 }
],
{ session }
);
// update an item
let result1 = await OrderItems.updateOne(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ session }
);
log(result1);
// commit
await session.commitTransaction();
// start another
session.startTransaction();
// Update and abort
let result2 = await OrderItems.findOneAndUpdate(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ 'new': true, session }
);
log(result2);
await session.abortTransaction();
/*
* $lookup join - expect Milk to be price: 4
*
*/
let joined = await Order.aggregate([
{ '$match': { _id: order._id } },
{ '$lookup': {
'from': OrderItems.collection.name,
'foreignField': 'order',
'localField': '_id',
'as': 'orderitems'
}}
]);
log(joined);
} catch(e) {
console.error(e)
} finally {
mongoose.disconnect()
}
})()
That is really just a simple listing with the class Order and related OrderItems. There really is nothing special in the code and you should see that it's basically the same as most listing examples you will see with a few small changes.
Notably we initialize a session and also session.startTransaction() as an indicator that a transaction should be in progress. Note that session would generally have a wider scope where you would typically re-use that object for more than just a few operations.
Now you have session and the transaction is started, this is simply added to the "options" of the various statements being executed:
let [order] = await Order.create([{ name: 'Bill' }], { session });
let items = await OrderItems.insertMany(
[
{ order: order._id, itemName: 'Cheese', price: 1 },
{ order: order._id, itemName: 'Bread', price: 2 },
{ order: order._id, itemName: 'Milk', price: 3 }
],
{ session }
);
Admittedly this is a brief example that does not fully cover all write error possibilities and how to handle that within separate try..catch blocks. But as a very basic example should any error occur before the session.commitTransaction() is called, then none of the operations since the transaction was started will actually be persisted within the session.
Also there is "causal consistency" in that once a normal write acknowledgement has been confirmed, then within the scope of the session the data appears written to the respective collections right up until the transaction commit or rollback.
In the event of a rollback ( as demonstrated in the final operation ):
// Update and abort
let result2 = await OrderItems.findOneAndUpdate(
{ order: order._id, itemName: 'Milk' },
{ $inc: { price: 1 } },
{ 'new': true, session }
);
log(result2);
await session.abortTransaction();
These writes though reported to be made as seen in the operation result, are indeed "rolled back" and further operations see the state of the data before these changes were made.
The full example code demonstrates this by adding the items with another update action in one transaction, then beginning another to alter data and read it then abort the transaction. The final data state shows of course only what was actually committed.
NOTE Operations like find() and findOne() or anything that retrieves data must include the session whilst a transaction is active in order to see the current state, just in the same way that write operations are doing as shown in the listing.
Without including the session, these changes in state are not visible in the "global" scope until the transaction is resolved.
Listing Outputs
Code listings given produce the following output when run, for reference.
fawndemo
Mongoose: ones.deleteMany({}, {})
Mongoose: twos.deleteMany({}, {})
Mongoose: ojlinttaskcollections.deleteMany({}, {})
Mongoose: ojlinttaskcollections.insertOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a"), steps: [ { dataStore: [], _id: ObjectId("5bf765f7e5c71c5fae77030d"), index: 0, type: 'save', state: 0, name: 'One', data: { name: 'Bill' } }, { dataStore: [], _id: ObjectId("5bf765f7e5c71c5fae77030c"), index: 1, type: 'save', state: 0, name: 'Two', data: { counter: 0 } }, { dataStore: [], _id: ObjectId("5bf765f7e5c71c5fae77030b"), index: 2, type: 'update', state: 0, name: 'Two', data: { '*_**ojlint**escape$*__tx__00***___string$inc': { counter: 1 } } } ], __v: 0 })
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.0.state': 1 } })
Mongoose: ones.insertOne({ _id: ObjectId("5bf765f7e5c71c5fae77030e"), name: 'Bill', __v: 0 })
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.0.state': 2 } })
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.1.state': 1 } })
Mongoose: twos.insertOne({ _id: ObjectId("5bf765f7e5c71c5fae77030f"), counter: 0, __v: 0 })
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.1.state': 2 } })
Mongoose: twos.find({})
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.2.state': 1 } })
Mongoose: twos.update({}, { '$inc': { counter: 1 } }, {})
(node:24494) DeprecationWarning: collection.update is deprecated. Use updateOne, updateMany, or bulkWrite instead.
Mongoose: ojlinttaskcollections.updateOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") }, { '$set': { 'steps.2.state': 2 } })
Mongoose: ojlinttaskcollections.deleteOne({ _id: ObjectId("5bf765f7e5c71c5fae77030a") })
[
{
"_id": "5bf765f7e5c71c5fae77030e",
"name": "Bill",
"__v": 0
},
{
"_id": "5bf765f7e5c71c5fae77030f",
"counter": 0,
"__v": 0
},
{
"n": 1,
"nModified": 1,
"opTime": {
"ts": "6626877488230301707",
"t": 139
},
"electionId": "7fffffff000000000000008b",
"ok": 1,
"operationTime": "6626877488230301707",
"$clusterTime": {
"clusterTime": "6626877488230301707",
"signature": {
"hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
"keyId": 0
}
}
}
]
Mongoose: ones.find({}, { projection: {} })
[
{
"_id": "5bf765f7e5c71c5fae77030e",
"name": "Bill",
"__v": 0
}
]
Mongoose: twos.find({}, { projection: {} })
[
{
"_id": "5bf765f7e5c71c5fae77030f",
"counter": 1,
"__v": 0
}
]
Mongoose: ojlinttaskcollections.find({}, { projection: {} })
[]
transdemo
Mongoose: orders.deleteMany({}, {})
Mongoose: orderitems.deleteMany({}, {})
Mongoose: orders.insertOne({ _id: ObjectId("5bf7661c3f60105fe48d076e"), name: 'Bill', __v: 0 }, { session: ClientSession("e146c6074bb046faa7b70ed787e1a334") })
Mongoose: orderitems.insertMany([ { _id: 5bf7661c3f60105fe48d076f, order: 5bf7661c3f60105fe48d076e, itemName: 'Cheese', price: 1, __v: 0 }, { _id: 5bf7661c3f60105fe48d0770, order: 5bf7661c3f60105fe48d076e, itemName: 'Bread', price: 2, __v: 0 }, { _id: 5bf7661c3f60105fe48d0771, order: 5bf7661c3f60105fe48d076e, itemName: 'Milk', price: 3, __v: 0 } ], { session: ClientSession("e146c6074bb046faa7b70ed787e1a334") })
Mongoose: orderitems.updateOne({ order: ObjectId("5bf7661c3f60105fe48d076e"), itemName: 'Milk' }, { '$inc': { price: 1 } }, { session: ClientSession("e146c6074bb046faa7b70ed787e1a334") })
{
"n": 1,
"nModified": 1,
"opTime": {
"ts": "6626877647144091652",
"t": 139
},
"electionId": "7fffffff000000000000008b",
"ok": 1,
"operationTime": "6626877647144091652",
"$clusterTime": {
"clusterTime": "6626877647144091652",
"signature": {
"hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAA=",
"keyId": 0
}
}
}
Mongoose: orderitems.findOneAndUpdate({ order: ObjectId("5bf7661c3f60105fe48d076e"), itemName: 'Milk' }, { '$inc': { price: 1 } }, { session: ClientSession("e146c6074bb046faa7b70ed787e1a334"), upsert: false, remove: false, projection: {}, returnOriginal: false })
{
"_id": "5bf7661c3f60105fe48d0771",
"order": "5bf7661c3f60105fe48d076e",
"itemName": "Milk",
"price": 5,
"__v": 0
}
Mongoose: orders.aggregate([ { '$match': { _id: 5bf7661c3f60105fe48d076e } }, { '$lookup': { from: 'orderitems', foreignField: 'order', localField: '_id', as: 'orderitems' } } ], {})
[
{
"_id": "5bf7661c3f60105fe48d076e",
"name": "Bill",
"__v": 0,
"orderitems": [
{
"_id": "5bf7661c3f60105fe48d076f",
"order": "5bf7661c3f60105fe48d076e",
"itemName": "Cheese",
"price": 1,
"__v": 0
},
{
"_id": "5bf7661c3f60105fe48d0770",
"order": "5bf7661c3f60105fe48d076e",
"itemName": "Bread",
"price": 2,
"__v": 0
},
{
"_id": "5bf7661c3f60105fe48d0771",
"order": "5bf7661c3f60105fe48d076e",
"itemName": "Milk",
"price": 4,
"__v": 0
}
]
}
]
const { Rentals, validateRentals } = require("../models/rentals");
const { Movie } = require("../models/movie");
const { Customer } = require("../models/customer");
const Fawn = require("fawn");
const express = require("express");
const router = express.Router();
Fawn.init("mongodb://127.0.0.1:27017/vidly");
router.get("/", async (req, res) => {
const rentals = await Rentals.find().sort("-dateOut");
res.send(rentals);
});
router.get("/:id", async (req, res) => {
const rentals = await Rentals.findById(req.params.id);
if (!rentals)
return res.status(404).send("The rental with the given ID was not found.");
res.send(rentals);
});
router.delete("/:id", async (req, res) => {
const rentals = await Rentals.findByIdAndRemove(req.params.id);
if (!rentals)
return res.status(404).send("The rental with the given ID was not found.");
res.send(rentals);
});
router.post("/", async (req, res) => {
const { error } = validateRentals(req.body);
if (error) return res.status(400).send(error.detais[0].message);
const movie = await Movie.findById(req.body.movieId);
if (!movie)
return res.status(404).send("The rental with the given ID was not found.");
const customer = await Customer.findById(req.body.customerId);
if (!customer)
return res.status(404).send("The rental with the given ID was not found.");
if (movie.numberInStock === 0)
return res.status(400).send("Movie not in stock");
let rentals = new Rentals({
customer: {
_id: customer._id,
name: customer.name,
isGold: customer.isGold,
phone: customer.phone,
},
movie: {
_id: movie._id,
title: movie.title,
dailyRentalRate: movie.dailyRentalRate,
},
});
try {
new Fawn.Task()
.save("rentals", rentals)
.update("movies", { _id: movie._id }, { $inc: { numberInStock: -1 } })
.run();
res.send(rentals);
} catch (ex) {
res.status(500).send("Something failed");
}
// rentals = await rentals.save();
// movie.numberInStock--;
// movie.save();
// res.send(rentals);
//implementing transaction
});
router.put("/:id", async (req, res) => {
const { error } = validateRentals(req.body);
if (error) return res.status(400).send(error.detais[0].message);
const movie = await Movie.findById(req.body.movieId);
if (!movie)
return res.status(404).send("The rental with the given ID was not found.");
const customer = await Customer.findById(req.body.customerId);
if (!customer)
return res.status(404).send("The rental with the given ID was not found.");
let rentals = await Rentals.findByIdAndUpdate(
req.params.id,
{
customer: {
_id: customer._id,
name: customer.name,
isGold: customer.isGold,
phone: customer.phone,
},
movie: {
_id: movie._id,
title: movie.title,
dailyRentalRate: movie.dailyRentalRate,
},
},
{ new: true }
);
if (!rentals)
return res.status(404).send("The rentals with the given ID was not found.");
res.send(rentals);
});
module.exports = router;
Instead of using
Fawn.init(mongoose)
try using
Fawn.init('mongodb://localhost/yourDataBaseName')
Related
Pushing into Nested array in mongodb
This is my mongoose Schema
const notesSchema = mongoose.Schema({
college: {type:String},
year: {type:String},
sem: {type:String},
branch:{type:String},
subjects: [
{
subject_id:{type:String},
name:{type:String},
codename: {type:String},
notes: [
{
notes_id:{type:String},
title: {type:String},
material: [
{
material_id:{type:String},
heading:{type:String},
link: {type:String}
}
]
}
]
}
]
})
I want to insert a object into 'material' array which is inside notes array and notes is inside subjects array.
I tried many syntax, but none of them worked for me. Recently I tried this.
try {
notes.updateOne({
$and:[ {_id: '62a61949dc0f920ae99fc687'}, {'subjects.notes.notes_id':'221fad-f35c-ee2a-65b3-8531dbfcf732'}]
},
{$push:{'subjects.0.notes.$.material':
[{ material_id: "hfklahfhoabfoab", heading: "Prime", link: "wwo.prime.com" }]
}}
This is full function code:-
router.get('/populate',async (req,res)=>{
// const {data} = req.body
// const link = "www.wiki.com"
try {
notes.updateOne({
_id: '62a61949dc0f920ae99fc687',
'subjects.notes.notes_id': '221fad-f35c-ee2a-65b3-8531dbfcf732',
},
{$push:{'subjects.0.notes.$.material':
[{ material_id: "hfklahfhoabfoab", heading: "Prime", link: "wwo.prime.com" }]
}}
)
console.log("posted")
const alreadyexist = await notes.find({$and:[{"year":'3'},{"sem":'2'}]})
res.send(alreadyexist)
// console.log(updata)
} catch (error) {
console.log(error)
}
} )
This is my current Database status.
enter image description here
Try to change your code like this:
router.get('/populate', async (req, res) => {
try {
await notes.updateOne(
{
_id: '62a61949dc0f920ae99fc687',
'subjects.notes.notes_id': '221fad-f35c-ee2a-65b3-8531dbfcf732',
},
{
$push: {
'subjects.0.notes.$.material': {
material_id: 'hfklahfhoabfoab',
heading: 'Prime',
link: 'wwo.prime.com',
},
},
}
);
console.log('posted');
const alreadyexist = await notes.find({ year: '3', sem: '2' });
res.send(alreadyexist);
} catch (error) {
console.log(error);
}
});
trying to calc average with $avg and got: arguments must be aggregate pipeline operators
got this error: arguments must be aggregate pipeline operators
this is the course 'Schema" that got bootcamp as objectId.
const CourseSchema = new mongoose.Schema({
bootcamp: {
type: mongoose.Schema.ObjectId,
ref: 'Bootcamp',
required: true
}
});
the aggregation :
//static method to get avg of course tuitions
CourseSchema.statics.getAverageCost = async function (bootcampId) {
console.log('calculating avg cost... with bootcampId:' + bootcampId);
const obj = await this.aggragate([{
$match: { bootcamp: bootcampId },
$group: {
_id: '$bootcamp',
averageCost: { $avg: '$tuition' }
}
}]);
console.log(obj);
}
calling for the aggregation before saving or removing:
...
// Call getAvarageCost after save
CourseSchema.post('save', function () {
this.constructor.getAverageCost(this.bootcamp);
})
// Call getAvarageCost before remove
CourseSchema.post('remove', function () {
this.constructor.getAverageCost(this.bootcamp);
})
...
$match and $group must be in different pipeline operations
const cursor = this.aggregate([
{ $match: { bootcamp: bootcampId } },
{
$group: {
_id: '$bootcamp',
averageCost: { $avg: '$tuition' },
},
},
])
console.log(await cursor.toArray())
How to update embedded document in array in mongoose where operation should be atomic, should update on the latest one and return updated document?
Currently working on node.js and MongoDB with mongoose.
//Document in mongoDB
{
name: 'first',
_id: '1000'
phases: [{
_id: 1,
phaseName: 'phase1',
stories: [{
_id: s1,
sname: 'sfirst'
},
{
_id: s2,
sname: 'ssecond'
},
{
_id: s3,
sname: 'sthird'
}
],
},
{
_id: 2,
phaseName: 'phase2',
stories: [{
_id: s1,
sname: 'sfirst'
},
{
_id: s2,
sname: 'ssecond'
},
{
_id: s3,
sname: 'sthird'
}
],
}
],
}
All I have is
id=1000 (to find this document in mongodb)
phaseId=1 and remstoryId=s2 (i.e delete s2 from stories from phase with id 1)
phaseId=2, storyId={ _id: s4, sname: fourth}, position: 1 (add story in stories at index 1 in phase 2)
Main requirement is this operation should be
Atomic
No other user can perform operation on this same document while performing above operation. I mean lock this document.
I can we can do this operation using findOne() and save()
Model.findOne({
id
}, (err, plan) => {
if (!err) {
//REMOVE THE STORY FROM PHASE
const phase = plan.phases.find((phase) => phase._id === delPhaseID);
phase.stories = phase.stories.filter((story) => story._id !== delStoryId);
//ADD STORY IN STORIES
const addPhase = plan.phases.find((phase) => phase._id === addPhaseID);
addPhase.stories.splice(pos, 0, story);
//SAVE PLAN
plan.save((err, doc) => {
if(!err)
console.log(doc) // UPDATED DOCUMENT
});
}
})
but this operation is not atomic as per below image taken from masteringjs. How can we solve this problem
You can try findOneAndUpdate() with arrayFilters,
$pull to remove in p1 id's stories is s1
$push to specific position for p2 use $position
new: true to return updated document
Model.findOneAndUpdate(
{ _id: "1000" },
{
$pull: {
"phases.$[p1].stories": {
_id: "s1"
}
},
$push: {
"phases.$[p2].stories": {
$each: [{
_id: "s4",
sname: "fourth"
}],
$position: 1
}
}
},
{
arrayFilters: [
{ "p1._id": 1 },
{ "p2._id": 2 }
],
new: true
}
)
Mongoose Saving All Updates Apart From One
I have a Mongo database which uses the following Schema:
const userSchema = {
first_login: {
type: Date,
default: Date.now
},
last_login: {
type: Date,
default: Date.now
},
provider_use_count: {
logins: {
type: Object,
required: true
}
},
total_login_count: {
type: Number,
required: true,
default: 0
},
login_hours: [
{
type: Number,
required: true
}
],
playlists: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Playlist'
}
],
genres: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Genre'
}
],
artists: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Artist'
}
],
auth_providers: [
{
type: mongoose.Schema.Types.ObjectId,
ref: 'Provider'
}
],
platform_ids: [
{
type: Object,
required: true,
}
]
}
I am updating it with the following function, which is in a class:
async updateUser(user, playlists, currentProvider) {
return new Promise((resolve, reject) => {
// Add one to the use count for the current provider
user.provider_use_count.logins[currentProvider]++;
user.total_login_count++;
user.last_login = new Date();
user.login_hours.push((new Date()).getHours() + 1);
return user.save((err, doc) => {
if(err){
return reject(err);
} else {
console.log(doc);
return resolve(doc);
}
});
});
}
However, when I use the .save() function it is not storing the updated value for provider_use_count.logins. I have tried everything I can think of: using ++, copying the object and then incrementing it, getting a variable of the current count and then saving it, removing the required field from the Schema, using + 1, using += 1, using an array which houses an object that has a field for count and provider.
I am out of ideas as the save function clearly works -- It saves all other updates. I feel like this could be due to the Schema and it is something which I am either just overlooking or haven't experienced before.
EDIT:
The exact field which is not being saved is:
user.provider_use_count.logins[currentProvider]++;
Also, I am not relying on the console.log(doc) block, I have checked in Atlas and it seems that field just remains at 0 while all others update.
As I have not implemented any other auth providers so the field in question should be identical to total_login_count
An example of the user object which is passed to the function is as follows:
{
total_login_count: 7,
login_hours: [
20, 20, 20, 20,
21, 21, 20, 20
],
playlists: [],
genres: [],
artists: [],
auth_providers: [],
platform_ids: [ { provider: 'spotify', id: 'makingstuffs' } ],
_id: 5dadfe7c9a25cc0c6a4d387c,
first_login: 2019-10-21T18:52:44.814Z,
last_login: 2019-10-21T19:23:30.580Z,
provider_use_count: { logins: { spotify: 0 } },
__v: 7
}
The issue was with the fact that mongoose has issues with saving mixed data types as they are considered schema-less. This means that you have to make a call to person.markModified('name_of_the_update_field') prior to calling the .save() function.
My updateUser() function now looks as follows:
async updateUser(user, playlists, currentProvider) {
return new Promise((resolve, reject) => {
// Add one to the use count for the current provider
user.provider_use_count.logins[currentProvider]++;
user.markModified('provider_use_count');
user.total_login_count++;
user.last_login = new Date();
user.login_hours.push((new Date()).getHours());
return user.save((err, doc) => {
if(err)
return reject(err);
return resolve(doc);
});
});
}
MongoDB find() with dot notation does not work [duplicate]
I'm pretty new to Mongoose and MongoDB in general so I'm having a difficult time figuring out if something like this is possible:
Item = new Schema({
id: Schema.ObjectId,
dateCreated: { type: Date, default: Date.now },
title: { type: String, default: 'No Title' },
description: { type: String, default: 'No Description' },
tags: [ { type: Schema.ObjectId, ref: 'ItemTag' }]
});
ItemTag = new Schema({
id: Schema.ObjectId,
tagId: { type: Schema.ObjectId, ref: 'Tag' },
tagName: { type: String }
});
var query = Models.Item.find({});
query
.desc('dateCreated')
.populate('tags')
.where('tags.tagName').in(['funny', 'politics'])
.run(function(err, docs){
// docs is always empty
});
Is there a better way do this?
Edit
Apologies for any confusion. What I'm trying to do is get all Items that contain either the funny tag or politics tag.
Edit
Document without where clause:
[{
_id: 4fe90264e5caa33f04000012,
dislikes: 0,
likes: 0,
source: '/uploads/loldog.jpg',
comments: [],
tags: [{
itemId: 4fe90264e5caa33f04000012,
tagName: 'movies',
tagId: 4fe64219007e20e644000007,
_id: 4fe90270e5caa33f04000015,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
},
{
itemId: 4fe90264e5caa33f04000012,
tagName: 'funny',
tagId: 4fe64219007e20e644000002,
_id: 4fe90270e5caa33f04000017,
dateCreated: Tue, 26 Jun 2012 00:29:36 GMT,
rating: 0,
dislikes: 0,
likes: 0
}],
viewCount: 0,
rating: 0,
type: 'image',
description: null,
title: 'dogggg',
dateCreated: Tue, 26 Jun 2012 00:29:24 GMT
}, ... ]
With the where clause, I get an empty array.
With a modern MongoDB greater than 3.2 you can use $lookup as an alternate to .populate() in most cases. This also has the advantage of actually doing the join "on the server" as opposed to what .populate() does which is actually "multiple queries" to "emulate" a join.
So .populate() is not really a "join" in the sense of how a relational database does it. The $lookup operator on the other hand, actually does the work on the server, and is more or less analogous to a "LEFT JOIN":
Item.aggregate(
[
{ "$lookup": {
"from": ItemTags.collection.name,
"localField": "tags",
"foreignField": "_id",
"as": "tags"
}},
{ "$unwind": "$tags" },
{ "$match": { "tags.tagName": { "$in": [ "funny", "politics" ] } } },
{ "$group": {
"_id": "$_id",
"dateCreated": { "$first": "$dateCreated" },
"title": { "$first": "$title" },
"description": { "$first": "$description" },
"tags": { "$push": "$tags" }
}}
],
function(err, result) {
// "tags" is now filtered by condition and "joined"
}
)
N.B. The .collection.name here actually evaluates to the "string" that is the actual name of the MongoDB collection as assigned to the model. Since mongoose "pluralizes" collection names by default and $lookup needs the actual MongoDB collection name as an argument ( since it's a server operation ), then this is a handy trick to use in mongoose code, as opposed to "hard coding" the collection name directly.
Whilst we could also use $filter on arrays to remove the unwanted items, this is actually the most efficient form due to Aggregation Pipeline Optimization for the special condition of as $lookup followed by both an $unwind and a $match condition.
This actually results in the three pipeline stages being rolled into one:
{ "$lookup" : {
"from" : "itemtags",
"as" : "tags",
"localField" : "tags",
"foreignField" : "_id",
"unwinding" : {
"preserveNullAndEmptyArrays" : false
},
"matching" : {
"tagName" : {
"$in" : [
"funny",
"politics"
]
}
}
}}
This is highly optimal as the actual operation "filters the collection to join first", then it returns the results and "unwinds" the array. Both methods are employed so the results do not break the BSON limit of 16MB, which is a constraint that the client does not have.
The only problem is that it seems "counter-intuitive" in some ways, particularly when you want the results in an array, but that is what the $group is for here, as it reconstructs to the original document form.
It's also unfortunate that we simply cannot at this time actually write $lookup in the same eventual syntax the server uses. IMHO, this is an oversight to be corrected. But for now, simply using the sequence will work and is the most viable option with the best performance and scalability.
Addendum - MongoDB 3.6 and upwards
Though the pattern shown here is fairly optimized due to how the other stages get rolled into the $lookup, it does have one failing in that the "LEFT JOIN" which is normally inherent to both $lookup and the actions of populate() is negated by the "optimal" usage of $unwind here which does not preserve empty arrays. You can add the preserveNullAndEmptyArrays option, but this negates the "optimized" sequence described above and essentially leaves all three stages intact which would normally be combined in the optimization.
MongoDB 3.6 expands with a "more expressive" form of $lookup allowing a "sub-pipeline" expression. Which not only meets the goal of retaining the "LEFT JOIN" but still allows an optimal query to reduce results returned and with a much simplified syntax:
Item.aggregate([
{ "$lookup": {
"from": ItemTags.collection.name,
"let": { "tags": "$tags" },
"pipeline": [
{ "$match": {
"tags": { "$in": [ "politics", "funny" ] },
"$expr": { "$in": [ "$_id", "$$tags" ] }
}}
]
}}
])
The $expr used in order to match the declared "local" value with the "foreign" value is actually what MongoDB does "internally" now with the original $lookup syntax. By expressing in this form we can tailor the initial $match expression within the "sub-pipeline" ourselves.
In fact, as a true "aggregation pipeline" you can do just about anything you can do with an aggregation pipeline within this "sub-pipeline" expression, including "nesting" the levels of $lookup to other related collections.
Further usage is a bit beyond the scope of what the question here asks, but in relation to even "nested population" then the new usage pattern of $lookup allows this to be much the same, and a "lot" more powerful in it's full usage.
Working Example
The following gives an example using a static method on the model. Once that static method is implemented the call simply becomes:
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
Or enhancing to be a bit more modern even becomes:
let results = await Item.lookup({
path: 'tags',
query: { 'tagName' : { '$in': [ 'funny', 'politics' ] } }
})
Making it very similar to .populate() in structure, but it's actually doing the join on the server instead. For completeness, the usage here casts the returned data back to mongoose document instances at according to both the parent and child cases.
It's fairly trivial and easy to adapt or just use as is for most common cases.
N.B The use of async here is just for brevity of running the enclosed example. The actual implementation is free of this dependency.
const async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/looktest');
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt,callback) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
this.aggregate(pipeline,(err,result) => {
if (err) callback(err);
result = result.map(m => {
m[opt.path] = m[opt.path].map(r => rel(r));
return this(m);
});
callback(err,result);
});
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
function log(body) {
console.log(JSON.stringify(body, undefined, 2))
}
async.series(
[
// Clean data
(callback) => async.each(mongoose.models,(model,callback) =>
model.remove({},callback),callback),
// Create tags and items
(callback) =>
async.waterfall(
[
(callback) =>
ItemTag.create([{ "tagName": "movies" }, { "tagName": "funny" }],
callback),
(tags, callback) =>
Item.create({ "title": "Something","description": "An item",
"tags": tags },callback)
],
callback
),
// Query with our static
(callback) =>
Item.lookup(
{
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
},
callback
)
],
(err,results) => {
if (err) throw err;
let result = results.pop();
log(result);
mongoose.disconnect();
}
)
Or a little more modern for Node 8.x and above with async/await and no additional dependencies:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
dateCreated: { type: Date, default: Date.now },
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
});
itemSchema.statics.lookup = function(opt) {
let rel =
mongoose.model(this.schema.path(opt.path).caster.options.ref);
let group = { "$group": { } };
this.schema.eachPath(p =>
group.$group[p] = (p === "_id") ? "$_id" :
(p === opt.path) ? { "$push": `$${p}` } : { "$first": `$${p}` });
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": opt.path,
"localField": opt.path,
"foreignField": "_id"
}},
{ "$unwind": `$${opt.path}` },
{ "$match": opt.query },
group
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [opt.path]: m[opt.path].map(r => rel(r)) })
));
}
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.create(
["movies", "funny"].map(tagName =>({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
const result = (await Item.lookup({
path: 'tags',
query: { 'tags.tagName' : { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
mongoose.disconnect();
} catch (e) {
console.error(e);
} finally {
process.exit()
}
})()
And from MongoDB 3.6 and upward, even without the $unwind and $group building:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/looktest';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const itemTagSchema = new Schema({
tagName: String
});
const itemSchema = new Schema({
title: String,
description: String,
tags: [{ type: Schema.Types.ObjectId, ref: 'ItemTag' }]
},{ timestamps: true });
itemSchema.statics.lookup = function({ path, query }) {
let rel =
mongoose.model(this.schema.path(path).caster.options.ref);
// MongoDB 3.6 and up $lookup with sub-pipeline
let pipeline = [
{ "$lookup": {
"from": rel.collection.name,
"as": path,
"let": { [path]: `$${path}` },
"pipeline": [
{ "$match": {
...query,
"$expr": { "$in": [ "$_id", `$$${path}` ] }
}}
]
}}
];
return this.aggregate(pipeline).exec().then(r => r.map(m =>
this({ ...m, [path]: m[path].map(r => rel(r)) })
));
};
const Item = mongoose.model('Item', itemSchema);
const ItemTag = mongoose.model('ItemTag', itemTagSchema);
const log = body => console.log(JSON.stringify(body, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// Clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create tags and items
const tags = await ItemTag.insertMany(
["movies", "funny"].map(tagName => ({ tagName }))
);
const item = await Item.create({
"title": "Something",
"description": "An item",
tags
});
// Query with our static
let result = (await Item.lookup({
path: 'tags',
query: { 'tagName': { '$in': [ 'funny', 'politics' ] } }
})).pop();
log(result);
await mongoose.disconnect();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
what you are asking for isn't directly supported but can be achieved by adding another filter step after the query returns.
first, .populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } ) is definitely what you need to do to filter the tags documents. then, after the query returns you'll need to manually filter out documents that don't have any tags docs that matched the populate criteria. something like:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags.length;
})
// do stuff with docs
});
Try replacing
.populate('tags').where('tags.tagName').in(['funny', 'politics'])
by
.populate( 'tags', null, { tagName: { $in: ['funny', 'politics'] } } )
Update: Please take a look at the comments - this answer does not correctly match to the question, but maybe it answers other questions of users which came across (I think that because of the upvotes) so I will not delete this "answer":
First: I know this question is really outdated, but I searched for exactly this problem and this SO post was the Google entry #1. So I implemented the docs.filter version (accepted answer) but as I read in the mongoose v4.6.0 docs we can now simply use:
Item.find({}).populate({
path: 'tags',
match: { tagName: { $in: ['funny', 'politics'] }}
}).exec((err, items) => {
console.log(items.tags)
// contains only tags where tagName is 'funny' or 'politics'
})
Hope this helps future search machine users.
After having the same problem myself recently, I've come up with the following solution:
First, find all ItemTags where tagName is either 'funny' or 'politics' and return an array of ItemTag _ids.
Then, find Items which contain all ItemTag _ids in the tags array
ItemTag
.find({ tagName : { $in : ['funny','politics'] } })
.lean()
.distinct('_id')
.exec((err, itemTagIds) => {
if (err) { console.error(err); }
Item.find({ tag: { $all: itemTagIds} }, (err, items) => {
console.log(items); // Items filtered by tagName
});
});
#aaronheckmann 's answer worked for me but I had to replace return doc.tags.length; to return doc.tags != null; because that field contain null if it doesn't match with the conditions written inside populate.
So the final code:
query....
.exec(function(err, docs){
docs = docs.filter(function(doc){
return doc.tags != null;
})
// do stuff with docs
});