We Keep Coding

How to filter with Mongo and Node ? using GridFs

I am making an Api with Node and Mongo that receives large volumes of data, I was receiving an error because the size of the records that were stored in mongo exceeded 16 MB. So I opted for the alternative offered by mongo in its gridFS documentation, to insert the records, which I had no problems with. But I am having conflicts to insert and filter since I don't know how to do it, I read the documentation and there are several ways. But I can't figure out how to filter (find a record by its field) and how to update.
The function to create a record works but it performs some necessary steps such as storing the json it receives in a file and then reading it and with that creating the record, I would have liked to find a more practical solution such as only inserting the json it receives without having to create a file with its content and then get the information from that file I attach the code to see if you can tell me how to solve this problem:
const { MongoClient, ObjectId, GridFSBucket,} = require('mongodb');
const { config } = require('../../config');
//const USER = encodeURIComponent(config.noRelDbUser);
//const PASSWORD = encodeURIComponent(config.noRelDbPassword);
const DB_NAME = config.noRelDbName;
const fs = require('fs');
const removeFile = require('../../modules/results/utils/RemoveFile');
// const MONGO_URI = `mongodb://${USER}:${PASSWORD}#${config.dbHost}:${config.dbPort}/admin?retryWrites=true&w=majority`
const MONGO_URI = `mongodb://${config.noRelDbHost}:${config.noRelDbPort}`;
class MongoLib {
constructor() {
this.client = new MongoClient(MONGO_URI, { useNewUrlParser: true, useUnifiedTopology: true });
this.dbName = DB_NAME;
}
connect() {
if (!MongoLib.connection) {
MongoLib.connection = new Promise((resolve, reject) => {
this.client.connect((err) => {
if (err) {
reject(err);
}
resolve(this.client.db(this.dbName));
});
});
}
return MongoLib.connection;
}
create(collection, data) {
return this.connect()
.then((db) => {
return db.collection(collection).insertOne(data);
})
.then((result) => result.insertedId);
}
async createWithForBigData(collection, data, vr_id , remove=false){
let vrule_id = vr_id;
return this.connect().then((db)=>{
try{
var bucket = new GridFSBucket(db, {
bucketName: collection,
chunkSizeBytes: 260000 ,
});
bucket
bucket.find()
let uploadStream = fs.createReadStream(data).pipe(bucket.openUploadStream(`resultsdetail${vrule_id}`));
let id = uploadStream.id;
uploadStream.on('error', (err) => {
console.log({ message: "Error uploading file" });
throw new Error(err);
});
bucket.find()
uploadStream.on('finish', () => {
console.log({ message: "File uploaded successfully, stored under Mongo ObjectID: " + id });
if(remove === true){
console.log('remueve archivo archivo de directorio storebigdata');
removeFile(data);
}
return id;
});
}catch(err){
console.log('ocurrió un error al almacenar big data',err);
throw new Error(err);
}
})
}
findBigData(){
//
}
UpdateBigData(){
//
}
}
module.exports = MongoLib;

TypeError deleting a file with gridfs mongoose and node.js

I am developing an application that allows uploading and downloading music.
I can upload files, send them to the client... however, I have problems when it comes to deleting a bucket file...
I'am using "mongoose": "^6.2.1".
My controller, where podcastId is a ObjectId:
const connection = require('../database')
const mongoose = require('mongoose')
const Users = require('../models/Users')
const PodcastInfo = require('../models/PodcastInfo')
ctrPod.deletePodcast = async (req, res, next) => {
try {
const id = req.params.idPodInfo
const info = await PodcastInfo.findById(id)
const { userId, podcastId } = info
const gridFsBucket = new mongoose.mongo.GridFSBucket(connection, {
bucketName: 'podcasts',
});
gridFsBucket.delete(podcastId, (err) => {
console.log(err)
})
.
.
.
I get this error:
TypeError: Cannot use 'in' operator to search for 'client' in undefined
at getTopology
The problem appears here, \node_modules\mongodb\lib\utils.js:363:23) :
function getTopology(provider) {
if (`topology` in provider && provider.topology) {
return provider.topology;
}
else if ('client' in provider.s && provider.s.client.topology) {
return provider.s.client.topology;
}
else if ('db' in provider.s && provider.s.db.s.client.topology) {
return provider.s.db.s.client.topology;
}
throw new error_1.MongoNotConnectedError('MongoClient must be connected to perform this operation');
}
////////////////////////
delete(id, callback) {
return (0, utils_1.executeLegacyOperation)((0, utils_1.getTopology)(this.s.db), _delete, [this, id, callback], {
skipSessions: true
});
}
/////////////////////////////////////
What am I doing wrong?

I think the problem lies here:
const gridFsBucket = new mongoose.mongo.GridFSBucket(connection, {
bucketName: 'podcasts',
});
new mongoose.mongo.GridFSBucket(db,{bucketName}) takes in a db not a connection. Try:
const gridFsBucket = new mongoose.mongo.GridFSBucket(connection.db, {
bucketName: 'podcasts',
});

How to push data with Mongoose to a nested array in MongoDB

I'm trying to push data to a nested array in mongodb. I'm using mongoose as well.
This is just mock code to see if i can get it working:
User model:
import mongoose from "mongoose";
const CoinSchema = new mongoose.Schema({
coinID: { type: String },
});
const CoinsSchema = new mongoose.Schema({
coin: [CoinSchema],
});
const WatchlistSchema = new mongoose.Schema({
watchlistName: { type: String },
coins: [CoinsSchema],
});
const NameSchema = new mongoose.Schema({
firstName: { type: String },
lastName: { type: String },
username: { type: String },
});
const UserSchema = new mongoose.Schema({
name: [NameSchema],
watchlists: [WatchlistSchema],
test: String,
});
const User = mongoose.model("User", UserSchema);
export default User;
route:
fastify.put("/:id", async (request, reply) => {
try {
const { id } = request.params;
const newCoin = request.body;
const updatedUser = await User.findByIdAndUpdate(id, {
$push: { "watchlists[0].coins[0].coin": newCoin },
});
await updatedUser.save();
// console.dir(updatedUser, { depth: null });
reply.status(201).send(updatedUser);
} catch (error) {
reply.status(500).send("could not add to list");
}
});
request.body // "coinID": "test"
I've tried a lot of different ways to push this data but still no luck. I still get 201 status codes in my terminal which indicates something has been pushed to the DB, but when I check nothing new is there.
Whats the correct way to target nested arrays and push data to them?

It's not perfect but you could get the user document, update the user's watchlist, and then save the updated watchlist like so:
fastify.put("/:id", async (request, reply) => {
try {
const { id } = request.params;
const newCoin = request.body;
// get the user
let user = await User.findById(id);
// push the new coin to the User's watchlist
user.watchlists[0].coins[0].coin.push(newCoin);
//update the user document
const updatedUser = await User.findOneAndUpdate({ _id: id },
{
watchlists: user.watchlists,
},
{
new: true,
useFindAndModify: false
}
);
reply.status(201).send(updatedUser);
} catch (error) {
reply.status(500).send("could not add to list");
}
});

How to make a model in another file

I'm beginner and tried to transfer the model to another file, it didn't work for me, suggest me how to do it correctly. The question may seem silly, but if I knew the answer, I would not ask it.
file todo.controller.js
const fs = require("fs");
const { v4: uuidv4 } = require("uuid");
const data = fs.readFileSync("./data/data.json");
let todos = JSON.parse(data);
class todoController {
async createTodo(req, res) {
req.on("data", (data) => {
const jsondata = JSON.parse(data);
const title = jsondata.title;
const description = jsondata.description;
if ((title, description)) {
todos.push({
id: uuidv4(),
title,
description,
dateOfCreate: new Date(),
lastModified: new Date(),
check: new Boolean(false),
});
fs.writeFile(
"./data/data.json",
JSON.stringify(todos, null, 2),
(err) => {
if (err) throw error;
}
);
}
});
}}
file todo.router.js
const url = require("url");
const todoController = require("../controllers/todo.controller");
const todoRouter = (req, res) => {
const urlparse = url.parse(req.url, true);
if (urlparse.pathname == "/todos" && req.method == "POST") {
todoController.createTodo(req, res);
}
};
module.exports = todoRouter;
here is file data.json
data.json

You have two separate problems here, separating your code to a different file and also saving or persisting that data somewhere, in this case a file.
You have to create something like a data model and then you have to import it in your other code.
// data.js
export const get = async () => {} // we will implement this just now
export const set = async (data) => {} // we will implement this just now
...
// controller.js
import {get, set} from './data.js' // import the methods we just created
...
const createTodo = async (req, res) => {
req.on("data", (data) => {
// here you can use get() if you want to use the data
set(JSON.stringify(data)) // send data to your data model
}
}
Then we also have to actually do something with those methods.
// data.js
export const get = async () => {
// may need to use JSON.parse here depending on how you'll use it
return fs.readFile('./data.json')
}
export const set = async (data) => {
fs.writeFile('data.json', JSON.stringify(data))
}
So the idea is to have a model responsible for managing the data, retrieving it and saving it, then importing and using those methods in the main controller. The code above isn't perfect, it's just to show you how to think about it.

MongoDB collection.findOne() returns undefined value

const Location = require("../models/locations");
getLocation = async(req, res) => {
await Location.findOne(
{ name: req.query.locationName }, // req.query.locationName is "Gurgaon"
{ restaurantIds: 1 },
(err, location) => {
if (err) throw err;
else {
console.log(location);
/*
{
_id: 6004f9cff6ae9921f89f0f81,
restaurantIds: [ 6004fb53f6ae9921f89f0f83, 600792321b229bae25a66497 ]
}
*/
console.log(location._id); // 6004f9cff6ae9921f89f0f81
console.log(location.restaurantIds); // undefined
return location;
}
}
);
}
module.exports = { getLocation };
Screenshot of the output
This is how the locations collection looks like.
{
"_id" : ObjectId("6004f9cff6ae9921f89f0f81"),
"name" : "Gurgaon",
"restaurantIds" : [
ObjectId("6004fb53f6ae9921f89f0f83"),
ObjectId("600792321b229bae25a66497")
]
}
Here is the locations schema.
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const Locations = new Schema({}, { strict: false });
module.exports = mongoose.model("locations", Locations);
I don't know the reason why location.restaurantIds is returning me undefined. Please help me with this. I am new to mongodb.

There will be some reasons might be you have not specified this field in your mongoose schema, try adding field in your schema and you are able to access this field in your query.
Second option if you don't want to specify that field in schema then try lean(),
By default, Mongoose queries return an instance of the Mongoose Document class. Enabling the lean option tells Mongoose to skip instantiating a full Mongoose document and just give you the POJO.
await Location.findOne(.. your query).lean();

restaurantIds is a nested object , you must populate it :
const Location = require("../models/locations");
getLocation = async(req, res) => {
await Location.findOne(
{ name: req.query.locationName },
{ restaurantIds: 1 })
.populate('restaurantIds').then(location => {
console.log(location);
console.log(location._id);
console.log(location.restaurantIds);
return location;
})
.catch(err => {
throw err;
})
);
}
module.exports = { getLocation };

It's look like your restaurantIds is an array, so when you print it, you must use as array. Start by change:
console.log(location.restaurantIds);
to:
console.log(location.restaurantIds[0]);
In this example, you will be printing the first object in the array, and use it in your code as array, it will be OK.
Edit:
After added the restaurantIds, and now we know it's array of ObjectID, and this kind of array cannot be printed.