How to dynamically nest $or operator into $and Mongoose statement? - javascript

Suppose I have 3 MongDB entries in a movie database:
"books": [
{
"_id": "xxxxxx",
"title": "Fast Five",
"rating": 6
},
{
"_id": "yyyyyy",
"title": "Kill Bill",
"rating": 8
},
{
"_id": "zzzzzzzz",
"title": "Jumanji",
"rating": 5
}
]
I use GraphQL to retrieve the id of multiple movies if their title and rating match the criteria:
query{
getMovies(itemFilterInput: {titles:["Fast Five", "Jumanji"], rating:6}){
_id
title
}
}
This should return Fast Five.
I want this to translate into a mongoDB query like this:
{$and: [{rating:6}, {$or: [{title:"Fast Five", title:"Jumanji"}]}]}
In the backend, I use NodeJS and Mongoose to handle this query and I use something like this:
getMovies: async args => {
try{
let d = args.itemFilterInput;
let filters = new Array();
const keys = Object.keys(d);
keys.forEach((key) => {
// The titles case
if(d[key] instanceof Array){
var sObj = {};
var sObjArray = [];
d[key].map(prop => {
sObj[key] = prop;
sObjArray.push(sObj);
});
filters.push({$or: sObjArray});
}else{
// The rating case
var obj = {};
obj[key] = d[key];
filters.push(obj);
}
});
const items = await Item.find({$and: filters});
return items.map(item => {
return item;
});
} catch (err) {
throw err;
}
}
Unfortunately, this approach does not work.
What is the correct way to nest $or parameters in $and params?

Answer:
First of all, it is better to use the $in operator instead of a $and/$or combination.
And to solve the error: use quotes everywhere to declare the query.

Related

How to return specified data in function? JavaScript

I have the following data structure in an array from Neo4j.
DATA: [
{
"keys": [
"J"
],
"length": 1,
"_fields": [
{
"identity": {
"low": 9,
"high": 0
},
"labels": [
"Journal"
],
"properties": {
"body": "123"
},
"elementId": "9"
}
],
"_fieldLookup": {
"J": 0
}
},
{
"keys": [
"J"
],
"length": 1,
"_fields": [
{
"identity": {
"low": 6,
"high": 0
},
"labels": [
"Journal"
],
"properties": {
"name": "Journal 221204",
"body": "<p>Test!</p>",
"lastEdit": "221204_03:53:02 PM",
"createdOn": "221204_03:45:33 PM"
},
"elementId": "6"
}
],
"_fieldLookup": {
"J": 0
}
}
]
I am trying to loop over the array, add specific data to a new array (finalList), and return the new array with this code.
// Allow require
import { createRequire } from "module";
const require = createRequire(import.meta.url);
require('dotenv').config()
var neo4j = require('neo4j-driver')
let finalList = [];
export default async function Database(query) {
const Neo4jUser = process.env.Neo4jUser;
const Neo4jPass = process.env.Neo4jPass;
const uri = "";
try {
const driver = neo4j.driver(uri, neo4j.auth.basic(Neo4jUser, Neo4jPass));
let session = driver.session();
let result = await session.run(query);
let records = Object.values(result)[0];
console.log(Object.keys(records).length);
if (Object.keys(records).length === 1) {
let record = Object.values(records)[0];
let fields = record._fields;
let fields2 = fields[0];
let properties = fields2.properties;
console.log(properties);
return properties;
};
if (Object.keys(records).length >= 2) {
let count = Object.keys(records).length;
console.log(2);
console.log(count);
let get_properties = async (records, countTimes) => {
let record = Object.values(records)[countTimes];
let fields = record._fields;
let fields2 = fields[0];
let properties = fields2.properties;
console.log(properties);
return properties;
};
let countTimes = 0;
while (countTimes < count) {
let node = get_properties(records, countTimes);
finalList.concat(node.then());
countTimes++
};
console.log(finalList);
console.log(`DATA: ${records}`);
return finalList
};
}
catch (err) {
if (err.name == "Neo4jError") {
node = "No Database found"
return node;
}
else {
console.log(err);
}
};
};
Every time I run the code, finalList is returned empty.
How can I add the appropriate data to an array and have it returned inside the function?
I understand that I am having a problem with the scope of finalArray inside the function, as I am not able to have finalList.concat "save" the data, but I am unable to find a solution to my problem after my research.
Also, can anyone suggest a more efficient way of sorting the data than what I have above?
Any help is appreciated!
The problem is that node.then() doesn't return anything, so there is nothing to concat. Instead you would have to either pass a callback to .then() which will handle concatenating the lists or use await to wait till the value returned from get_properties() is resolved and then concat the lists.
I would suggest something like this,
// Function to asynchronously concat the lists
const async_concat = async(oldList, value) => {
return oldList.concat(value);
}
// Make sure finalList is not const
var finalList = [];
while (countTimes < count) {
// await the value of node and asynchronously concat the lists
let node = await get_properties(records, countTimes);
finalList = await async_concat(finalList, node);
countTimes++
};

Undefined when searching with Find Mongoose Express

I receive an array of products that I need to search for with Mongoose, so I use Find passing the _id of each product. But when going through the array and trying to search, I only receive the data of the first product, for the others I always receive undefined.
This is my code:
const calc = async (details) => {
let grandSubtotal = 0; }
console.log(details);
for (let i = 0; i < details.length; i++) {
let verifyProduct = await Product.find({ _id: details[i]._id});
console.log(verifyProduct[i].salePrice); //It only shows the first product, the others are undefined
.......
}
In my MongoDB database I have the products all saved with salePrice always, in this way:
{
"_id": "628fa841cde1d960c675ee24",
"barCode": "0000075053765",
"idProduct": "03",
"name": "MALBORO ARTESANAL 20",
"desc": "PAQUETE 20 CIGARROS",
"presentation": "PIECES",
"salePrice": 550,
"purchasePrice": 526,
"stock": 0,
"available": true,
"img": [],
"status": false
}
How can I obtain the salePrice information of all the products that I receive since for now I only receive information on the first one and the others are always undefined?
this because, you are using .find({})
let verifyProduct = await Product.find({ _id: details[i]._id});
You're querying using .find({ _id: details[i]._id}), you will always have result in the form of [{..onevalue at 0..}] because .find() returns result in an []
so, when you execute the loop for the first time, your i will be 0 and so when you access the verifyProduct[0].salePrice it will have value. But when your i become 1, your verifyProduct will still have result at 0 position only.
Fix:
const calc = async (details) => {
let grandSubtotal = 0;
console.log(details);
for (let i = 0; i < details.length; i++) {
let verifyProduct = await Product.findById(details[i]._id);
// no array, so access it directly
console.log(verifyProduct.salePrice);
}
}
since you are querying by _id, you can use .findById({}) instead of .find().
Database query in a for loop is super inefficient. You should use the $in MongoDB operator to select multiple documents at once.
Example
const arrayOfIds = ["631318a217f73aa43a58855d", "63132ba7525da531e171c964"];
Product.find({ _id: { $in: arrayOfIds }});

How to loop through objects and count unique values of key?

I have logs of json files which are objects that look like
{
"logs": [
{
"id": "12321321321321",
"email": "test#email.com",
"message": "ahahaha"
},
{
"id": "12321321312",
"email": "test#email.com",
"message": "hahahaha."
},
"id": "12321321321"
}
I need to return a new object that contains
{
"hello_id": outer id of the json file,
"array": [
{
"email": "test#me.com",
"total": 2
}
]
}
So far I am looping through the json files and have
jsonsInDirectory.forEach((file) => {
const fileData = fs.readFileSync(path.join("./logs", file), "utf8");
const jsonData = JSON.parse(fileData);
}
});
The key is "logs" and "id" and the values are the objects in the "logs" and the value of "id"
How can I count and return a new object at the same time?
You can try this approach: make a hash object that counts emails. Then just map it to an array of objects.
const data = {
logs: [{
id: "89004ef9-e825-4547-a83a-c9e9429e8f95",
email: "noah.sanchez#me.com",
message: "successfully handled skipped operation."
},
{
id: "89004ef9-e825-4547-a83a-c9e9429e8f95",
email: "noah.sanchez#me.com",
message: "successfully handled skipped operation."
},
{
id: "89004ef9-e825-4547-a83a-c9e9429e8f95",
email: "noname#me.com",
message: "successfully handled skipped operation."
}],
id: "56f83bed-3705-4115-9067-73930cbecbc0",
};
const emails = data.logs.reduce((acc, { email }) => {
acc[email] = (acc[email] ?? 0) + 1;
return acc;
}, {});
const tally = Object.entries(emails)
.map(([email, total]) => ({ email, total }));
const result = { logs_id: data.id, tally };
console.log(result)
.as-console-wrapper { max-height: 100% !important; top: 0 }
When you do const jsonData = JSON.parse(fileData);, you get the file data as a JSON and knowing the struct of that JSON you can easily get the info.
I have created a example https://codesandbox.io/s/stackoverflow-logs-count-example-jys2vg?file=/src/index.js
It may not solve exactly wath you want.
To solve this problem with the most time efficiency, you can create a tally object by firstly creating a map of the occurences of each mail with the email as the key and no of occurences as the value, since this will take constant (O(1)) time to execute, afterwhich you can create the tally array from the map as given below
output = []
jsonsInDirectory.forEach((file) => {
const fileData = fs.readFileSync(path.join("./logs", file), "utf8");
const jsonData = JSON.parse(fileData);
var map = {}
jsonData.logs.forEach((log) => {
if(log.email in map){
map[log.email] += 1
}
else {
map[log.email] = 1
}
});
var tally = []
for(var email in map){
tally.push({email: email, total: map[email]})
}
output.push({logs_id: jsonData['id'], tally: tally});
})

How to await inside 2 map functions and retrieve documents from mongoDB

I need to perform some modification on an API that returns something like this:
[
{
"_id": "0000000000000000001",
"name": "Random name",
"categories": [
"5eb8548330550e017f8902e6",
"5eb2548630550e117f8909eb",
"5eb6548930550e017f8909e9"
]
},
{...},
]
each results' categories are returned as ids of their respective documents.
I need to manipulate the result so each object's categories field has an array of objects and each object has the id and the name of its category.
I make an example of what the outcome should be:
[
{
"_id": "0000000000000000001",
"name": "Random name",
"categories": [
{"_id": "5eb8548330550e017f8902e6",
"name": "Category 1"},
{"_id": "5eb2548630550e117f8909eb",
"name": "Category 2"},
{"_id": "5eb6548930550e017f8909e9",
"name": "Category 3"},
]
},
{...},
]
I need to do this using plain JS and for so far this is what I have done but it returns an array of undefined:
const resultArray = await Promise.all(
searchResults.map(async (item) => {
item.categories.map(async (categoryId) => {
return await CategoryType.find({ _id: categoryId});
});
})
);
For the moment I am trying to get each category document for each id in the categories field.
I bet the reason why I am getting an array of undefined is that I am handling async in the wrong way but cannot figure out how.
To strictly answer your question: you are missing synchronization (because Array.prototype.map 'ignores' async):
const resultArray = await Promise.all(
searchResults.map(async (item) => {
const promises = item.categories.map(async (categoryId) => {
// you dont want find, but findOne btw
return await CategoryType.findOne({ _id: categoryId});
});
const categories = await Promise.all(promises)
item.categories = categories
return item
})
);
This can be simplified down as
const resultArray = await Promise.all(
searchResults.map(async item => {
item.categories = await Promise.all(item.categories.map(categoryId => {
return CategoryType.findOne({ _id: categoryId})
}))
return item
})
);
But the proper way of doing it is likely to use populate
const mongoose = require('mongoose')
mongoose.connect('mongodb://localhost:27017/dummy')
const Category = mongoose.model('Category', {
name:String,
}, 'categories');
const X = mongoose.model('X', {
name:String,
categories: [{type: mongoose.Types.ObjectId, ref: 'Category'}]
}, 'xs');
;(async()=>{
try {
mongoose.set('debug', true)
const xs = await X.find().populate('categories')
console.log('xs : ', xs[0])
} finally {
mongoose.disconnect()
}
})()
You will notice by the way, that mongoose, under the hood uses find({ _id: {$in:[]}}) which makes only one request (so better) than doing multiple findOne (as you do)

Getting a list of unique fields within a collection including nested [duplicate]

I'd like to get the names of all the keys in a MongoDB collection.
For example, from this:
db.things.insert( { type : ['dog', 'cat'] } );
db.things.insert( { egg : ['cat'] } );
db.things.insert( { type : [] } );
db.things.insert( { hello : [] } );
I'd like to get the unique keys:
type, egg, hello
You could do this with MapReduce:
mr = db.runCommand({
"mapreduce" : "my_collection",
"map" : function() {
for (var key in this) { emit(key, null); }
},
"reduce" : function(key, stuff) { return null; },
"out": "my_collection" + "_keys"
})
Then run distinct on the resulting collection so as to find all the keys:
db[mr.result].distinct("_id")
["foo", "bar", "baz", "_id", ...]
With Kristina's answer as inspiration, I created an open source tool called Variety which does exactly this: https://github.com/variety/variety
You can use aggregation with the new $objectToArray aggregation operator in version 3.4.4 to convert all top key-value pairs into document arrays, followed by $unwind and $group with $addToSet to get distinct keys across the entire collection. (Use $$ROOT for referencing the top level document.)
db.things.aggregate([
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$unwind":"$arrayofkeyvalue"},
{"$group":{"_id":null,"allkeys":{"$addToSet":"$arrayofkeyvalue.k"}}}
])
You can use the following query for getting keys in a single document.
db.things.aggregate([
{"$match":{_id: "<<ID>>"}}, /* Replace with the document's ID */
{"$project":{"arrayofkeyvalue":{"$objectToArray":"$$ROOT"}}},
{"$project":{"keys":"$arrayofkeyvalue.k"}}
])
A cleaned up and reusable solution using pymongo:
from pymongo import MongoClient
from bson import Code
def get_keys(db, collection):
client = MongoClient()
db = client[db]
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
Usage:
get_keys('dbname', 'collection')
>> ['key1', 'key2', ... ]
If your target collection is not too large, you can try this under mongo shell client:
var allKeys = {};
db.YOURCOLLECTION.find().forEach(function(doc){Object.keys(doc).forEach(function(key){allKeys[key]=1})});
allKeys;
If you are using mongodb 3.4.4 and above then you can use below aggregation using $objectToArray and $group aggregation
db.collection.aggregate([
{ "$project": {
"data": { "$objectToArray": "$$ROOT" }
}},
{ "$project": { "data": "$data.k" }},
{ "$unwind": "$data" },
{ "$group": {
"_id": null,
"keys": { "$addToSet": "$data" }
}}
])
Here is the working example
Try this:
doc=db.thinks.findOne();
for (key in doc) print(key);
Using python. Returns the set of all top-level keys in the collection:
#Using pymongo and connection named 'db'
reduce(
lambda all_keys, rec_keys: all_keys | set(rec_keys),
map(lambda d: d.keys(), db.things.find()),
set()
)
Here is the sample worked in Python:
This sample returns the results inline.
from pymongo import MongoClient
from bson.code import Code
mapper = Code("""
function() {
for (var key in this) { emit(key, null); }
}
""")
reducer = Code("""
function(key, stuff) { return null; }
""")
distinctThingFields = db.things.map_reduce(mapper, reducer
, out = {'inline' : 1}
, full_response = True)
## do something with distinctThingFields['results']
I am surprise, no one here has ans by using simple javascript and Set logic to automatically filter the duplicates values, simple example on mongo shellas below:
var allKeys = new Set()
db.collectionName.find().forEach( function (o) {for (key in o ) allKeys.add(key)})
for(let key of allKeys) print(key)
This will print all possible unique keys in the collection name: collectionName.
I think the best way do this as mentioned here is in mongod 3.4.4+ but without using the $unwind operator and using only two stages in the pipeline. Instead we can use the $mergeObjects and $objectToArray operators.
In the $group stage, we use the $mergeObjects operator to return a single document where key/value are from all documents in the collection.
Then comes the $project where we use $map and $objectToArray to return the keys.
let allTopLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$map": {
"input": { "$objectToArray": "$array" },
"in": "$$this.k"
}
}
}
}
];
Now if we have a nested documents and want to get the keys as well, this is doable. For simplicity, let consider a document with simple embedded document that look like this:
{field1: {field2: "abc"}, field3: "def"}
{field1: {field3: "abc"}, field4: "def"}
The following pipeline yield all keys (field1, field2, field3, field4).
let allFistSecondLevelKeys = [
{
"$group": {
"_id": null,
"array": {
"$mergeObjects": "$$ROOT"
}
}
},
{
"$project": {
"keys": {
"$setUnion": [
{
"$map": {
"input": {
"$reduce": {
"input": {
"$map": {
"input": {
"$objectToArray": "$array"
},
"in": {
"$cond": [
{
"$eq": [
{
"$type": "$$this.v"
},
"object"
]
},
{
"$objectToArray": "$$this.v"
},
[
"$$this"
]
]
}
}
},
"initialValue": [
],
"in": {
"$concatArrays": [
"$$this",
"$$value"
]
}
}
},
"in": "$$this.k"
}
}
]
}
}
}
]
With a little effort, we can get the key for all subdocument in an array field where the elements are object as well.
This works fine for me:
var arrayOfFieldNames = [];
var items = db.NAMECOLLECTION.find();
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}
Maybe slightly off-topic, but you can recursively pretty-print all keys/fields of an object:
function _printFields(item, level) {
if ((typeof item) != "object") {
return
}
for (var index in item) {
print(" ".repeat(level * 4) + index)
if ((typeof item[index]) == "object") {
_printFields(item[index], level + 1)
}
}
}
function printFields(item) {
_printFields(item, 0)
}
Useful when all objects in a collection has the same structure.
To get a list of all the keys minus _id, consider running the following aggregate pipeline:
var keys = db.collection.aggregate([
{ "$project": {
"hashmaps": { "$objectToArray": "$$ROOT" }
} },
{ "$group": {
"_id": null,
"fields": { "$addToSet": "$hashmaps.k" }
} },
{ "$project": {
"keys": {
"$setDifference": [
{
"$reduce": {
"input": "$fields",
"initialValue": [],
"in": { "$setUnion" : ["$$value", "$$this"] }
}
},
["_id"]
]
}
}
}
]).toArray()[0]["keys"];
I know I am late to the party, but if you want a quick solution in python finding all keys (even the nested ones) you could do with a recursive function:
def get_keys(dl, keys=None):
keys = keys or []
if isinstance(dl, dict):
keys += dl.keys()
list(map(lambda x: get_keys(x, keys), dl.values()))
elif isinstance(dl, list):
list(map(lambda x: get_keys(x, keys), dl))
return list(set(keys))
and use it like:
dl = db.things.find_one({})
get_keys(dl)
if your documents do not have identical keys you can do:
dl = db.things.find({})
list(set(list(map(get_keys, dl))[0]))
but this solution can for sure be optimized.
Generally this solution is basically solving finding keys in nested dicts, so this is not mongodb specific.
Based on #Wolkenarchitekt answer: https://stackoverflow.com/a/48117846/8808983, I write a script that can find patterns in all keys in the db and I think it can help others reading this thread:
"""
Python 3
This script get list of patterns and print the collections that contains fields with this patterns.
"""
import argparse
import pymongo
from bson import Code
# initialize mongo connection:
def get_db():
client = pymongo.MongoClient("172.17.0.2")
db = client["Data"]
return db
def get_commandline_options():
description = "To run use: python db_fields_pattern_finder.py -p <list_of_patterns>"
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-p', '--patterns', nargs="+", help='List of patterns to look for in the db.', required=True)
return parser.parse_args()
def report_matching_fields(relevant_fields_by_collection):
print("Matches:")
for collection_name in relevant_fields_by_collection:
if relevant_fields_by_collection[collection_name]:
print(f"{collection_name}: {relevant_fields_by_collection[collection_name]}")
# pprint(relevant_fields_by_collection)
def get_collections_names(db):
"""
:param pymongo.database.Database db:
:return list: collections names
"""
return db.list_collection_names()
def get_keys(db, collection):
"""
See: https://stackoverflow.com/a/48117846/8808983
:param db:
:param collection:
:return:
"""
map = Code("function() { for (var key in this) { emit(key, null); } }")
reduce = Code("function(key, stuff) { return null; }")
result = db[collection].map_reduce(map, reduce, "myresults")
return result.distinct('_id')
def get_fields(db, collection_names):
fields_by_collections = {}
for collection_name in collection_names:
fields_by_collections[collection_name] = get_keys(db, collection_name)
return fields_by_collections
def get_matches_fields(fields_by_collections, patterns):
relevant_fields_by_collection = {}
for collection_name in fields_by_collections:
relevant_fields = [field for field in fields_by_collections[collection_name] if
[pattern for pattern in patterns if
pattern in field]]
relevant_fields_by_collection[collection_name] = relevant_fields
return relevant_fields_by_collection
def main(patterns):
"""
:param list patterns: List of strings to look for in the db.
"""
db = get_db()
collection_names = get_collections_names(db)
fields_by_collections = get_fields(db, collection_names)
relevant_fields_by_collection = get_matches_fields(fields_by_collections, patterns)
report_matching_fields(relevant_fields_by_collection)
if __name__ == '__main__':
args = get_commandline_options()
main(args.patterns)
As per the mongoldb documentation, a combination of distinct
Finds the distinct values for a specified field across a single collection or view and returns the results in an array.
and indexes collection operations are what would return all possible values for a given key, or index:
Returns an array that holds a list of documents that identify and describe the existing indexes on the collection
So in a given method one could do use a method like the following one, in order to query a collection for all it's registered indexes, and return, say an object with the indexes for keys (this example uses async/await for NodeJS, but obviously you could use any other asynchronous approach):
async function GetFor(collection, index) {
let currentIndexes;
let indexNames = [];
let final = {};
let vals = [];
try {
currentIndexes = await collection.indexes();
await ParseIndexes();
//Check if a specific index was queried, otherwise, iterate for all existing indexes
if (index && typeof index === "string") return await ParseFor(index, indexNames);
await ParseDoc(indexNames);
await Promise.all(vals);
return final;
} catch (e) {
throw e;
}
function ParseIndexes() {
return new Promise(function (result) {
let err;
for (let ind in currentIndexes) {
let index = currentIndexes[ind];
if (!index) {
err = "No Key For Index "+index; break;
}
let Name = Object.keys(index.key);
if (Name.length === 0) {
err = "No Name For Index"; break;
}
indexNames.push(Name[0]);
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function ParseFor(index, inDoc) {
if (inDoc.indexOf(index) === -1) throw "No Such Index In Collection";
try {
await DistinctFor(index);
return final;
} catch (e) {
throw e
}
}
function ParseDoc(doc) {
return new Promise(function (result) {
let err;
for (let index in doc) {
let key = doc[index];
if (!key) {
err = "No Key For Index "+index; break;
}
vals.push(new Promise(function (pushed) {
DistinctFor(key)
.then(pushed)
.catch(function (err) {
return pushed(Promise.resolve());
})
}))
}
return result(err ? Promise.reject(err) : Promise.resolve());
})
}
async function DistinctFor(key) {
if (!key) throw "Key Is Undefined";
try {
final[key] = await collection.distinct(key);
} catch (e) {
final[key] = 'failed';
throw e;
}
}
}
So querying a collection with the basic _id index, would return the following (test collection only has one document at the time of the test):
Mongo.MongoClient.connect(url, function (err, client) {
assert.equal(null, err);
let collection = client.db('my db').collection('the targeted collection');
GetFor(collection, '_id')
.then(function () {
//returns
// { _id: [ 5ae901e77e322342de1fb701 ] }
})
.catch(function (err) {
//manage your error..
})
});
Mind you, this uses methods native to the NodeJS Driver. As some other answers have suggested, there are other approaches, such as the aggregate framework. I personally find this approach more flexible, as you can easily create and fine-tune how to return the results. Obviously, this only addresses top-level attributes, not nested ones.
Also, to guarantee that all documents are represented should there be secondary indexes (other than the main _id one), those indexes should be set as required.
We can achieve this by Using mongo js file. Add below code in your getCollectionName.js file and run js file in the console of Linux as given below :
mongo --host 192.168.1.135 getCollectionName.js
db_set = connect("192.168.1.135:27017/database_set_name"); // for Local testing
// db_set.auth("username_of_db", "password_of_db"); // if required
db_set.getMongo().setSlaveOk();
var collectionArray = db_set.getCollectionNames();
collectionArray.forEach(function(collectionName){
if ( collectionName == 'system.indexes' || collectionName == 'system.profile' || collectionName == 'system.users' ) {
return;
}
print("\nCollection Name = "+collectionName);
print("All Fields :\n");
var arrayOfFieldNames = [];
var items = db_set[collectionName].find();
// var items = db_set[collectionName].find().sort({'_id':-1}).limit(100); // if you want fast & scan only last 100 records of each collection
while(items.hasNext()) {
var item = items.next();
for(var index in item) {
arrayOfFieldNames[index] = index;
}
}
for (var index in arrayOfFieldNames) {
print(index);
}
});
quit();
Thanks #ackuser
Following the thread from #James Cropcho's answer, I landed on the following which I found to be super easy to use. It is a binary tool, which is exactly what I was looking for:
mongoeye.
Using this tool it took about 2 minutes to get my schema exported from command line.
I know this question is 10 years old but there is no C# solution and this took me hours to figure out. I'm using the .NET driver and System.Linq to return a list of the keys.
var map = new BsonJavaScript("function() { for (var key in this) { emit(key, null); } }");
var reduce = new BsonJavaScript("function(key, stuff) { return null; }");
var options = new MapReduceOptions<BsonDocument, BsonDocument>();
var result = await collection.MapReduceAsync(map, reduce, options);
var list = result.ToEnumerable().Select(item => item["_id"].ToString());
This one lines extracts all keys from a collection into a comma separated sorted string:
db.<collection>.find().map((x) => Object.keys(x)).reduce((a, e) => {for (el of e) { if(!a.includes(el)) { a.push(el) } }; return a}, []).sort((a, b) => a.toLowerCase() > b.toLowerCase()).join(", ")
The result of this query typically looks like this:
_class, _id, address, city, companyName, country, emailId, firstName, isAssigned, isLoggedIn, lastLoggedIn, lastName, location, mobile, printName, roleName, route, state, status, token
I extended Carlos LM's solution a bit so it's more detailed.
Example of a schema:
var schema = {
_id: 123,
id: 12,
t: 'title',
p: 4.5,
ls: [{
l: 'lemma',
p: {
pp: 8.9
}
},
{
l: 'lemma2',
p: {
pp: 8.3
}
}
]
};
Type into the console:
var schemafy = function(schema, i, limit) {
var i = (typeof i !== 'undefined') ? i : 1;
var limit = (typeof limit !== 'undefined') ? limit : false;
var type = '';
var array = false;
for (key in schema) {
type = typeof schema[key];
array = (schema[key] instanceof Array) ? true : false;
if (type === 'object') {
print(Array(i).join(' ') + key+' <'+((array) ? 'array' : type)+'>:');
schemafy(schema[key], i+1, array);
} else {
print(Array(i).join(' ') + key+' <'+type+'>');
}
if (limit) {
break;
}
}
}
Run:
schemafy(db.collection.findOne());
Output
_id <number>
id <number>
t <string>
p <number>
ls <object>:
0 <object>:
l <string>
p <object>:
pp <number>
I was trying to write in nodejs and finally came up with this:
db.collection('collectionName').mapReduce(
function() {
for (var key in this) {
emit(key, null);
}
},
function(key, stuff) {
return null;
}, {
"out": "allFieldNames"
},
function(err, results) {
var fields = db.collection('allFieldNames').distinct('_id');
fields
.then(function(data) {
var finalData = {
"status": "success",
"fields": data
};
res.send(finalData);
delteCollection(db, 'allFieldNames');
})
.catch(function(err) {
res.send(err);
delteCollection(db, 'allFieldNames');
});
});
After reading the newly created collection "allFieldNames", delete it.
db.collection("allFieldNames").remove({}, function (err,result) {
db.close();
return;
});
I have 1 simpler work around...
What you can do is while inserting data/document into your main collection "things" you must insert the attributes in 1 separate collection lets say "things_attributes".
so every time you insert in "things", you do get from "things_attributes" compare values of that document with your new document keys if any new key present append it in that document and again re-insert it.
So things_attributes will have only 1 document of unique keys which you can easily get when ever you require by using findOne()

Categories