Trying to merge multiple collection using nodejs and mongoose but not working.Anyone can find solution for this.
Getting this error:
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
product.model.js:
module.exports = mongoose.model('Test1', userSchemaTest1, 'test1');
module.exports = mongoose.model('Test2', userSchemaTest2, 'test2');
module.exports = mongoose.model('Test3', userSchemaTest3, 'test3');
module.exports = mongoose.model('Test4', userSchemaTest4, 'test4');
module.exports = mongoose.model('Test5', userSchemaTest5, 'test5');
product.controller.js:
module.exports.getAllProducts = (req, res, next) => {
let collection = req.query.collection;
var newVal = collection.split(',');
var allP = [];
var getAllp;
allP.push(...newVal);
allP.forEach((element) => {
let tabledatas = mongoose.model(element);
tabledatas.find({}, function(err, docs) {
if (err) {
console.log('ss' + err);
return
}
getAllp = [...getAllp, res.json(docs)];
})
})
return getAllp;
}
api call
http://localhost:3000/api/getAllProducts?collection=Test1,Test2,Test3,Test4,Test5
There are several issues with your code, I'll focus only on how to combine results of multiple queries, which is actually how to deal with async function results in general.
But you should take notes of other issues mentioned here as well
Calling res.json() in a loop, you'll certainly get "Headers already sent" errors
Putting the result of res.json() in to an array; res.json() returns ServerResponse, which has nothing to do with what we want.
Accepting user inputs (collection names) and pass them directly to the database operations is dangerous.
Handling async functions:
// models would be a list of model names
const models = collection.split(',')
// map array of model names to array of Promises (result of async operations)
const resultPromises = models.map(modelName => {
return mongoose.model(modelName).find({}).exec() // use Promise result instead of passing callback
})
// wait on all Promises to be fulfilled and resolved to actual results.
const combinedResults = await Promise.all(resultPromises)
// now you will get a result of this form [[item1, item2, ...], [itemX, itemY, ...], ...]
// but what you want is [item1, item2, ..., itemX, itemY, ...], we can use array.flat()
cont results = combinedResults.flat()
Note that you can only use await in an async function, so you'll have to modify your function like this
module.exports.getAllProducts = async (req, res, next) => { // add async keyword
// ...
}
more on array.flat()
Related
I am using the code given below to run multiple https GET request for Wikipedia API.
app.get("/data_results", (req, res) => {
const articlesData = names.map(nameObj => {
let name = nameObj.name;
let articleExtract = "";
let contentURL =
`https://en.wikipedia.org/w/api.php?
action=query&titles=${name}&prop=extracts&format=json&exintro=1&explaintext=false&origin=*`;
// Getting article content
https.get(contentURL, response => {
response.on("data", async data => {
const wikiArticle = JSON.parse(data);
// Mapping the keys of the JSON data of query to its values.
articleExtract = await Object.keys(wikiArticle.query.pages).map(key => wikiArticle.query.pages[key])[0].extract;
nameObj.article = articleExtract.substring(0,350);
})
});
return nameObj;
});
res.send(articlesData);});
This is my names array
[
{ name: 'Indus%20Valley%20Civilisation' },
{ name: 'Ramayana' },
{ name: 'Mahabharata' },
{ name: 'Gautama%20Buddha' }
]
My aim :-
Run the HTTPS GET request for every value in the names array sequentially.
Add the article extract with its name and save in it in an objects array.
You could also suggest me better ways of doing this.
Try this solution. it creates an object like
{
Mahabharata: response
}
The code
const getValues = async () => {
const name_array = [
{name: 'Indus%20Valley%20Civilisation'},
{name: 'Ramayana'},
{name: 'Mahabharata'},
{name: 'Gautama%20Buddha'},
];
const namePromises = name_array.map(value =>
fetch('baseurl' + value.name).then(res => res.json()),
);
const response = await Promise.all(namePromises);
return response.reduce((obj, responseResults, index) => {
obj[name_array[index].name] = responseResults;
}, {});
};
I would suggest you use fetch and promise. all. Map over your array of names and create promises as input. and then return the array of resolved promises.
something like this
const namePromises = [
{ name: 'Indus%20Valley%20Civilisation' },
{ name: 'Ramayana' },
{ name: 'Mahabharata' },
{ name: 'Gautama%20Buddha' }
].map((value) => fetch(baseurl + value.name).then(res=>res.json()));
Promise.all(namePromises).then((response) => {
console.log(response);
});
Problems:
So, you have several things going on here.
http.get() is non-blocking and asynchronous. That means that the rest of your code continues to run while your multiple http.get() operations are running. That means you end up calling res.send(articlesData); before any of the http requests are complete.
.map() is not promise-aware or asynchronous-aware so it will not wait its loop for any asynchronous operation. As such, you're actually running ALL your http requests in parallel (they are all in-flight at the same time). The .map() loop starts them all and then they all finish some time later.
The data event for http.get() is not guaranteed to contain all your data or even a whole piece of data. It may just be a partial chunk of data. So, while your code may seem to get a complete response now, different network or server conditions could change all that and reading the result of your http requests may stop functioning correctly.
You don't show where names comes from in your request handler. If it's a statically declared array of objects declared in a higher-scoped variable, then this code has a problem that you're trying to modify the objects in that array in a request handler and multiple requests to this request handler can be conflicting with one another.
You don't show any error handling for errors in your http requests.
You're using await in a place it doesn't belong. await only does something useful when you are awaiting a promise.
Solutions:
Managing asynchronous operations in Javascript, particularly when you have more than one to coordinate) is a whole lot easier when using promises instead of the plain callback that http.get() uses. And, it's a whole lot easier to use promises when you use an http request interface that already supports promises. My goto library for built-in promise support for http requests in nodejs is got(). There are many good choices shown here.
You can use got() and promises to control the asynchronous flow and error handling like this:
const got = require('got');
app.get("/data_results", (req, res) => {
Promise.all(names.map(nameObj => {
let name = nameObj.name;
// create separate result object
let result = { name };
let contentURL =
`https://en.wikipedia.org/w/api.php?
action=query&titles=${name}&prop=extracts&format=json&exintro=1&explaintext=false&origin=*`;
return got(contentURL).json().then(wikiArticle => {
// Mapping the keys of the JSON data of query to its values.
let articleExtract = Object.keys(wikiArticle.query.pages).map(key => wikiArticle.query.pages[key])[0].extract;
result.article = articleExtract.substring(0, 350);
return result;
});
})).then(articlesData => {
res.send(articlesData);
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
});
This code attempts to fix all six of the above mentioned problems while still running all your http requests in parallel.
If your names array was large, running all these requests in parallel may consume too many resources, either locally or on the target server. Or, it may run into rate limiting on the target server. If that was an issue, you can run sequence the http requests to run them one at a time like this:
const got = require('got');
app.get("/data_results", async (req, res) => {
try {
let results = [];
for (let nameObj of names) {
let name = nameObj.name;
let result = { name };
let contentURL =
`https://en.wikipedia.org/w/api.php?
action=query&titles=${name}&prop=extracts&format=json&exintro=1&explaintext=false&origin=*`;
const wikiArticle = await got(contentURL).json();
let articleExtract = Object.keys(wikiArticle.query.pages).map(key => wikiArticle.query.pages[key])[0].extract;
result.article = articleExtract.substring(0, 350);
results.push(result);
}
res.send(results);
} catch (e) {
console.log(e);
res.sendStatus(500);
}
});
If you really want to stick with https.get(), then you can "promisify" it and use it in place of got():
function myGet(url, options = {}) {
return new Promise((resolve, reject) => {
https.get(url, options, (res) => {
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
resolve(parsedData);
} catch (e) {
reject(e);
}
});
}).on('error', reject);
});
}
app.get("/data_results", async (req, res) => {
try {
let results = [];
for (let nameObj of names) {
let name = nameObj.name;
let result = { name };
let contentURL =
`https://en.wikipedia.org/w/api.php?
action=query&titles=${name}&prop=extracts&format=json&exintro=1&explaintext=false&origin=*`;
const wikiArticle = await myGet(contentURL);
let articleExtract = Object.keys(wikiArticle.query.pages).map(key => wikiArticle.query.pages[key])[0].extract;
result.article = articleExtract.substring(0, 350);
results.push(result);
}
res.send(results);
} catch (e) {
console.log(e);
res.sendStatus(500);
}
});
I am trying to execute the following code, however "// go do something else" keeps happening before "// do stuff with things"
It appears that my code is not waiting for mongoose.model('things').find() to finish before moving on. I've tried different variations of async/await and nothing seems to work.
Not getting errors an everything executes, just out of order.
const asyncFunction = async () => {
mongoose.connect(`mongodb+srv://...`);
mongoose.model('things', {data:String});
mongoose.model('things').find((err, things)=>{
// do stuff with things
}
console.log('something');
}
const otherAsyncFunction = async () {
await asyncFunction();
// go do something else
}
otherAsyncFunction();
Your asyncFunction doesn't return anything, so there's no point awaiting it.
You have no Mongoose schema.
The syntax to create a Mongoose model (doc) is :
const thingsShema = new mongoose.Schema({ data: 'string' });
const Thing = mongoose.model('thing', thingsShema );
You are creating your model, but then your model isn't stored in a variable and you're not using it at all.
mongoose.model('things') : this line creates a new Mongoose model but you're not passing it any schema. Anyway you already did that on the previous line.
.find() is asynchronous, and you're not returning its value.
You aren't passing any argument to .find(), only a callback function.
This code should work better :
const asyncFunction = async () => {
await mongoose.connect(`mongodb+srv://...`);
const thingsSchema = new mongoose.Schema({ data: 'string' });
const Things = mongoose.model('things', thingsSchema);
const foundThings = await Things
.find({}) // Query conditions (here, querying for 'everything', an empty object)
.lean() // Returns simple JSON and not a collection of Mongoose objects
.exec(); // returns a true Promise and not just a thenable
console.log('foundThings = ', foundThings);
return foundThings; // this returns a Promise, because 'async' functions always do
}
const otherAsyncFunction = async () => {
const result = await asyncFunction();
console.log("Result = ", result); // Normally this will log the same thing as 'foundThings'
// go do something else
}
otherAsyncFunction();
async functions return Promise object. When you exec await asyncFunction(), 'await' wait status asyncFunction() - resolve or reject.But in your code there are no resolve() or reject() functions. So you need to do like this:
const asyncFunction = async (resolve) => {
mongoose.connect(`mongodb+srv://...`);
mongoose.model('things', {data:String});
mongoose.model('things').find((err, things)=>{
// do stuff with things
});
console.log('something');
resolve()
}
const otherAsyncFunction = async () => {
await asyncFunction();
// go do something else
}
otherAsyncFunction();
I'm trying to fetch data from an S3 object and put it into an array. I plan to map through this array and display the data on a React front end in grid/list whatever. I'm struggling with nested functions though, so I'd appreciate some help.
const dataFromS3 = async (bucket, file) => {
let lines = [];
const options = {
Bucket: bucket,
Key: file
};
s3.getObject(options, (err, data) => {
if (err) {
console.log(err);
} else {
let objectData = data.Body.toString('utf-8');
lines.push(objectData);
console.log(lines);
return lines;
}
});
};
Formatting is a bit weird but this is my function to get data from s3. I want to take the output of this function in the form of an array and pass it to my '/' route which I'm testing:
app.get('/', async (req, res, next) => {
try {
let apolloKey = await dataFromS3(s3Bucket, apolloKeywords);
res.send(apolloKey);
} catch (err) {
console.log('Error: ', err);
}
});
It seems that the return value of lines in the s3.getObject function needs to be returned within the first function so that I can access it in app.get but I can't seem to do it after some attempts. The value in lines turns into an empty array if I return it at the end of datafromS3() and I can't find a way to return it. I've tried using promises also using a method found here - How to get response from S3 getObject in Node.js? but I get a TypeError: Converting Circular Structure to JSON...
Thank you
You need to make your dataFromS3 func like htis. You were not returning anything from that. AWS also provided promise based function.
const dataFromS3 = async (bucket, file) => {
const lines = [];
const options = {
"Bucket": bucket,
"Key": file
};
const data = await s3.getObject(options).promise();
const objectData = data.Body.toString("utf-8");
lines.push(objectData); // You might need to conversion here using JSON.parse(objectData);
console.log(lines);
return lines;
};
I'm new to learning Node.js, so I'm still getting used to asynchronous programming and callbacks. I'm trying to query a MS SQL Server database and return the recordset to display in my view.
The mssql query is working correctly when printed to console.log. My problem is not knowing how to properly return the data.
Here is my mssql query:
var config = require('../../db/config');
async function getJobList(activeJD) {
const sql = require('mssql')
let sqlResult = '';
try {
await sql.connect(config)
const result = await sql.query(`select top 10 [jobid], [title] from OrgJobs where ActiveJD = ${activeJD}`);
console.log(result); // working correctly
sqlResult = result;
} catch (err) {
// ... error checks
}
return sqlResult;
}
module.exports = getJobList;
Using express, I'm trying to have my route call this sql query and pass it's result to my view. Here is my route code:
const express = require('express');
//....
const app = express();
//....
app.get('/jds', (req, res) => {
const getJobList = require("../models/jds/list");
let jobList = getJobList(0);
res.render('jds/index', {
title: appName + ' | Job Descriptions',
header: 'Active Job Descriptions',
data: jobList
});
})
The recordset from getJobList() isn't being returned to jobList. I've read around and I believe this is because getJobList is asynchronous and jobList is being called before getJobList has returned it's value, right? If so, I believe I need to add a callback, but I'm not sure how to implement that into what I already have. Any advice is greatly appreciated!
An async method will always return a Promise. That means that jobList is a Promise.
let jobList = getJobList(0);
You can either use the await syntax to "unpack" and get sqlResult from the promise (you would need to make your callback method async to do this), or simply use .then to get the return value from your promise (ie: sqlResult):
app.get('/jds', (req, res) => {
const getJobList = require("../models/jds/list");
const jobList = getJobList(0); // jobList is a promise
jobList.then(result => { // result is sqlResult
res.render('jds/index', {
title: appName + ' | Job Descriptions',
header: 'Active Job Descriptions',
data: result
});
});
})
I'm trying to do something like this
function retrieveUser(uname) {
var user = User.find({uname: uname}, function(err, users) {
if(err)
console.log(err);
return null;
else
return users[0];
return user;
But this returns a document instead of a user object. The parameter users is an array of user objects matching the query, so how would I store one of the objects into a variable that my function could return?
The function User.find() is an asynchronous function, so you can't use a return value to get a resultant value. Instead, use a callback:
function retrieveUser(uname, callback) {
User.find({uname: uname}, function(err, users) {
if (err) {
callback(err, null);
} else {
callback(null, users[0]);
}
});
};
The function would then be used like this:
retrieveUser(uname, function(err, user) {
if (err) {
console.log(err);
}
// do something with user
});
Updated on 25th Sept. 2019
Promise chaining can also be used for better readability:
Model
.findOne({})
.exec()
.then((result) => {
// ... rest of the code
return Model2.findOne({}).exec();
})
.then((resultOfModel2FindOne) => {
// ... rest of the code
})
.catch((error) => {
// ... error handling
});
I was looking for an answer to the same question.
Hopefully, MongooseJS has released v5.1.4 as of now.
Model.find({property: value}).exec() returns a promise.
it will resolve to an object if you use it in the following manner:
const findObject = (value) => {
return Model.find({property: value}).exec();
}
mainFunction = async => {
const object = await findObject(value);
console.log(object); // or anything else as per your wish
}
Basically, MongoDB and NodeJS have asynchronous functions so we have to make it to synchronous functions then after it will work properly as expected.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = []; // Created Empty Array
await mdl_users.find({}, function(err, data) {
data.forEach(function(value) {
userData.push(value);
});
});
res.send(userData);
});
In Example, mdl_users is mongoose model and I have a user collection(table) for user's data in MongoDB database and that data storing on "userData" variable to display it.In this find function i have split all documents(rows of table) by function if you want just all record then use direct find() function as following code.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = await mdl_users.find();
res.send(userData);
});