I built an API for my mobile apps using NodeJs. I built this app to add contacts to my Mongo DB, but the app uses too much memory.
var ObjectID = require('mongodb').ObjectID
module.exports = function(app, db) {
const myDb = db.db('sycontacts')
app.post('/addcontacts', (req, res) => {
const jason = JSON.parse(req.body.body);
jason.forEach(function(value){
const contact = { n: value.n, p: value.p };
const details = { p: value.p };
var bool = true;
myDb.collection('contacts').find(details).toArray(function(err, item) {
if (err) {
res.send({ 'error': 'An error has occured' });
} else {
if(item.length>0){
item.forEach(function(value){
if(value.n == contact.n){
bool= false;
return;
}
if(!bool)return;
});
if(bool){
myDb.collection('contacts').insertOne(contact, (err) => {
if (err) {
res.send({ 'error': 'An error has occured' });
}
});
}
}else {
myDb.collection('contacts').insertOne(contact, (err) => {
if (err) {
res.send({ 'error': 'An error has occured' });
}
});
}
}
});
});
res.send('findEd');
});
};
My database has about 5000000 documents.. Can anyone help me to make this app perform better and reduce its memory usage?
There's some error on your code and some missing optimization :
You do async function inside forEach function, to loop wisth async function use async.eachOf or async.eachOfLimit
You retrieve all your document in your database to insert missing documents
Do something like this instead :
var async = require("async");
const jason = JSON.parse(req.body.body);
// process 10 document in same time at maximum
async.eachOfLimit(jason, 10, function(value, index, callback){
// upsert document, update all or insert if match not found
myDb.collection('contacts').update({ n: value.n, p: value.p }, {n: value.n, p: value.p}, {multi: false, upsert: true}, function(err){
return callback(err);
});
}, function(err, result){
// all is finished
if(err)
{
return res.status(500).json({err: err});
}
else
{
return res.status(200).json({err: null});
}
})
Related
I'm new to NodeJS so please apologize if below code is not up-to the standard. I would like to access isSuccess value outside of this function stepfunctions.listExecutions
I tried below code but I'm getting the value is undefined not getting the expected output. I did some internet search and came to know in NodeJS we can't set the value which is defined in globally but I've use case and I'm pretty sure this is a common case for others too - where I would like to access this isSuccess value after my execution.
const AWS = require('aws-sdk');
const stepfunctions = new AWS.StepFunctions({
region: process.env.AWS_REGION
});
var params = {
stateMachineArn: 'arn:aws:states:us-west-1:121:stateMachine:test',
maxResults: '2',
nextToken: null,
statusFilter: 'SUCCEEDED'
};
var isSuccess
stepfunctions.listExecutions(params, function (err, data) {
if (err) console.log(err, err.stack);
else
data.executions.forEach(function (result) {
let params = {
executionArn: result.executionArn
};
stepfunctions.describeExecution(params, function (err, data) {
if (err) console.log(err, err.stack);
else {
isSuccess = 'true'
}
});
});
console.log('isSuccess: ' +isSuccess)
});
Expected output:
isSuccess: true
But I'm getting
isSuccess: undefined
Could you please help me to resolve this issue. Appreciated your help and support on this.
This is how you can wrap it on promise
let isSuccess;
const listExecute = function(params) {
return new Promise((resolve, reject) => {
stepfunctions.listExecutions(params, function (err, data) {
if (err) reject(err);
else
data.executions.forEach(function (result) {
let params = {
executionArn: result.executionArn
};
stepfunctions.describeExecution(params, function (err, data) {
if (err) reject(err);
else {
resolve(true)
}
});
});
});
})
}
async function getOutout(params) {
try {
isSuccess = await listExecute(params);
console.log(isSuccess, 'Output')
} catch(e) {
console.log(e)
}
}
getOutout(params)
Also you can export the listExecute so that you can use this function outside of this file.
module.exports = {listExecute}
I am trying to store API results into an array.
The data is displayed in console, but on pushing the data into an array, the array is still empty.
Here's the code:
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
var favorites = [];
dbConn.then( function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function(err, result) {
if(err) throw err;
if(!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({msg:'record not found'});
}
else {
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', {id: tweet.id}, function(err, data, response) {
if(!err){
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
}
});
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e){console.log(e)})
});
It looks like you're defining the favorites array within the scope of the function callback. Try putting var favorites = []; above you app.post() call instead.
Also, keep in mind that it will only have a value after the callback is complete, so any synchronous code later down the line will only see the empty array value.
I've updated your code to get favorites by storing separately the promise and call it afterwards:
UPDATE
As you can see in the demo, i have 2x console.log at the bottom, the first one(C1) is contained in the promise favoritesPromise () and the second (C2) is after the promise.
Synchronous actions will never wait for asynchronus actions to take place, therefore in my example C2 will always be outputted before C1, even if console.log(1 ... ) is before console.log(2 ... ), they'll appear reversed in the console.
In the promise i added a setTimeout of 1ms to mock a request, it was all it took to achieve the current output. Another thing you can test is removing the setTimeout then output will change a bit, your promise becomes synchronus until it reaches resolve(favorites), that means favorites has all the data by now, but when resolve takes place, it becomes async, and in your console you will still see C2 first (but now with data) and C1 second.
In my earlier answer i tried to implement this reasoning to your code.
Keep it async folks!
var favorites = [];
var favoritesPromise = () => {
return new Promise((resolve, reject) => {
console.log('Retrieving data from the internet.');
// This timeout mocks your request to anything that is async or promie
setTimeout(() => {
console.log('Request done')
let resultFavorite_tweets = [{
id: 1,
name: 'a dog'
}, {
id: 2,
name: 'a cat'
}];
resultFavorite_tweets.forEach(item => {
favorites.push(item.name);
})
resolve(favorites);
// if you have an error use
// reject(err)
}, 1);
});
}
favoritesPromise().then(favList => {
console.log(1, 'this will always contain data from the internet, but will always be last', favList);
})
console.log(2, 'this will be empty (unless you remove setTimeout), but will always be first', favorites);
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
const favoritesPromise = () => {
return new Promise((resolve, reject) => {
var favorites = [];
dbConn.then(function(database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne({
_id: ObjectId(req.userId)
}, function(err, result) {
if (err) reject(err);
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({
msg: 'record not found'
});
} else {
result.favorite_tweets.forEach(function(tweet) {
T.get('statuses/show', {
id: tweet.id
}, function(err, data, response) {
if (!err) {
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
reject(err);
}
});
resolve(data);
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e) {
reject(e)
})
});
}
// Here you call the promise to retrieve "favorites"
favoritesPromise().then(favoritesList => {
console.log('your favorites array', favoritesList)
})
})
Try next code
app.post('/fetchFavoriteTweets/', verifyToken, function (req, res) {
var favorites = [];
dbConn.then(function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function (err, result) {
if (err) throw err;
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({ msg: 'record not found' });
}
else {
// Counter
let count = result.favorite_tweets.length;
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', { id: tweet.id }, function (err, data, response) {
// Decrease count
count -= 1;
if (!err) {
favorites.push(data);
// Check if count is zero
if (count === 0) {
console.log(favorites);
res.status(200).json({msg:'success', data:favorites});
}
} else {
console.log(err);
}
});
});
}
});
}).catch(function (e) { console.log(e) })
});
I'm creating a childprocess when an express api is called after saving some info in mongodb. This is the api part
router.get("/create", function(req, res) {
let Schema = new New({
Id: crypto.randomBytes(16).toString("hex"),
Status: "Online"
});
Schema.save(function(err) {
if (err) {
console.log(err);
} else {
let child = exec(`node ${modulepath}`, (error, stdout) => {
if (error) {
throw error;
}
console.log(child.stdout);
});
New.update({
"botId": Schema.Id
}, {
"ProcessId": child.pid
}).then((err) => {
if (err) {
console.log(err);
} else {
console.log("pid updated");
}
});
Id = Schema.Id;
module.exports = {
Id
};
}
});
});
What i'm trying to do here is, at the bottom i'm exporting an id which ideally should be exported to the childprocess instance but that is not happening. In the childprocess, if i import the id by const Id = require("./routes").Id; and try to access some info through it by
await New.findOne({
"Id": Id
}, function(err, b) {
if (err) {
throw err;
} else {
//some task
}
});
I'm not able to access the info as mongodb cant get the id. I know i'm going wrong in passing the value to the childprocess part but what is the solution?
I am writing this code as a project for a customer
and when i go to a show route i got this 500 internal server error
http.get('/files/:id', function(req, res) {
var vid;
var pap;
Videos.find({}, function(err, videos) {
if (err) {
console.log(err);
} else {
vid = videos;
}
});
Papers.find({}, function(err, file) {
if (err) {
console.log(err);
} else {
pap = file;
}
});
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
});
this is my show route code.
Note : if i reload the page the error is gone and the page open.
The reason is you need to wait for all the database queries to finish before rendering. In your code, it is possible for the page to render before the other two queries have completed and returned their data. The good news is that Mongoose supports Promises for asynchronous functions.
http.get('/files/:id', function(req, res) {
Promise.all([
Videos.find({}).exec(),
Papers.find({}).exec(),
Material.findById(req.params.id).exec()
]).then( ([video, paper, material]) => {
res.render('files', {
file: paper,
video: video,
current: material
});
}).catch( error => console.log(error) );
});
The functions you're using with Mongoose are asynchronous in nature; the variables vid and pap are not initialized when you run res.render. When you attempt to use those variables in your frontend (template like Jade, Handlebars EJS, I don't know what you're using), they are undefined, and subsequently cause the 500 error. You'll need to run the functions such that the results of all Mongoose queries are available to res.render when it runs; either using an async NodeJS library, or calling each function within one another and then calling res.render at the end.
Solution 1: Using async Node module
var async = require('async');
async.parallel([
// Each function in this array will execute in parallel
// The callback function is executed once all functions in the array complete
function (cb) {
Videos.find({}, function(err, videos) {
if (err) {
return cb(err);
} else {
return cb(null, videos);
}
});
},
function (cb) {
Papers.find({}, function(err, papers) {
if (err) {
return cb(err);
} else {
return cb(null, papers);
}
});
},
function (cb) {
Material.findById(req.params.id, function(err, found) {
if (err) {
return cb(err);
} else {
return cb(null, found);
}
});
}
], function (err, results) {
if (err) {
// If any function returns an error
// (first argument), it will be here
console.log(err);
}
else {
// Even though the functions complete asynchronously,
// the order in which they are declared in the array
// will correspond to the position in the array
// if it returns anything as a second argument.
var videos = results[0];
var files = results[1];
var found = results[2];
res.render('files', {
file: files,
video: videos,
current: found
});
}
});
Solution 2: Nested Callbacks
Videos.find({}, function(err, videos) {
var vid = videos;
if (err) {
console.log(err);
} else {
Papers.find({}, function(err, file) {
var pap = file;
if (err) {
console.log(err);
} else {
Material.findById(req.params.id, function(err, found) {
if (err) {
console.log(err);
} else {
res.render('files', {
file: pap,
video: vid,
current: found
});
}
});
}
});
}
});
So i have implemented a mongodb on my nodejs server. And what I have done is store users via:
function insertUser() {
var collection = dbb.collection('user');
var user1 = {name: user, token: token};
collection.insert(user1, function(err, result) {
if (err) {
console.log(err);
} else {
console.log(result);
}
});
}
function findUserByName(devName) {
var collection = dbb.collection('user');
collection.find({name: devName}).toArray(function (err, result) {
if (err) {
console.log(err);
} else if (result.length) {
console.log('Found: ', result);
selectedUserToken = result.token;
} else {
console.log('No document found');
insertUser();
}
dbb.close();
});
}
So result will equal:
Found: [ { _id: 57be1cadc281c03ea116c9ab,
name: 'Austin Hunter',
token: 'dJyXVjMJk08kXWrua8SUjKb....SxACihKZoR53y_wOZmcFNKMmD5q99QNvsp3flL' } ]
My question is, how can I get that token out to equal selectedUserToken so I can send a push notification with gcm? Right now result.token is undefined.
You should use findOne() instead of find() since you only expect a single result back:
function findUserByName(devName) {
var collection = dbb.collection('user');
collection.findOne({name: devName}, function (err, result) {
if (err) {
console.log(err);
} else if (result) {
console.log('Found: ', result);
selectedUserToken = result.token;
} else {
console.log('No document found');
insertUser();
}
dbb.close();
});
}
But if you wanted to leave your code as is with the find() you would just retrieve the first element of the resulting array retrieved by find()
function findUserByName(devName) {
var collection = dbb.collection('user');
collection.find({name: devName}).toArray(function (err, result) {
if (err) {
console.log(err);
} else if (result.length) {
console.log('Found: ', result);
selectedUserToken = result[0].token;
} else {
console.log('No document found');
insertUser();
}
dbb.close();
});
}
Maybe result[0].token, because result is an array of user items.