So I wanted to save the stream of data from Twitter Streaming API in MongoDB using Mongoose. Somehow it doesn't save all of the data and it only saves 1 document out of the infinite amount of data in the stream(that is not complete) when i run my code:
var mongoose = require('mongoose'),
TwitterStream = require('twitter-stream-api'),
fs = require('fs');
mongoose.connect('mongodb://localhost/insert_sample');
var keys = {
consumer_key : "",
consumer_secret : "",
token : "",
token_secret : ""
};
var Schema = mongoose.Schema;
// create a schema
var userSchema = new Schema({}, {"strict": false});
// the schema is useless so far
// we need to create a model using it
var User = mongoose.model('User', userSchema);
// make this available to our users in our Node applications
module.exports = User;
var Twitter = new TwitterStream(keys, false);
Twitter.stream('statuses/filter', {
track: 'travel'
});
// create a new user called twitter
var Twitter = new User();
// call the built-in save method to save to the database
Twitter.save(function(err) {
if (err) throw err;
console.log('User saved successfully!');
});
Output of the command line: User saved successfully!
When I try to query in MongoDB shell using db.users.find(), the result is this:
{ "_id" : ObjectId("5a60a5558f4f1d21bcadd72b"), "__v" : 0 }
When it should be like:
{
"created_at": "Fri Aug 04 05:42:03 +0000 2017",
"id": 893346273255866400,
"id_str": "893346273255866368",
..........and soooo many more attributes }
EDIT!!! Sorry I accidentally hit enter and posted it without the other needed things to be clear. My bad.
You are overwriting your Twitter Variable and saving the user data empty object.
// create a new user called twitter
var Twitter = new User();
// call the built-in save method to save to the database
Twitter.save(function(err) {
if (err) throw err;
Once you create a stream you need to listen for data event and use that data to save in your database.
Remove above lines and use something like;
// listen for data from stream
Twitter.on('data', function (obj) {
var data = obj.toString('utf8');
console.log(data);
var TwitterData = new User(data); // create object
TwitterData.save(); // save data to DB
});
Hope it Helps!
Related
Now I am using clmtrackr library to detect emotions from the webcam and I want to save this emotions Here is my node.js code to save the values in mongodb
exports.storeemotion = function (emotions, callback){
var eshema= new emotioncollection({
emotions: [emotions]
});
eshema.save(function(err) {
});
callback({"status":"emotion remote done"});
}
and the schema code is
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
//var bcrypt = require('bcrypt-nodejs');
// search results schema
var ResultaSchema = new Schema({
emotions:[{emotion: String, value: String}]
});
module.exports = mongoose.model('emotioncollection',ResultaSchema);
emotions should be like that (check this image).
but the mongo saved an empty array (check this image).
The emotions parameters of your storeemotion function is already an array, so you just have to pass that parameters as is, not in another array:
exports.storeemotion = function (emotions, callback) {
var eshema = new emotioncollection({
emotions: emotions // <= your schema already expects an array of objects
});
eshema.save(function(err) {
if (err) return callback({"status": "error"});
callback({"status": "emotion remote done"});
});
}
I have created a little application that implemented a JSON file as the data store. I'm trying to learn about MEAN, so I'm trying to convert it to a NODE.js app.
I have created a mongoDB importing my JSON file using mongoimport. The db is there and connected. When I put http://localhost:3000/api/characters into the browser it returns JSON for all the characters.
I have built a connection string and required it into my controller as follows...
// FROM node app characters.controller.js
var dbconn = require('../data/dbconnection.js');
var characterData = require('../data/characters.json');
module.exports.charactersGetAll = function(req, res) {
var db = dbconn.get();
var collection = db.collection('characters');
// Get the documents from the collection
// Need to use to Array() as its only a cursor if you don't
collection
.find()
.toArray(function(err, docs) {
console.log("Found characters!", docs);
res
.status(200)
.json(docs);
});
//console.log('db', db);
//console.log("GET the Characters");
//console.log(req.query);
res
.status(200)
.json(characterData);
};
module.exports.charactersGetOne = function(req, res) {
var charId = req.params.id;
var thisChar = characterData[charId];
console.log("GET characterId", charId);
res
.status(200)
.json(thisChar);
};
From the non-NODE version of the app, I call in the JSON data as follows in my main.js file:
//FROM TRADITIONAL HTML/JS application
// Get JSON and callback so JSON can be stored globally
$.getJSON('people.json', callback);
// Populate faces and names in HTML
function callback(data) {
/*optional stuff to do after success */
var $charNames = $('.char-names');
var $faceImage = $('.faces');
$.each(data, function(index, val) {
console.log("success");
/* iterate through array or object */
/* .eq() method constructs new object from one element from set */
$charNames.eq(index).text(val.name);
$faceImage.eq(index).attr('src', val.image);
//Push all JSON to array
allTheCharacters.push(val);
allTheCharactersComp.push(val);
});
}
What I want to ask is – Is there a simple way I can access the mongoDB in the non-NODE main.js file instead of using the $.getJSON method and how would I add/adapt this for the node characters.controller.js
Hopefully, I am making sense. Apologies in advance for any misunderstanding.
We have different clients and the idea is to keep their data separate from each other in the same application. We are using node.js with mongodb and mongoose is being used for querying.
This is 'index.js' file in models directory
var mongoose = require('mongoose');
var fs = require('fs');
var connectionUrl = 'mongodbserverlink/';
var companies = [{ db: 'comp1_db', comp_id: 'com1' }, { db: 'com2_db', comp_id: 'com2' }, { db: 'com3_db', compa_id: 'com3'}];
var connections = {};
var models = {};
fs.readdirSync(__dirname)
.forEach(function (file) {
var Schema = file.split('.js')[0];
if (Schema === 'index') return;
models[Schema] = require('./' + Schema);
});
companies.forEach(function (company) {
var conn = mongoose.createConnection(connectionUrl + company.db);
connections[company.company_id] = {};
Object.keys(models).forEach(function (Schema) {
connections[company.company_id][Schema] = conn.model(Schema, models[Schema]);
});
conn.on('error', console.error.bind(console, company.db + ' connection error in mongodb in the first step!'));
conn.once('open', function() {
console.log(company.db + " mongodb connected");
});
});
module.exports = connections;
Here the connection is being made with different databases. The models directory has this index file.
Now in the controller where application logic is being done, this is what we are doing.
var models = require('../models');
var comp_id = req.body.comp_id;
db.collectionname.find...(This is not the syntax for find, I just cut it short to keep it simple) // -> this is not working now
when we tried logging models object this is what we got
models object is: {"com1":{},"com2":{},"com3":{}}
and only db when logged gives {}
We are facing issues in grasping the complete work... it is because the person who wrote the major chunk is not with us and there is no documentation.
What are we doing wrong here?
It looks like you already have your models exported from the index file. So, in the controller you can do a var models = require('index');. From the connections object, you may be able to retrieve the corresponding model: var companyModel = models[comp_id];.
Hope it helps.
I have a JSON Object called parsedSong that I want to convert into a Mongo document using Mongoose.
var newSong = new Song(parsedSong);
However, this only provides me with a new document that only has an id attribute. How can I make a conversion that doesn't cut my data from parsedSong?
I assumed you have defined these variables in schema. Then you can make something like this:
var bestSong = {
artist: "Seether",
song: "Careless Whisper"
};
var song = new Song(bestSong);
song.save(function(err) {
if(err) throw err;
console.log('saved');
});
And now this song should be in database.
I am trying to pass data to Mongodb using Websocket and total.js.
In my homepage.html I can get the user input and connect to the server via websocket after clicking the save button.
In default.js is my server side code. At this point the app hat got the user input and connected to the server correctly, but how can I save data to mongodb now?
This is my homepage.html
<br />
<div>
<input type="text" name="message" placeholder="Service" maxlength="200" style="width:500px" />
<button name="send" >Save</div>
</div>
<br />
<script type="text/javascript">
var socket = null;
$(document).ready(function() {
connect();
$('button').bind('click', function() {
if (this.name === 'send') {
console.log(send());
return;
}
});
});
function connect() {
if (socket !== null)
return;
socket = new WebSocket('ws://127.0.0.1:8000/');
socket.onopen = function() {
console.log('open');
};
socket.onmessage = function(e) {
var el = $('#output');
var m = JSON.parse(decodeURIComponent(e.data)).message;
el.val(m + '\n' + el.val());
};
socket.onclose = function(e) {
// e.reason ==> total.js client.close('reason message');
console.log('close');
};
}
function send() {
var el = $('input[name="message"]');
var msg = el.val();
if (socket !== null && msg.length > 0)
socket.send(encodeURIComponent(JSON.stringify({ message: msg })));
el.val('');
return msg;
}
This is my default.js
exports.install = function(framework) {
framework.route('/', view_homepage);
framework.route('/usage/', view_usage);
framework.websocket('/', socket_homepage, ['json']);
};
function view_usage() {
var self = this;
self.plain(self.framework.usage(true));
}
function view_homepage() {
var self = this;
self.view('homepage');
}
function socket_homepage() {
var controller = this;
controller.on('open', function(client) {
console.log('Connect');
});
controller.on('message', function(client, message) {
console.log(message);
/*
var self = this;
var message = MODEL('message').schema;
var model = self.body;
var message = new message({ message: model.message }).save(function(err) {
if (err)
self.throw500(err);
// Read all messages
message.find(self.callback());
});
*/
});
controller.on('error', function(error, client) {
framework.error(error, 'websocket', controller.uri);
});
}
Any help Please!!!
This is complete project
---Update---
In this function i use to save data to MongoDB
but it didn't give any error.also Didnt save the data to database.i not sure my code is write or wrong
controller.on('message', function(client, message) {
console.log(message);
/*
var self = this;
var message = MODEL('message').schema;
var model = self.body;
var message = new message({ message: model.message }).save(function(err) {
if (err)
self.throw500(err);
// Read all messages
message.find(self.callback());
});
*/
});
This my mongoose.js
var mongoose = require('mongoose');
mongoose.connect('mongodb://totaldemo:123456#ds029979.mongolab.com:29979/totaldemo');
global.mongoose = mongoose;
This is my user.js
var userSchema = mongoose.Schema({ user: String})
exports.schema = mongoose.model('user', userSchema,'user');
exports.name = 'user';
I don't know totaljs framework at all, but i see some issues already with plain javascript.
First of all, i suggest You set up Your model like this:
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var userSchema = new Schema({
user: String
});
module.exports = mongoose.model('User', userSchema);
and then in controller, when You import:
var User = require('path/to/user/file')
You can use it like this straight away:
User.find()
Also - i totally dont get what are You doing later.
You defined user model and exported NOTHING MORE than a STRING. Only tthing it will do is, that when You import that user to variable User, the User.name will === to 'user' string. so in Your example it would be:
var User = require('path/to/user/model/file')
console.log(User.name) // prints 'user'
and nothing more! There are no models attached to that export. Maybe its how totaljs works, but i VERY doubt it.
Later on - You try to ... use message model. Where it comes from? You defined user model, not message model.
Another thing - as i stated - i dont know totaljs, but I doubt it ask YOu to define var model, and then never use variable model.
I strongly suggest using plain node with mongoose first, then try to integrate it with any fullstack.
For sure its not a solution, but maybe it points out some problems in Your code and will help.
EDIT:
I looked quickly in totaljs, and it looks that You really should export string (which is little weird and doing magic stuff:) ), but its NOT mongoose, and i guess will ONLY work with native totaljs model solution. You cant use mongoose and totaljs like that. I dont know how much not using native totaljs models system ruins framework other options, but its probably safer to use native one.
Honestly, i dont have time to look deeper into docs, but google says nothing about sql or even mongo inside of totaljs docs... so, You have to figure it out :)
EDIT2 i found https://github.com/totaljs/examples/tree/master/mongoose and it looks weird... check if that example works (looks like You seen it, Your code is similar :)). check if You're mongod is working, check if You can conenct from plain node...
Honestly sorry, i surrender. Totaljs has to much magic and abstraction for me to help You out with this :(. Hope You will find Your answer.