Mongo Error on Heroku / No Operations in Bulk - javascript

I have an app that works fine local but once on heroku, it does not work. My app has a scrape function that stores the retrieved articles from the website into a database. The problem is, this does not work on Heroku for some reason and I get the error MongoError: Invalid Operation, No operations in bulk. I tried using new Promise and ORM but I still get the same error. The weird part is, this worked once on heroku when I was using the new Promise, but after I went to a route called /delete the scrape stopped working for some reason. This route just deletes all the data from both my models. I am unsure now what the problem is. Could it be that when I am doing an insertMany into my Article model, there are empty properties that do not get assigned?
Update: I just put a console.log of the array scrappedStuff inside of the scraper function and it is showing the array is empty in the heroku logs. Why is it showing the array is empty? Shouldnt this call back function that is passed the array be called once the .each() is finished?
//MY MODEL
var ArticleSchema = new Schema({
title: {
type: String
},
link: {
type: String
},
description: {
type: String
},
saved: {
type: Boolean,
default: false
},
img: {
type: String
},
comment: [{
type: Schema.Types.ObjectId,
ref: 'Comment'
}]
});
//ROUTE FILE
var scrape = require('./scrape');
var scraper = function (response, scrapedStuff, displayScrape) {
console.log("INSIDE SCRAPER CALLBACK FUNCTION (INSERTMANY)", scrapedStuff)
console.log("TWOOOOOOOOOOOOOOOOOOOOOOOOOO")
Article.collection.insertMany(scrapedStuff, { ordered: false }, function(err, docs){
if(err){
console.log(err);
}else{
console.log("INSERT MANY DOCS", docs);
console.log("FINISHED SCRAPING ELSE")
// resolve();
displayScrape(response);
}
});
}
var displayScrape = function(response){
console.log("THREEEEEEEEEEEEEEEEEEEEEEE")
Article.find({}, function(err, doc){
if(err){
console.log(err)
}else{
if(doc.length > 0){
console.log("CHECK TO SEE IF DOCS IN HOME GOT UPDATE", doc)
var articleLength = [];
for(var x = 0; x < doc.length; x++){
if(doc[x].saved === false){
articleLength.push(doc[x]);
}
}
// var finalLength = articleLength.length;
response.render('home', {data: {articles: doc, length: articleLength.length, finishScrape: true}})
}else{
response.render('home');
}
}
});
}
router.get('/scrape', function(req, response){
scrape(response, scraper, displayScrape);
});
// SCRAPE FILE
var request = require('request');
var cheerio = require('cheerio');
var scrape = function(response, scraper, displayScrape) {
console.log("ONNNNNNNNNEEEEEEEEEEE")
request('http://www.gamespot.com/', function(err, res, html){
if(err)
{
console.log(err)
// reject(err);
}
// else
var scrapedStuff = [];
console.log("GOT RESPONSE FROM GAMESPOT");
var $ = cheerio.load(html);
// var articleCounter = 1;
$('article a').each(function(i, element){
// var scrapedStuff = {};
// scrapedStuff.articleID = articleCounter;
var scrpObj = {
title: $(this).attr('data-event-title'),
link: $(this).attr('href'),
img: $(this).children('figure').children('div.media-img').children('img').attr('src'),
description: $(this).children('div.media-body').children('p.media-deck').text()
}
scrapedStuff.push(scrpObj);
});
console.log("SCRAPED RESULTS", scrapedStuff);
scraper(response, scrapedStuff, displayScrape);
});
}
module.exports = scrape;

Related

Node Js cannot read property of undefined

i am trying to upload multiple images from a form to a webpage but the problem is that it is creating new entries at each step of the for loop if i take it out of the loop then the req.body.ca.video1 becomes undefined. how do i create the entry only one time with all the images.
the post route
router.post("/n/v/n", upload.array('images'), middleware.isloggedin, async function(req, res) {
try {
req.body.ca.video1 = [];
for (var i = 0; i < req.files.length; i++) {
console.log(req.files[i].path);
cloudinary.v2.uploader.upload(req.files[i].path, { resource_type: "auto" }, function(error, result) {
req.body.ca.video1.push({
url: result.secure_url,
format: result.format
});
req.body.ca.author = {
id: req.user._id,
username: req.user.username
}
req.body.ca.created = new Date();
////if i get out of this then req.body.ca becomes undefined
const campground = await Campground.create(req.body.ca);
});
/////here the video1 goes undefined
}
req.flash("success", "it is uploading");
res.redirect("/c");
} catch (err) {
req.flash("error", err.message);
res.redirect("back");
}
});
That's because cloudinary.upload is async. Outside of the loop is executed before the request ends.
You need to wait for the request to finish to be able to access it.
Also you shouldn't declare new variables on the request body, unless you need them on another middleware on the pipeline.

Node.JS send response after api call in loop

I'm trying to retrieve a list of objects and send them back to my mobile app. I'm just having some difficulty actually sending them after the forEach loop is over.
I tried appending that variable "data" to an array and sending it outside of the loop but the array is empty. Obviously, there is data being retrieved, but it doesn't get pushed into the array on time.
How can I make sure the loop is over before I call res.send() ? I reduced the code as much as I could to make it as simple as possible.
var stripe = require("stripe")("stripe_key");
exports.fetchTransactions = function(req, res) {
var account = req.body.account;
stripe.transfers.list({ destination: account}, function(err, transactions) {
transactions.data.forEach(function(item) {
stripe.transfers.retrieve(item.id, function(err, transfer) {
if (err) {
console.log(err);
};
var data = {
amount: item.amount,
created: item.created
};
// Can't call res.send(data) inside the loop because res.send() can only be called once.
// But if I call res.send(array) outside of the loop, the array is still empty
});
});
});
};
Keep track of the responses from API. Call res.send when all responses have been received.
var stripe = require("stripe")("stripe_key");
exports.fetchTransactions = function(req, res) {
var account = req.body.account;
stripe.transfers.list({
destination: account
}, function(err, transactions) {
var pending= transactions.data.length;
transactions.data.forEach(function(item) {
stripe.transfers.retrieve(item.id, function(err, transfer) {
pending--;
if (err) {
return console.log(err);
};
var data = {
amount: item.amount,
created: item.created
};
if (pending == 0) {
res.send(array);
}
});
});
});
};

Sending an AlchemyData News query using Node.js (watson-developer-cloud module)

I'm currently working with Node.js using the watson-developer-cloud Node.js SDK and I'm having problems when sending a query that includes an entity.
This is my code:
// require watson's node sdk and fs
var watson = require('watson-developer-cloud');
var fs = require('fs');
// Define output file
var outputJSONFile = '/home/vagrant/Desktop/node/dir/data.json';
// Create alchemy_data_news object using our api_key
var alchemy_data_news = watson.alchemy_data_news({
api_key: ''
});
// Define params for the query and what values to return
// Accepted returne values:
// docs.alchemyapi.com/v1.0/docs/full-list-of-supported-news-api-fields
var params = {
start: 'now-1m',
end: 'now',
count: 2,
qs: ['q.enriched.url.enrichedTitle.entities.entity.text=apple'],
return: ['enriched.url.url,enriched.url.title']
};
// Call getNews method and return json
alchemy_data_news.getNews(params, function (err, news) {
if (err) {
console.log('error:', err);
} else {
fs.writeFile(outputJSONFile, JSON.stringify(news, null, 2), function(err) {
if (err) {
console.log('WriteFile Error:', err);
} else {
console.log("JSON saved to " + outputJSONFile);
}
});
}
});
I'm still trying to figure out how to send the entities parameters using the params object.
While digging up through some code I came across qs so I have been using that to test but I haven't had success at all.
Any suggestions are greatly appreciated.
P.S: I'm trying to pass:
q.enriched.url.enrichedTitle.entities.entity.text=apple
q.enriched.url.enrichedTitle.entities.entity.type=company
If you look at the node-sdk source code for AlchemyDataNews, you will see that the top level parameters are being sent as query strings.
Then params map should be:
var params = {
start: 'now-1m',
end: 'now',
count: 2,
return: ['enriched.url.url,enriched.url.title'],
// fields here
'q.enriched.url.enrichedTitle.entities.entity.text': 'apple',
'q.enriched.url.enrichedTitle.entities.entity.type': 'company'
};

Javascript nodejs tedious mssql is there a way to get json?

I'm using nodejs and tedious connector to get data from mssql server. In documentation, I only see this one way to retrieve data
var request = new Request("select Name, Value, Article_Id from [tableone] where Id = '1'", function (err, rowCount, rows) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
});
request.on('row', function (rows) {
...
bigArrat.push(JSON.stringify(rows));
});
But in my example I want all rows, not only one property but more. Currently, it return in separate row one cell eg. rows[0].value will return Name, rows[1].value Value ... for me it is rubbish.
I want to get all information in json array of object not all metadata or one property. There is a way to do this or there is a better connector for nodejs and sqlserver ?
The rows value sent to your initial callback is the array of rows being sent back:
var request = new Request("select Name, Value, Article_Id from [tableone] where Id = '1'", function (err, rowCount, rows) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
console.log(rows) // this is the full array of row objects
// it just needs some manipulating
jsonArray = []
rows.forEach(function (columns) {
var rowObject ={};
columns.forEach(function(column) {
rowObject[column.metadata.colName] = column.value;
});
jsonArray.push(rowObject)
});
return callback(null, rowCount, jsonArray);
});
In Sql Server 2016 you can format query results as JSON text using FOR JSON option, see https://msdn.microsoft.com/en-us/library/dn921882.aspx
You just need to read JSON fragments returned by query.
Add this to your config.
rowCollectionOnRequestCompletion: true
var config = {
userName: '', // update me
password: '', // update me
server: '', // update me
options: {
database: '', // update me
encrypt: true,
rowCollectionOnRequestCompletion: true
}
}
Then on your query you can now get the data of rows.
var executeQuery = (res,query) => {
request = new Request(query, (err, rowCount, rows) => {
console.log("Rows: ", rows);
res.send(rows);
});
connection.execSql(request);
}
I learned it from:
http://tediousjs.github.io/tedious/api-request.html
EDIT
Update not to have metadata:
var data = []
request = new Request(query, (err, rowCount, rows) => {
if(err) {
console.log(err)
res.send({ status: 500, data: null, message: "internal server error."})
} else {
console.log(rowCount+' row(s) returned')
res.send({ status: 200, data: data, message: "OK"})
}
})
request.on('row', function(row){
data.push({
last_name: row[0].value,
first_name: row[1].value
})
})
connection.execSql(request)
If you are using express on server side I can recommend using express4-tedious (see https://www.npmjs.com/package/express4-tedious). It allows to easily write apis for SQL connections with small code and streams json result to response.
Connection:
var express = require('express');
var tediousExpress = require('express4-tedious');
var app = express();
app.use(function (req, res, next) {
req.sql = tediousExpress(req, {connection object});
next();
});
Example Api:
/* GET from tableone, streams json result into response */
router.get('/', function (req, res) {
req.sql("select Name, Value, Article_Id from [tableone] where Id = '1' for json path")
.into(res);
});
You can then call these apis e.g. from frontend.
I tried that way but it did not work for me perhaps my knowledge of js and callbacks is not good enough. So, here is my solution. I had to add things to my config of connection to make rows of request work. You would also have to do this. Go to: at the end of new Request section, and to the rows.
here
Second thing, I did is pretty simple.
var jsonArray = [];
var rowObject= {};
var request = new Request("SELECT TOP 5 * FROM tableName",function(err,rowCounts,rows)
{
if (err)
{
console.log(err);
}
else
{
console.log(rowCounts + " rows returned");
}
//Now parse the data from each of the row and populate the array.
for(var i=0; i < rowCounts; i++)
{
var singleRowData = rows[i];
//console.log(singleRowData.length);
for(var j =0; j < singleRowData.length; j++)
{
var tempColName = singleRowData[j].metadata.colName;
var tempColData = singleRowData[j].value;
rowObject[tempColName] = tempColData;
}
jsonArray.push(rowObject);
}
//This line will print the array of JSON object.
console.log(jsonArray);
and to show you how my connection.config looks like:
static config: any =
{
userName: 'username',
password: 'password',
server: 'something.some.some.com',
options: { encrypt: false, database: 'databaseName' ,
rowCollectionOnRequestCompletion: true }
};//End: config
and this is how I am passing it to connection.
static connection = new Connection(Server.config);
Complementing the answer from #Jovan MSFT:
var request = new Request('select person_id, name from person for json path', function(err) {
if (err) {
console.log(err);
}
connection.close();
});
And, finally, in the row event:
request.on('row', function(columns) {
var obj = JSON.parse(columns[0].value);
console.log(obj[0].name);
});
P.S.: the code above does not iterate over columns parameter because for json path returns a single array of objects in a single row and column.
Applying map-reduce function in returned rows:
rows.map(r=>{
return r.reduce((a,k)=>{
a[k.metadata.colName]=k.value
return a
}
,{})
})
This is a combination of a few responses above. This uses FOR JSON AUTO in the SELECT statement and parses the "column" as JSON. The row/column nomenclature may be a bit misleading for folks unfamiliar with this API. In this case, the first "columns" value will be an array of the rows in your table:
var request = new Request("SELECT Name, Value, Article_Id FROM [tableone] WHERE Id = '1' FOR JSON AUTO", function (err, rowCount, rows) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
});
request.on('row', (columns) => {
const json = JSON.parse(columns[0].value);
});

Meteor insert uploaded into the console but not MongoDB

I've configured a FB graph call that would retrieve data from the API, however I'm having trouble inserting it into MongoDb. Right now if I run Photos.find().count(); in the browser it shows that there are photos, however if I run db.Photos.find().count(); in MongoDb it shows nothing. Also, if I run db.users.find(); in MongoDb it returns results from the FB user account, so MongoDb is talking to the API to some extent.
Any thoughts on what might be causing the issue?
Here is my code:
Client:
Template.test.events({
'click #btn-user-data': function(e) {
Meteor.call('getUserData', function(err, data) {
if(err) console.error(err);
});
}
});
Template.facebookphoto.helpers({
pictures: function () {
return Photos.find();
}
});
Server:
function Facebook(accessToken) {
this.fb = Meteor.require('fbgraph');
this.accessToken = accessToken;
this.fb.setAccessToken(this.accessToken);
this.options = {
timeout: 3000,
pool: {maxSockets: Infinity},
headers: {connection: "keep-alive"}
}
this.fb.setOptions(this.options);
}
Facebook.prototype.query = function(query, method) {
var self = this;
var method = (typeof method === 'undefined') ? 'get' : method;
var data = Meteor.sync(function(done) {
self.fb[method](query, function(err, res) {
done(null, res);
});
});
return data.result;
}
Facebook.prototype.getUserData = function() {
return this.query('me/photos');
}
Meteor.methods({
getUserData: function() {
var fb = new Facebook(Meteor.user().services.facebook.accessToken);
var data = fb.getUserData();
_.forEach(data.data, function(photo) {
if(Photos.findOne({id: photo.id})) return;
Photos.insert(photo, function(err) {
if(err) console.error(err);
});
});
}
});
Collection:
Photos = new Meteor.Collection('picture');
Thanks in advance!
Instead of db.Photos.find().count();, try db.picture.find().count();
Photos is just the name you gave to the JavaScript variable. The actual name of the collection in MongoDB is whatever you use when you initialized the Collection - in this case, picture.

Categories