Diffbot: "Where I can pass stats argument in analyze API?" - javascript

I am using Diffbot analyze API for detecting the page type and I want result like
this
{"stats":{"times": {"docParseTime":0,"docGlobalsTime":0,"fetchAndRenderTime":586,"typeTime":0},"fromCache":true,"types":{"recipe":0,"discussion":0,"audio":0,"error":0,"location":0,"faq":0,"image":0,"job":0,"download":0,"game":0,"product":0,"frontpage":0,"document":1,"article":0,"event":0,"chart":0,"serp":0,"reviewslist":0,"video":0,"profile":0}},"request":{"pageUrl":"http://www.irs.gov/pub/irs-pdf/fw4.pdf","api":"analyze","version":3,"options":["stats"]},"type":"other","objects":[]}
but currently I am getting like
this
{"request":{"pageUrl":"http://static.nfl.com/static/content/public/image/rulebook/pdfs/2013%20-%20Rule%20Book.pdf","api":"analyze","version":3},"type":"other","objects":[]}
I have to pass 'stats' argument in request.
But where in request, I can pass this argument.
Thanks,

Hi I got it and here is the solution, just customize the Diffbot lib file or write in your file its up to you, and here is the code
var diffbot = new Diffbot('xxxxxxxxxxxxxxxxx');
diffbot.analyze({
uri: "http://www.visitcalifornia.in/media/pages/getting_around/maps/ORANGE-COUNTY.pdf",
html: true,
comments: true,
stats: true
}, function(err, response) {
}
And here is the customize library code
Diffbot.prototype.analyze = function (options, callback) {
for (var i in options) {
this[i] = options[i];
}
var options = this;
// support 'url'
if (options.url) {
options.uri = options.url;
delete options.url;
}
if (!options.uri) {
throw new Error("the URI is required.");
}
var diffbot_url = "http://api.diffbot.com/v3/analyze?token=" + this.token + "&url=" + encodeURIComponent(options.uri)+"&fields=stats";
if (options.stats) {
diffbot_url += "&stats=1";
}
request({uri: diffbot_url}, function(error, response, body) {
if (error) {
callback(error, undefined);
} else {
callback(false, JSON.parse(body));
}
});
}
it works as charm!

Related

Node.js error in reading file

I am writing code in node.js where i want to read from a file and then export it to a web api. The problem is that I get an error on the code when i am using let.
The error appears to be in my function "render_html my views.js file:
"use strict";
const fs = require('fs');
const model = require('./model');
exports.car = (request, response, params) => {
if (Object.keys(params).length === 0) {
render_JSON(response, model.cars())
}else{
render_JSON(response, model.cars(parseInt(params.number)))
}
};
function render_html(response, file) {
fs.readFile(file, (err, data) => {
if (err) {
console.error(err)
} else {
response.write(data);
response.end();
}
});
}
function render_JSON(response, object) {
const responseJSON = JSON.stringify(object);
response.write(responseJSON);
response.end()
}
I also have problem in "function setHeaders" in router.js file:
"use strict";
const views = require('./views');
const url = require('url');
const routes = [
{
url: ['/api/cars'],
view: views.car,
methods: ['GET'],
headers: {'Content-Type': 'application/json; charset=UTF-8'} // application/json as per RFC4627
}];
function setHeaders(response, headers = {'Content-Type': 'text/plain'}, code = 200) {
response.writeHeader(code, headers);
}
// Filters trailing slash in the url
// for example allowing /api/cars and /api/cars/ to be treated equally by removing trailing slash in the second case
function filterURL(requestURL) {
if (requestURL.pathname.endsWith('/')) {
requestURL.pathname = requestURL.pathname.replace(/\/$/, '');
}
}
exports.route = (request, response) => {
for (let r of routes) {
const requestURL = url.parse(request.url, true);
// url matched and correct method
//if requestURL.pathname
filterURL(requestURL);
if (r.url.includes(requestURL.pathname) && r.methods.includes(request.method)) {
if (r.headers) {
setHeaders(response, r.headers);
} else {
setHeaders(response)
}
r.view(request, response, requestURL.query);
return;
}// if unsupported HTTP method
else if (r.url.includes(requestURL.pathname) && !r.methods.includes(request.method)) {
setHeaders(response, undefined, 405);
response.end();
return;
}
}
// if route not found respond with 404
setHeaders(response, undefined, 404);
response.end('404 Not Found!')
};
Somebody knows what the problem could be?
thanks.
about your problem in "render_html" function I think the problem is you are missing the encoding of the file, as fs doc says if you dont set a encoding the result will be a buffer. You can easy fix it using:
fs.readFile(file, 'utf8', callback)
(Assuming that you are using utf8 as encoding)
And I think your problem in "router.js" file is you should use "writeHead" instead "writeHeader" you can check it in http doc.
I hope it solves your issue, greetings.

Create my own Model with node who allow new Model and Model.find()

I'm trying to create a Model with node and I want to be able to use it like that :
Require the models
var Example = require('./models/example');
Create new object
The model can create a new object using new
var example = new Example({ data:'example' });
Find
The model can find object using method
Example.find({ data: 'mongoQuery' }, function(err, examples) {});
Save
The model can help to find object using method, who return object with methods.
Example.findOne({ query: 'example' }, function(err, example) {
example.set({ data: 'another data' });
example.save();
});
Example of use
I want to be able to create, find (one or multiple), uptade and delete a token (for example) using the model like that. I'll use that on controller or lib for example.
var Token = require('./models/token);
// Create and save a new token
var new_token = new Token({ key: 'abcdef' });
new_token.save();
// find multiple tokens, update, and save
var token = Token.find({key: 'abcderf'}, function(tokens) {
for(var i in tokens) {
tokens[i].set({key: '123456'});
tokens[i].save();
}
});
I already try lots of thing, but I can't create a module who allow the new Example and Example.find() in the same time.
Backbone :
I have try using Backbone, the new Token({ /* attributes */ }) work but the find function return an error : TypeError: Object function (){ return parent.apply(this, arguments); } has no method 'find'
var Token = Backbone.Model.extend({
initialize: function(attributes) {
console.log("Token create");
console.log(" attributes : ", attributes);
},
find : function(query, callback) {
console.log("Token find");
console.log(" query : ", query);
callback(null, [ /* tokens */]);
}
});
module.exports = Token;
Object
When I'm trying to use an object, the find function work well, but I can't use it with new returning the error : TypeError: object is not a function
module.exports = {
find : function(query, callback) {
console.log("[Step] models/token.js - find");
console.log(" query : ", query);
callback(null, []);
}
};
What the best way to create a Model to handle all objects on controller ?
Thank for helping !
Here you go :
YourModule = function(data){
this.data = data; // this.data is now a property of the object you create with New.
};
YourModule.prototype.find = function(query, callback) {
console.log("[Step] models/token.js - find");
console.log(" query : ", query);
callback(null, []);
return this.data; // this.data is available here.
};
YourModule.prototype.save = function(data) {
this.data = data; // this.data is available here to.
};
module.exports = YourModule;
You cannot instantiate an object using new keyword, you can however do Object.create to do so. i prefer my example though.
So... I create a model base on the #Neta Meta reply :
./models/token.js
var Backbone = require('backbone');
var db = require('../lib/database');
var Token = Backbone.Model.extend({
initialize: function(attributes) {
},
save: function(callback) {
db.collection('tokens').insert(self.attributes, function(err) {
if (typeof callback === 'function') {
callback(err, self);
}
});
}
});
module.exports.create = function(attr, callback) {
callback(null, new Token(attr));
};
module.exports.findOne = function(query, callback) {
db.collection('tokens').findOne(query, function(err, result) {
callback(err, new Token(result));
});
};
So now I can use it like that :
./controllers/example.js
var Token = require('../models/token');
// Create one :
Token.create({key: 'new data'}, function(err, token) {
token.set({key: 'updated data'});
token.save();
});
// Find one :
Token.findOne({key: 'updated data'}, function(err, token) {
token.set({key: 're-updated data'});
token.save(function(err) {
console.log("Token saved");
});
});

Proper pattern to set a variable via Node async call

I'm doing an HTTP post, which returns single 10-15 character string. I want to assign this value to a temporary variable and use it while building up another larger string which will contain said 10-15 digit character string.
I've gotten this to work by making the temp variable ("ticket") global, which I don't like:
var ticket; // Global. Lame.
...stuff happens
getTicket("someUser", function(err) {
if(err)
{console.log("problem");}
else
{console.log(ticket);}
});
...other stuff happens
// Helper functions down here....
var getTicket = function (userName, callback) {
var user = userName;
request.post(
{
url: 'http://somewhere',
form: { 'username': user }
},
function(err, response, body) {
if(err) {
callback(err);
return;
} else {
ticket = body;
}
callback(null);
});
}
Can someone point me to the proper pattern to return the variable ticket in the POST's callback (or the body variable, whatever) ala:
_ticket = getTicket("someuser")
Thanks much.
You'd pass ticket as a parameter into your callback:
getTicket("someUser", function(err, ticket) {
if(err)
{console.log("problem");}
else
{console.log(ticket);}
});
function getTicket (userName, callback) {
var user = userName;
request.post(
{
url: 'http://somewhere',
form: { 'username': user }
},
function(err, response, body) {
if(err) {
callback(err);
return;
}
callback(null, body); // body contains the ticket
});
}
request.post is async, so there isn't a pattern that will get you something like:
_ticket = getTicket("someuser")

Creating a simple oEmbed system in Nodejs

For a personal project I'm trying to create a simple oEmbed system using Nodejs.
My route looks like this:
app.get('/oembed/:url', function (req, res) {
oembed.get(req.params.url, function (error, result) {
return res.json(200, {message: "OK"});
});
});
and oembed is exposed using var oembed = require('../oembed');
For the oembed.js itself I have:
var request = require('request');
exports.get = function(url, callback) {
//this bit will be developed to call different functions depending on the URL
html = vimeo(url)
};
function vimeo(url) {
var videoUrl = url;
var endpoint = 'http://www.vimeo.com/api/oembed.json';
var url = endpoint + '?url=' + encodeURIComponent(videoUrl) + '&width=640';
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var video = JSON.parse(body);
return video.html
}
})
}
So far, the vimeo function returns the desired html to the function call but I'm a bit lost from here. I'm not sure how I return the html to the initial request (the oembed.get part) and utilise the callback's error and result parameters.
Any help (and advice) would be much appreciated.
It seems you just don't understand how callback functions work. You need something like this in your oembed.js file:
var request = require('request');
exports.get = function(url, callback) {
var endpoint = 'http://www.vimeo.com/api/oembed.json';
request(endpoint + '?url=' + encodeURIComponent(url) + '&width=640', function (error, response, body) {
if (!error && response.statusCode == 200) {
try {
callback(null, JSON.parse(body).html);
} catch (error) {
callback(error);
}
}
});
};
And your route should look like this:
app.get('/oembed/:url', function (req, res) {
oembed.get(req.params.url, function (error, result) {
if (error) {
res.json(500, {message: error});
} else {
res.json(200, {message: "OK"});
}
});
});

Meteor insert uploaded into the console but not MongoDB

I've configured a FB graph call that would retrieve data from the API, however I'm having trouble inserting it into MongoDb. Right now if I run Photos.find().count(); in the browser it shows that there are photos, however if I run db.Photos.find().count(); in MongoDb it shows nothing. Also, if I run db.users.find(); in MongoDb it returns results from the FB user account, so MongoDb is talking to the API to some extent.
Any thoughts on what might be causing the issue?
Here is my code:
Client:
Template.test.events({
'click #btn-user-data': function(e) {
Meteor.call('getUserData', function(err, data) {
if(err) console.error(err);
});
}
});
Template.facebookphoto.helpers({
pictures: function () {
return Photos.find();
}
});
Server:
function Facebook(accessToken) {
this.fb = Meteor.require('fbgraph');
this.accessToken = accessToken;
this.fb.setAccessToken(this.accessToken);
this.options = {
timeout: 3000,
pool: {maxSockets: Infinity},
headers: {connection: "keep-alive"}
}
this.fb.setOptions(this.options);
}
Facebook.prototype.query = function(query, method) {
var self = this;
var method = (typeof method === 'undefined') ? 'get' : method;
var data = Meteor.sync(function(done) {
self.fb[method](query, function(err, res) {
done(null, res);
});
});
return data.result;
}
Facebook.prototype.getUserData = function() {
return this.query('me/photos');
}
Meteor.methods({
getUserData: function() {
var fb = new Facebook(Meteor.user().services.facebook.accessToken);
var data = fb.getUserData();
_.forEach(data.data, function(photo) {
if(Photos.findOne({id: photo.id})) return;
Photos.insert(photo, function(err) {
if(err) console.error(err);
});
});
}
});
Collection:
Photos = new Meteor.Collection('picture');
Thanks in advance!
Instead of db.Photos.find().count();, try db.picture.find().count();
Photos is just the name you gave to the JavaScript variable. The actual name of the collection in MongoDB is whatever you use when you initialized the Collection - in this case, picture.

Categories