We Keep Coding

local PDF file scraping in node.js

I have uploaded a pdf via a MEAN stack web application using fs. I want to extract certain fields from the pdf and display them on the web app. I have looked at a couple npm packages like pdf.js, pdf2json. I can't figure out the documentation and javascript callbacks used in the examples available. Please help!

I hope I can help answer your question. Using pdf2json can be used to parse a pdf and extract the text. There are a couple of steps that need to be taken to get it working. I have adapted the example from https://github.com/modesty/pdf2json.
The setup is to install pdf2json in the node app, and also underscore. The example page didn't explain the need to define your own callback functions. It also used self instead of this to register them. So, with the appropriate changes the code to extract all the text from the pdf will be something like this:
// Get the dependencies that have already been installed
// to ./node_modules with `npm install <dep>`in the root director
// of your app
var _ = require('underscore'),
PDFParser = require('pdf2json');
var pdfParser = new PDFParser();
// Create a function to handle the pdf once it has been parsed.
// In this case we cycle through all the pages and extraxt
// All the text blocks and print them to console.
// If you do `console.log(JSON.stringify(pdf))` you will
// see how the parsed pdf is composed. Drill down into it
// to find the data you are looking for.
var _onPDFBinDataReady = function (pdf) {
console.log('Loaded pdf:\n');
for (var i in pdf.data.Pages) {
var page = pdf.data.Pages[i];
for (var j in page.Texts) {
var text = page.Texts[j];
console.log(text.R[0].T);
}
}
};
// Create an error handling function
var _onPDFBinDataError = function (error) {
console.log(error);
};
// Use underscore to bind the data ready function to the pdfParser
// so that when the data ready event is emitted your function will
// be called. As opposed to the example, I have used `this` instead
// of `self` since self had no meaning in this context
pdfParser.on('pdfParser_dataReady', _.bind(_onPDFBinDataReady, this));
// Register error handling function
pdfParser.on('pdfParser_dataError', _.bind(_onPDFBinDataError, this));
// Construct the file path of the pdf
var pdfFilePath = 'test3.pdf';
// Load the pdf. When it is loaded your data ready function will be called.
pdfParser.loadPDF(pdfFilePath);

I am running the code out of my server side controller.
module.exports = (function() {
return {
add: function(req, res) {
var tmp_path = req.files.pdf.path;
var target_path = './uploads/' + req.files.pdf.name;
fs.rename(tmp_path, target_path, function(err) {
if (err) throw err;
// delete the temporary file, so that the explicitly set temporary upload dir does not get filled with unwanted files
fs.unlink(tmp_path, function() {
if (err) throw err;
//edit here pdf parser
res.redirect('#/');
});
})
},
show: function(req, res) {
var pdfParser = new PDFParser();
var _onPDFBinDataReady = function (pdf) {
console.log('Loaded pdf:\n');
for (var i in pdf.data.Pages) {
var page = pdf.data.Pages[i];
// console.log(page.Texts);
for (var j in page.Texts) {
var text = page.Texts[j];
// console.log(text.R[0].T);
}
}
console.log(JSON.stringify(pdf));
};
// Create an error handling function
var _onPDFBinDataError = function (error) {
console.log(error);
};
pdfParser.on('pdfParser_dataReady', _.bind(_onPDFBinDataReady, this));
// Register error handling function
pdfParser.on('pdfParser_dataError', _.bind(_onPDFBinDataError, this));
// Construct the file path of the pdf
var pdfFilePath = './uploads/Invoice_template.pdf';
// Load the pdf. When it is loaded your data ready function will be called.
pdfParser.loadPDF(pdfFilePath);
},
//end controller
}

Better place to setup an initialisation in SailsJS

I'm new to Sails and don't know exactly where to put the initialisation of an object to be unique in all the app. After reading the docs I assumed that I can have it in the global sails object, but not sure if is the better place.
I'm using the new Appcelerator ArrowDB to store my users and objects. Docs talk about declare the appropriate vars and use it, with the APP_KEY.
var ArrowDB = require('arrowdb'),
arrowDBApp = new ArrowDB('<App Key>');
function login(req, res) {
var data = {
login: req.body.username,
password: req.body.password,
// the req and res parameters are optional
req: req,
res: res
};
arrowDBApp.usersLogin(data, function(err, result) {
if (err) {
console.error("Login error:" + (err.message || result.reason));
} else {
console.log("Login successful!");
console.log("UserInfo: " + JSON.stringify(result.body.response.users[0]));
}
});
}
But I will need to use constantly that arrowDBApp var to create, update, delete objects in the database, so I think the best way is to initialize it in the starting script app.js and share across the app.
I tried it, but I was not able to store it in the sails var, it seems that this var is not available (or lose its config) until sails.lift() is executed.
This code (app.js file) shows nothing in the console:
// Ensure we're in the project directory, so relative paths work as expected
// no matter where we actually lift from.
process.chdir(__dirname);
// Ensure a "sails" can be located:
(function() {
var sails;
try {
sails = require('sails');
} catch (e) {
console.error('To run an app using `node app.js`, you usually need to have a version of `sails` installed in the same directory as your app.');
console.error('To do that, run `npm install sails`');
console.error('');
console.error('Alternatively, if you have sails installed globally (i.e. you did `npm install -g sails`), you can use `sails lift`.');
console.error('When you run `sails lift`, your app will still use a local `./node_modules/sails` dependency if it exists,');
console.error('but if it doesn\'t, the app will run with the global sails instead!');
return;
}
// Try to get `rc` dependency
var rc;
try {
rc = require('rc');
} catch (e0) {
try {
rc = require('sails/node_modules/rc');
} catch (e1) {
console.error('Could not find dependency: `rc`.');
console.error('Your `.sailsrc` file(s) will be ignored.');
console.error('To resolve this, run:');
console.error('npm install rc --save');
rc = function () { return {}; };
}
}
// My own code
var APP_KEY = 'mykey';
var ArrowDB = require('arrowdb');
sails.arrowDBApp = new ArrowDB(APP_KEY);
console.log("Hi" + JSON.stringify(sails));
// Start server
sails.lift(rc('sails'));
console.log("Finish");
})();
No "HI" and no "Finish" are printed. If I try to use sails.arrowDBApp in another controller, it is undefined.
Tips are welcome.

It's not advisable to modify app.js unless you really need to.
The usual space to save all configuration information (e.g. the APP_KEY) is in the config directory in your project root.
One-time initializations (e.g. ArrowDB initialization) can be added to config/bootstrap.js.
Update
In config/arrowdb.js (you need to create this file yourself):
module.exports.arrowdb = {
APP_KEY: 'yourappkey',
ArrowDBApp: null
};
In config/bootstrap.js:
var ArrowDB = require('arrowdb');
module.exports.bootstrap = function(next){
sails.config.arrowdb['ArrowDBApp'] = new ArrowDB(sails.config.arrowdb['APP_KEY']);
next(); // Don't forget to add this
};
In your controller:
'task': function(req, res, next) {
sails.config.arrowdb['ArrowDBApp'].usersLogin(...);
// and so on.
// You could also add something like
// var ADB = sails.config.arrowdb['ArrowDBApp'];
// at the top in case you need to use it on and on.
}

Use config/bootstrap.js to initialize something before Sails lifted. Sometimes if we want to put something in global variable, this approach is good to use, like define/ override native Promise with Bluebird Promise.
Use api/services to put some method or other things that you will use regularly in your code (controllers, models, etc.), like Mail Service, that handle sending email within your application.
Use config at config folder to predefined something at sails.config[something]. It can be an object, function, or whatever in order to become configurable, like put Twitter API Key to use Twitter REST API.
To achieve what you wanted, I'll try to use service and bootstrap.js. Try this example.
Create service file at api/services/ArrowDBService.js
Put with this code:
var ArrowDB = require('arrowdb'),
arrowDBApp = new ArrowDB('<App Key>');
module.exports = {
arrowDBApp : arrowDBApp,
login : function (req, res) {
var data = {
login: req.body.username,
password: req.body.password,
// the req and res parameters are optional
req: req,
res: res
};
arrowDBApp.usersLogin(data, function(err, result) {
if (err) {
console.error("Login error:" + (err.message || result.reason));
} else {
console.log("Login successful!");
console.log("UserInfo: " + JSON.stringify(result.body.response.users[0]));
}
});
}
};
Now you can use it by sails.services.arrowdbservice.login(req,res) or simply ArrowDBService.login(req,res) (notice about case sensitive thing). Since I don't know about ArrowDB, so you may explore by yourself about login method that your example provide.

replicate pouchDB document with couchDB

I have used pouchDB in one application and now I want to introduce couchDB to sync the document to remote server. Hence i followed this link http://pouchdb.com/getting-started.html i used the below code to replicate the data to couchDB
var db2 = new PouchDB('todos');
var remoteCouch = 'http://localhost:5984/_utils/database.html?couchdb_sample';
db2.changes({
since: 'now',
live: true
}).on('change', showTodos);
sync();
function sync() {
//alert("sync");
//syncDom.setAttribute('data-sync-state', 'syncing');
//var opts = {live: true};
db2.replicate.to(remoteCouch).on('complete', function () {
console.log("done");
}).on('error', function (err) {
console.log(err);
});
function addTodo(text) {
var todo = {
_id: $("#eid").val()+$("#version").val(),
title: text,
name: $("#nameid").val(),
version: $("#version").val(),
completed: false
};
db2.put(todo, function callback(err, result) {
if (!err) {
console.log('Successfully posted a todo!');
}
else{
console.log(err);
}
});}
here the title has an xml string as value. But i am facing below error
SyntaxError: Unexpected token <
at Object.parse (native)
for this line db2.replicate.to(remoteCouch). I manually created a new document in couchDb database and entered the same data it gave no error but when i try replicating it shows syntax error. Can anyone please hint me where I have gone wrong

http://localhost:5984/_utils/database.html?couchdb_sample
Points to a HTML site (copied over from the browsers address bar, right?). Remove the middle part:
http://localhost:5984/couchdb_sample

It look like you have not defined the remote database in the way PouchDb is expecting. You should use the "new PouchDb" call. The second line of your code is:
var remoteCouch = 'http://localhost:5984/_utils/database.html?couchdb_sample';
but I think it should be like this:
var remoteCouch = new PouchDB('http://localhost:5984/couchdb_sample');
I am not clear from your code what the name of the remote database is, but it would not normally end in ".html" as Ingo Radatz pointed out, so I have assumed it is couchdb_sample above. There is more information about replication on the PouchDb site.

Upload file from Javascript to Google Cloud Endpoint

I'm creating a web app using only HTML5 + Javascript + jQueryMobile and I wanted to upload a file to a Google App Engine web application using a Google Cloud Endpoint, also created by me.
As I control both sides, I can (and want to) create the simplest interaction possible.
As for the Endpoint, I thought of creating a method like this:
#ApiMethod(
name = "uploadFile",
path = "upload_file",
httpMethod = HttpMethod.POST
)
public void uploadFile(File file) {
//process the file
}
This File class could contain a field fileData of type Blob, or byte[] or something like that, repersenting the file data... Something like:
public class File {
private String fileName;
private long fileSize;
private Blob fileData;
//getters and setters
}
So the first question would be: what's the most suitable type for this field fileData?
And, taking into account the type selected for the field, how could I create the necessary POST request for that endpoint method form Javascript/jQuery?
Basically I need to create a POST request to http://myappid.appspot.com/_ah/api/files/v1/upload_file adding the File object in the POST data.
Note: I'm sorry I haven't tried anything for the Javascript code because I'm not familiar at all with this technologies, so I'd appreciate any help...

Edit: The answer below targes python version of AppEngine
It is a common demand with no clear solution. Till now, gae-init-upload is a demonstration of how you can achieve that with AppEngine and CoffeeScript. Worth having a look, CoffeeScript is being compiled into JavaScript in case you are not familiar.
The JavaScript solution you are looking for is under
/main/static/src/coffee/common/upload.coffee

I eventually used this code in my AMD Javascript application. I'm sorry I cannot explain it too much because I've written a big amount of code since I wrote this project, and as you can see I didn't comment the code properly (fail!!), anyway maybe you can get some ideas...
Note that there's something about getting navigator position because I wanted to store the location where the file was uploaded from, but it's not necessary at all!
Controller.js
uploadFile: function(request, render) {
var self = this;
var file = $("#file").get(0).files[0];
var reader = new FileReader();
reader.onload = function (evt) {
var upload = {
provider: self.folder.provider,
folderIdentifier: self.folder.id,
fileName: file.name,
fileSize: file.size,
base64Data: btoa(evt.target.result),
location: {
latitude: self.position.coords.latitude,
longitude: self.position.coords.longitude
}
}
var uploadFilePromise = self.connector.uploadFile(self.sessionToken.token, upload);
uploadFilePromise.done(function (file) {
render("file", {
result: "DONE",
file: file
});
});
uploadFilePromise.fail(function (error) {
render("file", {
result: "FAIL"
});
});
}
navigator.geolocation.getCurrentPosition(function(position) {
self.position = position;
reader.readAsBinaryString(file);
});
}
Connector.js
uploadFile: function (sessionToken, upload) {
var self = this;
var promise = new Promise();
gapi.client.load('upload', 'v1', function() {
var request = gapi.client.upload.uploadFile({
session_token: sessionToken,
resource: upload
});
request.execute(function(response) {
if (response.error) {
promise.reject(response.error);
}
else {
var file = File.create(response.result.provider,
response.result.type,
response.result.identifier,
response.result.name,
response.result.description,
response.result.created,
response.result.size,
response.result.link,
{
latitude: response.result.location.latitude,
longitude: response.result.location.longitude
});
promise.resolve(file);
}
});
}, self.api);
return promise;
}
Endpoint.java
#Api(name="upload")
public class UploadEndpoint {
#ApiMethod(
name = "uploadFile",
path = "upload_file",
httpMethod = HttpMethod.POST
)
public File uploadFile (
#Named("session_token") String token,
Upload upload) throws InternalServerErrorException {
File file = new UploadController().uploadFile(token, upload);
return file;
}
}

Node.js - Organising code and closures - SFTP/Inotify

I was hoping I could get some advice on why my nodejs program is behaving in the way it is.
I am using two modules, node-sftp and node-inotify. I have setup node-inotify to watch a directory and call a function when something is written there, the function being an sftp upload.
Now the problem I have is that processing one file at a time is fine but when I drop 4 files in one go there, the function is called four times but only one sftp upload goes through.
Do I need to order my code in a particular way to ensure that the sftp upload occurs x times, is this something to do with closures perhaps?
This is a basic version of my code...
"event_handler" is called when something happens on a "watched" directory
"check_event" figures out if this type of event is one we want, in this case it's a "write"
"ftp_to_server" prepare connection details
"do_ftp" basically uses the node-sftp module to perform the sftp upload
event_handler = function(event){
var supplier;
check_event(event, supplier, type, ftp_to_server);
};
=================
function check_event(event, handler)
{
if (event.type === 'xxxxxx') {
var file_to_process_name = 'abc';
var file_to_process_dir = 'abc';
var remote_dir = 'abc';
handler(file_to_process_name, file_to_process_dir, remote_dir);
}
}
function ftp_to_server(file_to_process_name, file_to_process_dir, remote_dir) {
var connection_details = conf.ftp.connections
do_ftp(connection_details, file_to_process_name, file_to_process_dir, remote_dir);
}
function do_ftp(connection_details, file_to_process_name, file_to_process_dir, remote_dir) {
var credentials = {
// FTP settings here
};
var local_file = file_to_process_dir + file_to_process_name;
var remote_file = remote_dir + file_to_process_name;
connection = new sftp(credentials, function(err) {
if (err){
throw err;
}
connection.writeFile(remote_file, fs.readFileSync(local_file, "utf8"), null, function(err) {
if (err) {
throw err;
}
console.info('FTP PUT DONE');
});
});
};

Your "connection = new sftp(credentials, function(err) {"
should be
var connection = new sftp(credentials, function(err) {
The way you currently have it coded, "connection" is a global and you are writing over it.