Node Js: Using a WebWorker to resize images - javascript

I am making a node.js application that can resize images. I am able to do this successfully with jimp. However resizing an image is not asynchronous, and freezes the UI while resizing. I want to avoid this, so I tried using a webworker.
// main.js
var worker = new Worker(__dirname + '\\worker.js');
worker.addEventListener('message', function(e) {
if (e.data == 'done') { worker.terminate() } // Done
}, false);
worker.postMessage({'buff': buf, 'filename': filename}); // Start the worker
|
// worker.js
const Jimp = require('jimp'); // Oops, this doesn't work
self.addEventListener('message', function(e) {
resize(e.data.buf, e.data.filename);
}, false);
function resize(buf, filename) {
Jimp.read(buf).then(image => {
image.resize(1920, Jimp.AUTO);
image.writeAsync(filename).then(cb => { self.postMessage('done') });
});
}
What I found is that I cannot use node.js functions like require() in a webworker. How can I use Jimp in a webworker or resize an image in a different asynchronous way?
Edit: I am trying to use webworkify, copying the answer from #RubyJunk. When I try to create the worker I get the error Uncaught TypeError: Cannot convert undefined or null to object at Function.keys (<anonymous>). Does anyone know how to fix that?
Edit 2: I am using electron to create my Node.js app. They have a window property (nodeIntegrationInWorker) that makes it seem like I can run Node.js functions in a Web Worker, but when I try to use require() it still tells me it is not a function.

Try using webworkify, and create your worker like so instead
var work = require('webworkify');
var worker = work(require('./worker.js');
and put your worker method into module.exports like so
const Jimp = require('jimp');
module.exports = function(self) {
self.addEventListener('message', function(e) {
resize(e.data.buf, e.data.filename);
}, false);
function resize(buf, filename) {
Jimp.read(buf).then(image => {
image.resize(1920, Jimp.AUTO);
image.writeAsync(filename).then(cb => { self.postMessage('done') });
});
}
}

Related

Ifc.js: wasm streaming compile failed: LinkError: import object field 'a' is not a Memory

I am reading IFC files and ifc.js seemed like a solid option although I am not so experienced with javascript but I thought it could be a good opportunity to learn about it.
I followed the documentation example that can be food here ``https://ifcjs.github.io/info/docs/Hello%20world```.
I packed the app inside of a django project and everything is fine until I try to load up a file.
I am getting the following error:
RuntimeError: abort(LinkError: import object field 'a' is not a Memory). Build with -s ASSERTIONS=1 for more info.
On my browser debugger, the error links to the following class of my bundle.js file
class IFCLoader extends Loader {
constructor(manager) {
super(manager);
this.ifcManager = new IFCManager();
}
load(url, onLoad, onProgress, onError) {
const scope = this;
const loader = new FileLoader(scope.manager);
this.onProgress = onProgress;
loader.setPath(scope.path);
loader.setResponseType('arraybuffer');
loader.setRequestHeader(scope.requestHeader);
loader.setWithCredentials(scope.withCredentials);
loader.load(url, async function (buffer) {
try {
if (typeof buffer == 'string') {
throw new Error('IFC files must be given as a buffer!');
}
onLoad(await scope.parse(buffer));
} catch (e) {
if (onError) {
onError(e);
} else {
console.error(e);
}
I have no clue how to correct this issue and any help would be highly appreciated. I am happy to post additional files or code if needed. Thanks

How to use HTML DOM using npm test (CLI)

I am trying to find a way to run npm test using mocha over a HTML DOM. In this case, I am using the global document to retrieve a table out of the DOM. However, when I run npm test I get something like the error:
ReferenceError: document is not defined
at /home/luiz/Projects/linguist-unknown/src/scripts/ling-loader.js:92:61
at extFunc (/home/luiz/Projects/linguist-unknown/src/scripts/ling-loader.js:49:11)
at Array.every (native)
at Utilities.tryMatchUrlExtension (/home/luiz/Projects/linguist-unknown/src/scripts/ling-loader.js:60:25)
at Utilities.<anonymous> (/home/luiz/Projects/linguist-unknown/src/scripts/ling-loader.js:90:16)
at xhr.onload (/home/luiz/Projects/linguist-unknown/src/scripts/ling-loader.js:24:11)
at dispatchEvent (/home/luiz/Projects/linguist-unknown/node_modules/xmlhttprequest/lib/XMLHttpRequest.js:591:25)
at setState (/home/luiz/Projects/linguist-unknown/node_modules/xmlhttprequest/lib/XMLHttpRequest.js:614:14)
at IncomingMessage.<anonymous> (/home/luiz/Projects/linguist-unknown/node_modules/xmlhttprequest/lib/XMLHttpRequest.js:447:13)
at emitNone (events.js:91:20)
at IncomingMessage.emit (events.js:185:7)
at endReadableNT (_stream_readable.js:974:12)
at _combinedTickCallback (internal/process/next_tick.js:80:11)
at process._tickCallback (internal/process/next_tick.js:104:9)
1) should refresh table
16 passing (3s)
1 failing
1) Loader Utilities should refresh table:
Error: Timeout of 2000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves.
I understand that the document is undefined and that I need to, somehow, create one myself, however, I believe that my main problems are:
My first time using npm and mocha and I cannot find anything related to it in their documentation.
Mostly, all problems people have regarding that are related to webbrowsers // I am using CLI, it will be tested with Travis on Github
In my code below you'll see that I solved a similar problem with XMLHttpRequest. However, I just can't figure out the best approach for including the document variable properly into my tests.
Thus, pardon me asking that shall this answer be already there on stackoverflow
My code is the following:
test-utilities.js
...
global.XMLHttpRequest = require('xmlhttprequest').XMLHttpRequest;
global.jsyaml = require('../src/scripts-min/js-yaml.min.js');
global.LinguistHighlighter = require('../src/scripts/ling-highlighter.js').LinguistHighlighter;
var LinguistLoader = require('../src/scripts/ling-loader.js').LinguistLoader;
describe('Loader', function () {
var utilities = new LinguistLoader.Utilities();
it('should refresh table', function(done) {
var location = {
hostname: "github.com",
href: "https://github.com/github-aux/linguist-unknown/blob/chrome/examples/Brain/human_jump.brain",
pathname: "/github-aux/linguist-unknown/blob/chrome/examples/Brain/human_jump.brain"
};
// check if it is not breaking
utilities.refresh(location, function(langObj, table){
done();
});
});
});
...
utilities.js:
...
Utilities.prototype.refresh = function(location, callback) {
var new_url = location.href;
if (new_url === current_url || !this.isGithub(location)) {
return;
}
current_url = new_url;
if (linguistObj === null) {
linguistObj = {
path: this.getPossibleFilepath(location)
};
}
setTimeout(function() {
var downloadHelper = new DownloadHelper();
downloadHelper.load(linguistObj.path, function(objs){
this.tryMatchUrlExtension(current_url, objs, function(langObj){
var table = document.getElementsByClassName("blob-wrapper")[0]
.getElementsByTagName("table")[0];
new LinguistHighlighter.Highlighter(langObj).draw(table);
// callback for tests purposes only
if (callback) {
callback(langObj, table);
}
});
}.bind(this));
}.bind(this), 100);
};
...
Any help is appreciated. Thank you!
I found a very good tool: JSDOM. Its goal is to emulate a subset of a web browser, such as the DOM. With that, I could implement my test-utilities.js file without even touching my utilities.js file, which is pretty much what I wanted.
Here goes the resolution of the file test-utilities.js
const jsdom = require("jsdom");
const { JSDOM } = jsdom;
global.XMLHttpRequest = require('xmlhttprequest').XMLHttpRequest;
global.jsyaml = require('../src/scripts-min/js-yaml.min.js');
global.LinguistHighlighter = require('../src/scripts/ling-highlighter.js').LinguistHighlighter;
var LinguistLoader = require('../src/scripts/ling-loader.js').LinguistLoader;
describe('Loader', function () {
var utilities = new LinguistLoader.Utilities();
it('should refresh the code table', function(done) {
// Download the HTML string and parse it to JSDOM
JSDOM.fromURL("https://github.com/github-aux/linguist-unknown/blob/chrome/examples/Brain/human_jump.brain").then(dom => {
// JSDOM does not support 'innerText' and that is why I am creating this property for all objects.
var o = Object.prototype;
Object.defineProperty(o, "innerText", {
get: function jaca() {
if (this.innerHTML === undefined)
return "";
return this.innerHTML;
}
});
var location = {
hostname: "github.com",
href: "https://github.com/github-aux/linguist-unknown/blob/chrome/examples/Brain/human_jump.brain",
pathname: "/github-aux/linguist-unknown/blob/chrome/examples/Brain/human_jump.brain"
};
// check if it is not breaking
utilities.refresh(location, function(langObj, table) {
done();
});
});
});
That is working properly now! I hope it helps anyone! :D

`importScript` unable to resolve dependency before serviceWorker push event

Unable to resolve dependency using importScript before serviceWorker push promise resolved; can somebody suggest better way to get this sorted?
ServiceWorker fetch event:
var iDB; // Global Variable
self.addEventListener("fetch", function(e) {
self.importScripts("dexie.min.js");
var o = new Dexie('database_name');
o.version(1).stores({
iDBStore: "++id,adspotkey"
iDB = o
});
});
Service Worker push Event
self.addEventListener("push", function(e) {
e.waitUntil(getDetails().then(function(ex) {
try {
e = wpSDK.iDB;
return e.iDBStore.where("id").above(0).toArray().then(function(ox) {
console.log(ox);
}
}));
});
Note:
On push console throws Error as shown below
TypeError: Cannot read property 'iDBStore' of undefined(…)
Using Wrapper for IndexedDB - Dexie.js
You should run importScripts on the background like this.
self.importScripts("dexie.min.js");
var iDB; // Global Variable
self.addEventListener("fetch", function(e) {
var o = new Dexie('database_name');
o.version(1).stores({
iDBStore: "++id,adspotkey"
iDB = o
});
});
importScripts is a synchronous method. As a result, when you run this method on background, it will block ServiceWorker from being active until the script is completely loaded.

local PDF file scraping in node.js

I have uploaded a pdf via a MEAN stack web application using fs. I want to extract certain fields from the pdf and display them on the web app. I have looked at a couple npm packages like pdf.js, pdf2json. I can't figure out the documentation and javascript callbacks used in the examples available. Please help!
I hope I can help answer your question. Using pdf2json can be used to parse a pdf and extract the text. There are a couple of steps that need to be taken to get it working. I have adapted the example from https://github.com/modesty/pdf2json.
The setup is to install pdf2json in the node app, and also underscore. The example page didn't explain the need to define your own callback functions. It also used self instead of this to register them. So, with the appropriate changes the code to extract all the text from the pdf will be something like this:
// Get the dependencies that have already been installed
// to ./node_modules with `npm install <dep>`in the root director
// of your app
var _ = require('underscore'),
PDFParser = require('pdf2json');
var pdfParser = new PDFParser();
// Create a function to handle the pdf once it has been parsed.
// In this case we cycle through all the pages and extraxt
// All the text blocks and print them to console.
// If you do `console.log(JSON.stringify(pdf))` you will
// see how the parsed pdf is composed. Drill down into it
// to find the data you are looking for.
var _onPDFBinDataReady = function (pdf) {
console.log('Loaded pdf:\n');
for (var i in pdf.data.Pages) {
var page = pdf.data.Pages[i];
for (var j in page.Texts) {
var text = page.Texts[j];
console.log(text.R[0].T);
}
}
};
// Create an error handling function
var _onPDFBinDataError = function (error) {
console.log(error);
};
// Use underscore to bind the data ready function to the pdfParser
// so that when the data ready event is emitted your function will
// be called. As opposed to the example, I have used `this` instead
// of `self` since self had no meaning in this context
pdfParser.on('pdfParser_dataReady', _.bind(_onPDFBinDataReady, this));
// Register error handling function
pdfParser.on('pdfParser_dataError', _.bind(_onPDFBinDataError, this));
// Construct the file path of the pdf
var pdfFilePath = 'test3.pdf';
// Load the pdf. When it is loaded your data ready function will be called.
pdfParser.loadPDF(pdfFilePath);
I am running the code out of my server side controller.
module.exports = (function() {
return {
add: function(req, res) {
var tmp_path = req.files.pdf.path;
var target_path = './uploads/' + req.files.pdf.name;
fs.rename(tmp_path, target_path, function(err) {
if (err) throw err;
// delete the temporary file, so that the explicitly set temporary upload dir does not get filled with unwanted files
fs.unlink(tmp_path, function() {
if (err) throw err;
//edit here pdf parser
res.redirect('#/');
});
})
},
show: function(req, res) {
var pdfParser = new PDFParser();
var _onPDFBinDataReady = function (pdf) {
console.log('Loaded pdf:\n');
for (var i in pdf.data.Pages) {
var page = pdf.data.Pages[i];
// console.log(page.Texts);
for (var j in page.Texts) {
var text = page.Texts[j];
// console.log(text.R[0].T);
}
}
console.log(JSON.stringify(pdf));
};
// Create an error handling function
var _onPDFBinDataError = function (error) {
console.log(error);
};
pdfParser.on('pdfParser_dataReady', _.bind(_onPDFBinDataReady, this));
// Register error handling function
pdfParser.on('pdfParser_dataError', _.bind(_onPDFBinDataError, this));
// Construct the file path of the pdf
var pdfFilePath = './uploads/Invoice_template.pdf';
// Load the pdf. When it is loaded your data ready function will be called.
pdfParser.loadPDF(pdfFilePath);
},
//end controller
}

Scraping fully rendered webpage with nodejs

I am trying to get amazon pricing information with nodejs.
Here's the target url:
http://aws.amazon.com/ec2/pricing/
But the content of the pricing tables which I am reading in nodejs is not fully rendered and there are only javascripts.
So far I have used jsdom, jquerygo and phantom but I was not successful. Even setting timeouts does not help. Can anyone please provide me with a working solution for this specific case?
Thanks and best regards.
There are different ways to scrape a web page using node.js
I was inspired by spookjs
var Spooky = require('spooky');
var spooky = new Spooky({
child: {
transport: 'http'
},
casper: {
logLevel: 'debug',
verbose: true
}
}, function (err) {
if (err) {
e = new Error('Failed to initialize SpookyJS');
e.details = err;
throw e;
}
spooky.start(
'http://en.wikipedia.org/wiki/Spooky_the_Tuff_Little_Ghost');
spooky.then(function () {
this.emit('hello', 'Hello, from ' + this.evaluate(function () {
return document.title;
}));
});
spooky.run();
});
spooky.on('error', function (e, stack) {
console.error(e);
if (stack) {
console.log(stack);
}
});
spooky.on('console', function (line) {
console.log(line);
});
spooky.on('hello', function (greeting) {
console.log(greeting);
});
spooky.on('log', function (log) {
if (log.space === 'remote') {
console.log(log.message.replace(/ \- .*/, ''));
}
});
Note: Gives flexibility to run casperjs and phantom js using node.js
This solved my issue:
I noticed that when installing phantom module in node, it was complaining about version of phantomjs (version 2) and was downloading version (1.9.8) in some temporary location.
Thus I installed version 1.9.8 instead and set the PATH variable to that. And it worked!
Also must note that inside page.open(...) function you must setTimeout for quite a long time (in my case about 35 seconds) so that the whole page is fully loaded and rendered.

Categories