Node render process gone while downloading files - javascript

I'm programming a small Electron program, that (currently) is able to download multiple big files from Google Drive. Some files are too big to be scanned so it tricks the server into believing the "Download anyway" button was clicked.
The problem is, that after 3, 7 or 12 files, the Chrome Dev Tools window shows "Render process gone".
3 files:
- download unblocking turned on while trying to download the same blocked file 3 times in a row. The 4th file crashes.
7 files:
- mixture of 5 non-blocked files and 2 blocked ones with download unblocking turned on. The 8th file (non-blocked) request crashes
12 files:
- downloading 9 non-blocked files and 3 blocked ones while download unblocking turned off
Therefore, I came to conclude, that "unblocking" a file takes up 3 of the 12 possible "slots". Then, everything (or at least the numbers) would make sense.
Unblocking works by sending another request with the same cookie char and a ?confirm=xxxx at the same url.
Note that the whole thing doesn't crash after 12 files if I use another file provider with 100mb test files.
It would be extremely helpful if you could point me in the right direction.
Here's a really simplified version of the code that's used. Note that a lot of variable declarations and other stuff is missing, but these parts seem to be the problematic ones:
// downloadmanager.js
function DownloadManager(pack) {
var _this = this;
this.downloadpackages = function (package, data, cb) {
sync.fiber(function () {
...
Object.keys(ht_distinct_urls).forEach(function (url) {
localfile = sync.await(_this.download(remotefile, sync.defer()));
console.log("Downloaded:" + localfile);
});
});
}
this.dl = function (remotefile, cb) {
request(request_options, (err, response, body) => {
// cb() in this location makes it crash at the 13th file
cb(null, "");
});
// cb() in this location doesnt make it crash (but also not download anything)
//cb(null, "");
}
this.download = function (remotefile, cb) {
// Try to download
_this.dl(remotefile, function (err, data) {
if (data.downloaded) { // It worked
cb(err, data);
} else if (data.unblocked) { // It was able to get a code to unblock it
_this.dl(data, cb); // Try again with the new cookie and the code
} else {
// Fck it, just return the data anyway for debugging
cb(err, data);
}
});
};
}
// renderer.js
sync.fiber(function () {
var pack = getPackage();
var dm = new DownloadManager(pack);
var download_result = sync.await(dm.downloadpackages(pack, ht_distinct_urls, sync.defer()));
console.log(download_result);
});

Okay, it seems like I found a solution while posting this question. I'll just post it anyway, maybe it helps someone else...
It seems like google blocks instant downloading. Adding a timeout of 5 seconds between downloads solved my problem. Haven't tested how low the timeout can be, but maybe the problem was solved by the settimeout callback anyway...

Related

copyFileSync not copying file and not throwing error

I'm running a function which I've written in JavaScript inside a nodejs/Electron client.
This function is meant to copy a file from the users flash drive to their c:/Windows/System32 (The file is being copied there so that it can be ran from Command Prompt manually next time the computer is touched without having to switch directories)
The problem is, the files are not being copied, and copyFileSync is not throwing an error.
Here is the code I'm specifically having a problem with:
try {
console.log('copying t.bat');
fs.copyFileSync(remote.app.getAppPath() + '\\app\\files\\scripts\\files\\t.bat', 'C:\\Windows\\System32\\t.bat');
} catch(err) {
console.log('could not copy t.bat', err);
$('#mfail_title').text('Could not copy t.bat file');
$('#mfail_data').text(err);
UIkit.modal("#master_fail").show();
return false;
}
As you can see, I have copyFileSync inside a TRY CATCH block. I know this code is running because in the console I get copying t.bat, but nothing else.
How can I get my files to copy, or at least throw an error when it cannot?
This client is running inside OOBE mode on various Windows 10 machines, therefore always has administrator access.
I've tried updating to the async version of copyFile, but I'm having the same issue. Here is my code
var source = remote.app.getAppPath() + '\\app\\files\\scripts\\files\\t.bat';
var destination = 'C:\\Windows\\System32\\t.bat';
fs.copyFile(source, destination, (err) => {
if (err) {
console.log(err);
} else {
source = remote.app.getAppPath() + '\\app\\files\\scripts\\files\\p.bat';
destination = 'C:\\Windows\\System32\\p.bat';
fs.copyFile(source, destination, (err) => {
if (err) {
console.log(err);
} else {
source = remote.app.getAppPath() + '\\app\\files\\scripts\\files\\p.bat';
destination = 'C:\\Windows\\System32\\p.bat';
child = spawn("powershell.exe",['-ExecutionPolicy', 'ByPass', '-File', remote.app.getAppPath() + '\\app\\files\\scripts\\' + type + '.ps1']);
}
});
}
});
This should copy a file, then when it's complete it should copy another file, once that is complete, it should run a powershell script.
Each copyFile checks for an error before moving on, but it never throws an error, and the file is never copied.
I had a similar issue earlier, In which an Antivirus(Comodo) was not allowing electron app to access the hard drive.
Copy and other file operations were successful in that case as well, because electron in such case access the corresponding sandbox
Please check this is not the case with you.
You can actually access 'fs' in console from electron and check other things in the file system.
Looks to me as if you're using fs on then renderer process (client side) which will not work (assuming that your fs is the node.js fs module and (*)). Your first script seems to use jQuery (hints for renderer) and the second one uses remote in the first line.
fs can only (*) be used on the main process and you'll need to create an IRC channel and do something like:
ircRenderer.sendSync('copy-file-sync', {from: '/from/path', to: '/to/path'})
and, of course, implement the handler for that quickly invented 'copy-file' channel on the main process.
(*) Edit: I haven't played around a lot with nodeIntegration = true, so fs may or may not work on the renderer process with that flag set on the BrowserWindow. But the irc messaging should definitely work and if not, the problem is outside electron, probably related to file permissions.

Electron - Invalid package on unzip

For around 3 weeks I've been working on an Electron app and finally decided to get around to adding update checking. For my research, the standard way to do this in Electron (using Squirrel) requires the user to physically install the application onto their computer. I would rather not do this, and keep everything as portable as possible. I then decided to try making my own update script by having the program download the update.zip, and extract it to overwrite the existing files. This works well, up until the very end. At the very end of the extraction, I receive a Invalid package error, and the actual app.asar file is missing, rendering the application useless.
I am using this to download and extract the updates:
function downloadFile(url, target, fileName, cb) { // Downloads
var req = request({
method: 'GET',
uri: url
});
var out = fs.createWriteStream(target+'/'+fileName);
req.pipe(out);
req.on('end', function() {
unzip(target+'/'+fileName, target, function() {
if (cb) {
cb();
}
});
});
}
function unzip(file, target, cb) { // Unzips
var out = fs.createReadStream(file);
out.pipe(unzipper.Extract({ path: target })).on('finish', function () {
dialog.showMessageBox({
type: 'question',
message: 'Finished extracting to `'+target+'`'
});
if (cb) {
cb();
}
});
}
And call it with:
downloadFile('http://example.com/update.zip', path.join(__dirname, './'), 'update.zip', function() { // http://example.com/update.zip is not the real source
app.relaunch();
app.quit();
});
And I use the unzipper NPM package (https://www.npmjs.com/package/unzipper).
The code works perfectly for all other zips, but it fails when trying to extract a zip containing an Electron app.
Anything I'm doing wrong, or maybe a different package that properly supports extracting zips with .asar files?
Edit 1
I just found https://www.npmjs.com/package/electron-basic-updater, which does not throw the same JavaScript error however it still does not extract the .asar files correctly, and will throw it's own error. Since the .asar is still missing, the app is still useless after the "update"
Thanks to your link to electron-basic-updater, I have found this issue mentioned there: https://github.com/TamkeenLMS/electron-basic-updater/issues/4.
They refer to the issue in the electron app: https://github.com/electron/electron/issues/9304.
Finally, in the end of the second topic there's a solution:
This is due to the electron fs module treating asar files as directories rather than files. To make the unzip process work you need to do one of two things:
Set process.noAsar = true
Use original-fs instead of fs
I have seen the people working with original-fs. But it looked like a big trouble to me.
So I tried setting process.noAsar = true (and then process.noAsar = false after unzipping) - and that worked like a charm.

Retrieve html content of a page several seconds after it's loaded

I'm coding a script in nodejs to automatically retrieve data from an online directory.
Knowing that I had never done this, I chose javascript because it is a language I use every day.
I therefore from the few tips I could find on google use request with cheerios to easily access components of dom of the page.
I found and retrieved all the necessary information, the only missing step is to recover the link to the next page except that the one is generated 4 seconds after loading of page and link contains a hash so that this step Is unavoidable.
What I would like to do is to recover dom of page 4-5 seconds after its loading to be able to recover the link
I looked on the internet, and much advice to use PhantomJS for this manipulation, but I can not get it to work after many attempts with node.
This is my code :
#!/usr/bin/env node
require('babel-register');
import request from 'request'
import cheerio from 'cheerio'
import phantom from 'node-phantom'
phantom.create(function(err,ph) {
return ph.createPage(function(err,page) {
return page.open(url, function(err,status) {
console.log("opened site? ", status);
page.includeJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function(err) {
//jQuery Loaded.
//Wait for a bit for AJAX content to load on the page. Here, we are waiting 5 seconds.
setTimeout(function() {
return page.evaluate(function() {
var tt = cheerio.load($this.html())
console.log(tt)
}, function(err,result) {
console.log(result);
ph.exit();
});
}, 5000);
});
});
});
});
but i get this error :
return ph.createPage(function (page) {
^
TypeError: ph.createPage is not a function
Is what I am about to do is the best way to do what I want to do? If not what is the simplest way? If so, where does my error come from?
If You dont have to use phantomjs You can use nightmare to do it.
It is pretty neat library to solve problems like yours, it uses electron as web browser and You can run it with or without showing window (You can also open developer tools like in Google Chrome)
It has only one flaw if You want to run it on server without graphical interface that You must install at least framebuffer.
Nightmare has method like wait(cssSelector) that will wait until some element appears on website.
Your code would be something like:
const Nightmare = require('nightmare');
const nightmare = Nightmare({
show: true, // will show browser window
openDevTools: true // will open dev tools in browser window
});
const url = 'http://hakier.pl';
const selector = '#someElementSelectorWitchWillAppearAfterSomeDelay';
nightmare
.goto(url)
.wait(selector)
.evaluate(selector => {
return {
nextPage: document.querySelector(selector).getAttribute('href')
};
}, selector)
.then(extracted => {
console.log(extracted.nextPage); //Your extracted data from evaluate
});
//this variable will be injected into evaluate callback
//it is required to inject required variables like this,
// because You have different - browser scope inside this
// callback and You will not has access to node.js variables not injected
Happy hacking!

ng-webworker - IE 11 errors

So I'm using a library called ng-webworker and attempting to run a very simple long running task.
$scope.onParallelDownload = function() {
function doubler(num) {
return num * 2;
}
var myWorker = webWorker.create(doubler);
myWorker.run(3).then(function(result) {
alert("Answer: " + result);
}, function(error) {
var err = error;
});
}
This works perfectly in Chrome and shows the alert, but when run in Internet Explorer 11, where I am debugging it the error function is hit, which was still promising, however, there is no data given in the error payload which is problematic because I've absolutely no idea what is causing the web worker to fail on that particular browser.
Most likely you did not set the path to the file worker_wrapper.min.js (or worker_wrapper.js). This file is required for IE (see below). Adjust your app config to the following:
angular.module('myApp', [
// your dependencies
'ngWebworker'
])
.config(['WebworkerProvider', function (WebworkerProvider) {
WebworkerProvider.setHelperPath("./bower_components/ng-webworker/src/worker_wrapper.min.js"); // adjust path
}]);
This code assumes you installed ngWebworker with bower. You might still have to adjust the path, depending on the path you are in.
If you've already set the helper path but it still does not work, check if helper file is being loaded in the developer tools (you might have set a wrong path and get a 404).
Details
When passing a function to Webworker, it transforms this function into a blob which is then executed by the web worker as if it were an independent file. However, Internet Explorer treats these blobs as cross-domain, so this does not work. The workaround that ngWebworker uses is to run an independent JavaScript file (the worker_wrapper.min.js we set above). The web worker then runs that file and ngWebworker passes your stringified function to the worker where it is evaluated.
Note that if you're not on IE, this file will not be used.

Read files from SD Card with mosync/phonegap on Android with pure Javascript

I know this question was asked many times before but I cannot figure it out anyhow.
I'm able to write a text file to a directory on the sdcard (android 4), but no way to read it back again, using this code:
function get_file () {
window.requestFileSystem(LocalFileSystem.PERSISTENT, 0, getFilesystem, fail);
}
function getFilesystem(fs) {
alert("getFilesystem -> backup.txt"); // OK !
// alert("filesystem.name: "+fileSystem.name); // = persistent
// alert("filesystem.root.name: "+fileSystem.root.name); // = long number
//
fs.root.getFile("../../../../../../sdcard/test/backup.txt", {create: false, exclusive: false},
function(fileEntry) {
alert(fileEntry.fullPath); // shows that my path is appended to "data/.."
fileEntry.file(function(file) {
var reader = new FileReader();
reader.onloadend = function(evt) {
alert(+evt.target.result); // NOT SHOWING !
};
reader.readAsText(file);
}, fail);
}, fail);
}
I wrote the file using the writer function into the directory sdcard/test using this sequence of ../ - this is an ugly code but working !
But fs.root.getFile does not work in the same way - the fullPath information it returns shows that my path given is APPENDED to "/data/data/com.appname/files" but does not replace it !
The onloadend function obviously isn't working since i never got the alert message, neither did I get an error message.
Change of path to "file:///sdcard/test" or "sdcard/test" has no effect either.
Any help is highly appreciated - thank you in advance !
Chris
Please refer PHONEGAP DOCUMENT
From this you will get working example of getting file from sdcard.
And if you have problem in downloading a file please refer LINK
I think in Place of "/sdcard/backup.txt" try to use only "backup.txt".
It seems to work fine.
PhoneGap takes care of the details of the path ,also works with directories.
So finally I could figure it out:
To access the directory (on Android 4.0, Samsung Note 8) I had to go up literally all directories from /data/data/com.app-name/files/apps/(random no.)/ back to the sdcard by choosing:
fs.root.getFile("../../../../../../sdcard/test/backup.txt", ...
in mosync Reload client.
"Compiling" the app with mosync apk would require only 4 times "../" because the file hierarchy there is lower than in the Reload Client. In any case have to check in file explorer (root access required).
Different from that is when you compile in the cloud with phonegap - then indeed the root path is sdcard ! You then cannot go to the data directory as far as I could find out.
This path works with write and read in mosync - my mistake was that I had line break character "\n" inside the text file. In this case I could write it, but the reader stops without error message.
Hope that will help someone else as well !
Chris

Categories