My issue is downloading images with unknown extension( it may be 'png' or 'jpg' or 'bmp' or etc...).And I have some troubles with function chekHead's returning value:
var fs = require('fs'),
request = require('request');
var processImg = function (uri,filename){
if(checkHead(uri + 'png') > 2000){
download(uri + 'png', filename + '.png', function(){
console.log(uri + 'png' + " - downloaded")
})
}else if(checkHead(uri + 'jpg') > 2000){
download(uri + 'jpg', filename + '.jpg', function(){
console.log(uri + 'jpg' + " - downloaded")
})
}else if(checkHead(uri + 'bmp') > 2000) {
download(uri + 'bmp', filename + '.bmp', function () {
console.log(uri + 'bmp' + " - downloaded")
})
}
}
var checkHead = function(uri){
var length;
request.head(uri, function(err, res, body){
if(err) return console.log("Error");
length = res.headers['content-length'];
console.log(length);
});
return length;
}
var download = function(uri, filename, callback){
request(uri).pipe(fs.createWriteStream('./static/' + filename).on('close', callback));
};
So in checkHead function return length; always returns 'underfined', but console.log returns valid number; Why?
NodeJS executes your code in an asynchronous way using callbacks. Your return could happen before (in this case it's probably always the case) the callback is completed. The variable length is at the return undefined, because it hasn't received any values.
You could use promises to chain the function or you structure your code in another way.
For promises see e.g.:
async
q
var checkHead = function(uri){
var length;
// Callback is probably invoked after the return
request.head(uri, function(err, res, body){
if(err) return console.log("Error");
length = res.headers['content-length'];
console.log(length);
});
// gets executed directly
return length;
}
You have to use a callback so that is works the way you want it:
var checkHead = function(uri,callback){
request.head(uri, function(err, res, body){
if(err) return console.log("Error");
var length = res.headers['content-length'];
console.log(length);
callback(length);
});
};
Unfortunately because of your if-else logic I see no way at the moment to use promises(jquery) instead of callbacks and nested callbacks that can lead to callback-hell that is kind of bad pattern so I say sorry for this:
checkHead(uri + 'png',function(length){
if(length > 2000){
download(uri + 'png', filename + '.png', function(){
console.log(uri + 'png' + " - downloaded")
});
}
else{
checkHead(uri + 'jpg',function(length){
if(length > 2000){
download(uri + 'jpg', filename + '.jpg', function(){
console.log(uri + 'jpg' + " - downloaded")
});
}
else{
checkHead(uri + 'bmp',function(length){
if(length > 2000){
download(uri + 'jpg', filename + '.jpg', function(){
console.log(uri + 'jpg' + " - downloaded")
});
}
});
}
});
}
});
BUT
EcamScript 6 takes care of that. This is a good article about Generator Functions. And the main idea is to use yield for asynchronous methods or functions like request.head:
var checkHead = function*(uri){
var length = yield request.head(uri);
};
And use next to get length:
checkHead.next();//{value: 123, done: true}
This is only my concept i did not prove this but Generator Functions notations work in this way :)
Related
I have my selenium test set up to take screenshots, but they are not saving to the directory which I have specified. Can anybody show me what I am missing?
Here is how I am configuring the screenshots in the test:
function writeScreenshot(data, name) {
var fs = require('fs');
name = name || 'ss.png';
var screenshotPath = mkdirp(configuration.readSettings('screenshotDirectory') + fileNameURL + "/", function(err){});
fs.writeFileSync(screenshotPath + name, data, 'base64');
};
and then I take the screenshot:
driver.takeScreenshot().then(function(data) {
var screenshotFile = os + '_' + osVersion + '_' + browser + '_' + browserVersion + '.png';
writeScreenshot(data, screenshotFile);
});
The screenshots end up being saved instead in the projects root directory and with the file name preceded by 'undefined'. (ex. undefinedWindows_8_chrome_46.png)
It does, however, create the folders shown here: var screenshotPath = mkdirp(configuration.readSettings('screenshotDirectory') + fileNameURL + "/", function(err){});
So why is this happening?
mkdirp() is an async method. That is why you pass a callback. You will need to change your code to something like the following:
function writeScreenshot(data, name) {
var fs = require('fs');
name = name || 'ss.png';
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
mkdirp(screenshotPath, function(err){
if (err) {
// something else happened while creating the dir. You decide what to do
return;
}
// Otherwise (if dir was created)
fs.writeFileSync(screenshotPath + name, data, 'base64');
});
};
mkdirp() function is asynchronous - it creates a directory and returns nothing - this is why you having that leading undefined in the filename.
Save the file in the callback:
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
mkdirp(screenshotPath, function (err) {
if (!err) {
fs.writeFileSync(screenshotPath + name, data, 'base64');
} else {
// handle error
}
});
Or, synchronously create the directory and write to it this way:
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
if (mkdirp.sync(screenshotPath)) {
fs.writeFileSync(screenshotPath + name, data, 'base64');
}
I got some troubles with my image downloader script, so I have array with images names (size around 5000 elements), and I do for cycle in its array, and for every iteration download image with requests module.
All works fine but only then I have arrays with size not bigger then 500+- elements.
If I run script with 5000+ elements, I see many errors spamed from request module(err or underfined responce object) , and finnaly all application could fails with EMPTY FILE ERROR. I think that there some async troubles cause NODE.JS didnt handles so many operation at time.
Maybe I can solve it by splitting my large 5000 size array by 300 items, and dont iterate over(and dont call fetchImage()) on the next chunck before previous chunk. Or maybe there exists more nicer way to solving my problem.?
products.map(function (product) {
fetchImage(product.imageUrl,'./static/' + product.filename + '_big.', 0, 0);
return;
});
function fetchImage(url, localPath, index, iteration) {
var extensions = ['jpg', 'png', 'jpeg', 'bmp' , ''];
if (iteration > 2 || index === extensions.length) { // try another extensions
iteration++;
if(iteration < 3) {
setTimeout(function(){
fetchImage(url, localPath, 0, iteration);
}, 3000);
}else{
console.log('Fetching ' + url + ' failed or no image exists ');
return;
}
return;
}
var fileExtension;
if(extensions[index] === '' ) {
fileExtension = extensions[0];
}else{
fileExtension = extensions[index];
}
request.get(url + extensions[index], function(err,response,body) {
if(err || undefined === response){ // if err try after 3 sec timeout
setTimeout(function(){
console.log('Error URL : ' + url + extensions[index]);
fetchImage(url, localPath, index, iteration);
}, 3000);
return;
}else{
if(response.statusCode === 200) {
request(url + extensions[index])
.on('error', function(err) {
console.log("ERRRRRROR " + url + extensions[index] + " " + err);
setTimeout(function(){
console.log('Error URL : ' + url + extensions[index]);
fetchImage(url, localPath, index, iteration);
}, 3000);
return;
})
.pipe(fs.createWriteStream(localPath + fileExtension ));// write image to file
console.log('Successfully downloaded file ' + localPath + fileExtension);
return;
}else {
fetchImage(url, localPath, index + 1, iteration);
}
}
});
};
Fixed using setTimeOut beetween each request
setTimeout(
function () {
fetchImage(imageUrl,'./static/' + filename + '_big.', 0, 0);
},
300 * (i + 1) // where they will each progressively wait 300 msec more each
);
i'm using a recursive readdir to read the whole file tree of a directory to put it into a database.
My problem is that i'm trying to stop the next lines of code while the whole readdir / insert to the database isn't finished.
I was looking for a solution in promises but at the first call of my function (so in the first folder of the tree) the promise is fulfilled...
Any idea ?
function readsousdir(path, db, db2) {
var Datastore = require('nedb');
var fs = require('fs');
fs.readdir(path + '\\', function (err, files) {
files.forEach(function (file) {
fs.stat(path + '\\' + file, function (err, stats) {
var foldertrue = stats.isDirectory();
var filetrue = stats.isFile() == true;
if (foldertrue == true) {
var doc;
doc = folderdb(path + '\\' + file);
db2.insert(doc);
readsousdir(path + '\\' + file, db, db2);
}
if (filetrue) {
doc = pistedb(path + '\\' + file, []);
db.insert(doc);
}
});
});
});
}
Using BlueBird, you could use reduce:
var fs = Promise.promisifyAll(require("fs"));
function readsousdir(path, db, db2) {
var Datastore = require('nedb');
return fs.readdirAsync(path + '\\').reduce(function(_, file){
return fs.statAsync(path + '\\' + file)
.then(function(stats){
var foldertrue = stats.isDirectory();
var filetrue = stats.isFile() == true;
if (foldertrue == true) {
var doc;
doc = folderdb(path + '\\' + file);
db2.insert(doc);
return readsousdir(path + '\\' + file, db, db2)
}
if (filetrue) {
doc = pistedb(path + '\\' + file, []);
db.insert(doc);
}
});
});
}
Supposing your db library returns promises and you want to wait for the insertion, you would do
function readsousdir(path, db, db2) {
var Datastore = require('nedb');
return fs.readdirAsync(path + '\\').reduce(function(_, file){
return fs.statAsync(path + '\\' + file)
.then(function(stats){
var foldertrue = stats.isDirectory();
var filetrue = stats.isFile() == true;
if (foldertrue == true) {
var doc;
doc = folderdb(path + '\\' + file);
return db2.insert(doc).then(function(){
return readsousdir(path + '\\' + file, db, db2)
});
}
if (filetrue) {
doc = pistedb(path + '\\' + file, []);
return db.insert(doc);
}
});
});
}
i am developing a way to get callbacks in the browser page, following a emit to the socketio server.
server code:
/** exec php file and retrieve the stdout */
socket.on('php', function(func, uid, data) {
console.log('php'.green + ' act:' + func + ' uid:' + uid);
runner.exec("php " + php_[func].exec,
function(err, stdout, stderr) {
if (err == null) {
socket.emit('php', {uid: uid, o: stdout});
console.log('emitted');
} else {
console.log('err '.red + stdout + ' ' + stderr);
}
});
});
this code executes a php page and retrieves the output to display or parse in the browser
it receives an id to echo back to the page, so I can know what function to execute
browser code to execute callbacks:
function log(text) {
$('.out').append(text + '<br />');
}
window.callbacks = [];
function setcb(c) {
var max = 0;
$.each(window.callbacks, function(index) {max = (index > max ? index : max);});
window.callbacks[max+1] = c;
return max+1;
};
function C(k){return(document.cookie.match('(^|; )'+k+'=([^;]*)')||0)[2]}
var s = io.connect("http://"+ window.location.host +":8088");
//s.emit('debug', 'here');
s.on('getid', function(){
console.log('getid cookieis: ' + C('igr') );
s.emit('setid', C('igr'));
});
s.emit('php', 'test',
setcb(
function () {
var data = JSON.parse(this);
log('callback passed' + this);
}
), null
);
s.on('php', function(data) {
//window.callbacks[j.uid].call(j.o);
log('rec' + JSON.stringify(data));
//var jsn = JSON.parse(data);
console.log(data);
console.log(window.callbacks[data.uid]);
window.callbacks[data.uid].call(data.o);
delete window.callbacks[data.uid];
window.callbacks.splice(data.uid, 1);
console.log(window.callbacks);
});
this is working, but when I try to make two requests at the same time, it doesn't run like expected, leaving one callback to execute and in the callbacks array.
test code:
s.emit('php', 'test',
setcb(
function (data) {log('callback passed' + this);}
), null
);
s.emit('php', 'test',
setcb(
function (data) {log('callback passed' + this);}
), null
);
I want to eliminate this error, and for each event received, execute the callback I define.
This is way more simple than I've imagined
You can pass by reference the callback.
server side code:
/** exec php file and retrieve the stdout */
socket.on('php', function(func, data, callback) {
console.log('php'.green + ' act:' + func);
runner.exec("php " + php_[func].exec,
function(err, stdout, stderr) {
if (err == null) {
callback(stdout);
//socket.emit('php', {uid: uid, o: stdout});
console.log('emitted');
} else {
console.log('err '.red + stdout + ' ' + stderr);
}
});
});
client side code:
function log(text) {
$('.out').append(text + '<br />');
}
function C(k){return(document.cookie.match('(^|; )'+k+'=([^;]*)')||0)[2]}
var s = io.connect("http://"+ window.location.host +":8088");
//s.emit('debug', 'here');
s.on('getid', function(){
console.log('getid cookieis: ' + C('igr') );
s.emit('setid', C('igr'));
});
s.emit('php', 'test', null,
function(data) { log('d: ' + JSON.stringify(data)); }
);
I'm working on an app which needs to download some files from a webserver and store them on the device so they can be used offline. The code I'm using works fine in Android and iOS, however I'm having difficulty downloading more than one file on blackberry 10.
From looking at the web console it seems that the success callback is executed successfully for the first file, but not for any of the subsequent files. The fail callback is not being called for the subsequent files either.
I'm using cordova 2.9, and I've included all the required webworks plugins. I have set up access to my domain in config.xml and have set the access_shared permission.
I've also edited the cordova FileTransfer.download function to call the webworks download api "blackberry.io.filetransfer.download".
Below is the JavaScript code that I've written.
// Wait for device API libraries to load
//
document.addEventListener("deviceready", onDeviceReady, false);
// device APIs are available
//
function onDeviceReady() {
//replace list with your files you wish to download
var files = ["files/file1.txt", "files/file2.txt", "files/dir1/file3.txt"];
initiateDownload(files);
//getFSRoot(files);
}
var filesToDownload = 0;
var fileList = new Array();
var failedFileList;
var numFailedFiles = 0;
var currentFileIndex;
var retryCount = 0;
var root;
function isOnline(){
var bIsOnline = false;
if (navigator.connection.type != Connection.NONE && navigator.connection.type != Connection.UNKNOWN)
{
bIsOnline = true;
}
return bIsOnline;
}
function initiateDownload(files){
alert("initiate download");
failedFileList = new Array();
filesToDownload = files.length;
blackberry.io.sandbox = false;
for (i = 0; i < files.length; i++)
{
currentFileIndex = i;
console.log("initiate download of file index " + i + " File Name: " + files[i]);
getFile(files[i]);
}
}
function getFile(filePath)
{
//TODO: Do we need to make this function recursive to serialize asynchronous downloads?
console.log("START of function getFile() for " + filePath);
window.requestFileSystem(
LocalFileSystem.PERSISTENT, 0,
function onFileSystemSuccess(fileSystem)
{
console.log("Success getting filesystem for filePath: " + filePath);
createDirs(fileSystem.root, filePath, -1);
},
function(error){
console.log("Failed to get the filesystem for filePath: " + filePath);
}
);
}
function createDirs(parentDir, filePath, index)
{
console.log("createDirs params ===> parentDir=" + parentDir.toURL() + " filePath=" + filePath + " index=" + index);
arrDirs = filePath.split("/");
if (index >= (arrDirs.length - 1))
{
createFile(arrDirs[index], parentDir, filePath);
}
else
{
dirName = "myapp";
if (index >= 0)
{
dirName = arrDirs[index];
}
//if device is Blackberry, build up a full directory path as we are trying to install outside of sandbox
var path, dirToCreate = ""
if(device.platform = "BlackBerry"){
path = "myapp/";
console.log("JHPaths ======> arrDirs = " + arrDirs + " index = " +index);
for (i = 0; i <= index; i++){
path += arrDirs[i] + "/";
console.log("path = " + path + " i = " + i + " index = " + index);
}
dirToCreate = blackberry.io.home + "/" + path;
dirToCreate = dirToCreate.substring(0, dirToCreate.length - 1);
console.log("JHPaths Trying to create " + dirToCreate);
dirName = dirToCreate;
}
parentDir.getDirectory(dirName, {create: true, exclusive: false},
function (directoryEntry) {
console.log("Got directory " + directoryEntry.fullPath);
createDirs(directoryEntry, filePath, index + 1);
},
function (error) {console.log("Failed to get directory " + dirName + " Error code : " + error.code);});
}
}
function createFile(fileName, parentDir, filePath)
{
parentDir.getFile(
fileName, {create: true, exclusive: false},
function gotFileEntry(fileEntry)
{
localPath = fileEntry.fullPath;
if (isOnline())
{
console.log("Before remove");
fileEntry.remove(
function(){
console.log("FileEntry remove() was successful");
},
function(){
console.log("FileEntry remove() was failed");
}
);
console.log("After remove");
var fileTransfer = new FileTransfer();
//replace URL with the URL code you wish to download from
var baseURL = "http://<ip-address>/WebFolder/";
var uri = encodeURI(baseURL + filePath);
fullFilePath = blackberry.io.home + "/myapp/" + filePath;
fileTransfer.download(
uri,
/*localPath,*/
fullFilePath,
function(entry) {
console.log("download complete: " + entry.fullPath);
/*if(currentFileIndex < fileList.length){
currentFileIndex++;
}*/
if (device.platform == "iOS")
{
console.log("Setting file metadata");
parentDir.getFile(
fileName, {create:false, exclusive: false},
function gotFileEntry(fileEntry)
{
fileEntry.setMetaData(
function(){console.log("Set metadata for " + fileEntry.fullPath)},
function(){console.log("Set metadata failed for " + fileEntry.fullPath)},
{"com.apple.MobileBackup": 1}
);
},
function getFileFailed(fileError)
{
console.log("Get file failed:" + fileError.code);
}
);
}
filesToDownload--;
if (filesToDownload == 0 && failedFileList.length == 0)
{
//Call to retry failed Files
}
else if(failedFileList.length > 0){
if(retryCount < 1){
retryFailedFiles();
}
else{
}
}
},
function(error) {
console.log("Cache file download error source " + error.source);
console.log("Cache file download error target " + error.target);
console.log("Cache file download error code " + error.code);
//failedFileList[numFailedFiles++] = filePath;
failedFileList[numFailedFiles++] = filePath;
filesToDownload--;
if (filesToDownload == 0 && failedFileList.length == 0)
{
}
else if(failedFileList.length > 0){
if(retryCount < 1){
retryFailedFiles();
}
else{
}
}
}
);
}
},
function getFileFailed(fileError)
{
console.log("Create file failed:" + fileError.code);
filesToDownload--;
/*if (filesToDownload == 0)
{
callbackDeferred.resolve();
}*/
}
);
}
function retryFailedFiles(){
console.log("Retrying failed files");
retryCount++;
initiateDownload(failedFileList);
}