I'm using the following open source for unzip the files which is working OK.
Now I want to create some integration test for it to see that the file was saved,the problem is that the process is trowing events and I don't know for sure when the process end,any idea how to solve this?
ex
var DecompressZip = require('decompress-zip');
var unzipper = new DecompressZip(filename)
unzipper.on('error', function (err) {
console.log('Caught an error');
});
unzipper.on('extract', function (log) {
console.log('Finished extracting');
});
unzipper.on('progress', function (fileIndex, fileCount) {
console.log('Extracted file ' + (fileIndex + 1) + ' of ' + fileCount);
});
unzipper.extract({
path: 'some/path',
filter: function (file) {
return file.type !== "SymbolicLink";
}
});
I use mocha & chai as test framework.
and this is the open source
https://github.com/bower/decompress-zip
Mocha support asynchronous code you need to call test case with done argument and then once all checks is completed call done function.
Example:
var DecompressZip = require('decompress-zip');
describe("Unzipper test", function() {
it("should extract folder without error", function(done) {
var unzipper = new DecompressZip(filename);
unzipper.on('error', done); // Pass error to done callback
unzipper.on('extract', function (log) {
// Add your checks
done();
});
});
unzipper.extract({
path: 'some/path',
filter: function (file) {
return file.type !== "SymbolicLink";
}
});
});
Related
I'm new to Node.js and jasmine, and my JavaScript experience is old and rusty, so I'm a newbie there too. I finished Manuel Kiessling's book, The Node Beginner Book, and I am working my way through his second book, The Node Craftsman Book. I'm stuck on the FilesizeWatcher tutorial. I've been able to run earlier tests but this one is not working. There is a similar question on SO: No output from jasmine-node but the answer isn't working for me.
I'll post my code here and hopefully somebody can tell me what I'm doing wrong.
FilesizeWatcherSpec.js:
'use strict';
var FilesizeWatcher = require('./FilesizeWatcher');
var exec = require('child_process').exec;
describe('FilesizeWatcher', function() {
var watcher;
afterEach(function() {
watcher.stop();
});
it('should fire a "grew" event when the file grew in size', function(done) {
var path = './var/tmp/filesizewatcher.test';
exec('rm -f ' + path + ' ; touch ' + path, function() {
watcher = new FilesizeWatcher(path);
watcher.on('grew', function(gain) {
expect(gain).toBe(5);
done();
});
exec('echo "test" > ' + path, function(){});
});
});
it('should fire a "shrank" event when the file shrank in size', function(done) {
var path = './var/tmp/filesizewatcher.test';
exec('rm -f ' + path + ' ; echo "test" > ' + path, function() {
watcher = new FilesizeWather(path);
watcher.on('shrank', function(loss) {
expect(loss).toBe(3);
done();
});
exec('echo "a" > ' + path, function(){});
});
});
it('should fire an "error" if path does not start', function(done) {
var path = 'var/tmp/filesizewatcher.test';
watcher = new FilesizeWather(path);
watcher.on('error', function(err) {
expect(err).toBe('Path does not start with a slash');
done();
});
});
});
FilesizeWatcher.js:
'use strict';
var fs = require('fs');
var util = require('util');
var EventEmitter = require('events').EventEmitter;
var FilesizeWatcher = function (path) {
var self = this;
if (/^\//.test(path) === false) {
process.nextTick(function() {
self.emit('error', 'Path does not start with a slash');
});
return;
}
fs.stat(path, function (err, stats) {
console.log('stats= ' + stats);
self.lastfilesize = stats.size;
});
self.interval = setInterval(
function () {
console.log('We are in function()');
fs.stat(path, function (err, stats) {
if (stats.size > self.lastfilesize) {
self.emit('grew', stats.size - self.lastfilesize);
self.lastfilesize = stats.size;
}
if (stats.size < self.lastfilesize) {
self.emit('shrank', self.lastfilesize - stats.size);
self.lastfilesize = stats.size;
}
}, 1000);
});
};
util.inherits(FilesizeWatcher, EventEmitter);
FilesizeWatcher.prototype.stop = function () {
clearInterval(this.interval);
};
module.exports = FilesizeWatcher;
Console output:
C:\Users\pdl\Projects\NodeCraftsman>jasmine-node ./FilesizeWatcherSpec.js
C:\Users\pdl\Projects\NodeCraftsman>
Other tests run fine:
C:\Users\pdl\Projects\NodeCraftsmanTestDrivenDevelopment>jasmine-node spec\greetSpec.js
..
Finished in 0.006 seconds
2 tests, 2 assertions, 0 failures, 0 skipped
C:\Users\pdl\Projects\NodeCraftsmanTestDrivenDevelopment>
I added --captureExceptions to see if I could get any information and I got the TypeError: self.callbacks.error is not a function.
My first problem was as Eppilo suggested below, that I needed to use process.nextTick on self.callbacks'error'. Mixing async code with sync code causes the error event to be fired before the error handler is registered. So I made the changes and am now using the EventEmitter but I'm still getting the following errors:
If I include the "." in the path: var path = './var/tmp/filesizewatcher.test'; then the file gets written. Otherwise, it does not.
If the file does NOT get written, stats= undefined and I receive this error:
TypeError: Cannot read property 'size' of undefined
at C:\Users\pdl\Projects\NodeCraftsman\FilesizeWatcher.js:19:34
at FSReqWrap.oncomplete (fs.js:82:15)
If the file DOES get written, then I receive this error:
Error: Uncaught, unspecified "error" event. (Path does not start with a slash)
at emit (events.js:144:17)
at C:\Users\pdl\Projects\NodeCraftsman\FilesizeWatcher.js:12:18
at nextTickCallbackWith0Args (node.js:419:9)
at process._tickCallback (node.js:348:13)
Of course, it's not supposed to start with a slash. That is the test. But when I remove the --captureExceptions from the command, I still get no output.
First of all try and run Jasmine on verbose mode and capture exceptions:
jasmine-node ./FilesizeWatcherSpec.js --verbose --captureExceptions
Link: https://github.com/mhevery/jasmine-node/wiki/Command-Line-Usage
Also try to make the error checking asynchronous:
if (/^\//.test(path) === false) {
process.nextTick(function() {
self.callbacks['error']('Path does not start with a slash');
});
return;
}
Newbie as well, with not enough reputation to comment.
I got the same no output on my Mac too, and was able to get the test to work with this.
There is an error in FilesizeWatcher.js.
Currently:
self.interval = setInterval(
function (){
...
fs.stat(path, function (err, stats) {
...
}, 1000);
});
It should instead be:
self.interval = setInterval(
function (){
...
fs.stat(path, function (err, stats) {
...
});
},1000);
Just sharing my findings, cheers.
I am unsuccessfully trying to write to the file system of an aws lambda instance. The docs say that a standard lambda instance has 512mb of space available at /tmp/. However the following code that runs on my local machine isn't working at all on the lambda instance:
var fs = require('fs');
fs.writeFile("/tmp/test.txt", "testing", function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
The code in the anonymous callback function is never getting called on the lambda instance. Anyone had any success doing this? Thanks so much for your help.
It's possible that this is a related question. Is it possible that there is some kind of conflict going on between the s3 code and what I'm trying to do with the fs callback function? The code below is what's currently being run.
console.log('Loading function');
var aws = require('aws-sdk');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });
var fs = require('fs');
exports.handler = function(event, context) {
//console.log('Received event:', JSON.stringify(event, null, 2));
// Get the object from the event and show its content type
var bucket = event.Records[0].s3.bucket.name;
var key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
var params = {
Bucket: bucket,
Key: key
};
s3.getObject(params, function(err, data) {
if (err) {
console.log(err);
var message = "Error getting object " + key + " from bucket " + bucket +
". Make sure they exist and your bucket is in the same region as this function.";
console.log(message);
context.fail(message);
} else {
//console.log("DATA: " + data.Body.toString());
fs.writeFile("/tmp/test.csv", "testing", function (err) {
if(err) {
context.failed("writeToTmp Failed " + err);
} else {
context.succeed("writeFile succeeded");
}
});
}
});
};
Modifying your code into the Lambda template worked for me. I think you need to assign a function to exports.handler and call the appropriate context.succeed() or context.fail() method. Otherwise, you just get generic errors.
var fs = require("fs");
exports.handler = function(event, context) {
fs.writeFile("/tmp/test.txt", "testing", function (err) {
if (err) {
context.fail("writeFile failed: " + err);
} else {
context.succeed("writeFile succeeded");
}
});
};
So the answer lies in the context.fail() or context.succeed() functions. Being completely new to the world of aws and lambda I was ignorant to the fact that calling any of these methods stops execution of the lambda instance.
According to the docs:
The context.succeed() method signals successful execution and returns
a string.
By eliminating these and only calling them after I had run all the code that I wanted, everything worked well.
I ran into this, and it seems like AWS Lambda may be using an older (or modified) version of fs. I figured this out by logging the response from fs.writeFile and noticed it wasn't a promise.
To get around this, I wrapped the call in a promise:
var promise = new Promise(function(resolve, reject) {
fs.writeFile('/tmp/test.txt', 'testing', function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
Hopefully this helps someone else :hug-emoji:
Here is my code :
function aCallbackInLoop(dataArray) {
dataArray.forEach(function (item, index) {
fs.appendFile(fileName, JSON.stringify(item) + "\r\n", function (err) {
if (err) {
console.log('Error writing data ' + err);
} else {
console.log('Data written');
}
});
});
}
I get random errors :
Data written
Data written
.
.
Error writing data Error: UNKNOWN, open 'output/mydata.json'
Error writing data Error: UNKNOWN, open 'output/mydata.json'
.
.
Data written
Error writing data Error: UNKNOWN, open 'output/mydata.json'
The function (aCallbackInLoop) is a callback for a web-service request, which returns chunks of data in dataArray. Multiple web-service requests are being made in a loop, so this callback is perhaps being called in parallel. I doubt it's some file lock issue, but I am not sure how to resolve.
PS: I have made sure it's not a data issue (I am logging all items in dataArray)
Edit : Code after trying write stream :
function writeDataToFile(fileName, data) {
try {
var wStream = fs.createWriteStream(fileName);
wStream.write(JSON.stringify(data) + "\r\n");
wStream.end();
} catch (err) {
console.log(err.message);
}
}
function aCallbackInLoop(dataArray){
dataArray.forEach(function(item, index){
writeDataToFile(filename, item); //filename is global var
});
}
As you have observed, multiple appendFile calls are not able to proceed because of the previous appendFile calls. In this particular case, it would be better to create a write stream.
var wstream = fs.createWriteStream(fileName);
dataArray.forEach(function (item) {
wstream.write(JSON.stringify(item + "\r\n");
});
wstream.end();
If you want to know when all the data is written, then you can register a function with the finish event, like this
var wstream = fs.createWriteStream(fileName);
wstream.on("finish", function() {
// Writing to the file is actually complete.
});
dataArray.forEach(function (item) {
wstream.write(JSON.stringify(item + "\r\n");
});
wstream.end();
Try using the synchronous version of appendFile - https://nodejs.org/api/fs.html#fs_fs_appendfilesync_filename_data_options
I am trying to write a newer watch module that uses the fs.watch method instead of the watchFile approach.
So far, it works beautifully, but only when I run it outside of mocha. I can't figure out why my unit test is throwing a tantrum, maybe someone here can?
Here is my class code:
/**
* requirements
*/
var fs, path, events;
fs = require('fs');
path = require('path');
events = require('events');
/**
* private
*/
var Monitor = function(directory, options) {
this.directory = directory;
this.options = options || {};
(this.options.lazy && this.empty()) || this.walk(this.directory);
this.watch(this.directory);
};
Monitor.prototype = new events.EventEmitter();
Monitor.prototype.watch = function(directory, stats) {
var stats = stats || {};
if (!this.directories[directory]) {
var w = fs.watch(directory, this.options, this.justlookatit.bind(this));
}
this.directories[directory] = { 'stats': stats, 'w': w };
};
Monitor.prototype.directories = function() {
if (!Object.keys(this.directories).length) {
this.walk(this.directory);
}
return this.directories;
};
Monitor.prototype.files = function() {
if (!Object.keys(this.files).length) {
this.walk(this.directory);
}
return this.files;
};
Monitor.prototype.unwatch = function() {
if (!Object.keys(this.directories).length) {
for (var dir in this.directories) {
dir.w.close();
}
}
};
Monitor.prototype.empty = function() {
this.unwatch();
this.files = {};
this.directories = {};
};
Monitor.prototype.walk = function(directory) {
var monitor = this;
this.empty();
fs.readdir(directory, function(err, files) {
if (err) return;
for (var file in files) {
var fullname = path.resolve(files[file]);
if (!monitor.options.filter || monitor.options.filter(fullname)) {
fs.stat(fullname, function(err, stats) {
if (err) return;
if (stats.isDirectory()) {
monitor.walk(fullname);
monitor.watch(fullname, stats);
} else {
monitor.files[fullname] = stats;
}
});
}
}
});
};
Monitor.prototype.justlookatit = function(action, file) {
var monitor = this;
var fullname = path.resolve(file);
if (this.options.filter && !this.options.filer(fullname)) return;
fs.exists(fullname, function(exists) {
if (exists) {
fs.stat(fullname, function(err, stats) {
if (stats.isDirectory()) {
monitor.watch(fullname, stats);
} else {
if (monitor.files[fullname]) {
if (stats.mtime.getTime() > monitor.files[fullname].mtime.getTime()) {
monitor.emit('modified', fullname, stats);
}
} else {
monitor.emit('added', fullname, stats);
}
monitor.files[fullname] = stats;
}
});
} else {
if (monitor.files[fullname]) {
delete monitor.files[fullname];
monitor.emit('deleted', fullname);
} else if (monitor.directories[fullname]) {
monitor.directories[fullname].w.close();
delete monitor.directories[fullname];
}
}
});
};
/**
* exports
*/
exports.start = function(directory, options) {
return new Monitor(directory, options);
};
Here is my Working external test code:
var watch = require("./watch.js");
var fs = require('fs');
monitor = watch.start(__dirname);
monitor.on('added', function(file, stats) {
console.log("Caught Added: " + file);
});
monitor.on('modified', function(file, stats) {
console.log("Caught Modified: " + file);
});
monitor.on('deleted', function(file) {
console.log("Caught deleted: " + file);
});
// try creating a file immediately
fs.openSync('v.md', 'w');
The first test file runs perfectly fine, and I've tried both openSync and open. Finally, here is a version of the same test code, wrapped in a mocha unit test which is timing out:
/**
* requirements
*/
var watch, Q, fs, path, mocha, chai, assert;
watch = require('../lib/watch.js');
Q = require('q');
fs = require('fs');
path = require('path');
mocha = require('mocha');
chai = require('chai');
assert = chai.assert;
/**
* variables
*/
var watch_directory = path.join(__dirname, './watch');
/**
* tests
*/
describe('test watch', function() {
it('should create a monitor and run callbacks after fs changes', function(done) {
// I had planned to implement promises that chained the three callbacks
// but couldn't get one of them working in general
var added = function(file, stats) {
console.log("added");
done();
};
var modified = function(file, stats) {
console.log("modified");
};
var deleted = function(file, stats) {
console.log("deleted");
};
// create our service
var monitor = watch.start(watch_directory);
// assert it is defined
assert.isDefined(monitor);
// establish a listener
monitor.on('added', added);
monitor.on('modified', modified);
monitor.on('deleted', deleted);
// here is a file name using the current date to prevent duplication during tests
var file = path.join(watch_directory, (new Date()).getTime() + '.md');
// let's create the file, then delete it
fs.open(file, 'w+', function(err, fileDescriptor) {
// this prints before console output from the watch.js's `justlookatit` method
console.log(err);
console.log("writing to file");
// we probably don't want to try closing the fileDescriptor if the open failed
if (err) return;
// close the file descriptor
fs.close(fileDescriptor, function() {
// delete the file we just created
// fs.unlink(file, function() { /* not a big deal */ });
});
});
// modify a known-existing test file
fs.open('test.md', 'w+', function() {/* we don't care about this */});
})
});
I checked with console.log(fullname) inside the justlookatit method on the watch code, and it spits out the correct file name, matching the one generated by the unit test.
However, it then proceeds to return false when I run fs.exists. As I undestand it, that means the file system is notifying me that a file exists before it exists, which doesn't make sense really. So I tried adding an additional delay by wrapping my fs.exists method in a setTimeout, and that didn't change the results. I have also tried using both openSync and existsSync, and that made no difference.
I'm stumped, does anyone have any ideas why the mocha code isn't working?
So, the solution was to go for a walk. I came back, looked at the code again and figured out the cause of the problem with mocha, and also identified many other bugs.
The problem was the lack of context. The justlookatit method does not have a context, and in the test.js scenario it is watching the current directory, while the mocha test is watching a sub-directory.
The path.resolve was receiving only the file name, not the directory, and therefore merged it with the default (executables) directory, so the level of test.js, or watch_test.js for mocha. It proceeded to fail to locate any of the files in the mocha test case because they were all one level below the executable.
I won't go into detail about all the other bugs, but I may come back and post the repository link when I get to a point that I want to push it online.
You're missing the callback return(done); at the end of your test. Unless you call that callback, Mocha will time out every time.
Which is the simplest way to compare a hash of a file without storing it in a database?
For example:
var filename = __dirname + '/../public/index.html';
var shasum = crypto.createHash('sha1');
var s = fs.ReadStream(filename);
s.on('data', function(d) {
shasum.update(d);
});
s.on('end', function() {
var d = shasum.digest('hex');
console.log(d + ' ' + filename);
fs.writeFile(__dirname + "/../public/log.txt", d.toString() + '\n', function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved!");
}
});
});
The above code returns the hash of the HTML file. If I edit the file how can I know if it has been changed? In other words, how can I know if the hash has been changed?
Any suggestions?
Edited
Now the hash is being saved in the log file. How can I retrieve the hash from the file and match it with the new generated one? A code example would be awesome to give me a better understanding.
There is no difference with this question, but it isn't clear for me yet how to implement it.
If you're looking for changes on a file, then you can use one of Node's filesystem functions, fs.watch. This is how it's used:
fs.watch(filename, function (event, filename) {
//event is either 'rename' or 'change'
//filename is the name of the file which triggered the event
});
The watch function is however not very consistent, so you can use fs.watchFile as an alternative. fs.watchFile uses stat polling, so it's quite a bit slower than fs.watch, which detects file changes instantly.
Watching a file will return an instance of fs.FSWatcher, which has the events change and error. Calling .close will stop watching for changes on the file.
Here's an example relating to your code:
var filename = __dirname + '/../public/index.html';
var shasum = crypto.createHash('sha1');
var oldhash = null;
var s = fs.ReadStream(filename);
s.on('data', function(d) {
shasum.update(d);
});
s.on('end', function() {
var d = shasum.digest('hex');
console.log(d + ' ' + filename);
oldhash = d.toString();
fs.writeFile(__dirname + "/../public/log.txt", d.toString() + '\n', function(err) {
if(err) {
console.log(err);
}
else {
console.log("The file was saved!");
}
});
});
//watch the log for changes
fs.watch(__dirname + "/../public/log.txt", function (event, filename) {
//read the log contents
fs.readFile(__dirname + "/../public/log.txt", function (err, data) {
//match variable data with the old hash
if (data == oldhash) {
//do something
}
});
});
What's the difference between this question and the previous one you asked? If you're not wanting to store it in a database, then store it as a file. If you want to save the hash for multiple files, then maybe put them in a JSON object and write them out as a .json file so they're easy to read/write.
EDIT
Given what you added to your question, it should be pretty simple. You might write a function to do check and re-write:
function updateHash (name, html, callback) {
var sha = crypto.createHash('sha1');
sha.update(html);
var newHash = sha.digest('hex');
var hashFileName = name + '.sha';
fs.readFile(hashFileName, 'utf8', function (err, oldHash) {
var changed = true;
if (err)
console.log(err); // probably indicates the file doesn't exist, but you should consider doing better error handling
if (oldHash === newHash)
changed = false;
fs.writeFile(hashFileName, newHash, { encoding: 'utf8' }, function (err) {
callback(err, changed);
});
});
}
updateHash('index.html', "<html><head><title>...", function (err, isChanged) {
// do something with this information ?
console.log(isChanged);
});