Skip to content

Commit feb2fd5

Browse files
heath-guidewireheath-guidewire
authored
Merge pull request #1 from heath-guidewire/add-ignore-files-and-streaming
Added support for an additional `ignoreCallback` function and reading files using streams
2 parents 12dee17 + 85ecd68 commit feb2fd5

File tree

6 files changed

+206
-46
lines changed

6 files changed

+206
-46
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.idea
2+
node_modules

‎.jshintrc‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
{
2+
// JSHint Default Configuration File (as on JSHint website)
3+
// See http://jshint.com/docs/ for more details
4+
5+
"maxerr" : 50, // {int} Maximum error before stopping
6+
7+
// Enforcing
8+
"bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
9+
"camelcase" : false, // true: Identifiers must be in camelCase
10+
"curly" : false, // true: Require {} for every new block or scope
11+
"eqeqeq" : true, // true: Require triple equals (===) for comparison
12+
"forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty()
13+
"freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
14+
"immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
15+
"latedef" : false, // true: Require variables/functions to be defined before being used
16+
"newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()`
17+
"noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
18+
"noempty" : false, // true: Prohibit use of empty blocks
19+
"nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters.
20+
"nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment)
21+
"plusplus" : false, // true: Prohibit use of `++` and `--`
22+
"quotmark" : false, // Quotation mark consistency:
23+
// false : do nothing (default)
24+
// true : ensure whatever is used is consistent
25+
// "single" : require single quotes
26+
// "double" : require double quotes
27+
"undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
28+
"unused" : true, // Unused variables:
29+
// true : all variables, last function parameter
30+
// "vars" : all variables only
31+
// "strict" : all variables, all function parameters
32+
"strict" : false, // true: Requires all functions run in ES5 Strict Mode
33+
"maxparams" : false, // {int} Max number of formal params allowed per function
34+
"maxdepth" : false, // {int} Max depth of nested blocks (within functions)
35+
"maxstatements" : false, // {int} Max number statements per function
36+
"maxcomplexity" : false, // {int} Max cyclomatic complexity per function
37+
"maxlen" : false, // {int} Max number of characters per line
38+
"varstmt" : false, // true: Disallow any var statements. Only `let` and `const` are allowed.
39+
40+
// Relaxing
41+
"asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
42+
"boss" : false, // true: Tolerate assignments where comparisons would be expected
43+
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
44+
"eqnull" : false, // true: Tolerate use of `== null`
45+
"esversion" : 5, // {int} Specify the ECMAScript version to which the code must adhere.
46+
"moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
47+
// (ex: `for each`, multiple try/catch, function expression…)
48+
"evil" : false, // true: Tolerate use of `eval` and `new Function()`
49+
"expr" : false, // true: Tolerate `ExpressionStatement` as Programs
50+
"funcscope" : false, // true: Tolerate defining variables inside control statements
51+
"globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
52+
"iterator" : false, // true: Tolerate using the `__iterator__` property
53+
"lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
54+
"laxbreak" : false, // true: Tolerate possibly unsafe line breakings
55+
"laxcomma" : false, // true: Tolerate comma-first style coding
56+
"loopfunc" : false, // true: Tolerate functions being defined in loops
57+
"multistr" : false, // true: Tolerate multi-line strings
58+
"noyield" : false, // true: Tolerate generator functions with no yield statement in them.
59+
"notypeof" : false, // true: Tolerate invalid typeof operator values
60+
"proto" : false, // true: Tolerate using the `__proto__` property
61+
"scripturl" : false, // true: Tolerate script-targeted URLs
62+
"shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
63+
"sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
64+
"supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
65+
"validthis" : false, // true: Tolerate using this in a non-constructor function
66+
67+
// Environments
68+
"browser" : true, // Web Browser (window, document, etc)
69+
"browserify" : false, // Browserify (node.js code in the browser)
70+
"couch" : false, // CouchDB
71+
"devel" : true, // Development/debugging (alert, confirm, etc)
72+
"dojo" : false, // Dojo Toolkit
73+
"jasmine" : false, // Jasmine
74+
"jquery" : false, // jQuery
75+
"mocha" : true, // Mocha
76+
"mootools" : false, // MooTools
77+
"node" : true, // Node.js
78+
"nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
79+
"phantom" : false, // PhantomJS
80+
"prototypejs" : false, // Prototype and Scriptaculous
81+
"qunit" : false, // QUnit
82+
"rhino" : false, // Rhino
83+
"shelljs" : false, // ShellJS
84+
"typed" : false, // Globals for typed array constructions
85+
"worker" : false, // Web Workers
86+
"wsh" : false, // Windows Scripting Host
87+
"yui" : false, // Yahoo User Interface
88+
89+
// Custom Globals
90+
"globals" : {} // additional predefined global variables
91+
}

‎README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,37 @@ returned `Hashes` object, a `hash` which is the hash of hashes, and `files`,
7575
which is a key/value pairing of filename to hash. Each file might be another
7676
object, which indicates there was a directory tree encountered.
7777

78+
### Skipping Files/Directories Usage
79+
80+
There is an additional parameter to the method, an `ignoreCallback` function that can be provided to allow the algorithm to skip over certain files or directories.
81+
This callback has the signature: `function(path, filename)`. An example is as follows:
82+
83+
var dirsum = require('../lib/dirsum');
84+
85+
dirsum.digest('/your/tree', 'sha1', function(err, hashes) {
86+
if (err) throw err;
87+
console.log(JSON.stringify(hashes, null, 2));
88+
}, function(path, filename) {
89+
// ignore any files starting with '.'
90+
if (filename.indexOf('.') === 0) {
91+
return true;
92+
}
93+
return false;
94+
});
95+
96+
### NOTES
97+
98+
If the `method` parameter (i.e. the second parameter) is not passed as a string, the `callback` and `ignoreCallback` are effectively shifted left respectively.
99+
In other words the signature for the `digest` method looks either like this:
100+
101+
dirsum.digest(path : string, method : string, callback: function(err, hashes), ignoreCallback : function(path, filename));
102+
103+
OR
104+
105+
dirsum.digest(path : string, callback : function(err, hashes), ignoreCallback : function(path, filename));
106+
107+
Finally, the file read implementation uses streams so that large files are NOT read-entirely into memory.
108+
78109
## Installation
79110

80111
npm install dirsum

‎lib/dirsum.js‎

Lines changed: 65 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,70 @@ function _summarize(method, hashes) {
2424
return obj;
2525
}
2626

27-
function digest(root, method, callback) {
27+
function doDigesting(dirName, method, callback, ignoreCallback) {
28+
var hashes = {};
29+
fs.readdir(dirName, function(err, files) {
30+
if (err) return callback(err);
31+
32+
if (files.length === 0) {
33+
return callback(undefined, { hash: '', files: {} });
34+
}
35+
36+
var processed = 0;
37+
var total = files.length;
38+
function bookkeeping() {
39+
if (++processed >= total) {
40+
callback(undefined, _summarize(method, hashes));
41+
}
42+
}
43+
files.forEach(function(f) {
44+
var path = dirName + '/' + f;
45+
if (!ignoreCallback || !ignoreCallback(path, f)) {
46+
fs.stat(path, function(err, stats) {
47+
if (err) return callback(err);
48+
49+
if (stats.isDirectory()) {
50+
return doDigesting(path, method, function(err, hash) {
51+
if (err) return callback(err);
52+
53+
hashes[f] = hash;
54+
bookkeeping();
55+
}, ignoreCallback, hashes);
56+
} else if (stats.isFile()) {
57+
var hash = crypto.createHash(method);
58+
var stream = fs.createReadStream(path, { encoding: 'utf8' });
59+
stream.on('data', function(data) {
60+
hash.update(data);
61+
});
62+
stream.on('end', function() {
63+
hashes[f] = hash.digest('hex');
64+
bookkeeping();
65+
});
66+
stream.on('error', function(err) {
67+
callback(err);
68+
});
69+
} else {
70+
console.error('Skipping hash of %s', f);
71+
bookkeeping();
72+
}
73+
});
74+
} else {
75+
console.log('Skipping: ', path);
76+
bookkeeping();
77+
}
78+
});
79+
});
80+
}
81+
82+
function digest(root, method, callback, ignoreCallback) {
2883
if (!root || typeof(root) !== 'string') {
2984
throw new TypeError('root is required (string)');
3085
}
3186
if (method) {
3287
if (typeof(method) === 'string') {
3388
// NO-OP
3489
} else if (typeof(method) === 'function') {
90+
ignoreCallback = callback;
3591
callback = method;
3692
method = 'md5';
3793
} else {
@@ -44,51 +100,15 @@ function digest(root, method, callback) {
44100
throw new TypeError('callback is required (function)');
45101
}
46102

47-
var hashes = {};
48-
49-
fs.readdir(root, function(err, files) {
50-
if (err) return callback(err);
51-
52-
if (files.length === 0) {
53-
return callback(undefined, {hash: '', files: {}});
54-
}
55-
56-
var hashed = 0;
57-
files.forEach(function(f) {
58-
var path = root + '/' + f;
59-
fs.stat(path, function(err, stats) {
60-
if (err) return callback(err);
61-
62-
if (stats.isDirectory()) {
63-
return digest(path, method, function(err, hash) {
64-
if (err) return hash;
65-
66-
hashes[f] = hash;
67-
if (++hashed >= files.length) {
68-
return callback(undefined, _summarize(method, hashes));
69-
}
70-
});
71-
} else if (stats.isFile()) {
72-
fs.readFile(path, 'utf8', function(err, data) {
73-
if (err) return callback(err);
74-
75-
var hash = crypto.createHash(method);
76-
hash.update(data);
77-
hashes[f] = hash.digest('hex');
103+
if (ignoreCallback && typeof(ignoreCallback) !== 'function') {
104+
throw new TypeError('ignoreCallback is required to be a function');
105+
} else {
106+
console.log('ignoreCallback: ', ignoreCallback);
107+
}
78108

79-
if (++hashed >= files.length) {
80-
return callback(undefined, _summarize(method, hashes));
81-
}
82-
});
83-
} else {
84-
console.error('Skipping hash of %s', f);
85-
if (++hashed > files.length) {
86-
return callback(undefined, _summarize(method, hashes));
87-
}
88-
}
89-
});
90-
});
91-
});
109+
doDigesting(root, method, function(err, hashes) {
110+
callback(err, hashes);
111+
}, ignoreCallback);
92112
}
93113

94114
module.exports = {

‎package.json‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "dirsum",
33
"description": "A small library that computes checksums of directory trees",
4-
"version": "0.1.1",
4+
"version": "0.1.2-gwre",
55
"repository": {
66
"type": "git",
77
"url": "git://github.com/mcavage/node-dirsum.git"
@@ -11,6 +11,12 @@
1111
"scripts": {
1212
"test": "jshint lib tst/dirsum.test.js && node tst/dirsum.test.js"
1313
},
14+
"maintainers": [
15+
{
16+
"name": "Heath Chiavettone",
17+
"url": "https://github.com/heath-guidewire"
18+
}
19+
],
1420
"devDependencies": {
1521
"jshint": ">=0.1.9"
1622
},

‎tst/dirsum.test.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,13 @@ dirsum.digest(process.cwd() + '/tst/openldap', function(err, hashes) {
99
assert.ok(hashes);
1010
console.log(JSON.stringify(hashes, null, 2));
1111
});
12+
13+
dirsum.digest(process.cwd() + '/tst', function(err, hashes) {
14+
assert.ok(!err);
15+
assert.ok(hashes);
16+
assert.ok(hashes.files);
17+
assert.equal(hashes.files['dirsum.test.js'], undefined);
18+
console.log(JSON.stringify(hashes, null, 2));
19+
}, function(path, filename) {
20+
return filename === 'dirsum.test.js';
21+
});

0 commit comments

Comments
 (0)