Allow Filter To Persist For Warm Initial Boot

This incorporates the ideas in
https://github.com/stefanpenner/broccoli-persistent-filter and to make
via an optional `persist` flag. This currently does not work on windows
however it is not detrimental to the end user as it is opt in.

index.js

@@ -13,6 +13,9 @@ var copyDereferenceSync = require('copy-dereference').sync;
 var Cache = require('./lib/cache');
 var debugGenerator = require('debug');
 var keyForFile = require('./lib/key-for-file');
+var PersistentCache = require('async-disk-cache');
+var hashForDep = require('hash-for-dep');
+var md5Hex = require('md5-hex');
 
 module.exports = Filter;
 
@@ -43,6 +46,16 @@ function Filter(inputTree, options) {
       this.inputEncoding = options.inputEncoding;
     if (options.outputEncoding != null)
       this.outputEncoding = options.outputEncoding;
+    if (options.persist) {
+      if (/^win/.test(process.platform)) {
+        console.log('Unfortunately persistent cache is currently not available on windows based systems.');
+      } else {
+        this.persistent = options.persist;
+        this._peristentCache = new PersistentCache(this.cacheKey(), {
+          compression: 'deflate'
+        });
+      }
+    }
   }
 
   this._cache = new Cache();
@@ -75,6 +88,36 @@ Filter.prototype.build = function build() {
   });
 };
 
+/*
+ * @private
+ *
+ *
+ * @method cachKey
+ * @return {String} this filters top-level cache key
+ */
+Filter.prototype.cacheKey = function() {
+  return hashForDep(this.baseDir());
+};
+
+/* @public
+ *
+ * @method baseDir
+ * @returns {String} absolute path to the root of the filter...
+ */
+Filter.prototype.baseDir = function() {
+  throw Error('Filter must implement prototype.baseDir');
+};
+
+/*
+ * @public
+ *
+ * @method cacheKeyProcessString
+ * @return {String} this filters top-level cache key
+ */
+Filter.prototype.cacheKeyProcessString = function(string) {
+  return md5Hex(string);
+};
+
 Filter.prototype.canProcessFile =
     function canProcessFile(relativePath) {
   if (this.extensions == null || !this.extensions.length) return false;
@@ -158,25 +201,37 @@ Filter.prototype.processAndCacheFile =
   }
 };
 
-Filter.prototype.processFile =
-    function processFile(srcDir, destDir, relativePath) {
+Filter.prototype.processFile = function processFile(srcDir, destDir, relativePath) {
   var self = this;
   var inputEncoding = this.inputEncoding;
   var outputEncoding = this.outputEncoding;
   if (inputEncoding === void 0) inputEncoding = 'utf8';
   if (outputEncoding === void 0) outputEncoding = 'utf8';
   var contents = fs.readFileSync(
       srcDir + '/' + relativePath, { encoding: inputEncoding });
+  var promise;
 
-  return Promise.resolve(this.processString(contents, relativePath)).
-      then(function asyncOutputFilteredFile(outputString) {
-        var outputPath = internalGetDestFilePath(self, relativePath);
-        outputPath = destDir + '/' + outputPath;
-        mkdirp.sync(path.dirname(outputPath));
-        fs.writeFileSync(outputPath, outputString, {
-          encoding: outputEncoding
-        });
-      });
+  if (this.persistent) {
+    var key = this.cacheKeyProcessString(contents, relativePath);
+    promise = this._peristentCache.get(key).then(function(entry) {
+      return entry.isCached ? entry.value : self.processString(contents, relativePath);
+    });
+  } else {
+    promise = Promise.resolve(this.processString(contents, relativePath));
+  }
+
+  return promise.then(function asyncOutputFilteredFile(outputString) {
+    var outputPath = internalGetDestFilePath(self, relativePath);
+    outputPath = destDir + '/' + outputPath;
+    mkdirp.sync(path.dirname(outputPath));
+    fs.writeFileSync(outputPath, outputString, {
+      encoding: outputEncoding
+    });
+
+    if (self.persistent) {
+      return self._peristentCache.set(key, outputString);
+    }
+  });
 };
 
 Filter.prototype.processString =

package.json

@@ -25,10 +25,13 @@
     "cache"
   ],
   "dependencies": {
+    "async-disk-cache": "^1.0.0",
     "broccoli-kitchen-sink-helpers": "^0.2.7",
     "broccoli-plugin": "^1.0.0",
     "copy-dereference": "^1.0.0",
     "debug": "^2.2.0",
+    "hash-for-dep": "0.0.3",
+    "md5-hex": "^1.0.2",
     "mkdirp": "^0.5.1",
     "promise-map-series": "^0.2.1",
     "rsvp": "^3.0.18",

test/test.js

@@ -43,6 +43,10 @@ ReplaceFilter.prototype.processString = function(contents, relativePath) {
   return result;
 };
 
+ReplaceFilter.prototype.baseDir = function() {
+  return '../';
+};
+
 function IncompleteFilter(inputTree, options) {
   if (!this) return new IncompleteFilter(inputTree, options);
   Filter.call(this, inputTree, options);
@@ -315,6 +319,61 @@ describe('Filter', function() {
         to.equal('utf8');
   });
 
+
+  describe('persistent cache', function() {
+    var f;
+    function F(inputTree, options) { Filter.call(this, inputTree, options); }
+    inherits(F, Filter);
+    F.prototype.baseDir = function() {
+      return '../';
+    };
+
+    beforeEach(function() {
+      f = new F(fixturePath, { persist: true });
+    });
+
+    it('cache is initialized', function() {
+      expect(f._peristentCache).to.be.ok;
+    });
+
+    it('default `baseDir` implementation throws an Unimplemented Exception', function() {
+      function F(inputTree, options) { Filter.call(this, inputTree, options); }
+      inherits(F, Filter);
+      expect(function() {
+        new F(fixturePath, { persist: true });
+      }).to.throw(/Filter must implement prototype.baseDir/);
+    });
+
+    it('`cacheKey` returns correct second level file cache', function() {
+      expect(f.cacheKey()).to.eql('d359afdd76b6eea24a00210bce4974e7');
+    });
+
+    it('`cacheKeyProcessString` return correct first level file cache', function() {
+      expect(f.cacheKeyProcessString('foo-bar-baz', 'relative-path')).to.eql('4c43793687f9a7170a9149ad391cbf70');
+    });
+
+    it('filter properly reads file tree', function() {
+      var builder = makeBuilder(ReplaceFilter, fixturePath, function(awk) {
+        return awk;
+      });
+
+      return builder('dir', {
+        persist: true,
+        glob: '**/*.md',
+        search: 'dogs',
+        replace: 'cats'
+      }).then(function(results) {
+        expect(results.files).to.deep.eql([
+          'a/',
+          'a/README.md',
+          'a/bar/',
+          'a/bar/bar.js',
+          'a/foo.js'
+        ]);
+      });
+    });
+  });
+
   describe('processFile', function() {
     beforeEach(function() {
       sinon.spy(fs, 'mkdirSync');