Allow Filter To Persist For Warm Boot

README.md

@@ -92,6 +92,48 @@ var node = new Awk('docs', 'ES6', 'ECMAScript 2015');
 module.exports = node;
 ```
 
+## Persistent Cache
+
+__Note: This feature is experimental and is only available on Unix based systems.__
+
+Adding persist flag allows a subclass to persist state across restarts. This exists to mitigate the upfront cost of some more expensive transforms on warm boot. __It does not aim to improve incremental build performance, if it does, it should indicate something is wrong with the filter or input filter in question.__
+
+### How does it work?
+
+It does so but establishing a 2 layer file cache. The first layer, is the entire bucket.
+The second, `cacheKeyProcessString` is a per file cache key.
+
+Together, these two layers should provide the right balance of speed and sensibility.
+
+The bucket level cacheKey must be stable but also never become stale. If the key is not
+stable, state between restarts will be lost and performance will suffer. On the flip-side,
+if the cacheKey becomes stale changes may not be correctly reflected.
+
+It is configured by subclassing and refining `cacheKey` method. A good key here, is
+likely the name of the plugin, its version and the actual versions of its dependencies.
+
+```js
+Subclass.prototype.cacheKey = function() {
+ return md5(Filter.prototype.call(this) + inputOptionsChecksum + dependencyVersionChecksum);
+}
+```
+
+The second key, represents the contents of the file. Typically the base-class's functionality
+is sufficient, as it merely generates a checksum of the file contents. If for some reason this
+is not sufficient, it can be re-configured via subclassing.
+
+```js
+Subclass.prototype.cacheKeyProcessString = function(string, relativePath) {
+  return superAwesomeDigest(string);
+}
+```
+
+It is recommended that persistent re-builds is opt-in by the consumer as it does not currently work on all systems.
+
+```js
+var myTree = new SomePlugin('lib', { persist: true });
+```
+
 ## FAQ
 
 ### Upgrading from 0.1.x to 1.x

index.js

@@ -13,9 +13,14 @@ var copyDereferenceSync = require('copy-dereference').sync;
 var Cache = require('./lib/cache');
 var debugGenerator = require('debug');
 var keyForFile = require('./lib/key-for-file');
+var PersistentCache = require('async-disk-cache');
+var md5Hex = require('md5-hex');
+var Processor = require('./lib/processor');
+var defaultProccessor = require('./lib/strategies/default');
 
 module.exports = Filter;
 
+
 Filter.prototype = Object.create(Plugin.prototype);
 Filter.prototype.constructor = Filter;
 function Filter(inputTree, options) {
@@ -33,6 +38,9 @@ function Filter(inputTree, options) {
 
   Plugin.call(this, [inputTree]);
 
+  this.processor = new Processor(options);
+  this.processor.setStrategy(defaultProccessor);
+
   /* Destructuring assignment in node 0.12.2 would be really handy for this! */
   if (options) {
     if (options.extensions != null)
@@ -43,8 +51,13 @@ function Filter(inputTree, options) {
       this.inputEncoding = options.inputEncoding;
     if (options.outputEncoding != null)
       this.outputEncoding = options.outputEncoding;
+    if (options.persist) {
+      this.processor.setStrategy(require('./lib/strategies/persistent'));
+    }
   }
 
+  this.processor.init(this);
+
   this._cache = new Cache();
   this._canProcessCache = Object.create(null);
   this._destFilePathCache = Object.create(null);
@@ -75,6 +88,26 @@ Filter.prototype.build = function build() {
   });
 };
 
+/* @public
+ *
+ * @method baseDir
+ * @returns {String} absolute path to the root of the filter...
+ */
+Filter.prototype.baseDir = function() {
+  throw Error('Filter must implement prototype.baseDir');
+};
+
+/**
+ * @public
+ *
+ * optionally override this to build a more rhobust cache key
+ * @param  {String} string The contents of a file that is being processed
+ * @return {String}        A cache key
+ */
+Filter.prototype.cacheKeyProcessString = function(string /*, relativePath*/) {
+  return md5Hex(string);
+};
+
 Filter.prototype.canProcessFile =
     function canProcessFile(relativePath) {
   return !!this.getDestFilePath(relativePath);
@@ -94,7 +127,7 @@ Filter.prototype.getDestFilePath = function getDestFilePath(relativePath) {
     }
   }
   return null;
-}
+};
 
 Filter.prototype.processAndCacheFile =
     function processAndCacheFile(srcDir, destDir, relativePath) {
@@ -129,7 +162,7 @@ Filter.prototype.processAndCacheFile =
         e.file = relativePath;
         e.treeDir = srcDir;
         throw e;
-      })
+      });
 
   function copyToCache() {
     var entry = {
@@ -161,18 +194,20 @@ Filter.prototype.processFile =
   var contents = fs.readFileSync(
       srcDir + '/' + relativePath, { encoding: inputEncoding });
 
-  return Promise.resolve(this.processString(contents, relativePath)).
-      then(function asyncOutputFilteredFile(outputString) {
-        var outputPath = self.getDestFilePath(relativePath);
-        if (outputPath == null) {
-          throw new Error('canProcessFile("' + relativePath + '") is true, but getDestFilePath("' + relativePath + '") is null');
-        }
-        outputPath = destDir + '/' + outputPath;
-        mkdirp.sync(path.dirname(outputPath));
-        fs.writeFileSync(outputPath, outputString, {
-          encoding: outputEncoding
-        });
-      });
+  return this.processor.processString(this, contents, relativePath).then(function asyncOutputFilteredFile(result) {
+    var outputString = result.string;
+    var outputPath = self.getDestFilePath(relativePath);
+    if (outputPath == null) {
+      throw new Error('canProcessFile("' + relativePath + '") is true, but getDestFilePath("' + relativePath + '") is null');
+    }
+    outputPath = destDir + '/' + outputPath;
+    mkdirp.sync(path.dirname(outputPath));
+    fs.writeFileSync(outputPath, outputString, {
+      encoding: outputEncoding
+    });
+
+    return self.processor.done(self, result);
+  });
 };
 
 Filter.prototype.processString =

lib/processor.js

@@ -0,0 +1,27 @@
+function Processor(options) {
+  options = options || {};
+  this.processor = {};
+  this.persistent = options.persist;
+}
+
+Processor.prototype.setStrategy = function(stringProcessor) {
+  if (this.persistent && /^win/.test(process.platform)) {
+    console.log('Unfortunately persistent cache is currently not available on windows based systems. Please see https://github.com/stefanpenner/hash-for-dep/issues/8.');
+    return;
+  }
+  this.processor = stringProcessor;
+};
+
+Processor.prototype.init = function(ctx) {
+  this.processor.init(ctx);
+};
+
+Processor.prototype.processString = function(ctx, contents, relativePath) {
+  return this.processor.processString(ctx, contents, relativePath);
+};
+
+Processor.prototype.done = function(ctx, result) {
+  return this.processor.done(ctx, result);
+};
+
+module.exports = Processor;

lib/strategies/default.js

@@ -0,0 +1,13 @@
+var Promise = require('rsvp').Promise;
+
+module.exports = {
+  init: function() {},
+
+  processString: function(ctx, contents, relativePath) {
+    return Promise.resolve({ string: ctx.processString(contents, relativePath) });
+  },
+
+  done: function(ctx) {
+    return Promise.resolve(ctx.outputPath);
+  }
+};

lib/strategies/persistent.js

@@ -0,0 +1,49 @@
+'use strict';
+
+var md5Hex = require('md5-hex');
+var PersistentCache = require('async-disk-cache');
+var hashForDep = require('hash-for-dep');
+
+module.exports = {
+
+  _peristentCache: {},
+
+  init: function(ctx) {
+    if (!ctx.constructor.cacheKey) {
+      ctx.constructor.cacheKey = this.cacheKey(ctx);
+    }
+
+    this._peristentCache = new PersistentCache(ctx.constructor.cacheKey, {
+      compression: 'deflate'
+    });
+  },
+
+  cacheKey: function(ctx) {
+    return hashForDep(ctx.baseDir());
+  },
+
+  processString: function(ctx, contents, relativePath) {
+    var key = ctx.cacheKeyProcessString(contents, relativePath);
+    return this._peristentCache.get(key).then(function(entry) {
+      var result;
+
+      if (entry.isCached) {
+        result = {
+          string: entry.value,
+          key: key
+        };
+      } else {
+        result = {
+          string: ctx.processString(contents, relativePath),
+          key: key
+        };
+      }
+
+      return result;
+    });
+  },
+
+  done: function(ctx, result) {
+    return this._peristentCache.set(result.key, result.string);
+  }
+};

package.json

@@ -25,10 +25,13 @@
     "cache"
   ],
   "dependencies": {
+    "async-disk-cache": "^1.0.0",
     "broccoli-kitchen-sink-helpers": "^0.2.7",
     "broccoli-plugin": "^1.0.0",
     "copy-dereference": "^1.0.0",
     "debug": "^2.2.0",
+    "hash-for-dep": "0.0.3",
+    "md5-hex": "^1.0.2",
     "mkdirp": "^0.5.1",
     "promise-map-series": "^0.2.1",
     "rsvp": "^3.0.18",

test/key-for-file-tests.js

@@ -8,7 +8,7 @@ describe('keyForFile', function () {
   describe('when given a path to a directory', function () {
     it('throws an error', function () {
       expect(function () {
-        keyForFile('./test/fixtures/directory');
+        keyForFile('./test/fixtures/dir');
       }).to.throw(/cannot diff directory/i);
     });
   });

test/test.js

@@ -45,6 +45,10 @@ ReplaceFilter.prototype.processString = function(contents, relativePath) {
   return result;
 };
 
+ReplaceFilter.prototype.baseDir = function() {
+  return '../';
+};
+
 function IncompleteFilter(inputTree, options) {
   if (!this) return new IncompleteFilter(inputTree, options);
   Filter.call(this, inputTree, options);
@@ -290,6 +294,58 @@ describe('Filter', function() {
         to.equal('utf8');
   });
 
+  if (!/^win/.test(process.platform)) {
+    describe('persistent cache', function() {
+      var f;
+      function F(inputTree, options) { Filter.call(this, inputTree, options); }
+      inherits(F, Filter);
+      F.prototype.baseDir = function() {
+        return '../';
+      };
+
+      beforeEach(function() {
+        f = new F(fixturePath, { persist: true });
+      });
+
+      it('cache is initialized', function() {
+        expect(f.processor.processor._peristentCache).to.be.ok;
+      });
+
+      it('default `baseDir` implementation throws an Unimplemented Exception', function() {
+        function F(inputTree, options) { Filter.call(this, inputTree, options); }
+        inherits(F, Filter);
+        expect(function() {
+          new F(fixturePath, { persist: true });
+        }).to.throw(/Filter must implement prototype.baseDir/);
+      });
+
+      it('`cacheKeyProcessString` return correct first level file cache', function() {
+        expect(f.cacheKeyProcessString('foo-bar-baz', 'relative-path')).to.eql('4c43793687f9a7170a9149ad391cbf70');
+      });
+
+      it('filter properly reads file tree', function() {
+        var builder = makeBuilder(ReplaceFilter, fixturePath, function(awk) {
+          return awk;
+        });
+
+        return builder('dir', {
+          persist: true,
+          glob: '**/*.md',
+          search: 'dogs',
+          replace: 'cats'
+        }).then(function(results) {
+          expect(results.files).to.deep.eql([
+            'a/',
+            'a/README.md',
+            'a/bar/',
+            'a/bar/bar.js',
+            'a/foo.js'
+          ]);
+        });
+      });
+    });
+  }
+
   describe('processFile', function() {
     beforeEach(function() {
       sinon.spy(fs, 'mkdirSync');