cut FS activity down dramatically

* better logging
* use mkdir-batch to group mkdirs together and do the less overall work (don’t attempt to re-create the same dirs repeatedly)
* rather then writing to output and then doing a copyDereferenceSync to the cache, we just write to cache and link to output

Author: stefanpenner

index.js

@@ -2,6 +2,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var mkdirpBulk = require('mkdirp-bulk');
 var mkdirp = require('mkdirp');
 var Promise = require('rsvp').Promise;
 var Plugin = require('broccoli-plugin');
@@ -50,28 +51,65 @@ function Filter(inputTree, options) {
   this._destFilePathCache = Object.create(null);
 }
 
+Filter.prototype._resetStats = function() {
+  this._stats = {
+    paths: 0,
+    stale: 0,
+    mkdirp: 0,
+    mkdirpCache: 0,
+    processed: 0,
+    unprocessed: 0,
+    hits: 0,
+    miss: 0,
+    prime: 0,
+    hash: 0,
+    processedTime: 0
+  };
+};
+
 Filter.prototype.build = function build() {
   var self = this;
+  var start = new Date();
+  this._resetStats();
   var srcDir = this.inputPaths[0];
   var destDir = this.outputPath;
-  var paths = walkSync(srcDir);
+  var paths = walkSync(srcDir).filter(Boolean);
+
+  self._stats.paths = paths.length;
+  self._stats.walkSyncTime = new Date() - start;
 
   this._cache.deleteExcept(paths).forEach(function(key) {
+    self._stats.stale++;
     fs.unlinkSync(this.cachePath + '/' + key);
   }, this);
 
+  self._stats.mkdirp += mkdirpBulk.sync(paths.map(function(p) {
+    return self.outputPath + '/' + p;
+  }));
+
+  if (this._cacheDirsPrimed === undefined) {
+    this._cacheDirsPrimed = true;
+    self._stats.mkdirp += mkdirpBulk.sync(paths.map(function(p) {
+      return self.cachePath + '/' + p;
+    }));
+  }
+
   return mapSeries(paths, function rebuildEntry(relativePath) {
     var destPath = destDir + '/' + relativePath;
     if (relativePath.slice(-1) === '/') {
-      mkdirp.sync(destPath);
     } else {
       if (self.canProcessFile(relativePath)) {
+        self._stats.processed++;
         return self.processAndCacheFile(srcDir, destDir, relativePath);
       } else {
+        self._stats.unprocessed++;
         var srcPath = srcDir + '/' + relativePath;
         symlinkOrCopySync(srcPath, destPath);
       }
     }
+  }).finally(function() {
+    self._cacheDirsPrimed = false;
+    self._debug('build %o in %dms', self._stats, new Date() - start);
   });
 };
 
@@ -98,29 +136,34 @@ Filter.prototype.getDestFilePath = function getDestFilePath(relativePath) {
 
 Filter.prototype.processAndCacheFile =
     function processAndCacheFile(srcDir, destDir, relativePath) {
+  var start = new Date();
   var self = this;
   var cacheEntry = this._cache.get(relativePath);
+  var result;
 
   if (cacheEntry) {
+    this._stats.hash++;
     var hashResult = hash(srcDir, cacheEntry.inputFile);
 
     if (cacheEntry.hash.hash === hashResult.hash) {
-      this._debug('cache hit: %s', relativePath);
+      this._stats.hits++;
 
-      return symlinkOrCopyFromCache(cacheEntry, destDir, relativePath);
+      symlinkOrCopySync(cacheEntry.cacheFile, destDir + '/' + relativePath);
+      this._stats.processedTime += new Date() - start;
+      return result;
     } else {
-      this._debug('cache miss: %s \n  - previous: %o \n  - next:     %o ', relativePath, cacheEntry.hash.key, hashResult.key);
+      this._stats.miss++;
     }
 
   } else {
-    this._debug('cache prime: %s', relativePath);
+    this._stats.prime++;
   }
 
   return Promise.resolve().
       then(function asyncProcessFile() {
         return self.processFile(srcDir, destDir, relativePath);
       }).
-      then(copyToCache,
+      then(linkFromCache,
       // TODO(@caitp): error wrapper is for API compat, but is not particularly
       // useful.
       // istanbul ignore next
@@ -129,23 +172,28 @@ Filter.prototype.processAndCacheFile =
         e.file = relativePath;
         e.treeDir = srcDir;
         throw e;
-      })
+      }).finally(function() {
+        self._stats.processedTime += new Date() - start;
+      });
+
+  function linkFromCache() {
+    self._stats.hash++;
 
-  function copyToCache() {
     var entry = {
       hash: hash(srcDir, relativePath),
       inputFile: relativePath,
       outputFile: destDir + '/' + self.getDestFilePath(relativePath),
       cacheFile: self.cachePath + '/' + relativePath
     };
 
-    if (fs.existsSync(entry.cacheFile)) {
-      fs.unlinkSync(entry.cacheFile);
-    } else {
+    if (fs.existsSync(entry.outputFile)) {
+      fs.unlinkSync(entry.outputFile);
+    } else if (self._cacheDirsPrimed === false) {
+      self._stats.mkdirpCache++;
       mkdirp.sync(path.dirname(entry.cacheFile));
     }
 
-    copyDereferenceSync(entry.outputFile, entry.cacheFile);
+    symlinkOrCopySync(entry.cacheFile, entry.outputFile);
 
     return self._cache.set(relativePath, entry);
   }
@@ -167,8 +215,7 @@ Filter.prototype.processFile =
         if (outputPath == null) {
           throw new Error('canProcessFile("' + relativePath + '") is true, but getDestFilePath("' + relativePath + '") is null');
         }
-        outputPath = destDir + '/' + outputPath;
-        mkdirp.sync(path.dirname(outputPath));
+        outputPath = self.cachePath + '/' + outputPath;
         fs.writeFileSync(outputPath, outputString, {
           encoding: outputEncoding
         });
@@ -196,9 +243,3 @@ function hash(src, filePath) {
     ])
   };
 }
-
-function symlinkOrCopyFromCache(entry, dest, relativePath) {
-  mkdirp.sync(path.dirname(entry.outputFile));
-
-  symlinkOrCopySync(entry.cacheFile, dest + '/' + relativePath);
-}

package.json

@@ -30,6 +30,7 @@
     "copy-dereference": "^1.0.0",
     "debug": "^2.2.0",
     "mkdirp": "^0.5.1",
+    "mkdirp-bulk": "^1.0.0",
     "promise-map-series": "^0.2.1",
     "rsvp": "^3.0.18",
     "symlink-or-copy": "^1.0.1",