New: Implement cache in order to only operate on changed files since previous run. (fixes #2998)

Author: royriojas

.gitignore

@@ -9,3 +9,5 @@ tmp/
 jsdoc/
 versions.json
 *.iml
+.eslintcache
+.cache

Makefile.js

@@ -390,10 +390,20 @@ target.all = function() {
 
 target.lint = function() {
     var errors = 0,
+        makeFileCache = " ",
+        jsCache = " ",
+        testCache = " ",
         lastReturn;
 
+    // using the cache locally to speed up linting process
+    if (!process.env.TRAVIS) {
+        makeFileCache = " --cache --cache-file .cache/makefile_cache ";
+        jsCache = " --cache --cache-file .cache/js_cache ";
+        testCache = " --cache --cache-file .cache/test_cache ";
+    }
+
     echo("Validating Makefile.js");
-    lastReturn = exec(ESLINT + MAKEFILE);
+    lastReturn = exec(ESLINT + makeFileCache + MAKEFILE);
     if (lastReturn.code !== 0) {
         errors++;
     }
@@ -411,13 +421,13 @@ target.lint = function() {
     }
 
     echo("Validating JavaScript files");
-    lastReturn = exec(ESLINT + JS_FILES);
+    lastReturn = exec(ESLINT + jsCache + JS_FILES);
     if (lastReturn.code !== 0) {
         errors++;
     }
 
     echo("Validating JavaScript test files");
-    lastReturn = exec(ESLINT + TEST_FILES);
+    lastReturn = exec(ESLINT + testCache + TEST_FILES);
     if (lastReturn.code !== 0) {
         errors++;
     }

docs/developer-guide/nodejs-api.md

@@ -122,6 +122,8 @@ The `CLIEngine` is a constructor, and you can create a new instance by passing i
 * `rules` - An object of rules to use (default: null). Corresponds to `--rule`.
 * `useEslintrc` - Set to false to disable use of `.eslintrc` files (default: true). Corresponds to `--no-eslintrc`.
 * `parser` - Specify the parser to be used (default: `espree`). Corresponds to `--parser`.
+* `cache` - Operate only on changed files (default: `false`). Corresponds to `--cache`.
+* `cacheFile` - Name of the file where the cache will be stored (default: `.eslintcache`). Corresponds to `--cache-file`.
 
 For example:
 

docs/user-guide/command-line-interface.md

@@ -30,6 +30,8 @@ Basic configuration:
   --ext [String]              Specify JavaScript file extensions - default: .js
   --global [String]           Define global variables
   --parser String             Specify the parser to be used - default: espree
+  --cache                     Only check changed files - default: false
+  --cache-file String         Path to the cache file - default: .eslintcache
 
 Specifying rules and plugins:
   --rulesdir [path::String]   Use additional rules from this directory
@@ -58,6 +60,7 @@ Miscellaneous:
   --init                      Run config initialization wizard - default: false
   -h, --help                  Show help
   -v, --version               Outputs the version number
+
 ```
 
 ### Basic configuration
@@ -275,6 +278,14 @@ Example:
 
     eslint -v
 
+### `--cache`
+
+Store the info about processed files in order to only operate on the changed ones.
+
+### `--cache-file`
+
+Path to the cache file. If none specified `.eslintcache` will be used. The file will be created in the directory where the `eslint` command is executed.
+
 ## Ignoring files from linting
 
 ESLint supports `.eslintignore` files to exclude files from the linting process when ESLint operates on a directory. Files given as individual CLI arguments will be exempt from exclusion. The `.eslintignore` file is a plain text file containing one pattern per line. It can be located in any of the target directory's ancestors; it will affect files in its containing directory as well as all sub-directories. Here's a simple example of a `.eslintignore` file:

lib/cli-engine.js

@@ -29,7 +29,8 @@ var fs = require("fs"),
     IgnoredPaths = require("./ignored-paths"),
     Config = require("./config"),
     util = require("./util"),
-    validator = require("./config-validator");
+    validator = require("./config-validator"),
+    fileEntryCache = require("file-entry-cache");
 
 var DEFAULT_PARSER = require("../conf/eslint.json").parser;
 
@@ -81,7 +82,9 @@ var defaultOptions = {
         extensions: [".js"],
         ignore: true,
         ignorePath: null,
-        parser: DEFAULT_PARSER
+        parser: DEFAULT_PARSER,
+        cache: false,
+        cacheFile: ".eslintcache"
     },
     loadedPlugins = Object.create(null);
 
@@ -302,6 +305,17 @@ function CLIEngine(options) {
      */
     this.options = assign(Object.create(defaultOptions), options || {});
 
+    /**
+     * cache used to not operate on files that haven't changed since last successful
+     * execution (e.g. file passed with no errors and no warnings
+     * @type {Object}
+     */
+    this._fileCache = fileEntryCache.create(path.resolve(this.options.cacheFile)); // eslint-disable-line no-underscore-dangle
+
+    if (!this.options.useCache) {
+        this._fileCache.destroy(); // eslint-disable-line no-underscore-dangle
+    }
+
     // load in additional rules
     if (this.options.rulePaths) {
         this.options.rulePaths.forEach(function(rulesdir) {
@@ -399,17 +413,20 @@ CLIEngine.prototype = {
      * @returns {Object} The results for all files that were linted.
      */
     executeOnFiles: function(files) {
-
         var results = [],
             processed = {},
             options = this.options,
+            fileCache = this._fileCache, // eslint-disable-line no-underscore-dangle
             configHelper = new Config(options),
             ignoredPaths = options.ignore ? IgnoredPaths.load(options).patterns : [],
             extensions = options.extensions.map(function(ext) {
                 return ext.charAt(0) === "." ? ext : "." + ext;
             }),
             globOptions,
-            stats;
+            stats,
+            startTime;
+
+        startTime = Date.now();
 
         files = files.map(processPath);
         ignoredPaths = ignoredPaths.map(processPath);
@@ -431,10 +448,52 @@ CLIEngine.prototype = {
                 return;
             }
 
+            if (options.useCache) {
+                // get the descriptor for this file
+                // with the metadata and the flag that determines if
+                // the file has changed
+                var descriptor = fileCache.getFileDescriptor(filename);
+                if (!descriptor.changed) {
+                    debug("Skipping file since hasn't changed: " + filename);
+
+                    // Adding the filename to the processed hashmap
+                    // so the reporting is not affected (showing a warning about .eslintignore being used
+                    // when it is not really used)
+
+                    processed[filename] = true;
+
+                    // Add the the cached results (always will be 0 error and 0 warnings)
+                    // cause we don't save to cache files that failed
+                    // to guarantee that next execution will process those files as well
+                    results.push(descriptor.meta.results);
+
+                    // move to the next file
+                    return;
+                }
+            }
+
             debug("Processing " + filename);
 
             processed[filename] = true;
-            results.push(processFile(filename, configHelper));
+            var res = processFile(filename, configHelper);
+
+            if (options.useCache) {
+                // if a file contains errors or warnings we don't want to
+                // store the file in the cache so we can guarantee that
+                // next execution will also operate on this file
+                if ( res.errorCount > 0 || res.warningCount > 0 ) {
+                    debug("File has problems, skipping it: " + filename);
+                    // remove the entry from the cache
+                    fileCache.removeEntry( filename );
+                } else {
+                    // since the file passed we store the result here
+                    // TODO: check this as we might not need to store the
+                    // successful runs as it will always should be 0 error 0 warnings
+                    descriptor.meta.results = res;
+                }
+            }
+
+            results.push(res);
         }
 
         files.forEach(function(pattern) {
@@ -453,6 +512,13 @@ CLIEngine.prototype = {
 
         stats = calculateStatsPerRun(results);
 
+        if (options.useCache) {
+            // persist the cache to disk
+            fileCache.reconcile();
+        }
+
+        debug("Linting complete in: " + (Date.now() - startTime) + "ms");
+
         return {
             results: results,
             errorCount: stats.errorCount,

lib/cli.js

@@ -49,7 +49,9 @@ function translateOptions(cliOptions) {
         configFile: cliOptions.config,
         rulePaths: cliOptions.rulesdir,
         useEslintrc: cliOptions.eslintrc,
-        parser: cliOptions.parser
+        parser: cliOptions.parser,
+        useCache: cliOptions.cache,
+        cacheFile: cliOptions.cacheFile
     };
 }
 

lib/options.js

@@ -57,6 +57,18 @@ module.exports = optionator({
             default: "espree",
             description: "Specify the parser to be used"
         },
+        {
+            option: "cache",
+            type: "Boolean",
+            default: "false",
+            description: "Only check changed files"
+        },
+        {
+            option: "cache-file",
+            type: "String",
+            default: ".eslintcache",
+            description: "Path to the cache file"
+        },
         {
             heading: "Specifying rules and plugins"
         },

package.json

@@ -48,6 +48,7 @@
     "globals": "^8.6.0",
     "handlebars": "^3.0.3",
     "inquirer": "^0.9.0",
+    "file-entry-cache": "^1.1.1",
     "is-my-json-valid": "^2.10.0",
     "is-resolvable": "^1.0.0",
     "js-yaml": "^3.2.5",

tests/fixtures/cache/src/fail-file.js

@@ -0,0 +1,2 @@
+var abc = 'some value'
+console.log(foo);

tests/fixtures/cache/src/test-file.js

@@ -0,0 +1,3 @@
+var abc = 2;
+
+console.log(abc);