Up-to-date index for Maven plugin (#935)

Author: nedtwigg

README.md

@@ -42,7 +42,7 @@ output = [
 '| Toggle with [`spotless:off` and `spotless:on`](plugin-gradle/#spotlessoff-and-spotlesson) | {{yes}}    | {{yes}}      | {{no}}       | {{no}}  |',
 '| [Ratchet from](https://github.com/diffplug/spotless/tree/main/plugin-gradle#ratchet) `origin/main` or other git ref | {{yes}}    | {{yes}}      | {{no}}       | {{no}}  |',
 '| Define [line endings using git](https://github.com/diffplug/spotless/tree/main/plugin-gradle#line-endings-and-encodings-invisible-stuff) | {{yes}}    | {{yes}}      | {{yes}}       | {{no}}  |',
-'| Fast incremental format and up-to-date check   | {{yes}}      | {{no}}       | {{no}}       | {{no}}  |',
+'| Fast incremental format and up-to-date check   | {{yes}}      | {{yes}}      | {{no}}       | {{no}}  |',
 '| Fast format on fresh checkout using buildcache | {{yes}}      | {{no}}       | {{no}}       | {{no}}  |',
 lib('generic.EndWithNewlineStep')                +'{{yes}}       | {{yes}}      | {{no}}       | {{no}}  |',
 lib('generic.IndentStep')                        +'{{yes}}       | {{yes}}      | {{no}}       | {{no}}  |',
@@ -81,7 +81,7 @@ extra('wtp.EclipseWtpFormatterStep')             +'{{yes}}       | {{yes}}
 | Toggle with [`spotless:off` and `spotless:on`](plugin-gradle/#spotlessoff-and-spotlesson) | :+1:    | :+1:      | :white_large_square:       | :white_large_square:  |
 | [Ratchet from](https://github.com/diffplug/spotless/tree/main/plugin-gradle#ratchet) `origin/main` or other git ref | :+1:    | :+1:      | :white_large_square:       | :white_large_square:  |
 | Define [line endings using git](https://github.com/diffplug/spotless/tree/main/plugin-gradle#line-endings-and-encodings-invisible-stuff) | :+1:    | :+1:      | :+1:       | :white_large_square:  |
-| Fast incremental format and up-to-date check   | :+1:      | :white_large_square:       | :white_large_square:       | :white_large_square:  |
+| Fast incremental format and up-to-date check   | :+1:      | :+1:      | :white_large_square:       | :white_large_square:  |
 | Fast format on fresh checkout using buildcache | :+1:      | :white_large_square:       | :white_large_square:       | :white_large_square:  |
 | [`generic.EndWithNewlineStep`](lib/src/main/java/com/diffplug/spotless/generic/EndWithNewlineStep.java) | :+1:       | :+1:      | :white_large_square:       | :white_large_square:  |
 | [`generic.IndentStep`](lib/src/main/java/com/diffplug/spotless/generic/IndentStep.java) | :+1:       | :+1:      | :white_large_square:       | :white_large_square:  |

gradle.properties

@@ -16,6 +16,7 @@ artifactIdGradle=spotless-plugin-gradle
 # Build requirements
 VER_JAVA=1.8
 VER_SPOTBUGS=4.5.0
+VER_JSR_305=3.0.2
 
 # Dependencies provided by Spotless plugin
 VER_SLF4J=[1.6,2.0[

gradle/java-setup.gradle

@@ -35,5 +35,5 @@ tasks.named('spotbugsMain') {
 dependencies {
 	compileOnly 'net.jcip:jcip-annotations:1.0'
 	compileOnly "com.github.spotbugs:spotbugs-annotations:${VER_SPOTBUGS}"
-	compileOnly 'com.google.code.findbugs:jsr305:3.0.2'
+	compileOnly "com.google.code.findbugs:jsr305:${VER_JSR_305}"
 }

plugin-maven/CHANGES.md

@@ -3,6 +3,8 @@
 We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (starting after version `1.27.0`).
 
 ## [Unreleased]
+### Added
+* Incremental up-to-date checking ([#935](https://github.com/diffplug/spotless/pull/935)).
 
 ## [2.17.7] - 2021-12-16
 ### Fixed

plugin-maven/README.md

@@ -900,6 +900,47 @@ If your project has not been rigorous with copyright headers, and you'd like to
 
 <a name="ratchet"></a>
 
+## Incremental up-to-date checking and formatting
+
+**This feature is turned off by default.**
+
+Execution of `spotless:check` and `spotless:apply` for large projects can take time.
+By default, Spotless Maven plugin needs to read and format each source file.
+Repeated executions of `spotless:check` or `spotless:apply` are completely independent.
+
+If your project has many source files managed by Spotless and formatting takes a long time, you can
+enable incremental up-to-date checking with the following configuration:
+
+```xml
+<configuration>
+  <upToDateChecking>
+    <enabled>true</enabled>
+  </upToDateChecking>
+  <!-- ... define formats ... -->
+</configuration>
+```
+
+With up-to-date checking enabled, Spotless creates an index file in the `target` directory.
+The index file contains source file paths and corresponding last modified timestamps.
+It allows Spotless to skip already formatted files that have not changed.
+
+**Note:** the index file is located in the `target` directory. Executing `mvn clean` will delete
+the index file, and Spotless will need to check/format all the source files.
+
+Spotless will remove the index file when up-to-date checking is explicitly turned off with the
+following configuration:
+
+```xml
+<configuration>
+  <upToDateChecking>
+    <enabled>false</enabled>
+  </upToDateChecking>
+  <!-- ... define formats ... -->
+</configuration>
+```
+
+Consider using this configuration if you experience issues with up-to-date checking.
+
 ## How can I enforce formatting gradually? (aka "ratchet")
 
 If your project is not currently enforcing formatting, then it can be a noisy transition.  Having a giant commit where every single file gets changed makes the history harder to read.  To address this, you can use the `ratchet` feature:

plugin-maven/build.gradle

@@ -79,6 +79,7 @@ dependencies {
 
 	compileOnly "org.apache.maven:maven-plugin-api:${VER_MAVEN_API}"
 	compileOnly "org.apache.maven.plugin-tools:maven-plugin-annotations:${VER_MAVEN_API}"
+	compileOnly "org.apache.maven:maven-core:${VER_MAVEN_API}"
 	compileOnly "org.eclipse.aether:aether-api:${VER_ECLIPSE_AETHER}"
 	compileOnly "org.eclipse.aether:aether-util:${VER_ECLIPSE_AETHER}"
 
@@ -95,6 +96,7 @@ dependencies {
 	testImplementation "org.apache.maven:maven-plugin-api:${VER_MAVEN_API}"
 	testImplementation "org.eclipse.aether:aether-api:${VER_ECLIPSE_AETHER}"
 	testImplementation "org.codehaus.plexus:plexus-resources:${VER_PLEXUS_RESOURCES}"
+	testImplementation "org.apache.maven:maven-core:${VER_MAVEN_API}"
 }
 
 task cleanMavenProjectDir(type: Delete) { delete MAVEN_PROJECT_DIR }
@@ -159,6 +161,7 @@ task createPomXml(dependsOn: installLocalDependencies) {
 			mavenApiVersion           : VER_MAVEN_API,
 			eclipseAetherVersion      : VER_ECLIPSE_AETHER,
 			spotlessLibVersion        : libVersion,
+			jsr305Version             : VER_JSR_305,
 			additionalDependencies    : additionalDependencies
 		]
 

plugin-maven/src/main/java/com/diffplug/spotless/maven/AbstractSpotlessMojo.java

@@ -22,12 +22,16 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -37,6 +41,7 @@
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugins.annotations.Component;
 import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
 import org.codehaus.plexus.resource.ResourceManager;
 import org.codehaus.plexus.resource.loader.FileResourceLoader;
 import org.codehaus.plexus.util.FileUtils;
@@ -54,6 +59,8 @@
 import com.diffplug.spotless.maven.generic.Format;
 import com.diffplug.spotless.maven.generic.LicenseHeader;
 import com.diffplug.spotless.maven.groovy.Groovy;
+import com.diffplug.spotless.maven.incremental.UpToDateChecker;
+import com.diffplug.spotless.maven.incremental.UpToDateChecking;
 import com.diffplug.spotless.maven.java.Java;
 import com.diffplug.spotless.maven.kotlin.Kotlin;
 import com.diffplug.spotless.maven.pom.Pom;
@@ -84,6 +91,9 @@ public abstract class AbstractSpotlessMojo extends AbstractMojo {
 	@Parameter(property = "spotless.check.skip", defaultValue = "false")
 	private boolean checkSkip;
 
+	@Parameter(defaultValue = "${project}", required = true, readonly = true)
+	private MavenProject project;
+
 	@Parameter(defaultValue = "${repositorySystemSession}", required = true, readonly = true)
 	private RepositorySystemSession repositorySystemSession;
 
@@ -147,13 +157,36 @@ public abstract class AbstractSpotlessMojo extends AbstractMojo {
 	@Parameter(property = LicenseHeaderStep.spotlessSetLicenseHeaderYearsFromGitHistory)
 	private String setLicenseHeaderYearsFromGitHistory;
 
-	protected abstract void process(Iterable<File> files, Formatter formatter) throws MojoExecutionException;
+	@Parameter
+	private UpToDateChecking upToDateChecking;
+
+	protected abstract void process(Iterable<File> files, Formatter formatter, UpToDateChecker upToDateChecker) throws MojoExecutionException;
 
 	@Override
 	public final void execute() throws MojoExecutionException {
+		if (shouldSkip()) {
+			getLog().info(String.format("Spotless %s skipped", goal));
+			return;
+		}
+
 		List<FormatterFactory> formatterFactories = getFormatterFactories();
+		FormatterConfig config = getFormatterConfig();
+
+		Map<FormatterFactory, Supplier<Iterable<File>>> formatterFactoryToFiles = new HashMap<>();
 		for (FormatterFactory formatterFactory : formatterFactories) {
-			execute(formatterFactory);
+			Supplier<Iterable<File>> filesToFormat = () -> collectFiles(formatterFactory, config);
+			formatterFactoryToFiles.put(formatterFactory, filesToFormat);
+		}
+
+		try (FormattersHolder formattersHolder = FormattersHolder.create(formatterFactoryToFiles, config);
+				UpToDateChecker upToDateChecker = createUpToDateChecker(formattersHolder.getFormatters())) {
+			for (Entry<Formatter, Supplier<Iterable<File>>> entry : formattersHolder.getFormattersWithFiles().entrySet()) {
+				Formatter formatter = entry.getKey();
+				Iterable<File> files = entry.getValue().get();
+				process(files, formatter, upToDateChecker);
+			}
+		} catch (PluginException e) {
+			throw e.asMojoExecutionException();
 		}
 	}
 
@@ -169,21 +202,7 @@ private boolean shouldSkip() {
 		return false;
 	}
 
-	private void execute(FormatterFactory formatterFactory) throws MojoExecutionException {
-		if (shouldSkip()) {
-			getLog().info(String.format("Spotless %s skipped", goal));
-			return;
-		}
-
-		FormatterConfig config = getFormatterConfig();
-		List<File> files = collectFiles(formatterFactory, config);
-
-		try (Formatter formatter = formatterFactory.newFormatter(files, config)) {
-			process(files, formatter);
-		}
-	}
-
-	private List<File> collectFiles(FormatterFactory formatterFactory, FormatterConfig config) throws MojoExecutionException {
+	private List<File> collectFiles(FormatterFactory formatterFactory, FormatterConfig config) {
 		Optional<String> ratchetFrom = formatterFactory.ratchetFrom(config);
 		try {
 			final List<File> files;
@@ -208,11 +227,11 @@ private List<File> collectFiles(FormatterFactory formatterFactory, FormatterConf
 					.filter(shouldInclude)
 					.collect(toList());
 		} catch (IOException e) {
-			throw new MojoExecutionException("Unable to scan file tree rooted at " + baseDir, e);
+			throw new PluginException("Unable to scan file tree rooted at " + baseDir, e);
 		}
 	}
 
-	private List<File> collectFilesFromGit(FormatterFactory formatterFactory, String ratchetFrom) throws MojoExecutionException {
+	private List<File> collectFilesFromGit(FormatterFactory formatterFactory, String ratchetFrom) {
 		MatchPatterns includePatterns = MatchPatterns.from(
 				withNormalizedFileSeparators(getIncludes(formatterFactory)));
 		MatchPatterns excludePatterns = MatchPatterns.from(
@@ -223,7 +242,7 @@ private List<File> collectFilesFromGit(FormatterFactory formatterFactory, String
 			dirtyFiles = GitRatchetMaven
 					.instance().getDirtyFiles(baseDir, ratchetFrom);
 		} catch (IOException e) {
-			throw new MojoExecutionException("Unable to scan file tree rooted at " + baseDir, e);
+			throw new PluginException("Unable to scan file tree rooted at " + baseDir, e);
 		}
 
 		List<File> result = new ArrayList<>();
@@ -237,8 +256,7 @@ private List<File> collectFilesFromGit(FormatterFactory formatterFactory, String
 		return result;
 	}
 
-	private List<File> collectFilesFromFormatterFactory(FormatterFactory formatterFactory)
-			throws MojoExecutionException, IOException {
+	private List<File> collectFilesFromFormatterFactory(FormatterFactory formatterFactory) throws IOException {
 		String includesString = String.join(",", getIncludes(formatterFactory));
 		String excludesString = String.join(",", getExcludes(formatterFactory));
 
@@ -256,11 +274,11 @@ private static String withTrailingSeparator(String path) {
 		return path.endsWith(File.separator) ? path : path + File.separator;
 	}
 
-	private Set<String> getIncludes(FormatterFactory formatterFactory) throws MojoExecutionException {
+	private Set<String> getIncludes(FormatterFactory formatterFactory) {
 		Set<String> configuredIncludes = formatterFactory.includes();
 		Set<String> includes = configuredIncludes.isEmpty() ? formatterFactory.defaultIncludes() : configuredIncludes;
 		if (includes.isEmpty()) {
-			throw new MojoExecutionException("You must specify some files to include, such as '<includes><include>src/**</include></includes>'");
+			throw new PluginException("You must specify some files to include, such as '<includes><include>src/**</include></includes>'");
 		}
 		return includes;
 	}
@@ -300,4 +318,12 @@ private List<FormatterStepFactory> getFormatterStepFactories() {
 				.filter(Objects::nonNull)
 				.collect(toList());
 	}
+
+	private UpToDateChecker createUpToDateChecker(Iterable<Formatter> formatters) {
+		if (upToDateChecking != null && upToDateChecking.isEnabled()) {
+			getLog().info("Up-to-date checking enabled");
+			return UpToDateChecker.forProject(project, formatters, getLog());
+		}
+		return UpToDateChecker.noop(project, getLog());
+	}
 }

plugin-maven/src/main/java/com/diffplug/spotless/maven/FormatterFactory.java

@@ -24,6 +24,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import org.apache.maven.plugins.annotations.Parameter;
@@ -71,10 +72,10 @@ public final Set<String> excludes() {
 		return excludes == null ? emptySet() : Sets.newHashSet(excludes);
 	}
 
-	public final Formatter newFormatter(List<File> filesToFormat, FormatterConfig config) {
+	public final Formatter newFormatter(Supplier<Iterable<File>> filesToFormat, FormatterConfig config) {
 		Charset formatterEncoding = encoding(config);
 		LineEnding formatterLineEndings = lineEndings(config);
-		LineEnding.Policy formatterLineEndingPolicy = formatterLineEndings.createPolicy(config.getFileLocator().getBaseDir(), () -> filesToFormat);
+		LineEnding.Policy formatterLineEndingPolicy = formatterLineEndings.createPolicy(config.getFileLocator().getBaseDir(), filesToFormat);
 
 		FormatterStepConfig stepConfig = stepConfig(formatterEncoding, config);
 		List<FormatterStepFactory> factories = gatherStepFactories(config.getGlobalStepFactories(), stepFactories);

plugin-maven/src/main/java/com/diffplug/spotless/maven/FormattersHolder.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2021 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.spotless.maven;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import com.diffplug.spotless.Formatter;
+
+class FormattersHolder implements AutoCloseable {
+
+	private final Map<Formatter, Supplier<Iterable<File>>> formatterToFiles;
+
+	FormattersHolder(Map<Formatter, Supplier<Iterable<File>>> formatterToFiles) {
+		this.formatterToFiles = formatterToFiles;
+	}
+
+	static FormattersHolder create(Map<FormatterFactory, Supplier<Iterable<File>>> formatterFactoryToFiles, FormatterConfig config) {
+		Map<Formatter, Supplier<Iterable<File>>> formatterToFiles = new HashMap<>();
+		try {
+			for (Entry<FormatterFactory, Supplier<Iterable<File>>> entry : formatterFactoryToFiles.entrySet()) {
+				FormatterFactory formatterFactory = entry.getKey();
+				Supplier<Iterable<File>> files = entry.getValue();
+
+				Formatter formatter = formatterFactory.newFormatter(files, config);
+				formatterToFiles.put(formatter, files);
+			}
+		} catch (RuntimeException openError) {
+			try {
+				close(formatterToFiles.keySet());
+			} catch (Exception closeError) {
+				openError.addSuppressed(closeError);
+			}
+			throw openError;
+		}
+
+		return new FormattersHolder(formatterToFiles);
+	}
+
+	Iterable<Formatter> getFormatters() {
+		return formatterToFiles.keySet();
+	}
+
+	Map<Formatter, Supplier<Iterable<File>>> getFormattersWithFiles() {
+		return formatterToFiles;
+	}
+
+	@Override
+	public void close() {
+		try {
+			close(formatterToFiles.keySet());
+		} catch (Exception e) {
+			throw new RuntimeException("Unable to close formatters", e);
+		}
+	}
+
+	private static void close(Set<Formatter> formatters) throws Exception {
+		Exception error = null;
+		for (Formatter formatter : formatters) {
+			try {
+				formatter.close();
+			} catch (Exception e) {
+				if (error == null) {
+					error = e;
+				} else {
+					error.addSuppressed(e);
+				}
+			}
+		}
+		if (error != null) {
+			throw error;
+		}
+	}
+}