diff --git a/README.md b/README.md index 5f9ec923b..f6681d344 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ apply plugin: 'com.palantir.baseline-eclipse' apply plugin: 'com.palantir.baseline-idea' apply plugin: 'org.inferred.processors' // installs the "processor" configuration needed for baseline-error-prone apply plugin: 'com.palantir.baseline-error-prone' +apply plugin: 'com.palantir.baseline-class-uniqueness' ``` - Run ``./gradlew baselineUpdateConfig`` to download the config files @@ -255,6 +256,13 @@ checks](https://errorprone.info): - Slf4jLogsafeArgs: Allow only com.palantir.logsafe.Arg types as parameter inputs to slf4j log messages. More information on Safe Logging can be found at [github.com/palantir/safe-logging](https://github.com/palantir/safe-logging). + +### Class Uniqueness Plugin (com.palantir.baseline-class-uniqueness) + +Run `./gradlew checkClassUniqueness` to scan all jars on the `runtime` classpath for identically named classes. +This task will run automatically as part of `./gradlew build`. + + ### Copyright Checks By default Baseline enforces Palantir copyright at the beginning of files. To change this, edit the template copyright diff --git a/gradle-baseline-java/build.gradle b/gradle-baseline-java/build.gradle index 3f5b34fad..573cd48b0 100644 --- a/gradle-baseline-java/build.gradle +++ b/gradle-baseline-java/build.gradle @@ -10,10 +10,10 @@ repositories { dependencies { compile gradleApi() compile 'net.ltgt.gradle:gradle-errorprone-plugin' + compile 'com.google.guava:guava' testCompile gradleTestKit() testCompile 'com.netflix.nebula:nebula-test' // for better temp directory junit rule only - testCompile 'com.google.guava:guava' testCompile 'net.lingala.zip4j:zip4j' testCompile 'org.apache.commons:commons-io' } @@ -54,6 +54,10 @@ pluginBundle { id = 'com.palantir.baseline-idea' displayName = 'Palantir Baseline IntelliJ Plugin' } + baselineClassUniquenessPlugin { + id = 'com.palantir.baseline-class-uniqueness' + displayName = 'Palantir Baseline Class Uniqueness Plugin' + } } } diff --git a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/Baseline.groovy b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/Baseline.groovy index a39ed4305..766ec7e58 100644 --- a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/Baseline.groovy +++ b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/Baseline.groovy @@ -30,5 +30,8 @@ class Baseline implements Plugin { project.plugins.apply BaselineEclipse project.plugins.apply BaselineIdea project.plugins.apply BaselineErrorProne + + // TODO(dfox): enable this when it has been validated on a few real projects + // project.plugins.apply BaselineClassUniquenessPlugin } } diff --git a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineClassUniquenessPlugin.java b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineClassUniquenessPlugin.java new file mode 100644 index 000000000..dfcc1cf5e --- /dev/null +++ b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineClassUniquenessPlugin.java @@ -0,0 +1,40 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.baseline.plugins; + +import com.palantir.baseline.tasks.CheckClassUniquenessTask; +import org.gradle.api.Project; + +/** + * This plugin is similar to https://github.com/nebula-plugins/gradle-lint-plugin/wiki/Duplicate-Classes-Rule + * but goes one step further and actually hashes any identically named classfiles to figure out if they're + * completely identical (and therefore safely interchangeable). + * + * The task only fails if it finds classes which have the same name but different implementations. + */ +public class BaselineClassUniquenessPlugin extends AbstractBaselinePlugin { + + @Override + public final void apply(Project project) { + project.getPlugins().withId("java", plugin -> { + project.getTasks().create("checkClassUniqueness", CheckClassUniquenessTask.class, task -> { + task.setConfiguration(project.getConfigurations().getByName("runtime")); + project.getTasks().getByName("check").dependsOn(task); + }); + }); + } +} diff --git a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/CheckClassUniquenessTask.java b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/CheckClassUniquenessTask.java new file mode 100644 index 000000000..4db3b37c8 --- /dev/null +++ b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/CheckClassUniquenessTask.java @@ -0,0 +1,121 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.baseline.tasks; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Comparator; +import java.util.Set; +import java.util.stream.Collectors; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +public class CheckClassUniquenessTask extends DefaultTask { + + private Configuration configuration; + + public CheckClassUniquenessTask() { + setGroup("Verification"); + setDescription("Checks that the given configuration contains no identically named classes."); + } + + @Input + public final Configuration getConfiguration() { + return configuration; + } + + public final void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + @TaskAction + public final void checkForDuplicateClasses() { + ClassUniquenessAnalyzer analyzer = new ClassUniquenessAnalyzer(getLogger()); + analyzer.analyzeConfiguration(getConfiguration()); + boolean success = analyzer.getDifferingProblemJars().isEmpty(); + writeResultFile(success); + + if (!success) { + analyzer.getDifferingProblemJars().forEach((problemJars) -> { + Set differingClasses = analyzer.getDifferingSharedClassesInProblemJars(problemJars); + getLogger().error("{} Identically named classes with differing impls found in {}: {}", + differingClasses.size(), problemJars, differingClasses); + }); + + throw new IllegalStateException(String.format( + "'%s' contains multiple copies of identically named classes - " + + "this may cause different runtime behaviour depending on classpath ordering.\n" + + "To resolve this, try excluding one of the following jars:\n\n%s", + configuration.getName(), + formatSummary(analyzer) + )); + } + } + + private static String formatSummary(ClassUniquenessAnalyzer summary) { + Collection> allProblemJars = summary.getDifferingProblemJars(); + + int maxLength = allProblemJars.stream().flatMap(Set::stream) + .map(ModuleVersionIdentifier::toString) + .map(String::length) + .max(Comparator.naturalOrder()).get(); + String format = "%-" + (maxLength + 1) + "s"; + + StringBuilder builder = new StringBuilder(); + + allProblemJars.forEach(problemJars -> { + int count = summary.getDifferingSharedClassesInProblemJars(problemJars).size(); + String countColumn = String.format("\t%-14s", "(" + count + " classes) "); + builder.append(countColumn); + + String jars = problemJars.stream().map(jar -> String.format(format, jar)).collect(Collectors.joining()); + builder.append(jars); + + builder.append('\n'); + }); + + return builder.toString(); + } + + /** + * This only exists to convince gradle this task is incremental. + */ + @OutputFile + public final File getResultFile() { + return getProject().getBuildDir().toPath() + .resolve(Paths.get("uniqueClassNames", configuration.getName())) + .toFile(); + } + + private void writeResultFile(boolean success) { + try { + File result = getResultFile(); + Files.createDirectories(result.toPath().getParent()); + Files.write(result.toPath(), Boolean.toString(success).getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException("Unable to write boolean result file", e); + } + } +} diff --git a/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/ClassUniquenessAnalyzer.java b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/ClassUniquenessAnalyzer.java new file mode 100644 index 000000000..402efc71f --- /dev/null +++ b/gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/ClassUniquenessAnalyzer.java @@ -0,0 +1,155 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.baseline.tasks; + +import static java.util.stream.Collectors.toSet; + +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.hash.HashingInputStream; +import com.google.common.io.ByteStreams; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarInputStream; +import java.util.stream.Collectors; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.artifacts.ResolvedArtifact; +import org.slf4j.Logger; + +public final class ClassUniquenessAnalyzer { + + private final Map, Set> jarsToClasses = new HashMap<>(); + private final Map> classToHashCodes = new HashMap<>(); + private final Logger log; + + public ClassUniquenessAnalyzer(Logger log) { + this.log = log; + } + + public void analyzeConfiguration(Configuration configuration) { + Instant before = Instant.now(); + Set dependencies = configuration + .getResolvedConfiguration() + .getResolvedArtifacts(); + + // we use these temporary maps to accumulate information as we process each jar, + // so they may include singletons which we filter out later + Map> classToJars = new HashMap<>(); + Map> tempClassToHashCodes = new HashMap<>(); + + dependencies.stream().forEach(resolvedArtifact -> { + File file = resolvedArtifact.getFile(); + if (!file.exists()) { + log.info("Skipping non-existent jar {}: {}", resolvedArtifact, file); + return; + } + + try (FileInputStream fileInputStream = new FileInputStream(file); + JarInputStream jarInputStream = new JarInputStream(fileInputStream)) { + JarEntry entry; + while ((entry = jarInputStream.getNextJarEntry()) != null) { + if (entry.isDirectory() || !entry.getName().endsWith(".class")) { + continue; + } + + String className = entry.getName().replaceAll("/", ".").replaceAll(".class", ""); + HashingInputStream inputStream = new HashingInputStream(Hashing.sha256(), jarInputStream); + ByteStreams.exhaust(inputStream); + + multiMapPut(classToJars, + className, + resolvedArtifact.getModuleVersion().getId()); + + multiMapPut(tempClassToHashCodes, + className, + inputStream.hash()); + } + } catch (IOException e) { + log.error("Failed to read JarFile {}", resolvedArtifact, e); + throw new RuntimeException(e); + } + }); + + // discard all the classes that only come from one jar - these are completely safe! + classToJars.entrySet().stream() + .filter(entry -> entry.getValue().size() > 1) + .forEach(entry -> multiMapPut(jarsToClasses, entry.getValue(), entry.getKey())); + + // figure out which classes have differing hashes + tempClassToHashCodes.entrySet().stream() + .filter(entry -> entry.getValue().size() > 1) + .forEach(entry -> + entry.getValue().forEach(value -> multiMapPut(classToHashCodes, entry.getKey(), value))); + + Instant after = Instant.now(); + log.info("Checked {} classes from {} dependencies for uniqueness ({}ms)", + classToJars.size(), dependencies.size(), Duration.between(before, after).toMillis()); + } + + /** + * Any groups jars that all contain some identically named classes. + * Note: may contain non-scary duplicates - class files which are 100% identical, so their + * clashing name doesn't have any effect. + */ + public Collection> getProblemJars() { + return jarsToClasses.keySet(); + } + + /** + * Class names that appear in all of the given jars. + */ + public Set getSharedClassesInProblemJars(Collection problemJars) { + return jarsToClasses.get(problemJars); + } + + /** + * Jars which contain identically named classes with non-identical implementations. + */ + public Collection> getDifferingProblemJars() { + return getProblemJars() + .stream() + .filter(jars -> getDifferingSharedClassesInProblemJars(jars).size() > 0) + .collect(Collectors.toSet()); + } + + /** + * Class names which appear in all of the given jars and also have non-identical implementations. + */ + public Set getDifferingSharedClassesInProblemJars(Collection problemJars) { + return getSharedClassesInProblemJars(problemJars).stream() + .filter(classToHashCodes::containsKey) + .collect(toSet()); + } + + private static void multiMapPut(Map> map, K key, V value) { + map.compute(key, (unused, collection) -> { + Set newCollection = collection != null ? collection : new HashSet<>(); + newCollection.add(value); + return newCollection; + }); + } +} diff --git a/gradle-baseline-java/src/main/resources/META-INF/gradle-plugins/com.palantir.baseline-class-uniqueness.properties b/gradle-baseline-java/src/main/resources/META-INF/gradle-plugins/com.palantir.baseline-class-uniqueness.properties new file mode 100644 index 000000000..550b78235 --- /dev/null +++ b/gradle-baseline-java/src/main/resources/META-INF/gradle-plugins/com.palantir.baseline-class-uniqueness.properties @@ -0,0 +1 @@ +implementation-class=com.palantir.baseline.plugins.BaselineClassUniquenessPlugin diff --git a/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineClassUniquenessPluginIntegrationTest.groovy b/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineClassUniquenessPluginIntegrationTest.groovy new file mode 100644 index 000000000..c4cdf2ed3 --- /dev/null +++ b/gradle-baseline-java/src/test/groovy/com/palantir/baseline/BaselineClassUniquenessPluginIntegrationTest.groovy @@ -0,0 +1,159 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.baseline + +import java.nio.file.Files +import java.util.stream.Stream +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.TaskOutcome + +class BaselineClassUniquenessPluginIntegrationTest extends AbstractPluginTest { + + def standardBuildFile = """ + plugins { + id 'java' + id 'com.palantir.baseline-class-uniqueness' + } + subprojects { + apply plugin: 'java' + } + repositories { + mavenCentral() + maven { url 'https://dl.bintray.com/palantir/releases' } + } + """.stripIndent() + + def 'Task should run as part of :check'() { + when: + buildFile << standardBuildFile + + then: + def result = with('check', '--stacktrace').build() + result.task(':checkClassUniqueness').outcome == TaskOutcome.SUCCESS + } + + def 'detect duplicates in two external jars'() { + when: + buildFile << standardBuildFile + buildFile << """ + dependencies { + compile group: 'javax.el', name: 'javax.el-api', version: '3.0.0' + compile group: 'javax.servlet.jsp', name: 'jsp-api', version: '2.1' + } + """.stripIndent() + BuildResult result = with('checkClassUniqueness').buildAndFail() + + then: + result.output.contains("26 Identically named classes with differing impls found in [javax.servlet.jsp:jsp-api:2.1, javax.el:javax.el-api:3.0.0]: [javax.") + result.getOutput().contains("'runtime' contains multiple copies of identically named classes") + result.getOutput().contains("(26 classes) javax.servlet.jsp:jsp-api:2.1 javax.el:javax.el-api:3.0.0"); + println result.getOutput() + } + + + def 'ignores duplicates when the implementations are identical'() { + when: + buildFile << standardBuildFile + buildFile << """ + dependencies { + compile 'com.palantir.tritium:tritium-api:0.9.0' + compile 'com.palantir.tritium:tritium-core:0.9.0' + } + """.stripIndent() + + then: + with('checkClassUniqueness').build() + } + + def 'task should be up-to-date when classpath is unchanged'() { + when: + buildFile << standardBuildFile + + then: + BuildResult result1 = with('checkClassUniqueness').build() + result1.task(':checkClassUniqueness').outcome == TaskOutcome.SUCCESS + + BuildResult result = with('checkClassUniqueness').build() + result.task(':checkClassUniqueness').outcome == TaskOutcome.UP_TO_DATE + } + + def 'passes when no duplicates are present'() { + when: + buildFile << standardBuildFile + buildFile << """ + dependencies { + compile 'com.google.guava:guava:19.0' + compile 'org.apache.commons:commons-io:1.3.2' + compile 'junit:junit:4.12' + compile 'com.netflix.nebula:nebula-test:6.4.2' + } + """.stripIndent() + BuildResult result = with('checkClassUniqueness', '--info').build() + + then: + result.task(":checkClassUniqueness").outcome == TaskOutcome.SUCCESS + println result.getOutput() + } + + def 'should detect duplicates from transitive dependencies'() { + when: + multiProject.addSubproject('foo', """ + dependencies { + compile group: 'javax.el', name: 'javax.el-api', version: '3.0.0' + } + """) + multiProject.addSubproject('bar', """ + dependencies { + compile group: 'javax.servlet.jsp', name: 'jsp-api', version: '2.1' + } + """) + + buildFile << standardBuildFile + buildFile << """ + dependencies { + compile project(':foo') + compile project(':bar') + } + """.stripIndent() + + then: + BuildResult result = with('checkClassUniqueness').buildAndFail() + result.output.contains("26 Identically named classes with differing impls found in [javax.servlet.jsp:jsp-api:2.1, javax.el:javax.el-api:3.0.0]: [javax.") + } + + def 'currently skips duplicates from user-authored code'() { + when: + Stream.of(multiProject.addSubproject('foo'), multiProject.addSubproject('bar')).forEach({ subproject -> + File myClass = new File(subproject, "src/main/com/something/MyClass.java") + Files.createDirectories(myClass.toPath().getParent()) + myClass << "package com.something; class MyClass {}" + }) + + buildFile << standardBuildFile + buildFile << """ + dependencies { + compile project(':foo') + compile project(':bar') + } + """.stripIndent() + + then: + BuildResult result = with('checkClassUniqueness', '--info').build() + println result.getOutput() + result.task(":checkClassUniqueness").outcome == TaskOutcome.SUCCESS // ideally should should say failed! + } +}