Plugin to validate all classes on the classpath are uniquely named (#267

)

* Added BaselineClasspathConflict plugin

* Rename to BaselineClasspathDuplicatesPlugin

* Add meta info properties file

* silentConflict -> checkClasspathIsDuplicateFree

* Factor out logic to a new CheckUniqueClassNamesTask

* apply plugin order shouldn't matter

* Checkstyle

* Ugh groovy

* Add some magic publishing stuff

* Try removing the 'final' modifier

* Convert to Java

* Remove 'final' modifier

* Verify up-to-date behaviour

* Slightly different syntax

* Don't use org.gradle.internal.impldep HashMultimap

* Checkstyle

* Test verifies duplicates can be detected

* test for up-to-date

* Sweet human readable output

* Factor out the crawling

* Get ready for parallel

* parallelStream over dependencies

* Test case for project dependencies

* README

* Rename to 'com.palantir.baseline-class-uniqueness'

* Don't enable straight away for everyone

* Add sweet formatted table with count of offending classes

* Sort table nicely

* Check runtime instead of testRuntime by default

* More comprehensive test

* Factor out ClassUniquenessAnalyzer

* Pull in guava

* Tests pass

* Checkstyle

* Some CR tweaks

* Add some javadoc

* Don't bother sorting the table

* Only report classes with differing impls

* Ensure we only log classes with different impls

* More minimal tests

* Javadoc mentioning netflix's impl

* Be better at programming

* Remove duplicate guava dep

README.md

@@ -51,6 +51,7 @@ apply plugin: 'com.palantir.baseline-eclipse'
 apply plugin: 'com.palantir.baseline-idea'
 apply plugin: 'org.inferred.processors'  // installs the "processor" configuration needed for baseline-error-prone
 apply plugin: 'com.palantir.baseline-error-prone'
+apply plugin: 'com.palantir.baseline-class-uniqueness'
 ```
 
 - Run ``./gradlew baselineUpdateConfig`` to download the config files
@@ -255,6 +256,13 @@ checks](https://errorprone.info):
 - Slf4jLogsafeArgs: Allow only com.palantir.logsafe.Arg types as parameter inputs to slf4j log messages. More information on
 Safe Logging can be found at [github.com/palantir/safe-logging](https://github.com/palantir/safe-logging).
 
+
+### Class Uniqueness Plugin (com.palantir.baseline-class-uniqueness)
+
+Run `./gradlew checkClassUniqueness` to scan all jars on the `runtime` classpath for identically named classes.
+This task will run automatically as part of `./gradlew build`.
+
+
 ### Copyright Checks
 
 By default Baseline enforces Palantir copyright at the beginning of files. To change this, edit the template copyright

gradle-baseline-java/build.gradle

@@ -10,10 +10,10 @@ repositories {
 dependencies {
     compile gradleApi()
     compile 'net.ltgt.gradle:gradle-errorprone-plugin'
+    compile 'com.google.guava:guava'
 
     testCompile gradleTestKit()
     testCompile 'com.netflix.nebula:nebula-test' // for better temp directory junit rule only
-    testCompile 'com.google.guava:guava'
     testCompile 'net.lingala.zip4j:zip4j'
     testCompile 'org.apache.commons:commons-io'
 }
@@ -54,6 +54,10 @@ pluginBundle {
             id = 'com.palantir.baseline-idea'
             displayName = 'Palantir Baseline IntelliJ Plugin'
         }
+        baselineClassUniquenessPlugin {
+            id = 'com.palantir.baseline-class-uniqueness'
+            displayName = 'Palantir Baseline Class Uniqueness Plugin'
+        }
     }
 }
 

gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/Baseline.groovy

@@ -30,5 +30,8 @@ class Baseline implements Plugin<Project> {
         project.plugins.apply BaselineEclipse
         project.plugins.apply BaselineIdea
         project.plugins.apply BaselineErrorProne
+
+        // TODO(dfox): enable this when it has been validated on a few real projects
+        // project.plugins.apply BaselineClassUniquenessPlugin
     }
 }

gradle-baseline-java/src/main/groovy/com/palantir/baseline/plugins/BaselineClassUniquenessPlugin.java

@@ -0,0 +1,40 @@
+/*
+ * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.baseline.plugins;
+
+import com.palantir.baseline.tasks.CheckClassUniquenessTask;
+import org.gradle.api.Project;
+
+/**
+ * This plugin is similar to https://github.com/nebula-plugins/gradle-lint-plugin/wiki/Duplicate-Classes-Rule
+ * but goes one step further and actually hashes any identically named classfiles to figure out if they're
+ * <i>completely</i> identical (and therefore safely interchangeable).
+ *
+ * The task only fails if it finds classes which have the same name but different implementations.
+ */
+public class BaselineClassUniquenessPlugin extends AbstractBaselinePlugin {
+
+    @Override
+    public final void apply(Project project) {
+        project.getPlugins().withId("java", plugin -> {
+            project.getTasks().create("checkClassUniqueness", CheckClassUniquenessTask.class, task -> {
+                task.setConfiguration(project.getConfigurations().getByName("runtime"));
+                project.getTasks().getByName("check").dependsOn(task);
+            });
+        });
+    }
+}

gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/CheckClassUniquenessTask.java

@@ -0,0 +1,121 @@
+/*
+ * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.baseline.tasks;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+public class CheckClassUniquenessTask extends DefaultTask {
+
+    private Configuration configuration;
+
+    public CheckClassUniquenessTask() {
+        setGroup("Verification");
+        setDescription("Checks that the given configuration contains no identically named classes.");
+    }
+
+    @Input
+    public final Configuration getConfiguration() {
+        return configuration;
+    }
+
+    public final void setConfiguration(Configuration configuration) {
+        this.configuration = configuration;
+    }
+
+    @TaskAction
+    public final void checkForDuplicateClasses() {
+        ClassUniquenessAnalyzer analyzer = new ClassUniquenessAnalyzer(getLogger());
+        analyzer.analyzeConfiguration(getConfiguration());
+        boolean success = analyzer.getDifferingProblemJars().isEmpty();
+        writeResultFile(success);
+
+        if (!success) {
+            analyzer.getDifferingProblemJars().forEach((problemJars) -> {
+                Set<String> differingClasses = analyzer.getDifferingSharedClassesInProblemJars(problemJars);
+                getLogger().error("{} Identically named classes with differing impls found in {}: {}",
+                        differingClasses.size(), problemJars, differingClasses);
+            });
+
+            throw new IllegalStateException(String.format(
+                    "'%s' contains multiple copies of identically named classes - "
+                            + "this may cause different runtime behaviour depending on classpath ordering.\n"
+                            + "To resolve this, try excluding one of the following jars:\n\n%s",
+                    configuration.getName(),
+                    formatSummary(analyzer)
+            ));
+        }
+    }
+
+    private static String formatSummary(ClassUniquenessAnalyzer summary) {
+        Collection<Set<ModuleVersionIdentifier>> allProblemJars = summary.getDifferingProblemJars();
+
+        int maxLength = allProblemJars.stream().flatMap(Set::stream)
+                .map(ModuleVersionIdentifier::toString)
+                .map(String::length)
+                .max(Comparator.naturalOrder()).get();
+        String format = "%-" + (maxLength + 1) + "s";
+
+        StringBuilder builder = new StringBuilder();
+
+        allProblemJars.forEach(problemJars -> {
+            int count = summary.getDifferingSharedClassesInProblemJars(problemJars).size();
+            String countColumn = String.format("\t%-14s", "(" + count + " classes) ");
+            builder.append(countColumn);
+
+            String jars = problemJars.stream().map(jar -> String.format(format, jar)).collect(Collectors.joining());
+            builder.append(jars);
+
+            builder.append('\n');
+        });
+
+        return builder.toString();
+    }
+
+    /**
+     * This only exists to convince gradle this task is incremental.
+     */
+    @OutputFile
+    public final File getResultFile() {
+        return getProject().getBuildDir().toPath()
+                .resolve(Paths.get("uniqueClassNames", configuration.getName()))
+                .toFile();
+    }
+
+    private void writeResultFile(boolean success) {
+        try {
+            File result = getResultFile();
+            Files.createDirectories(result.toPath().getParent());
+            Files.write(result.toPath(), Boolean.toString(success).getBytes(StandardCharsets.UTF_8));
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to write boolean result file", e);
+        }
+    }
+}

gradle-baseline-java/src/main/groovy/com/palantir/baseline/tasks/ClassUniquenessAnalyzer.java

@@ -0,0 +1,155 @@
+/*
+ * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.baseline.tasks;
+
+import static java.util.stream.Collectors.toSet;
+
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.hash.HashingInputStream;
+import com.google.common.io.ByteStreams;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+import java.util.stream.Collectors;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.artifacts.ResolvedArtifact;
+import org.slf4j.Logger;
+
+public final class ClassUniquenessAnalyzer {
+
+    private final Map<Set<ModuleVersionIdentifier>, Set<String>> jarsToClasses = new HashMap<>();
+    private final Map<String, Set<HashCode>> classToHashCodes = new HashMap<>();
+    private final Logger log;
+
+    public ClassUniquenessAnalyzer(Logger log) {
+        this.log = log;
+    }
+
+    public void analyzeConfiguration(Configuration configuration) {
+        Instant before = Instant.now();
+        Set<ResolvedArtifact> dependencies = configuration
+                .getResolvedConfiguration()
+                .getResolvedArtifacts();
+
+        // we use these temporary maps to accumulate information as we process each jar,
+        // so they may include singletons which we filter out later
+        Map<String, Set<ModuleVersionIdentifier>> classToJars = new HashMap<>();
+        Map<String, Set<HashCode>> tempClassToHashCodes = new HashMap<>();
+
+        dependencies.stream().forEach(resolvedArtifact -> {
+            File file = resolvedArtifact.getFile();
+            if (!file.exists()) {
+                log.info("Skipping non-existent jar {}: {}", resolvedArtifact, file);
+                return;
+            }
+
+            try (FileInputStream fileInputStream = new FileInputStream(file);
+                    JarInputStream jarInputStream = new JarInputStream(fileInputStream)) {
+                JarEntry entry;
+                while ((entry = jarInputStream.getNextJarEntry()) != null) {
+                    if (entry.isDirectory() || !entry.getName().endsWith(".class")) {
+                        continue;
+                    }
+
+                    String className = entry.getName().replaceAll("/", ".").replaceAll(".class", "");
+                    HashingInputStream inputStream = new HashingInputStream(Hashing.sha256(), jarInputStream);
+                    ByteStreams.exhaust(inputStream);
+
+                    multiMapPut(classToJars,
+                            className,
+                            resolvedArtifact.getModuleVersion().getId());
+
+                    multiMapPut(tempClassToHashCodes,
+                            className,
+                            inputStream.hash());
+                }
+            } catch (IOException e) {
+                log.error("Failed to read JarFile {}", resolvedArtifact, e);
+                throw new RuntimeException(e);
+            }
+        });
+
+        // discard all the classes that only come from one jar - these are completely safe!
+        classToJars.entrySet().stream()
+                .filter(entry -> entry.getValue().size() > 1)
+                .forEach(entry -> multiMapPut(jarsToClasses, entry.getValue(), entry.getKey()));
+
+        // figure out which classes have differing hashes
+        tempClassToHashCodes.entrySet().stream()
+                .filter(entry -> entry.getValue().size() > 1)
+                .forEach(entry ->
+                        entry.getValue().forEach(value -> multiMapPut(classToHashCodes, entry.getKey(), value)));
+
+        Instant after = Instant.now();
+        log.info("Checked {} classes from {} dependencies for uniqueness ({}ms)",
+                classToJars.size(), dependencies.size(), Duration.between(before, after).toMillis());
+    }
+
+    /**
+     * Any groups jars that all contain some identically named classes.
+     * Note: may contain non-scary duplicates - class files which are 100% identical, so their
+     * clashing name doesn't have any effect.
+     */
+    public Collection<Set<ModuleVersionIdentifier>> getProblemJars() {
+        return jarsToClasses.keySet();
+    }
+
+    /**
+     * Class names that appear in all of the given jars.
+     */
+    public Set<String> getSharedClassesInProblemJars(Collection<ModuleVersionIdentifier> problemJars) {
+        return jarsToClasses.get(problemJars);
+    }
+
+    /**
+     * Jars which contain identically named classes with non-identical implementations.
+     */
+    public Collection<Set<ModuleVersionIdentifier>> getDifferingProblemJars() {
+        return getProblemJars()
+                .stream()
+                .filter(jars -> getDifferingSharedClassesInProblemJars(jars).size() > 0)
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * Class names which appear in all of the given jars and also have non-identical implementations.
+     */
+    public Set<String> getDifferingSharedClassesInProblemJars(Collection<ModuleVersionIdentifier> problemJars) {
+        return getSharedClassesInProblemJars(problemJars).stream()
+                .filter(classToHashCodes::containsKey)
+                .collect(toSet());
+    }
+
+    private static <K, V> void multiMapPut(Map<K, Set<V>> map, K key, V value) {
+        map.compute(key, (unused, collection) -> {
+            Set<V> newCollection = collection != null ? collection : new HashSet<>();
+            newCollection.add(value);
+            return newCollection;
+        });
+    }
+}

gradle-baseline-java/src/main/resources/META-INF/gradle-plugins/com.palantir.baseline-class-uniqueness.properties

@@ -0,0 +1 @@
+implementation-class=com.palantir.baseline.plugins.BaselineClassUniquenessPlugin