add Tool to run avro code generation for entire projects (#24)

* add Tool to run avro code generation for entire projects

avro-codegen/build.gradle

@@ -7,25 +7,50 @@
 plugins {
     id "java-library"
     id "checkstyle"
+    id "com.github.johnrengelman.shadow"
 }
 
 dependencies {
-    compile project(":avro-migration-helper")
+    implementation project(":avro-migration-helper")
 
-    compile "commons-io:commons-io:2.6"
-    compile "org.apache.logging.log4j:log4j-api:2.11.2"
+    implementation "commons-io:commons-io:2.6"
+    implementation "org.apache.logging.log4j:log4j-api:2.11.2"
+    implementation "com.beust:jcommander:1.78"
+    implementation "org.apache.ant:ant:1.10.7"
 
-    //TODO - make these compile only
-    compile "org.apache.avro:avro:1.4.1"
-    //compile "org.apache.avro:avro-compiler:1.8.2"
+    compileOnly ("org.apache.avro:avro:1.4.1") {
+        exclude group: "org.mortbay.jetty"
+        exclude group: "org.apache.velocity"
+        exclude group: "commons-lang"
+        exclude group: "org.jboss.netty"
+        exclude group: "com.thoughtworks.paranamer", module: "paranamer-ant"
+        exclude group: "org.slf4j"
+    }
 
+    testCompile ("org.apache.avro:avro:1.4.1") {
+        exclude group: "org.mortbay.jetty"
+        exclude group: "org.apache.velocity"
+        exclude group: "commons-lang"
+        exclude group: "org.jboss.netty"
+        exclude group: "com.thoughtworks.paranamer", module: "paranamer-ant"
+        exclude group: "org.slf4j"
+    }
     testCompile 'net.openhft:compiler:2.3.1'
-    
+
     //redirect logging to log4j2 for tests
     testRuntimeOnly "org.apache.logging.log4j:log4j-core:2.11.2"
     testRuntimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.11.2"
 }
 
-test {
-    testLogging.showStandardStreams = true
+jar {
+    manifest {
+        attributes(
+            "Main-Class": "com.linkedin.avro.codegen.Tool"
+        )
+    }
+}
+
+shadowJar {
+    //needs to have a classifier to not overwrite the regular jar
+    classifier = "all"
 }

avro-codegen/src/main/java/com/linkedin/avro/codegen/StringToFileConverter.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avro.codegen;
+
+import com.beust.jcommander.IStringConverter;
+import java.io.File;
+
+
+public class StringToFileConverter implements IStringConverter<File> {
+  @Override
+  public File convert(String value) {
+    return new File(value);
+  }
+}

avro-codegen/src/main/java/com/linkedin/avro/codegen/Tool.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avro.codegen;
+
+import com.beust.jcommander.JCommander;
+
+
+public class Tool {
+
+  public static void main(String[] args) {
+    ToolArgs conf = parseArgs(args);
+
+    CodeGenerator codegen = new CodeGenerator();
+    codegen.setInputs(conf.getSrc());
+    codegen.setOutputFolder(conf.getOutputFolder());
+    codegen.setIncludes(conf.getIncludes());
+    codegen.setAllowClasspathLookup(conf.isAllowClasspathLookup());
+    codegen.setAllowReverseEngineeringClasspathSchemas(conf.isAllowReverseEngineeringClasspathSchemas());
+    codegen.setValidateSchemaNamespaceVsFilePath(conf.isValidateSchemaNamespaceVsFilePath());
+    codegen.setValidateSchemaNameVsFileName(conf.isValidateSchemaNameVsFileName());
+
+    codegen.generateCode();
+  }
+
+  private static ToolArgs parseArgs(String[] args) {
+    ToolArgs parsed = new ToolArgs();
+    JCommander.newBuilder()
+        .addObject(parsed)
+        .build()
+        .parse(args);
+    return parsed;
+  }
+}

avro-codegen/src/main/java/com/linkedin/avro/codegen/ToolArgs.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+package com.linkedin.avro.codegen;
+
+import com.beust.jcommander.Parameter;
+import com.linkedin.avro.compatibility.AvroVersion;
+import java.io.File;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+
+public class ToolArgs {
+  @Parameter(names = "-src", required = true, description = "files or folders to generate code for", converter = StringToFileConverter.class)
+  private Set<File> src;
+  @Parameter(names = "-include", description = "include patterns. default is \"**/*.avsc\"")
+  private Set<String> includes = new HashSet<>(Collections.singletonList("**/*.avsc"));
+  @Parameter(names = "-importable", description = "patterns for importable schema files. default is \"**/*.*\"")
+  private Set<String> importablePatterns = new HashSet<>(Collections.singletonList("**/*.*"));
+  //if true will go fishing on the classpath for generated specific record classes of missing schemas
+  @Parameter(names = "-allow-cp", description = "allow looking up undefined schemas on the classpath")
+  private boolean allowClasspathLookup = true;
+  /**
+   * if true, and we find a SpecificFixed or Enum class on the classpath (see allowClasspathLookup above)
+   * and this class we find was generated by horrible ancient avro and not made compatible (so we cant get
+   * its actual schema from it) we can try and "guess" the schema by the class' contents.
+   * this has an interesting side effect of allowing avro schemas to use just any java enum ...
+   */
+  @Parameter(names = "-allow-guessing", description = "allow generating schemas for classpath classes that dont have them")
+  private boolean allowReverseEngineeringClasspathSchemas = false;
+  //top-level schema a.b.c.X must be in a file who's path from root is a/b/c/*.avsc
+  @Parameter(names = "-validate-path")
+  private boolean validateSchemaNamespaceVsFilePath = true;
+  //top-level schema called a.b.c.X must be in a file called X.avsc
+  @Parameter(names = "-validate-name")
+  private boolean validateSchemaNameVsFileName = true;
+  //if != null, would post-process the code to make it runtime-compatible with the designated version and onwards
+  private AvroVersion minTargetAvroVersion = AvroVersion.AVRO_1_4;
+  //output root for java files (or null, in which case no results will be written to disk)
+  @Parameter(names = "-dest", required = true, description = "destination root folder", converter = StringToFileConverter.class)
+  private File outputFolder;
+
+  public Set<File> getSrc() {
+    return src;
+  }
+
+  public Set<String> getIncludes() {
+    return includes;
+  }
+
+  public Set<String> getImportablePatterns() {
+    return importablePatterns;
+  }
+
+  public boolean isAllowClasspathLookup() {
+    return allowClasspathLookup;
+  }
+
+  public boolean isAllowReverseEngineeringClasspathSchemas() {
+    return allowReverseEngineeringClasspathSchemas;
+  }
+
+  public boolean isValidateSchemaNamespaceVsFilePath() {
+    return validateSchemaNamespaceVsFilePath;
+  }
+
+  public boolean isValidateSchemaNameVsFileName() {
+    return validateSchemaNameVsFileName;
+  }
+
+  public AvroVersion getMinTargetAvroVersion() {
+    return minTargetAvroVersion;
+  }
+
+  public File getOutputFolder() {
+    return outputFolder;
+  }
+}

avro-codegen/src/test/java/com/linkedin/avro/codegen/CodeGeneratorTest.java

@@ -94,9 +94,9 @@ public void testProjectWithClasspathDependencies() throws Exception {
         Collection<AvroGeneratedSourceCode> firstModuleSources = generator.generateCode();
         Assert.assertEquals(firstModuleSources.size(), 3);
 
-        Class enumClass = null;
-        Class fixedClass = null;
-        Class recordClass = null;
+        Class<?> enumClass = null;
+        Class<?> fixedClass = null;
+        Class<?> recordClass = null;
 
         ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
         for (AvroGeneratedSourceCode code : firstModuleSources) {

helper/helper/build.gradle

@@ -6,8 +6,8 @@
 
 plugins {
   id "java-library"
-  id "com.github.johnrengelman.shadow"
   id "checkstyle"
+  id "com.github.johnrengelman.shadow"
 }
 
 dependencies {