retain builders in generated code if target avro >= 1.6 (#112)

* expending testTool to support minAvroVersion argument for code-gen * allow retaining builder support if codegen target avro is >= 1.6
linkedin · Nov 17, 2020 · dc3da5a · dc3da5a
1 parent e056996
commit dc3da5a
Show file tree

Hide file tree

Showing 23 changed files with 277 additions and 105 deletions.
diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java
@@ -29,6 +29,10 @@
  */
 public interface AvroAdapter {
 
+  //metadata
+
+  AvroVersion supporttedMajorVersion();
+
   //codecs
 
   BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse);

diff --git a/...helper-common/src/main/java/com/linkedin/avroutil1/compatibility/CodeTransformations.java b/...helper-common/src/main/java/com/linkedin/avroutil1/compatibility/CodeTransformations.java
@@ -76,6 +76,43 @@ public class CodeTransformations {
 
   private static final int MAX_STRING_LITERAL_SIZE = 65000; //just under 64k
 
+  /**
+   * applies all transformations to a java class generated by avro
+   * @param code raw java code for a class generated by avro
+   * @param generatedBy major version of avro that generated the input code
+   * @param minSupportedVersion minimum major avro version under which the result should "work"
+   * @param maxSupportedVersion maximum major avro version under which the result should "work"
+   * @return fixed-up code
+   */
+  public static String applyAll(String code, AvroVersion generatedBy, AvroVersion minSupportedVersion, AvroVersion maxSupportedVersion) {
+    String fixed = code;
+
+    //general fix-ups that are considered safe regardless of min or max avro version
+    fixed = CodeTransformations.transformFixedClass(fixed, minSupportedVersion, maxSupportedVersion);
+    fixed = CodeTransformations.transformEnumClass(fixed, minSupportedVersion, maxSupportedVersion);
+    fixed = CodeTransformations.transformParseCalls(fixed, generatedBy, minSupportedVersion, maxSupportedVersion);
+    fixed = CodeTransformations.addGetClassSchemaMethod(fixed, generatedBy, minSupportedVersion, maxSupportedVersion);
+
+    //1.6+ features
+    if (minSupportedVersion.earlierThan(AvroVersion.AVRO_1_6)) {
+      //optionally strip out builders
+      fixed = CodeTransformations.removeBuilderSupport(fixed, minSupportedVersion, maxSupportedVersion);
+    }
+
+    //1.7+ features
+    //this is considered harmless enough we can keep doing it?
+    fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minSupportedVersion, maxSupportedVersion);
+
+    //1.8+ features
+    fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minSupportedVersion, maxSupportedVersion);
+    fixed = CodeTransformations.transformExternalizableSupport(fixed, minSupportedVersion, maxSupportedVersion);
+
+    //1.9+ features
+    fixed = CodeTransformations.transformCustomCodersSupport(fixed, minSupportedVersion, maxSupportedVersion);
+
+    return fixed;
+  }
+
   /**
    * this method transforms the code for a generated class for an avro "fixed" type to make it compatible with a
    * range of avro versions.<br>

diff --git a/...r-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java b/...r-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java
@@ -94,6 +94,11 @@ private void tryInitializeCompilerFields() {
         }
     }
 
+    @Override
+    public AvroVersion supporttedMajorVersion() {
+        return AvroVersion.AVRO_1_10;
+    }
+
     @Override
     public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
         if (buffered) {
@@ -264,17 +269,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
     private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
         List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-        String fixed;
         for (AvroGeneratedSourceCode generated : avroGenerated) {
-            fixed = generated.getContents();
-            fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_9, minAvro, maxAvro);
-            fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-            fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+            String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
             transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
         }
         return transformed;

diff --git a/...lper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java b/...lper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java
@@ -78,6 +78,11 @@ public Avro14Adapter() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_4;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (reuse != null && reuse instanceof Avro18BufferedBinaryEncoder) {
@@ -236,18 +241,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_4, minAvro, maxAvro);
-      fixed = CodeTransformations.addGetClassSchemaMethod(fixed, AvroVersion.AVRO_1_4, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/...lper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java b/...lper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java
@@ -89,6 +89,11 @@ private void tryInitializeCompilerFields() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_5;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (buffered) {
@@ -261,18 +266,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_5, minAvro, maxAvro);
-      fixed = CodeTransformations.addGetClassSchemaMethod(fixed, AvroVersion.AVRO_1_5, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/...lper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java b/...lper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java
@@ -95,6 +95,11 @@ private void tryInitializeCompilerFields() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_6;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (buffered) {
@@ -270,18 +275,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_6, minAvro, maxAvro);
-      fixed = CodeTransformations.addGetClassSchemaMethod(fixed, AvroVersion.AVRO_1_6, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/...lper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java b/...lper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java
@@ -138,6 +138,11 @@ private void tryInitializeCompilerFields() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_7;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (buffered) {
@@ -331,18 +336,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_7, minAvro, maxAvro);
-      fixed = CodeTransformations.addGetClassSchemaMethod(fixed, AvroVersion.AVRO_1_7, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/...lper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java b/...lper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java
@@ -97,6 +97,11 @@ private void tryInitializeCompilerFields() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_8;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (buffered) {
@@ -269,17 +274,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_8, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/...lper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/...lper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java
@@ -97,6 +97,11 @@ private void tryInitializeCompilerFields() {
     }
   }
 
+  @Override
+  public AvroVersion supporttedMajorVersion() {
+    return AvroVersion.AVRO_1_9;
+  }
+
   @Override
   public BinaryEncoder newBinaryEncoder(OutputStream out, boolean buffered, BinaryEncoder reuse) {
     if (buffered) {
@@ -267,17 +272,8 @@ public Collection<AvroGeneratedSourceCode> compile(
 
   private Collection<AvroGeneratedSourceCode> transform(List<AvroGeneratedSourceCode> avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) {
     List<AvroGeneratedSourceCode> transformed = new ArrayList<>(avroGenerated.size());
-    String fixed;
     for (AvroGeneratedSourceCode generated : avroGenerated) {
-      fixed = generated.getContents();
-      fixed = CodeTransformations.transformFixedClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformEnumClass(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformParseCalls(fixed, AvroVersion.AVRO_1_9, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBuilderSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeBinaryMessageCodecSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.removeAvroGeneratedAnnotation(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformExternalizableSupport(fixed, minAvro, maxAvro);
-      fixed = CodeTransformations.transformCustomCodersSupport(fixed, minAvro, maxAvro);
+      String fixed = CodeTransformations.applyAll(generated.getContents(), supporttedMajorVersion(), minAvro, maxAvro);
       transformed.add(new AvroGeneratedSourceCode(generated.getPath(), fixed));
     }
     return transformed;

diff --git a/helper/tests/codegen-110/build.gradle b/helper/tests/codegen-110/build.gradle
@@ -53,7 +53,12 @@ task runCompatAvroCodegen {
       javaexec {
         classpath configurations.codegen
         main = 'com.linkedin.avroutil1.TestTool'
-        args = ["compile", file.getAbsolutePath(), "$buildDir/generated/sources/compat-avro/java/main"]
+        args = [
+            "-op", "compile",
+            "-in", file.getAbsolutePath(),
+            "-out", "$buildDir/generated/sources/compat-avro/java/main",
+            "-min", "AVRO_1_4"
+        ]
         debug = false
       }
     }

diff --git a/helper/tests/codegen-14/build.gradle b/helper/tests/codegen-14/build.gradle
@@ -53,7 +53,12 @@ task runCompatAvroCodegen {
       javaexec {
         classpath configurations.codegen
         main = 'com.linkedin.avroutil1.TestTool'
-        args = ["compile", file.getAbsolutePath(), "$buildDir/generated/sources/compat-avro/java/main"]
+        args = [
+            "-op", "compile",
+            "-in", file.getAbsolutePath(),
+            "-out", "$buildDir/generated/sources/compat-avro/java/main",
+            "-min", "AVRO_1_4"
+        ]
       }
     }
   }

diff --git a/helper/tests/codegen-15/build.gradle b/helper/tests/codegen-15/build.gradle
@@ -53,7 +53,12 @@ task runCompatAvroCodegen {
       javaexec {
         classpath configurations.codegen
         main = 'com.linkedin.avroutil1.TestTool'
-        args = ["compile", file.getAbsolutePath(), "$buildDir/generated/sources/compat-avro/java/main"]
+        args = [
+            "-op", "compile",
+            "-in", file.getAbsolutePath(),
+            "-out", "$buildDir/generated/sources/compat-avro/java/main",
+            "-min", "AVRO_1_4"
+        ]
       }
     }
   }

diff --git a/helper/tests/codegen-16/build.gradle b/helper/tests/codegen-16/build.gradle
@@ -53,7 +53,12 @@ task runCompatAvroCodegen {
       javaexec {
         classpath configurations.codegen
         main = 'com.linkedin.avroutil1.TestTool'
-        args = ["compile", file.getAbsolutePath(), "$buildDir/generated/sources/compat-avro/java/main"]
+        args = [
+            "-op", "compile",
+            "-in", file.getAbsolutePath(),
+            "-out", "$buildDir/generated/sources/compat-avro/java/main",
+            "-min", "AVRO_1_4"
+        ]
       }
     }
   }