new helper module, full avro 1.4-1.9 support (#22)

* new helper module, full avro 1.4-1.9 support this commit includes the following: - new helper module build has a sub-module per major version of avro which compiles natively vs the target avro version. this eliminates the need for reflection in _most_ use cases. cases involving optional avro-compiler jar still rely on reflection - helper code now implemented across all major avro versions from 1.4 to 1.9 inclusive - code generation support has been extended to cover above avro versions - tests are now performed with multiple classpaths, including variants with and without the avro-compiler jar (on avro 1.5+) Signed-off-by: Radai Rosenblatt <[email protected]>
linkedin · Feb 20, 2020 · 3365e27 · 3365e27
1 parent abbe76f
commit 3365e27
Show file tree

Hide file tree

Showing 199 changed files with 10,014 additions and 249 deletions.
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@ Avro-Util
 [![Build Status](https://travis-ci.org/linkedin/avro-util.svg?branch=master)](https://travis-ci.org/linkedin/avro-util)
 [![Download](https://api.bintray.com/packages/linkedin/maven/avro-util/images/download.svg)](https://bintray.com/linkedin/maven/avro-util/_latestVersion)
 
-A collection of utilitiesto allow java projects to better work with avro.
+A collection of utilities and libraries to allow java projects to better work with avro.
 
 ### Background ###
 
@@ -21,7 +21,7 @@ performance.
 
 The following modules are available in this project.
 
-## avro-migration-helper ##
+## helper ##
 
 This module provides utility functions which, when coded against, ensure 
 compatibility with every Avro supported version. This is achieved by offering 
@@ -50,10 +50,13 @@ LinkedIn.
 
 ### Supported versions of Avro ###
 
+The helper module supports avro 1.4 - 1.9 inclusive. for fastserde support: 
+
 | Version  | Serialization | Deserialization | Fast Serialization | Fast Deserialization |
 | -------- | ------------- | --------------- | ------------------ | -------------------- |
 | Avro 1.4 |      Yes      |      Yes        |      Yes           |      Yes             |
-| Avro 1.5 |      ???      |      ???        |      No            |      No              |
-| Avro 1.6 |      ???      |      ???        |      No            |      No              |
+| Avro 1.5 |      Yes      |      Yes        |      No            |      No              |
+| Avro 1.6 |      Yes      |      Yes        |      No            |      No              |
 | Avro 1.7 |      Yes      |      Yes        |      Yes           |      Yes             |
 | Avro 1.8 |      Yes      |      Yes        |      Yes           |      Yes             |
+| Avro 1.9 |      Yes      |      Yes        |      ???           |      ???             |
diff --git a/avro-codegen/README.md b/avro-codegen/README.md
@@ -0,0 +1,15 @@
+Avro Codegen
+=========
+
+This module contains utility classes for generating java classes out of avro schema (*.avsc) files 
+
+Major features include:
+
+* the ability to reuse ("import") schemas defined in either other *.avsc files in the same
+  code generation scope of schemas defined by classes on the classpath when generating code, 
+  allowing for schema reuse inside and between large projects
+* the ability to generate java code for specific records that is runtime-compatible with a
+  wide range of avro versions, allowing to reuse the same libraries of schemas between 
+  projects that use different versions of avro
+
+instructions and examples TBD     
diff --git a/avro-codegen/build.gradle b/avro-codegen/build.gradle
@@ -6,6 +6,7 @@
 
 plugins {
     id "java-library"
+    id "checkstyle"
 }
 
 dependencies {

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/ClassifiedIssue.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/ClassifiedIssue.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 class ClassifiedIssue {

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/ClasspathFishingResults.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/ClasspathFishingResults.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 import java.util.Map;

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/CodeGenerator.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/CodeGenerator.java
@@ -11,6 +11,21 @@
 import com.linkedin.avro.compatibility.AvroVersion;
 import com.linkedin.avro.compatibility.SchemaParseResult;
 import com.linkedin.avro.util.TemplateUtil;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringJoiner;
 import org.apache.avro.Schema;
 import org.apache.avro.SchemaParseException;
 import org.apache.avro.specific.FixedSize;
@@ -23,11 +38,9 @@
 import org.apache.tools.ant.DirectoryScanner;
 import org.apache.tools.ant.types.selectors.SelectorUtils;
 
-import java.io.*;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/FileParseIssue.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/FileParseIssue.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 import org.apache.avro.SchemaParseException;

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/IssueType.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/IssueType.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 enum IssueType {

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/SchemaDetails.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/SchemaDetails.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 import org.apache.avro.Schema;

diff --git a/avro-codegen/src/main/java/com/linkedin/avro/codegen/SchemaSource.java b/avro-codegen/src/main/java/com/linkedin/avro/codegen/SchemaSource.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2018 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package com.linkedin.avro.codegen;
 
 enum SchemaSource {

diff --git a/avro-codegen/src/test/java/com/linkedin/avro/codegen/CodeGeneratorTest.java b/avro-codegen/src/test/java/com/linkedin/avro/codegen/CodeGeneratorTest.java
@@ -8,19 +8,21 @@
 
 import com.linkedin.avro.codegen.testutil.CompilerHelper;
 import com.linkedin.avro.compatibility.AvroGeneratedSourceCode;
+import java.io.File;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
 import net.openhft.compiler.CompilerUtils;
 import org.apache.avro.specific.SpecificFixed;
 import org.apache.avro.specific.SpecificRecord;
 import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import java.io.File;
-import java.lang.reflect.Method;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.*;
-
 public class CodeGeneratorTest {
     private File sourceRoot;
     private Path outputRoot;

diff --git a/avro-codegen/src/test/java/com/linkedin/avro/codegen/testutil/CompilerHelper.java b/avro-codegen/src/test/java/com/linkedin/avro/codegen/testutil/CompilerHelper.java
@@ -6,20 +6,34 @@
 
 package com.linkedin.avro.codegen.testutil;
 
-import org.apache.commons.io.IOUtils;
-import org.testng.Assert;
-
-import javax.tools.*;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.*;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import javax.tools.Diagnostic;
+import javax.tools.DiagnosticListener;
+import javax.tools.FileObject;
+import javax.tools.ForwardingJavaFileManager;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+import org.apache.commons.io.IOUtils;
+import org.testng.Assert;
 
 /**
  * utility class for dealing with the java compiler in unit tests

diff --git a/avro-fastserde/README.md b/avro-fastserde/README.md
@@ -0,0 +1,12 @@
+avro-fastserde
+=========
+
+This is a fork of https://github.com/RTBHOUSE/avro-fastserde
+
+Notable features include:
+
+* The code has been altered to support multiple major versions of avro at runtime (by using 
+  the helper module)
+* various performance improvements to array fields?
+
+instructions and examples TBD     
diff --git a/avro-migration-helper/build.gradle b/avro-migration-helper/build.gradle
@@ -6,6 +6,7 @@
 
 plugins {
   id "java-library"
+  id "checkstyle"
 }
 
 configurations {

diff --git a/...gration-helper/src/main/java/com/linkedin/avro/compatibility/AvroCompatibilityHelper.java b/...gration-helper/src/main/java/com/linkedin/avro/compatibility/AvroCompatibilityHelper.java
@@ -18,8 +18,14 @@
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.generic.IndexedRecord;
-import org.apache.avro.io.*;
+import org.apache.avro.io.Avro14Adapter;
+import org.apache.avro.io.Avro17Adapter;
 import org.apache.avro.io.AvroAdapter;
+import org.apache.avro.io.BinaryDecoder;
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.io.JsonEncoder;
 import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.avro.specific.SpecificRecord;
 import org.codehaus.jackson.JsonGenerator;

diff --git a/avro-migration-helper/src/main/java/com/linkedin/avro/legacy/FixDefaultValueStep.java b/avro-migration-helper/src/main/java/com/linkedin/avro/legacy/FixDefaultValueStep.java
@@ -46,7 +46,7 @@ static ValueType fromLiteral(String literal) {
         Integer.parseInt(literal);
         return INT;
       } catch (NumberFormatException ignored) {
-
+        //empty
       }
       throw new IllegalStateException("unhandled: " + literal);
     }

diff --git a/avro-migration-helper/src/main/java/org/apache/avro/io/Avro14Adapter.java b/avro-migration-helper/src/main/java/org/apache/avro/io/Avro14Adapter.java
@@ -9,21 +9,26 @@
 import com.linkedin.avro.compatibility.AvroGeneratedSourceCode;
 import com.linkedin.avro.compatibility.AvroVersion;
 import com.linkedin.avro.compatibility.SchemaNormalization;
+import com.linkedin.avro.compatibility.SchemaParseResult;
+import com.linkedin.avro.util.TemplateUtil;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
-import com.linkedin.avro.compatibility.SchemaParseResult;
-import com.linkedin.avro.util.TemplateUtil;
+import org.apache.avro.Avro14SchemaAccessHelper;
 import org.apache.avro.AvroRuntimeException;
 import org.apache.avro.Schema;
-import org.apache.avro.Avro14SchemaAccessHelper;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.io.avro18.Avro18BufferedBinaryEncoder;

diff --git a/avro-migration-helper/src/main/java/org/apache/avro/io/Avro17Adapter.java b/avro-migration-helper/src/main/java/org/apache/avro/io/Avro17Adapter.java
@@ -9,17 +9,21 @@
 import com.linkedin.avro.compatibility.AvroCompatibilityHelper;
 import com.linkedin.avro.compatibility.AvroGeneratedSourceCode;
 import com.linkedin.avro.compatibility.AvroVersion;
+import com.linkedin.avro.compatibility.SchemaParseResult;
 import java.io.OutputStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
-import com.linkedin.avro.compatibility.SchemaParseResult;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.specific.SpecificData;

diff --git a/avro-migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BinaryData.java b/avro-migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BinaryData.java
@@ -1,3 +1,27 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.avro.io.avro18;
 
 

diff --git a/avro-migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BinaryEncoder.java b/avro-migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BinaryEncoder.java
@@ -1,3 +1,27 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.avro.io.avro18;
 
 import java.io.IOException;

diff --git a/...migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BufferedBinaryEncoder.java b/...migration-helper/src/main/java/org/apache/avro/io/avro18/Avro18BufferedBinaryEncoder.java
@@ -1,3 +1,9 @@
+/*
+ * Copyright 2020 LinkedIn Corp.
+ * Licensed under the BSD 2-Clause License (the "License").
+ * See License in the project root for license information.
+ */
+
 package org.apache.avro.io.avro18;
 
 /**