From d99099d3b5b5871b680b626cc7b58976ca9f0845 Mon Sep 17 00:00:00 2001 From: Radai Rosenblatt Date: Fri, 18 Jun 2021 13:56:47 -0700 Subject: [PATCH] initial support for field properties in FieldBuilder (#162) addresses #160 --- .../avroutil1/compatibility/AvroAdapter.java | 4 + .../avroutil1/compatibility/FieldBuilder.java | 1 + .../AvroCompatibilityHelper.java | 12 ++ .../compatibility/avro110/Avro110Adapter.java | 21 +++ .../avro110/FieldBuilder110.java | 25 ++-- .../compatibility/avro14/Avro14Adapter.java | 6 + .../compatibility/avro14/FieldBuilder14.java | 47 +++++-- .../compatibility/avro15/Avro15Adapter.java | 6 + .../compatibility/avro15/FieldBuilder15.java | 47 +++++-- .../compatibility/avro16/Avro16Adapter.java | 6 + .../compatibility/avro16/FieldBuilder16.java | 25 ++-- .../compatibility/avro17/Avro17Adapter.java | 18 +++ .../compatibility/avro17/Avro17Utils.java | 132 ++++++++++++++++++ .../compatibility/avro17/FieldBuilder17.java | 23 +-- .../compatibility/avro18/Avro18Adapter.java | 19 +++ .../compatibility/avro18/FieldBuilder18.java | 30 ++-- .../compatibility/avro19/Avro19Adapter.java | 21 +++ .../compatibility/avro19/FieldBuilder19.java | 25 ++-- .../compatibility/FieldBuilderTest.java | 44 ++++++ .../main/resources/RecordWithFieldProps.avsc | 21 +++ 20 files changed, 470 insertions(+), 63 deletions(-) create mode 100644 helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java create mode 100644 helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/FieldBuilderTest.java create mode 100644 helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java index a2a1bc82c..afcde7c5f 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java @@ -86,12 +86,16 @@ BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, Object getGenericDefaultValue(Schema.Field field); + //schema query and manipulation utils + boolean fieldHasDefault(Schema.Field field); FieldBuilder cloneSchemaField(Schema.Field field); FieldBuilder newFieldBuilder(String name); + String getFieldPropAsJsonString(Schema.Field field, String propName); + //code generation Collection compile( diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/FieldBuilder.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/FieldBuilder.java index 3dc5eca13..73dad43e4 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/FieldBuilder.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/FieldBuilder.java @@ -25,6 +25,7 @@ public interface FieldBuilder { FieldBuilder setOrder(Order order); + @Deprecated FieldBuilder copyFromField(); Schema.Field build(); diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java index f84c45938..5bfa79667 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java @@ -634,4 +634,16 @@ public static Schema.Field createSchemaField(String name, Schema schema, String public static Schema.Field createSchemaField(String name, Schema schema, String doc, Object defaultValue) { return createSchemaField(name, schema, doc, defaultValue, Schema.Field.Order.ASCENDING); } + + /** + * returns the value of the specified field prop as a json literal. + * returns null if the field has no such property. + * note that string values are returned quoted (as a proper json string literal) + * @param field the field who's property value we wish to get + * @param propName the name of the property + * @return field property value as json literal, or null if no such property + */ + public static String getFieldPropAsJsonString(Schema.Field field, String propName) { + return ADAPTER.getFieldPropAsJsonString(field, propName); + } } diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java index 801340517..11e572a35 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java @@ -6,6 +6,8 @@ package com.linkedin.avroutil1.compatibility.avro110; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.avroutil1.compatibility.AvroAdapter; import com.linkedin.avroutil1.compatibility.AvroGeneratedSourceCode; import com.linkedin.avroutil1.compatibility.AvroVersion; @@ -34,6 +36,7 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificData; +import org.apache.avro.util.internal.JacksonUtils; import java.io.IOException; import java.io.InputStream; @@ -54,6 +57,9 @@ public class Avro110Adapter implements AvroAdapter { + //doc says thread safe outside config methods + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private boolean compilerSupported; private Constructor specificCompilerCtr; private Method compilerEnqueueMethod; @@ -260,6 +266,21 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder110(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + Object val = field.getObjectProp(propName); + if (val == null) { + return null; + } + //the following is VERY rube-goldberg-ish, but will do until someone complains + JsonNode asJsonNode = JacksonUtils.toJsonNode(val); + try { + return OBJECT_MAPPER.writeValueAsString(asJsonNode); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java index 95e919a1d..45caf53f0 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/FieldBuilder110.java @@ -10,18 +10,24 @@ import org.apache.avro.Schema; import org.apache.avro.Schema.Field.Order; +import java.util.Map; + public class FieldBuilder110 implements FieldBuilder { private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private Object _defaultVal; private Order _order; + private Map _props; public FieldBuilder110(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultVal(); + _order = field.order(); + _props = field.getObjectProps(); } public FieldBuilder110(String name) { @@ -53,18 +59,19 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultVal(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + return result; } } diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java index 9714b3757..d8f8df65d 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java @@ -238,6 +238,12 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder14(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + String val = field.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/FieldBuilder14.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/FieldBuilder14.java index b503afd08..91f57703e 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/FieldBuilder14.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/FieldBuilder14.java @@ -12,18 +12,37 @@ import org.apache.avro.Schema.Field.Order; import org.codehaus.jackson.JsonNode; +import java.lang.reflect.Field; +import java.util.Map; + public class FieldBuilder14 implements FieldBuilder { + private final static Field SCHEMA_FIELD_PROPS_FIELD; + + static { + try { + Class fieldClass = Schema.Field.class; + SCHEMA_FIELD_PROPS_FIELD = fieldClass.getDeclaredField("props"); + SCHEMA_FIELD_PROPS_FIELD.setAccessible(true); //muwahahaha + } catch (Throwable issue) { + throw new IllegalStateException("unable to find/access Schema$Field.props", issue); + } + } + private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private JsonNode _defaultVal; private Order _order; + private Map _props; public FieldBuilder14(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultValue(); + _order = field.order(); + _props = getProps(field); } public FieldBuilder14(String name) { @@ -59,18 +78,28 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultValue(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null && !_props.isEmpty()) { + Map clonedProps = getProps(result); + clonedProps.putAll(_props); + } + return result; + } + + private Map getProps(Schema.Field field) { + try { + @SuppressWarnings("unchecked") + Map props = (Map) SCHEMA_FIELD_PROPS_FIELD.get(field); + return props; + } catch (Exception e) { + throw new IllegalStateException("unable to access props on Schema$Field " + field.name(), e); + } } } diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java index 7e15234cc..9ea884a26 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java @@ -256,6 +256,12 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder15(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + String val = field.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/FieldBuilder15.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/FieldBuilder15.java index 424de4144..d41f6bbaa 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/FieldBuilder15.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/FieldBuilder15.java @@ -12,18 +12,37 @@ import org.apache.avro.Schema.Field.Order; import org.codehaus.jackson.JsonNode; +import java.lang.reflect.Field; +import java.util.Map; + public class FieldBuilder15 implements FieldBuilder { + private final static Field SCHEMA_FIELD_PROPS_FIELD; + + static { + try { + Class fieldClass = Schema.Field.class; + SCHEMA_FIELD_PROPS_FIELD = fieldClass.getDeclaredField("props"); + SCHEMA_FIELD_PROPS_FIELD.setAccessible(true); //muwahahaha + } catch (Throwable issue) { + throw new IllegalStateException("unable to find/access Schema$Field.props", issue); + } + } + private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private JsonNode _defaultVal; private Order _order; + private Map _props; public FieldBuilder15(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultValue(); + _order = field.order(); + _props = getProps(field); } public FieldBuilder15(String name) { @@ -59,18 +78,28 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultValue(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null && !_props.isEmpty()) { + Map clonedProps = getProps(result); + clonedProps.putAll(_props); + } + return result; + } + + private Map getProps(Schema.Field field) { + try { + @SuppressWarnings("unchecked") + Map props = (Map) SCHEMA_FIELD_PROPS_FIELD.get(field); + return props; + } catch (Exception e) { + throw new IllegalStateException("unable to access props on Schema$Field " + field.name(), e); + } } } diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java index 9d02a8ffb..a579f29a4 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java @@ -268,6 +268,12 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder16(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + String val = field.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/FieldBuilder16.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/FieldBuilder16.java index bc27b1e44..948d00ba1 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/FieldBuilder16.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/FieldBuilder16.java @@ -12,18 +12,24 @@ import org.apache.avro.Schema.Field.Order; import org.codehaus.jackson.JsonNode; +import java.util.Map; + public class FieldBuilder16 implements FieldBuilder { private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private JsonNode _defaultVal; private Order _order; + private Map _props; public FieldBuilder16(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultValue(); + _order = field.order(); + _props = field.props(); } public FieldBuilder16(String name) { @@ -59,18 +65,19 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultValue(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + return result; } } diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java index 0ab835d2f..ed7797c5c 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java @@ -54,6 +54,8 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificData; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +63,9 @@ public class Avro17Adapter implements AvroAdapter { private final static Logger LOG = LoggerFactory.getLogger(Avro17Adapter.class); + //doc says thread safe outside config methods + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + //compiler-related fields and methods (if the compiler jar is on the classpath) private boolean compilerSupported; @@ -327,6 +332,19 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder17(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + JsonNode val = Avro17Utils.getJsonProp(field, propName); + if (val == null) { + return null; + } + try { + return OBJECT_MAPPER.writeValueAsString(val); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java new file mode 100644 index 000000000..fa6d0df34 --- /dev/null +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java @@ -0,0 +1,132 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro17; + +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.node.TextNode; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; + +/** + * utility code specific to avro 1.7 + */ +public class Avro17Utils { + private final static boolean IS_AT_LEAST_1_7_3 = isIsAtLeast173(); + private final static Method GET_JSON_PROPS_METHOD; + private final static Method GET_JSON_PROP_METHOD; + private final static Method ADD_JSON_PROP_METHOD; + + static { + if (IS_AT_LEAST_1_7_3) { + GET_JSON_PROPS_METHOD = findNewerGetPropsMethod(); + GET_JSON_PROP_METHOD = findNewerGetPropMethod(); + ADD_JSON_PROP_METHOD = findNewerAddPropMethod(); + } else { + GET_JSON_PROPS_METHOD = null; + GET_JSON_PROP_METHOD = null; + ADD_JSON_PROP_METHOD = null; + } + } + + static boolean isIsAtLeast173() { + try { + //this class was created as part of AVRO-1157 for 1.7.3 + Class.forName("org.apache.avro.JsonProperties"); + return true; + } catch (ClassNotFoundException nope) { + return false; + } + } + + static Method findNewerGetPropsMethod() { + try { + Class jsonPropertiesClass = Class.forName("org.apache.avro.JsonProperties"); + return jsonPropertiesClass.getDeclaredMethod("getJsonProps"); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + static Method findNewerGetPropMethod() { + try { + Class jsonPropertiesClass = Class.forName("org.apache.avro.JsonProperties"); + return jsonPropertiesClass.getDeclaredMethod("getJsonProp", String.class); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + static Method findNewerAddPropMethod() { + try { + Class jsonPropertiesClass = Class.forName("org.apache.avro.JsonProperties"); + return jsonPropertiesClass.getDeclaredMethod("addProp", String.class, JsonNode.class); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + static JsonNode getJsonProp(Schema.Field field, String name) { + if (GET_JSON_PROP_METHOD != null) { + try { + return (JsonNode) GET_JSON_PROP_METHOD.invoke(field, name); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + String strProp = field.getProp(name); + if (strProp == null) { + return null; + } + return new TextNode(strProp); + } + + static Map getProps(Schema.Field field) { + if (GET_JSON_PROPS_METHOD != null) { + try { + //noinspection unchecked + return (Map) GET_JSON_PROPS_METHOD.invoke(field); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + @SuppressWarnings("deprecation") + Map strProps = field.props(); + if (strProps == null) { + return null; + } + Map jsonProps = new HashMap<>(strProps.size()); + for (Map.Entry entry : strProps.entrySet()) { + jsonProps.put(entry.getKey(), new TextNode(entry.getValue())); + } + return jsonProps; + } + + static void setProps(Schema.Field field, Map jsonProps) { + if (ADD_JSON_PROP_METHOD != null) { + try { + for (Map.Entry entry : jsonProps.entrySet()) { + ADD_JSON_PROP_METHOD.invoke(field, entry.getKey(), entry.getValue()); + } + return; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + for (Map.Entry entry : jsonProps.entrySet()) { + JsonNode jsonValue = entry.getValue(); + if (jsonValue.isTextual()) { + field.addProp(entry.getKey(), jsonValue.getTextValue()); + } + } + } +} diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/FieldBuilder17.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/FieldBuilder17.java index a0e16fcf6..fa078a83d 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/FieldBuilder17.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/FieldBuilder17.java @@ -12,18 +12,24 @@ import org.apache.avro.Schema.Field.Order; import org.codehaus.jackson.JsonNode; +import java.util.Map; + public class FieldBuilder17 implements FieldBuilder { private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private JsonNode _defaultVal; private Order _order; + private Map _props; public FieldBuilder17(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultValue(); + _order = field.order(); + _props = Avro17Utils.getProps(field); } public FieldBuilder17(String name) { @@ -59,18 +65,17 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultValue(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null) { + Avro17Utils.setProps(result, _props); + } + return result; } } diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java index 24ffd42cc..c1e2e1ba8 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java @@ -50,6 +50,8 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificData; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +59,9 @@ public class Avro18Adapter implements AvroAdapter { private final static Logger LOG = LoggerFactory.getLogger(Avro18Adapter.class); + //doc says thread safe outside config methods + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private boolean compilerSupported; private Constructor specificCompilerCtr; private Method compilerEnqueueMethod; @@ -265,6 +270,20 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder18(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + @SuppressWarnings("deprecation") //this is faster + JsonNode val = field.getJsonProp(propName); + if (val == null) { + return null; + } + try { + return OBJECT_MAPPER.writeValueAsString(val); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/FieldBuilder18.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/FieldBuilder18.java index 30b9e9d69..ec5eadc7e 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/FieldBuilder18.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/FieldBuilder18.java @@ -12,18 +12,27 @@ import org.apache.avro.Schema.Field.Order; import org.codehaus.jackson.JsonNode; +import java.util.Map; + public class FieldBuilder18 implements FieldBuilder { private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private JsonNode _defaultVal; private Order _order; + private Map _props; public FieldBuilder18(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + //noinspection deprecation + _defaultVal = field.defaultValue(); //deprecated but faster + _order = field.order(); + //this is actually faster + //noinspection deprecation + _props = field.getJsonProps(); } public FieldBuilder18(String name) { @@ -59,18 +68,21 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultValue(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + @SuppressWarnings("deprecation") //deprecated but faster + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null) { + for (Map.Entry entry : _props.entrySet()) { + //noinspection deprecation + result.addProp(entry.getKey(), entry.getValue()); //deprecated but faster + } + } + return result; } } diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java index e7d14bc0c..00813fc6f 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java @@ -6,6 +6,8 @@ package com.linkedin.avroutil1.compatibility.avro19; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.avroutil1.compatibility.AvroAdapter; import com.linkedin.avroutil1.compatibility.AvroGeneratedSourceCode; import com.linkedin.avroutil1.compatibility.AvroVersion; @@ -50,6 +52,7 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificData; +import org.apache.avro.util.internal.JacksonUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +60,9 @@ public class Avro19Adapter implements AvroAdapter { private final static Logger LOG = LoggerFactory.getLogger(Avro19Adapter.class); + //doc says thread safe outside config methods + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private boolean compilerSupported; private Constructor specificCompilerCtr; private Method compilerEnqueueMethod; @@ -263,6 +269,21 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder19(name); } + @Override + public String getFieldPropAsJsonString(Schema.Field field, String propName) { + Object val = field.getObjectProp(propName); + if (val == null) { + return null; + } + //the following is VERY rube-goldberg-ish, but will do until someone complains + JsonNode asJsonNode = JacksonUtils.toJsonNode(val); + try { + return OBJECT_MAPPER.writeValueAsString(asJsonNode); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java index c3f5204f5..749cc0091 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/FieldBuilder19.java @@ -10,18 +10,24 @@ import org.apache.avro.Schema; import org.apache.avro.Schema.Field.Order; +import java.util.Map; + public class FieldBuilder19 implements FieldBuilder { private final String _name; - private Schema.Field _field; private Schema _schema; private String _doc; private Object _defaultVal; private Order _order; + private Map _props; public FieldBuilder19(Schema.Field field) { this(field.name()); - _field = field; + _schema = field.schema(); + _doc = field.doc(); + _defaultVal = field.defaultVal(); + _order = field.order(); + _props = field.getObjectProps(); } public FieldBuilder19(String name) { @@ -53,18 +59,19 @@ public FieldBuilder setOrder(Order order) { } @Override + @Deprecated public FieldBuilder copyFromField() { - if (_field == null) { - throw new NullPointerException("Field in FieldBuilder can not be empty!"); - } - _doc = _field.doc(); - _defaultVal = _field.defaultVal(); - _order = _field.order(); return this; } @Override public Schema.Field build() { - return new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + Schema.Field result = new Schema.Field(_name, _schema, _doc, _defaultVal, _order); + if (_props != null) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + return result; } } diff --git a/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/FieldBuilderTest.java b/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/FieldBuilderTest.java new file mode 100644 index 000000000..bfac7cf5f --- /dev/null +++ b/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/FieldBuilderTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility; + +import com.linkedin.avroutil1.TestUtil; +import org.apache.avro.Schema; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class FieldBuilderTest { + + @Test + public void testFieldPropSupportOnClone() throws Exception { + AvroVersion runtimeAvroVersion = AvroCompatibilityHelper.getRuntimeAvroVersion(); + String avsc = TestUtil.load("RecordWithFieldProps.avsc"); + Schema schema = Schema.parse(avsc); + Schema.Field originalField = schema.getField("stringField"); + + FieldBuilder builder = AvroCompatibilityHelper.cloneSchemaField(originalField); + Schema.Field newField = builder.build(); + + Assert.assertNotSame(newField, originalField); + Assert.assertNull(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "noSuchProp")); + Assert.assertEquals(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "stringProp"), "\"stringValue\""); + if (runtimeAvroVersion.earlierThan(AvroVersion.AVRO_1_7)) { + //sadly avro 1.4/5/6 do not preserve any other (non textual) props + return; + } + String intPropAsJsonLiteral = AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "intProp"); + if (runtimeAvroVersion.equals(AvroVersion.AVRO_1_7) && intPropAsJsonLiteral == null) { + //we must be under avro < 1.7.3. avro 1.7 only supports non-text props at 1.7.3+ + return; + } + Assert.assertEquals(intPropAsJsonLiteral, "42"); + Assert.assertEquals(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "floatProp"), "42.42"); + Assert.assertEquals(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "nullProp"), "null"); + Assert.assertEquals(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "boolProp"), "true"); + Assert.assertEquals(AvroCompatibilityHelper.getFieldPropAsJsonString(newField, "objectProp"), "{\"a\":\"b\",\"c\":\"d\"}"); + } +} diff --git a/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc b/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc new file mode 100644 index 000000000..08ac500d0 --- /dev/null +++ b/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc @@ -0,0 +1,21 @@ +{ + "type": "record", + "namespace": "com.acme", + "name": "RecordWithFieldProps", + "doc": "A perfectly normal record with field props", + "fields": [ + { + "name": "stringField", + "type": "string", + "stringProp": "stringValue", + "intProp": 42, + "floatProp": 42.42, + "nullProp": null, + "boolProp": true, + "objectProp": { + "a": "b", + "c": "d" + } + } + ] +} \ No newline at end of file