diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AbstractSchemaBuilder.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AbstractSchemaBuilder.java new file mode 100644 index 000000000..4d9075057 --- /dev/null +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AbstractSchemaBuilder.java @@ -0,0 +1,107 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility; + +import org.apache.avro.Schema; + +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractSchemaBuilder implements SchemaBuilder { + + protected final AvroAdapter _adapter; + protected Schema.Type _type; + protected String _name; + protected String _namespace; + protected String _doc; + protected boolean _isError; + protected List _fields; + + protected AbstractSchemaBuilder(AvroAdapter _adapter, Schema original) { + this._adapter = _adapter; + _type = original.getType(); + _name = original.getName(); + _namespace = original.getNamespace(); + _doc = original.getDoc(); + _isError = original.isError(); + //make a copy of fields so its mutable + _fields = new ArrayList<>(original.getFields()); + } + + @Override + public SchemaBuilder addField(Schema.Field field) { + checkNewField(field); + _fields.add(field); + return this; + } + + @Override + public SchemaBuilder addField(int position, Schema.Field field) { + checkNewField(field); + _fields.add(position, field); //will throw IOOB on bad positions + return this; + } + + @Override + public SchemaBuilder removeField(String fieldName) { + if (fieldName == null || fieldName.isEmpty()) { + throw new IllegalArgumentException("argument cannot be null or empty"); + } + int index = fieldPositionByName(fieldName, false); + if (index >= 0) { + _fields.remove(index); + } + return this; + } + + @Override + public SchemaBuilder removeField(int position) { + _fields.remove(position); //throws IOOB + return this; + } + + /** + * {@link Schema.Field} has a position ("pos") property that is set when its added to a schema. + * this means we need to clone fields to add them to another schema + * @param originals list of fields to clone + * @return list of cloned fields + */ + protected List cloneFields(List originals) { + List clones = new ArrayList<>(originals.size()); + for (Schema.Field original : originals) { + FieldBuilder fb = _adapter.cloneSchemaField(original); + Schema.Field clone = fb.build(); + clones.add(clone); + } + return clones; + } + + protected void checkNewField(Schema.Field field) { + if (field == null) { + throw new IllegalArgumentException("argument cannot be null"); + } + int otherIndex = fieldPositionByName(field.name(), false); + if (otherIndex >= 0) { + throw new IllegalArgumentException("schema already contains a field called " + field.name()); + } + } + + protected int fieldPositionByName(String name, boolean caseSensitive) { + for (int i = 0; i < _fields.size(); i++) { + Schema.Field candidate = _fields.get(i); + String cName = candidate.name(); + if (caseSensitive) { + if (cName.equals(name)) { + return i; + } + } else if (cName.equalsIgnoreCase(name)){ + return i; + } + } + return -1; + } +} diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java index afcde7c5f..9c4372516 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java @@ -94,8 +94,12 @@ BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, FieldBuilder newFieldBuilder(String name); + SchemaBuilder cloneSchema(Schema schema); + String getFieldPropAsJsonString(Schema.Field field, String propName); + String getSchemaPropAsJsonString(Schema schema, String propName); + //code generation Collection compile( diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/SchemaBuilder.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/SchemaBuilder.java new file mode 100644 index 000000000..107983394 --- /dev/null +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/SchemaBuilder.java @@ -0,0 +1,59 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility; + +import org.apache.avro.Schema; + +/** + * Builder for creating {@link Schema} instances at runtime + */ +public interface SchemaBuilder { + + /** + * add a field to the schema under construction. field is added + * to the end of the field list + * @param field new field to add + * @return the builder + * @throws IllegalArgumentException if a field by the same name + * (case INSENSITIVE) exists + */ + SchemaBuilder addField(Schema.Field field); + + /** + * add a field to the schema under construction at the specified position. + * existing fields starting from the specified position are "right shifted" + * @param position desired position for the field to be added, 0 based. + * @param field field to add + * @return the builder + * @throws IllegalArgumentException if a field by the same name + * (case INSENSITIVE) exists + * @throws IndexOutOfBoundsException if index is invalid + */ + SchemaBuilder addField(int position, Schema.Field field); + + /** + * removes a field by its (case INSENSITIVE) name, if such a field exists. + * @param fieldName name of field to be removed. required. + * @return the builder + * @throws IllegalArgumentException if argument is null or emoty + */ + SchemaBuilder removeField(String fieldName); + + /** + * removes a field by its position. + * @param position position (0 based) os the field to remove + * @return the builder + * @throws IndexOutOfBoundsException if position is invalid + */ + SchemaBuilder removeField(int position); + + /** + * constructs a {@link Schema} out of this builder + * @return a {@link Schema} + */ + Schema build(); +} diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java index 5bfa79667..eda0d9903 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java @@ -486,6 +486,11 @@ public static FieldBuilder cloneSchemaField(Schema.Field field) { return ADAPTER.cloneSchemaField(field); } + public static SchemaBuilder cloneSchema(Schema schema) { + assertAvroAvailable(); + return ADAPTER.cloneSchema(schema); + } + // code generation /** @@ -646,4 +651,8 @@ public static Schema.Field createSchemaField(String name, Schema schema, String public static String getFieldPropAsJsonString(Schema.Field field, String propName) { return ADAPTER.getFieldPropAsJsonString(field, propName); } + + public static String getSchemaPropAsJsonString(Schema schema, String propName) { + return ADAPTER.getSchemaPropAsJsonString(schema, propName); + } } diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java index 11e572a35..db41a21a0 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java @@ -14,6 +14,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; import com.linkedin.avroutil1.compatibility.SkipDecoder; @@ -266,19 +267,21 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder110(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder110(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { Object val = field.getObjectProp(propName); - if (val == null) { - return null; - } - //the following is VERY rube-goldberg-ish, but will do until someone complains - JsonNode asJsonNode = JacksonUtils.toJsonNode(val); - try { - return OBJECT_MAPPER.writeValueAsString(asJsonNode); - } catch (Exception issue) { - throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); - } + return objectPropToJsonString(val); + } + + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + Object val = schema.getObjectProp(propName); + return objectPropToJsonString(val); } @Override @@ -341,6 +344,19 @@ public Collection compile( } } + private String objectPropToJsonString(Object val) { + if (val == null) { + return null; + } + //the following is VERY rube-goldberg-ish, but will do until someone complains + JsonNode asJsonNode = JacksonUtils.toJsonNode(val); + try { + return OBJECT_MAPPER.writeValueAsString(asJsonNode); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + private Collection transform(List avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) { List transformed = new ArrayList<>(avroGenerated.size()); for (AvroGeneratedSourceCode generated : avroGenerated) { diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/SchemaBuilder110.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/SchemaBuilder110.java new file mode 100644 index 000000000..6620053ef --- /dev/null +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/SchemaBuilder110.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro110; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; + +import java.util.Map; + +public class SchemaBuilder110 extends AbstractSchemaBuilder { + + private Map _props; + + public SchemaBuilder110(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = original.getObjectProps(); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } +} diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java index d8f8df65d..d250dbba8 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java @@ -13,6 +13,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaNormalization; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; @@ -238,12 +239,23 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder14(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder14(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { String val = field.getProp(propName); return val == null ? null : "\"" + val + "\""; } + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + String val = schema.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/SchemaBuilder14.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/SchemaBuilder14.java new file mode 100644 index 000000000..e023fc222 --- /dev/null +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/SchemaBuilder14.java @@ -0,0 +1,66 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro14; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; + +import java.lang.reflect.Field; +import java.util.Map; + +public class SchemaBuilder14 extends AbstractSchemaBuilder { + private final static Field SCHEMA_PROPS_FIELD; + + static { + try { + Class schemaClass = Schema.class; + SCHEMA_PROPS_FIELD = schemaClass.getDeclaredField("props"); + SCHEMA_PROPS_FIELD.setAccessible(true); + } catch (Throwable issue) { + throw new IllegalStateException("unable to find/access Schema.props", issue); + } + } + + private Map _props; + + public SchemaBuilder14(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = getProps(original); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + getProps(result).putAll(_props); + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } + + private Map getProps(Schema schema) { + try { + @SuppressWarnings("unchecked") + Map props = (Map) SCHEMA_PROPS_FIELD.get(schema); + return props; + } catch (Exception e) { + throw new IllegalStateException("unable to access props on Schema " + schema.getFullName(), e); + } + } +} diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java index 9ea884a26..4d2b43eb2 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java @@ -14,6 +14,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaNormalization; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; @@ -256,12 +257,23 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder15(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder15(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { String val = field.getProp(propName); return val == null ? null : "\"" + val + "\""; } + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + String val = schema.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/SchemaBuilder15.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/SchemaBuilder15.java new file mode 100644 index 000000000..82eb63286 --- /dev/null +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/SchemaBuilder15.java @@ -0,0 +1,66 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro15; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; + +import java.lang.reflect.Field; +import java.util.Map; + +public class SchemaBuilder15 extends AbstractSchemaBuilder { + private final static Field SCHEMA_PROPS_FIELD; + + static { + try { + Class schemaClass = Schema.class; + SCHEMA_PROPS_FIELD = schemaClass.getDeclaredField("props"); + SCHEMA_PROPS_FIELD.setAccessible(true); + } catch (Throwable issue) { + throw new IllegalStateException("unable to find/access Schema.props", issue); + } + } + + private Map _props; + + public SchemaBuilder15(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = getProps(original); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + getProps(result).putAll(_props); + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } + + private Map getProps(Schema schema) { + try { + @SuppressWarnings("unchecked") + Map props = (Map) SCHEMA_PROPS_FIELD.get(schema); + return props; + } catch (Exception e) { + throw new IllegalStateException("unable to access props on Schema " + schema.getFullName(), e); + } + } +} diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java index a579f29a4..ed3cada06 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java @@ -13,6 +13,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaNormalization; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; @@ -268,12 +269,23 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder16(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder16(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { String val = field.getProp(propName); return val == null ? null : "\"" + val + "\""; } + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + String val = schema.getProp(propName); + return val == null ? null : "\"" + val + "\""; + } + @Override public Collection compile( Collection toCompile, diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/SchemaBuilder16.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/SchemaBuilder16.java new file mode 100644 index 000000000..4ae9e4c16 --- /dev/null +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/SchemaBuilder16.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro16; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; + +import java.util.Map; + +public class SchemaBuilder16 extends AbstractSchemaBuilder { + + private Map _props; + + public SchemaBuilder16(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = original.getProps(); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } +} diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java index ed7797c5c..39ea06133 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java @@ -13,6 +13,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; import com.linkedin.avroutil1.compatibility.SchemaValidator; @@ -332,17 +333,21 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder17(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder17(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { JsonNode val = Avro17Utils.getJsonProp(field, propName); - if (val == null) { - return null; - } - try { - return OBJECT_MAPPER.writeValueAsString(val); - } catch (Exception issue) { - throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); - } + return jsonNodeToJsonString(val); + } + + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + JsonNode val = Avro17Utils.getJsonProp(schema, propName); + return jsonNodeToJsonString(val); } @Override @@ -405,6 +410,17 @@ public Collection compile( } } + private String jsonNodeToJsonString(JsonNode val) { + if (val == null) { + return null; + } + try { + return OBJECT_MAPPER.writeValueAsString(val); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + private Collection transform(List avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) { List transformed = new ArrayList<>(avroGenerated.size()); for (AvroGeneratedSourceCode generated : avroGenerated) { diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java index fa6d0df34..bc4c9b5ac 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Utils.java @@ -88,6 +88,22 @@ static JsonNode getJsonProp(Schema.Field field, String name) { return new TextNode(strProp); } + static JsonNode getJsonProp(Schema schema, String name) { + if (GET_JSON_PROP_METHOD != null) { + try { + return (JsonNode) GET_JSON_PROP_METHOD.invoke(schema, name); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + String strProp = schema.getProp(name); + if (strProp == null) { + return null; + } + return new TextNode(strProp); + } + static Map getProps(Schema.Field field) { if (GET_JSON_PROPS_METHOD != null) { try { @@ -110,6 +126,28 @@ static Map getProps(Schema.Field field) { return jsonProps; } + static Map getProps(Schema schema) { + if (GET_JSON_PROPS_METHOD != null) { + try { + //noinspection unchecked + return (Map) GET_JSON_PROPS_METHOD.invoke(schema); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + @SuppressWarnings("deprecation") + Map strProps = schema.getProps();; + if (strProps == null) { + return null; + } + Map jsonProps = new HashMap<>(strProps.size()); + for (Map.Entry entry : strProps.entrySet()) { + jsonProps.put(entry.getKey(), new TextNode(entry.getValue())); + } + return jsonProps; + } + static void setProps(Schema.Field field, Map jsonProps) { if (ADD_JSON_PROP_METHOD != null) { try { @@ -129,4 +167,24 @@ static void setProps(Schema.Field field, Map jsonProps) { } } } + + static void setProps(Schema schema, Map jsonProps) { + if (ADD_JSON_PROP_METHOD != null) { + try { + for (Map.Entry entry : jsonProps.entrySet()) { + ADD_JSON_PROP_METHOD.invoke(schema, entry.getKey(), entry.getValue()); + } + return; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + //this must be < 1.7.3 + for (Map.Entry entry : jsonProps.entrySet()) { + JsonNode jsonValue = entry.getValue(); + if (jsonValue.isTextual()) { + schema.addProp(entry.getKey(), jsonValue.getTextValue()); + } + } + } } diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/SchemaBuilder17.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/SchemaBuilder17.java new file mode 100644 index 000000000..e089b5e41 --- /dev/null +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/SchemaBuilder17.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro17; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; + +import java.util.Map; + +public class SchemaBuilder17 extends AbstractSchemaBuilder { + + private Map _props; + + public SchemaBuilder17(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = Avro17Utils.getProps(original); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + Avro17Utils.setProps(result, _props); + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } +} diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java index c1e2e1ba8..f7e80c2d1 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java @@ -12,6 +12,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; import com.linkedin.avroutil1.compatibility.SkipDecoder; @@ -270,18 +271,23 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder18(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder18(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { @SuppressWarnings("deprecation") //this is faster JsonNode val = field.getJsonProp(propName); - if (val == null) { - return null; - } - try { - return OBJECT_MAPPER.writeValueAsString(val); - } catch (Exception issue) { - throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); - } + return JsonNodeToJsonString(val); + } + + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + @SuppressWarnings("deprecation") //this is faster + JsonNode val = schema.getJsonProp(propName); + return JsonNodeToJsonString(val); } @Override @@ -344,6 +350,17 @@ public Collection compile( } } + private String JsonNodeToJsonString(JsonNode val) { + if (val == null) { + return null; + } + try { + return OBJECT_MAPPER.writeValueAsString(val); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + private Collection transform(List avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) { List transformed = new ArrayList<>(avroGenerated.size()); for (AvroGeneratedSourceCode generated : avroGenerated) { diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/SchemaBuilder18.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/SchemaBuilder18.java new file mode 100644 index 000000000..16bb3033e --- /dev/null +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/SchemaBuilder18.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro18; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; + +import java.util.Map; + +public class SchemaBuilder18 extends AbstractSchemaBuilder { + + private Map _props; + + public SchemaBuilder18(AvroAdapter adapter, Schema original) { + super(adapter, original); + //noinspection deprecation + _props = original.getJsonProps(); //actually faster + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); //deprecated but faster + } + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } +} diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java index 00813fc6f..c1ab5011a 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java @@ -14,6 +14,7 @@ import com.linkedin.avroutil1.compatibility.CodeGenerationConfig; import com.linkedin.avroutil1.compatibility.CodeTransformations; import com.linkedin.avroutil1.compatibility.FieldBuilder; +import com.linkedin.avroutil1.compatibility.SchemaBuilder; import com.linkedin.avroutil1.compatibility.SchemaParseConfiguration; import com.linkedin.avroutil1.compatibility.SchemaParseResult; import com.linkedin.avroutil1.compatibility.SkipDecoder; @@ -269,19 +270,21 @@ public FieldBuilder newFieldBuilder(String name) { return new FieldBuilder19(name); } + @Override + public SchemaBuilder cloneSchema(Schema schema) { + return new SchemaBuilder19(this, schema); + } + @Override public String getFieldPropAsJsonString(Schema.Field field, String propName) { Object val = field.getObjectProp(propName); - if (val == null) { - return null; - } - //the following is VERY rube-goldberg-ish, but will do until someone complains - JsonNode asJsonNode = JacksonUtils.toJsonNode(val); - try { - return OBJECT_MAPPER.writeValueAsString(asJsonNode); - } catch (Exception issue) { - throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); - } + return objectPropToJsonString(val); + } + + @Override + public String getSchemaPropAsJsonString(Schema schema, String propName) { + Object val = schema.getObjectProp(propName); + return objectPropToJsonString(val); } @Override @@ -344,6 +347,19 @@ public Collection compile( } } + private String objectPropToJsonString(Object val) { + if (val == null) { + return null; + } + //the following is VERY rube-goldberg-ish, but will do until someone complains + JsonNode asJsonNode = JacksonUtils.toJsonNode(val); + try { + return OBJECT_MAPPER.writeValueAsString(asJsonNode); + } catch (Exception issue) { + throw new IllegalStateException("while trying to serialize " + val + " (a " + val.getClass().getName() + ")", issue); + } + } + private Collection transform(List avroGenerated, AvroVersion minAvro, AvroVersion maxAvro) { List transformed = new ArrayList<>(avroGenerated.size()); for (AvroGeneratedSourceCode generated : avroGenerated) { diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/SchemaBuilder19.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/SchemaBuilder19.java new file mode 100644 index 000000000..3afb4ee6f --- /dev/null +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/SchemaBuilder19.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility.avro19; + +import com.linkedin.avroutil1.compatibility.AbstractSchemaBuilder; +import com.linkedin.avroutil1.compatibility.AvroAdapter; +import org.apache.avro.Schema; + +import java.util.Map; + +public class SchemaBuilder19 extends AbstractSchemaBuilder { + + private Map _props; + + public SchemaBuilder19(AvroAdapter adapter, Schema original) { + super(adapter, original); + _props = original.getObjectProps(); + } + + @Override + public Schema build() { + if (_type == null) { + throw new IllegalArgumentException("type not set"); + } + Schema result; + //noinspection SwitchStatementWithTooFewBranches + switch (_type) { + case RECORD: + result = Schema.createRecord(_name, _doc, _namespace, _isError); + result.setFields(cloneFields(_fields)); + if (_props != null && !_props.isEmpty()) { + for (Map.Entry entry : _props.entrySet()) { + result.addProp(entry.getKey(), entry.getValue()); + } + } + break; + default: + throw new UnsupportedOperationException("unhandled type " + _type); + } + return result; + } +} diff --git a/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/SchemaBuilderTest.java b/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/SchemaBuilderTest.java new file mode 100644 index 000000000..0787b9d9d --- /dev/null +++ b/helper/tests/helper-tests-allavro/src/test/java/com/linkedin/avroutil1/compatibility/SchemaBuilderTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package com.linkedin.avroutil1.compatibility; + +import com.linkedin.avroutil1.TestUtil; +import org.apache.avro.Schema; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class SchemaBuilderTest { + + @Test + public void testSchemaPropSupportOnClone() throws Exception { + AvroVersion runtimeAvroVersion = AvroCompatibilityHelper.getRuntimeAvroVersion(); + String avsc = TestUtil.load("RecordWithFieldProps.avsc"); + Schema originalSchema = Schema.parse(avsc); + Assert.assertEquals(originalSchema.getFields().size(), 2); //got 2 fields + + SchemaBuilder builder = AvroCompatibilityHelper.cloneSchema(originalSchema); + Schema newSchema = builder.removeField("intField").build(); + + Assert.assertNotSame(newSchema, originalSchema); + Assert.assertEquals(newSchema.getFields().size(), 1); //got 1 field + Assert.assertNull(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "noSuchSchemaProp")); + Assert.assertEquals(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaStringProp"), "\"stringyMcStringface\""); + if (runtimeAvroVersion.earlierThan(AvroVersion.AVRO_1_7)) { + //sadly avro 1.4/5/6 do not preserve any other (non textual) props + return; + } + String intPropAsJsonLiteral = AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaIntProp"); + if (runtimeAvroVersion.equals(AvroVersion.AVRO_1_7) && intPropAsJsonLiteral == null) { + //we must be under avro < 1.7.3. avro 1.7 only supports non-text props at 1.7.3+ + return; + } + Assert.assertEquals(intPropAsJsonLiteral, "24"); + Assert.assertEquals(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaFloatProp"), "1.2"); + Assert.assertEquals(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaNullProp"), "null"); + Assert.assertEquals(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaBoolProp"), "false"); + Assert.assertEquals(AvroCompatibilityHelper.getSchemaPropAsJsonString(newSchema, "schemaObjectProp"), "{\"e\":\"f\",\"g\":\"h\"}"); + } +} diff --git a/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc b/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc index 08ac500d0..8b06010b5 100644 --- a/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc +++ b/helper/tests/helper-tests-common/src/main/resources/RecordWithFieldProps.avsc @@ -16,6 +16,19 @@ "a": "b", "c": "d" } + }, + { + "name": "intField", + "type": "int" } - ] + ], + "schemaStringProp": "stringyMcStringface", + "schemaIntProp": 24, + "schemaFloatProp": 1.2, + "schemaNullProp": null, + "schemaBoolProp": false, + "schemaObjectProp": { + "e": "f", + "g": "h" + } } \ No newline at end of file