Skip to content

Commit

Permalink
Added specific record tests for primitive arrays. (#81)
Browse files Browse the repository at this point in the history 
* Fixed bug in specific record support for primitive arrays.

Addressed bugs related to missing constructors in the PrimitiveArrayList
and its child classes. Also refactored those classes slightly to reduce
boilerplate.

Altered the meta code:

- List construction does not rely on Collections.emptyList() since that
  is incompatible with the primitive list classes.
- Fixed a case where list instantiation wasn't guarded by getShouldRead(),
  when schema evolution gets rid of a list field. This would previously
  cause a NPE during code gen.
- Refactored the list handling code so that list instance reuse occurs
  even though the particular payload being deserialized has zero elements.

* Code gen for PR #81
  • Loading branch information
@FelixGV
FelixGV authored Jul 23, 2020
1 parent ce0f422 commit 5469e25
Show file tree
Hide file tree
Showing 54 changed files with 1,400 additions and 1,585 deletions.
4 changes: 2 additions & 2 deletions avro-fastserde/regenerate_avro.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ FULL_CODE_GEN_PATH=(
)
FULL_CODE_GEN_ROOT_PATH="${CODE_GEN_PATH}/com/linkedin/avro/fastserde/generated/avro"

if [[ $# < 1 ]]; then
if [[ $# -lt 1 ]]; then
echo "Usage: $0 avro_tools_path"
echo ""
echo " avro_tools_path: full path to the avro-tools-1.x.x.jar file (required). If you use 'default_avro_140', it will take:"
Expand Down Expand Up @@ -93,7 +93,7 @@ for file in `ls ${FULL_CODE_GEN_ROOT_PATH}/*.java`; do
rm -f ${file}
done

echo "Finished deleting old files. About to generate new ones..."
echo "Finished deleting old files. About to generate new ones using $AVRO_TOOLS_JAR..."

for file in `ls ${AVRO_SCHEMAS_ROOT_PATH}/*.avsc`; do
echo ${file}
Expand Down
22 changes: 9 additions & 13 deletions ...VRO_1_4/Array_of_BOOLEAN_GenericDeserializer_5988037297456147388_5988037297456147388.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,18 @@ public List<Boolean> deserialize(List<Boolean> reuse, Decoder decoder)
{
PrimitiveBooleanList array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof PrimitiveBooleanList) {
array0 = ((PrimitiveBooleanList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readBoolean()));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof PrimitiveBooleanList) {
array0 = ((PrimitiveBooleanList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveBooleanArrayList(((int) chunkLen0));
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readBoolean()));
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
22 changes: 9 additions & 13 deletions ...AVRO_1_4/Array_of_DOUBLE_GenericDeserializer_6064316435611861740_6064316435611861740.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,18 @@ public List<Double> deserialize(List<Double> reuse, Decoder decoder)
{
PrimitiveDoubleList array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof PrimitiveDoubleList) {
array0 = ((PrimitiveDoubleList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readDouble()));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof PrimitiveDoubleList) {
array0 = ((PrimitiveDoubleList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveDoubleArrayList(((int) chunkLen0));
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readDouble()));
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
22 changes: 9 additions & 13 deletions ...on/AVRO_1_4/Array_of_INT_GenericDeserializer_3343716480540445685_3343716480540445685.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,18 @@ public List<Integer> deserialize(List<Integer> reuse, Decoder decoder)
{
PrimitiveIntList array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof PrimitiveIntList) {
array0 = ((PrimitiveIntList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveIntArrayList(((int) chunkLen0));
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readInt()));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof PrimitiveIntList) {
array0 = ((PrimitiveIntList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveIntArrayList(((int) chunkLen0));
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readInt()));
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
22 changes: 9 additions & 13 deletions ...n/AVRO_1_4/Array_of_LONG_GenericDeserializer_2055015354772118358_2055015354772118358.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,18 @@ public List<Long> deserialize(List<Long> reuse, Decoder decoder)
{
PrimitiveLongList array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof PrimitiveLongList) {
array0 = ((PrimitiveLongList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveLongArrayList(((int) chunkLen0));
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readLong()));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof PrimitiveLongList) {
array0 = ((PrimitiveLongList)(reuse));
array0 .clear();
} else {
array0 = new PrimitiveLongArrayList(((int) chunkLen0));
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
array0 .addPrimitive((decoder.readLong()));
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
44 changes: 20 additions & 24 deletions ...on/AVRO_1_4/Array_of_UNION_GenericDeserializer_585074122056792963_585074122056792963.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,33 +31,29 @@ public List<IndexedRecord> deserialize(List<IndexedRecord> reuse, Decoder decode
{
List<IndexedRecord> array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof List) {
array0 = ((List)(reuse));
array0 .clear();
} else {
array0 = new org.apache.avro.generic.GenericData.Array<IndexedRecord>(((int) chunkLen0), readerSchema);
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
Object arrayArrayElementReuseVar0 = null;
if ((reuse) instanceof GenericArray) {
arrayArrayElementReuseVar0 = ((GenericArray)(reuse)).peek();
}
int unionIndex0 = (decoder.readIndex());
if (unionIndex0 == 0) {
decoder.readNull();
} else {
if (unionIndex0 == 1) {
array0 .add(deserializerecord0(arrayArrayElementReuseVar0, (decoder)));
}
}
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof List) {
array0 = ((List)(reuse));
array0 .clear();
} else {
array0 = new org.apache.avro.generic.GenericData.Array<IndexedRecord>(((int) chunkLen0), readerSchema);
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
Object arrayArrayElementReuseVar0 = null;
if ((reuse) instanceof GenericArray) {
arrayArrayElementReuseVar0 = ((GenericArray)(reuse)).peek();
}
int unionIndex0 = (decoder.readIndex());
if (unionIndex0 == 0) {
decoder.readNull();
} else {
if (unionIndex0 == 1) {
array0 .add(deserializerecord0(arrayArrayElementReuseVar0, (decoder)));
}
}
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
30 changes: 13 additions & 17 deletions ...AVRO_1_4/Array_of_record_GenericDeserializer_1629046702287533603_1629046702287533603.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,22 @@ public List<IndexedRecord> deserialize(List<IndexedRecord> reuse, Decoder decode
{
List<IndexedRecord> array0 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if ((reuse) instanceof List) {
array0 = ((List)(reuse));
array0 .clear();
} else {
array0 = new org.apache.avro.generic.GenericData.Array<IndexedRecord>(((int) chunkLen0), readerSchema);
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
Object arrayArrayElementReuseVar0 = null;
if ((reuse) instanceof GenericArray) {
arrayArrayElementReuseVar0 = ((GenericArray)(reuse)).peek();
}
array0 .add(deserializerecord0(arrayArrayElementReuseVar0, (decoder)));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if ((reuse) instanceof List) {
array0 = ((List)(reuse));
array0 .clear();
} else {
array0 = new org.apache.avro.generic.GenericData.Array<IndexedRecord>(((int) chunkLen0), readerSchema);
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
Object arrayArrayElementReuseVar0 = null;
if ((reuse) instanceof GenericArray) {
arrayArrayElementReuseVar0 = ((GenericArray)(reuse)).peek();
}
array0 .add(deserializerecord0(arrayArrayElementReuseVar0, (decoder)));
}
chunkLen0 = (decoder.arrayNext());
}
return array0;
}

Expand Down
66 changes: 29 additions & 37 deletions ...ratorTest_shouldReadEnum_GenericDeserializer_7716892313569022782_7716892313569022782.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,52 +55,44 @@ public IndexedRecord deserializeFastGenericDeserializerGeneratorTest_shouldReadE
}
List<org.apache.avro.generic.GenericData.EnumSymbol> testEnumArray1 = null;
long chunkLen0 = (decoder.readArrayStart());
if (chunkLen0 > 0) {
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(2) instanceof List) {
testEnumArray1 = ((List) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(2));
testEnumArray1 .clear();
} else {
testEnumArray1 = new org.apache.avro.generic.GenericData.Array<org.apache.avro.generic.GenericData.EnumSymbol>(((int) chunkLen0), testEnumArray0);
}
do {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
testEnumArray1 .add(new org.apache.avro.generic.GenericData.EnumSymbol(testEnum0 .getEnumSymbols().get((decoder.readEnum()))));
}
chunkLen0 = (decoder.arrayNext());
} while (chunkLen0 > 0);
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(2) instanceof List) {
testEnumArray1 = ((List) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(2));
testEnumArray1 .clear();
} else {
testEnumArray1 = new org.apache.avro.generic.GenericData.Array<org.apache.avro.generic.GenericData.EnumSymbol>(((int) chunkLen0), testEnumArray0);
}
while (chunkLen0 > 0) {
for (int counter0 = 0; (counter0 <chunkLen0); counter0 ++) {
testEnumArray1 .add(new org.apache.avro.generic.GenericData.EnumSymbol(testEnum0 .getEnumSymbols().get((decoder.readEnum()))));
}
chunkLen0 = (decoder.arrayNext());
}
FastGenericDeserializerGeneratorTest_shouldReadEnum.put(2, testEnumArray1);
List<org.apache.avro.generic.GenericData.EnumSymbol> testEnumUnionArray1 = null;
long chunkLen1 = (decoder.readArrayStart());
if (chunkLen1 > 0) {
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3) instanceof List) {
testEnumUnionArray1 = ((List) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3));
testEnumUnionArray1 .clear();
} else {
testEnumUnionArray1 = new org.apache.avro.generic.GenericData.Array<org.apache.avro.generic.GenericData.EnumSymbol>(((int) chunkLen1), testEnumUnionArray0);
}
do {
for (int counter1 = 0; (counter1 <chunkLen1); counter1 ++) {
Object testEnumUnionArrayArrayElementReuseVar0 = null;
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3) instanceof GenericArray) {
testEnumUnionArrayArrayElementReuseVar0 = ((GenericArray) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3)).peek();
}
int unionIndex1 = (decoder.readIndex());
if (unionIndex1 == 0) {
decoder.readNull();
} else {
if (unionIndex1 == 1) {
testEnumUnionArray1 .add(new org.apache.avro.generic.GenericData.EnumSymbol(testEnum0 .getEnumSymbols().get((decoder.readEnum()))));
}
}
}
chunkLen1 = (decoder.arrayNext());
} while (chunkLen1 > 0);
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3) instanceof List) {
testEnumUnionArray1 = ((List) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3));
testEnumUnionArray1 .clear();
} else {
testEnumUnionArray1 = new org.apache.avro.generic.GenericData.Array<org.apache.avro.generic.GenericData.EnumSymbol>(((int) chunkLen1), testEnumUnionArray0);
}
while (chunkLen1 > 0) {
for (int counter1 = 0; (counter1 <chunkLen1); counter1 ++) {
Object testEnumUnionArrayArrayElementReuseVar0 = null;
if (FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3) instanceof GenericArray) {
testEnumUnionArrayArrayElementReuseVar0 = ((GenericArray) FastGenericDeserializerGeneratorTest_shouldReadEnum.get(3)).peek();
}
int unionIndex1 = (decoder.readIndex());
if (unionIndex1 == 0) {
decoder.readNull();
} else {
if (unionIndex1 == 1) {
testEnumUnionArray1 .add(new org.apache.avro.generic.GenericData.EnumSymbol(testEnum0 .getEnumSymbols().get((decoder.readEnum()))));
}
}
}
chunkLen1 = (decoder.arrayNext());
}
FastGenericDeserializerGeneratorTest_shouldReadEnum.put(3, testEnumUnionArray1);
return FastGenericDeserializerGeneratorTest_shouldReadEnum;
}
Expand Down
Loading

0 comments on commit 5469e25

Please sign in to comment.