Skip to content

Commit

Permalink
Optimize enum field deserialization (#499)
Browse files Browse the repository at this point in the history
BUG=FI-13182
Closes #302

ClassValue lazily caches result of Enum#values for the JVM to avoid unnecessary memory overhead.

odp download --event-id 1094224
* com.linkedin.followfeed.partitioner.graph.ContentQualityClassification[].values()
  Total Allocation: 20.5G (3.58%)
  • Loading branch information
kcepmada authored Jul 11, 2023
1 parent 2377f53 commit accc9ed
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 160 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1032,7 +1032,7 @@ private String getSerializedCustomDecodeBlock(SpecificRecordGenerationConfig con
case ENUM:
TypeName enumClassName = SpecificRecordGeneratorUtil.getTypeName(fieldSchema, AvroType.ENUM, true, config.getDefaultFieldStringRepresentation());
serializedCodeBlock =
String.format("%s = %s.values()[in.readEnum()]", fieldName, enumClassName.toString());
String.format("%s = com.linkedin.avroutil1.Enums.getConstant(%s.class, in.readEnum())", fieldName, enumClassName.toString());
break;
case FIXED:
codeBlockBuilder.beginControlFlow("if ($L == null)", fieldName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
package com.linkedin.avroutil1.codegen;

import com.linkedin.avroutil1.compatibility.StringUtils;
import com.linkedin.avroutil1.model.AvroEnumSchema;
import com.linkedin.avroutil1.model.AvroFixedSchema;
import com.linkedin.avroutil1.model.AvroNamedSchema;
import com.linkedin.avroutil1.model.AvroRecordSchema;
import com.linkedin.avroutil1.parser.avsc.AvscParseResult;
import com.linkedin.avroutil1.parser.avsc.AvscParser;
Expand All @@ -22,176 +21,43 @@


public class SpecificRecordClassGeneratorTest {

@Test
public void testSimpleEnum() throws Exception {
String avsc = TestUtil.load("schemas/SimpleEnum.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroEnumSchema enumSchema = (AvroEnumSchema) result.getTopLevelSchema();
Assert.assertNotNull(enumSchema);
JavaFileObject javaFileObject = generator.generateSpecificClass(enumSchema,
SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);

}

@Test
public void testHugeEnum() throws Exception {
String avsc = TestUtil.load("schemas/SimpleEnumWithHugeDoc.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroEnumSchema enumSchema = (AvroEnumSchema) result.getTopLevelSchema();
Assert.assertNotNull(enumSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(enumSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);

}

@Test
public void testSimpleFixed() throws Exception {
String avsc = TestUtil.load("schemas/SimpleFixed.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroFixedSchema fixedSchema = (AvroFixedSchema) result.getTopLevelSchema();
Assert.assertNotNull(fixedSchema);
JavaFileObject javaFileObject = generator.generateSpecificClass(fixedSchema,
SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);

}

@Test
public void testSimpleFixedWithHugeDoc() throws Exception {
String avsc = TestUtil.load("schemas/SimpleFixedWithHugeDoc.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroFixedSchema fixedSchema = (AvroFixedSchema) result.getTopLevelSchema();
Assert.assertNotNull(fixedSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(fixedSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);

}

@DataProvider
Object[][] testRecordWithArrayOfRecordsProvider() {
Object[][] schemaPaths() {
return new Object[][]{
{"schemas/ArrayOfStringRecord.avsc"},
{"schemas/ArrayOfRecords.avsc"},
{"schemas/TestCollections.avsc"}
{"schemas/ArrayOfRecords.avsc", false},
{"schemas/ArrayOfStringRecord.avsc", true},
{"schemas/BuilderTester.avsc", false},
{"schemas/DollarSignInDoc.avsc", true},
{"schemas/MoneyRange.avsc", false},
{"schemas/RecordDefault.avsc", false},
{"schemas/RecordWithRecordOfEnum.avsc", false},
{"schemas/RecordWithRecordOfRecord.avsc", false},
{"schemas/SimpleEnum.avsc", true},
{"schemas/SimpleEnumWithHugeDoc.avsc", true},
{"schemas/SimpleFixed.avsc", true},
{"schemas/SimpleFixedWithHugeDoc.avsc", true},
{"schemas/TestCollections.avsc", true},
{"schemas/TestProblematicRecord.avsc", true},
{"schemas/TestRecord.avsc", false}
};
}

@Test(dataProvider = "testRecordWithArrayOfRecordsProvider")
public void testRecordWithArrayOfRecords(String path) throws Exception {
@Test(dataProvider = "schemaPaths")
public void testSchema(String path, boolean checkCompiles) throws Exception {
String avsc = TestUtil.load(path);
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);

}

@DataProvider
Object[][] testSpecificWithInternalClassesProvider() {
return new Object[][]{
{"schemas/RecordWithRecordOfEnum.avsc"},
{"schemas/RecordWithRecordOfRecord.avsc"},
{"schemas/TestRecord.avsc"}
};
}

@Test(dataProvider = "testSpecificWithInternalClassesProvider")
public void testSpecificWithInternalClasses(String filePath) throws Exception {
String avsc = TestUtil.load(filePath);
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaSourceFile = generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
}

@Test
public void testSpecificWith$InDoc() throws Exception {
String avsc = TestUtil.load("schemas/DollarSignInDoc.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);
}

@Test
public void testSpecificRecordWithInternalDefinedTypeReuse() throws Exception {
String avsc = TestUtil.load("schemas/MoneyRange.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
}

@Test
public void testDefaultForRecord() throws Exception {
String avsc = TestUtil.load("schemas/RecordDefault.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
}

@Test
public void testRecord() throws Exception {
String avsc = TestUtil.load("schemas/BuilderTester.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema recordSchema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(recordSchema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(recordSchema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
}
AvroNamedSchema schema = (AvroNamedSchema) result.getTopLevelSchema();
Assert.assertNotNull(schema);

@Test
public void testProblematicRecord() throws Exception {
String avsc = TestUtil.load("schemas/TestProblematicRecord.avsc");
SpecificRecordClassGenerator generator = new SpecificRecordClassGenerator();
AvscParser parser = new AvscParser();
AvscParseResult result = parser.parse(avsc);
Assert.assertNull(result.getParseError());
AvroRecordSchema schema = (AvroRecordSchema) result.getTopLevelSchema();
Assert.assertNotNull(schema);
JavaFileObject javaFileObject =
generator.generateSpecificClass(schema, SpecificRecordGenerationConfig.BROAD_COMPATIBILITY);
CompilerHelper.assertCompiles(javaFileObject);

if (checkCompiles) {
CompilerHelper.assertCompiles(javaFileObject);
}
}

@Test
Expand Down
32 changes: 32 additions & 0 deletions helper/helper/src/main/java/com/linkedin/avroutil1/Enums.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright 2023 LinkedIn Corp.
* Licensed under the BSD 2-Clause License (the "License").
* See License in the project root for license information.
*/

package com.linkedin.avroutil1;

/**
* Consists exclusively of static methods that operate on or return {@link Enum}s.
*/
public class Enums {
private static final ClassValue<Enum<?>[]> ENUM_CONSTANTS = new ClassValue<Enum<?>[]>() {
@Override
protected Enum<?>[] computeValue(Class<?> type) {
// JVM creates array everytime invoked.
return (Enum<?>[]) type.getEnumConstants();
}
};

private Enums() {
} // utility

/**
* Returns shared enum constant for specified ordinal.
*
* Lazily caches result of Enum#values to avoid unnecessary memory overhead.
*/
public static <E extends Enum<?>> E getConstant(Class<E> type, int ordinal) {
return (E) ENUM_CONSTANTS.get(type)[ordinal];
}
}
30 changes: 30 additions & 0 deletions helper/helper/src/test/java/com/linkedin/avroutil1/EnumsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright 2023 LinkedIn Corp.
* Licensed under the BSD 2-Clause License (the "License").
* See License in the project root for license information.
*/

package com.linkedin.avroutil1;

import org.testng.Assert;
import org.testng.annotations.Test;


public class EnumsTest {
@Test(expectedExceptions = ArrayIndexOutOfBoundsException.class)
public void testGetEnumConstantWithNegativeOrdinal() {
Enums.getConstant(Thread.State.class, -1);
}

@Test(expectedExceptions = ArrayIndexOutOfBoundsException.class)
public void testGetEnumConstantWithLargeOrdinal() {
Enums.getConstant(Thread.State.class, Thread.State.values().length);
}

@Test
public void testGetEnumConstant() {
for (Thread.State state : Thread.State.values()) {
Assert.assertEquals(Enums.getConstant(Thread.State.class, state.ordinal()), state);
}
}
}

0 comments on commit accc9ed

Please sign in to comment.