From 80ecbc2edf44efadafb7d4729cc43398ddc96498 Mon Sep 17 00:00:00 2001 From: Piotr Jaczewski Date: Fri, 31 Jul 2020 21:22:10 +0200 Subject: [PATCH] logger verbosity changes (#75) * schema content is not logged upon successful fast(de)serializer compilation by default * success messages are logged after actual (de)serializer generation in FastSerdeCache --- .../avro/fastserde/FastSerdeBase.java | 4 +- .../avro/fastserde/FastSerdeCache.java | 67 +++++++++++++++---- .../AvroCompatibilityHelper.java | 30 +++++++-- 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java index 36e33a862..4f3cf1f23 100644 --- a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java +++ b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java @@ -106,11 +106,9 @@ protected Class compileClass(final String className, Set knownUsedFullyQ codeModel.build(destination); String filePath = destination.getAbsolutePath() + generatedSourcesPath + className + ".java"; - LOGGER.info("Generated source file: " + filePath); JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); String compileClassPathForCurrentFile = Utils.inferCompileDependencies(compileClassPath, filePath, knownUsedFullyQualifiedClassNameSet); - LOGGER.info("For source file: " + filePath + ", and the inferred compile class path: " + compileClassPathForCurrentFile); int compileResult; try { /* @@ -127,7 +125,9 @@ protected Class compileClass(final String className, Set knownUsedFullyQ * Keeping this config also does not bring any downgrade. * */ + LOGGER.info("Starting compilation for the generated source file: {} with the inferred compile class path: {}", filePath, compileClassPathForCurrentFile); compileResult = compiler.run(null, null, null, "-cp", compileClassPathForCurrentFile, filePath, "-XDuseUnsharedTable"); + LOGGER.info("Successfully compiled class {} defined at source file: {}", className, filePath); } catch (Exception e) { throw new FastSerdeGeneratorException("Unable to compile:" + className + " from source file: " + filePath, e); } diff --git a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeCache.java b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeCache.java index a0a61c201..fbbcc2e0a 100644 --- a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeCache.java +++ b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeCache.java @@ -1,5 +1,8 @@ package com.linkedin.avro.fastserde; +import static com.linkedin.avro.fastserde.Utils.getSchemaFingerprint; +import static com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper.getSchemaFullName; + import java.io.File; import java.io.IOException; import java.lang.reflect.ParameterizedType; @@ -15,6 +18,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; + import org.apache.avro.Schema; import org.apache.avro.generic.ColdGenericDatumReader; import org.apache.avro.generic.ColdSpecificDatumReader; @@ -292,8 +296,8 @@ public FastSerializer getFastGenericSerializer(Schema schema) { } private String getSchemaKey(Schema writerSchema, Schema readerSchema) { - return String.valueOf(Math.abs(Utils.getSchemaFingerprint(writerSchema))) + Math.abs( - Utils.getSchemaFingerprint(readerSchema)); + return String.valueOf(Math.abs(getSchemaFingerprint(writerSchema))) + Math.abs( + getSchemaFingerprint(readerSchema)); } /** @@ -308,9 +312,20 @@ public FastDeserializer buildFastSpecificDeserializer(Schema writerSchema, Sc FastSpecificDeserializerGenerator generator = new FastSpecificDeserializerGenerator<>(writerSchema, readerSchema, classesDir, classLoader, compileClassPath.orElseGet(() -> null)); - LOGGER.info("Generated class dir: {}, and generation of specific FastDeserializer is done for writer schema: " - + "[\n{}\n] and reader schema: [\n{}\n]", classesDir, writerSchema.toString(true), readerSchema.toString(true)); - return generator.generateDeserializer(); + FastDeserializer fastDeserializer = generator.generateDeserializer(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Generated classes dir: {} and generation of specific FastDeserializer is done for writer schema of type: {} with fingerprint: {}" + + " and content: [\n{}\n] and reader schema of type: {} with fingerprint: {} and content: [\n{}\n]", classesDir, getSchemaFullName(writerSchema), + writerSchema.toString(true), getSchemaFingerprint(writerSchema), getSchemaFullName(readerSchema), getSchemaFingerprint(readerSchema), + readerSchema.toString(true)); + } else { + LOGGER.info("Generated classes dir: {} and generation of specific FastDeserializer is done for writer schema of type: {} with fingerprint: {}" + + " and reader schema of type: {} with fingerprint: {}", classesDir, getSchemaFullName(writerSchema), getSchemaFingerprint(writerSchema), + getSchemaFullName(readerSchema), getSchemaFingerprint(readerSchema)); + } + + return fastDeserializer; } /** @@ -352,9 +367,21 @@ public FastDeserializer buildFastGenericDeserializer(Schema writerSchema, Sch FastGenericDeserializerGenerator generator = new FastGenericDeserializerGenerator<>(writerSchema, readerSchema, classesDir, classLoader, compileClassPath.orElseGet(() -> null)); - LOGGER.info("Generated classes dir: {} and generation of generic FastDeserializer is done for writer schema: " - + "[\n{}\n] and reader schema:[\n{}\n]", classesDir, writerSchema.toString(true), readerSchema.toString(true)); - return generator.generateDeserializer(); + + FastDeserializer fastDeserializer = generator.generateDeserializer(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Generated classes dir: {} and generation of generic FastDeserializer is done for writer schema of type: {} with fingerprint: {}" + + " and content: [\n{}\n] and reader schema of type: {} with fingerprint: {} and content: [\n{}\n]", classesDir, getSchemaFullName(writerSchema), + writerSchema.toString(true), getSchemaFingerprint(writerSchema), getSchemaFullName(readerSchema), getSchemaFingerprint(readerSchema), + readerSchema.toString(true)); + } else { + LOGGER.info("Generated classes dir: {} and generation of generic FastDeserializer is done for writer schema of type: {} with fingerprint: {}" + + " and reader schema of type: {} with fingerprint: {}", classesDir, getSchemaFullName(writerSchema), getSchemaFingerprint(writerSchema), + getSchemaFullName(readerSchema), getSchemaFingerprint(readerSchema)); + } + + return fastDeserializer; } /** @@ -393,8 +420,16 @@ public FastSerializer buildFastSpecificSerializer(Schema schema) { } FastSpecificSerializerGenerator generator = new FastSpecificSerializerGenerator<>(schema, classesDir, classLoader, compileClassPath.orElseGet(() -> null)); - LOGGER.info("Generated classes dir: {} and generation of specific FastSerializer is done for schema: [\n{}\n]", - classesDir, schema.toString(true)); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Generated classes dir: {} and generation of specific FastSerializer is done for schema of type: {}" + + " and fingerprint: {} and content: [\n{}\n]", classesDir, getSchemaFullName(schema), getSchemaFingerprint(schema), + schema.toString(true)); + } else { + LOGGER.info("Generated classes dir: {} and generation of specific FastSerializer is done for schema of type: {}" + + " and fingerprint: {}", classesDir, getSchemaFullName(schema), getSchemaFingerprint(schema)); + } + return generator.generateSerializer(); } @@ -429,8 +464,16 @@ public FastSerializer buildFastGenericSerializer(Schema schema) { } FastGenericSerializerGenerator generator = new FastGenericSerializerGenerator<>(schema, classesDir, classLoader, compileClassPath.orElseGet(() -> null)); - LOGGER.info("Generated classes dir: {} and generation of generic FastSerializer is done for schema: [\n{}\n]", - classesDir, schema.toString(true)); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Generated classes dir: {} and generation of generic FastSerializer is done for schema of type: {}" + + " and fingerprint: {} and content: [\n{}\n]", classesDir, getSchemaFullName(schema), getSchemaFingerprint(schema), + schema.toString(true)); + } else { + LOGGER.info("Generated classes dir: {} and generation of generic FastSerializer is done for schema of type: {}" + + " and fingerprint: {}", classesDir, getSchemaFullName(schema), getSchemaFingerprint(schema)); + } + return generator.generateSerializer(); } diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java index 3fcd6537d..823811aaa 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java @@ -6,12 +6,6 @@ package com.linkedin.avroutil1.compatibility; -import com.linkedin.avroutil1.compatibility.avro14.Avro14Adapter; -import com.linkedin.avroutil1.compatibility.avro15.Avro15Adapter; -import com.linkedin.avroutil1.compatibility.avro16.Avro16Adapter; -import com.linkedin.avroutil1.compatibility.avro17.Avro17Adapter; -import com.linkedin.avroutil1.compatibility.avro18.Avro18Adapter; -import com.linkedin.avroutil1.compatibility.avro19.Avro19Adapter; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -20,6 +14,7 @@ import java.io.OutputStream; import java.lang.reflect.Modifier; import java.util.Collection; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.IndexedRecord; @@ -29,6 +24,13 @@ import org.apache.avro.io.JsonEncoder; import org.apache.avro.specific.SpecificRecord; +import com.linkedin.avroutil1.compatibility.avro14.Avro14Adapter; +import com.linkedin.avroutil1.compatibility.avro15.Avro15Adapter; +import com.linkedin.avroutil1.compatibility.avro16.Avro16Adapter; +import com.linkedin.avroutil1.compatibility.avro17.Avro17Adapter; +import com.linkedin.avroutil1.compatibility.avro18.Avro18Adapter; +import com.linkedin.avroutil1.compatibility.avro19.Avro19Adapter; + /** * a Utility class for performing various avro-related operations under a wide range of avro versions at runtime. @@ -472,4 +474,20 @@ private static AvroVersion detectAvroVersion() { return AvroVersion.AVRO_1_9; } + + /** + * Get the full schema name for records or type name for primitives. This adds compatibility + * layer for {@link Schema#getFullName} implementation in avro 1.4, which defaults to throwing exception. + * + * @param schema the schema whose full name should be retrieved + * @return full schema name or primitive type name + */ + public static String getSchemaFullName(Schema schema) { + try { + return schema.getFullName(); + } catch (RuntimeException e) { + return schema.getType().name(); + } + } + }