From 6c686aee64c35a35107020f1e425ed5002e00773 Mon Sep 17 00:00:00 2001 From: minurajeeve Date: Tue, 5 Dec 2023 21:30:18 +0530 Subject: [PATCH 1/2] fix for using set-type when it's the first directive in the recipie --- .../io/cdap/directives/column/SetType.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java b/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java index 985a6719a..3b5754254 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java +++ b/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java @@ -134,33 +134,35 @@ public Schema getOutputSchema(SchemaResolutionContext context) { if (field.getName().equals(col)) { Integer outputScale = scale; Integer outputPrecision = precision; - Schema fieldSchema = field.getSchema().getNonNullable(); - Pair scaleAndPrecision = getPrecisionAndScale(fieldSchema); - Integer inputSchemaScale = scaleAndPrecision.getSecond(); - Integer inputSchemaPrecision = scaleAndPrecision.getFirst(); + if (type.equalsIgnoreCase("decimal") && field.getSchema().isNullable()) { + Schema fieldSchema = field.getSchema().getNonNullable(); + Pair scaleAndPrecision = getPrecisionAndScale(fieldSchema); + Integer inputSchemaScale = scaleAndPrecision.getSecond(); + Integer inputSchemaPrecision = scaleAndPrecision.getFirst(); - if (scale == null && precision == null) { - outputScale = inputSchemaScale; - outputPrecision = inputSchemaPrecision; - } else if (scale == null && inputSchemaScale != null) { - if (precision - inputSchemaScale < 1) { - throw new DirectiveParseException(String.format( - "Cannot set scale as '%s' and precision as '%s' when " - + "given precision - scale is less than 1 ", inputSchemaScale, - precision)); - } - outputScale = inputSchemaScale; - outputPrecision = precision; + if (scale == null && precision == null) { + outputScale = inputSchemaScale; + outputPrecision = inputSchemaPrecision; + } else if (scale == null && inputSchemaScale != null) { + if (precision - inputSchemaScale < 1) { + throw new DirectiveParseException(String.format( + "Cannot set scale as '%s' and precision as '%s' when " + + "given precision - scale is less than 1 ", inputSchemaScale, + precision)); + } + outputScale = inputSchemaScale; + outputPrecision = precision; - } else if (precision == null && inputSchemaPrecision != null) { - if (inputSchemaPrecision - scale < 1) { - throw new DirectiveParseException(String.format( - "Cannot set scale as '%s' and precision as '%s' when " - + "given precision - scale is less than 1 ", scale, - inputSchemaPrecision)); + } else if (precision == null && inputSchemaPrecision != null) { + if (inputSchemaPrecision - scale < 1) { + throw new DirectiveParseException(String.format( + "Cannot set scale as '%s' and precision as '%s' when " + + "given precision - scale is less than 1 ", scale, + inputSchemaPrecision)); + } + outputScale = scale; + outputPrecision = inputSchemaPrecision; } - outputScale = scale; - outputPrecision = inputSchemaPrecision; } return Schema.Field.of(col, ColumnConverter.getSchemaForType(type, outputScale, outputPrecision)); From 6ffcfaa5c2ece8a4315ec3e9293fa6f8ce55a2b5 Mon Sep 17 00:00:00 2001 From: minurajeeve Date: Tue, 2 Jan 2024 13:32:01 +0530 Subject: [PATCH 2/2] review comments -0 --- .../io/cdap/directives/column/SetType.java | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java b/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java index 3b5754254..0f49ad321 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java +++ b/wrangler-core/src/main/java/io/cdap/directives/column/SetType.java @@ -136,33 +136,10 @@ public Schema getOutputSchema(SchemaResolutionContext context) { Integer outputPrecision = precision; if (type.equalsIgnoreCase("decimal") && field.getSchema().isNullable()) { Schema fieldSchema = field.getSchema().getNonNullable(); - Pair scaleAndPrecision = getPrecisionAndScale(fieldSchema); - Integer inputSchemaScale = scaleAndPrecision.getSecond(); - Integer inputSchemaPrecision = scaleAndPrecision.getFirst(); - - if (scale == null && precision == null) { - outputScale = inputSchemaScale; - outputPrecision = inputSchemaPrecision; - } else if (scale == null && inputSchemaScale != null) { - if (precision - inputSchemaScale < 1) { - throw new DirectiveParseException(String.format( - "Cannot set scale as '%s' and precision as '%s' when " - + "given precision - scale is less than 1 ", inputSchemaScale, - precision)); - } - outputScale = inputSchemaScale; - outputPrecision = precision; - - } else if (precision == null && inputSchemaPrecision != null) { - if (inputSchemaPrecision - scale < 1) { - throw new DirectiveParseException(String.format( - "Cannot set scale as '%s' and precision as '%s' when " - + "given precision - scale is less than 1 ", scale, - inputSchemaPrecision)); - } - outputScale = scale; - outputPrecision = inputSchemaPrecision; - } + Pair scaleAndPrecision = getValidatedPrecisionAndScale( + fieldSchema, precision, scale); + outputScale = scaleAndPrecision.getSecond(); + outputPrecision = scaleAndPrecision.getFirst(); } return Schema.Field.of(col, ColumnConverter.getSchemaForType(type, outputScale, outputPrecision)); @@ -173,20 +150,50 @@ public Schema getOutputSchema(SchemaResolutionContext context) { } } ) - .collect(Collectors.toList()) + .collect(Collectors.toList()) ); } /** * extracts precision and scale from schema string */ - public static Pair getPrecisionAndScale(Schema fieldSchema) { - Integer precision = null; - Integer scale = null; - if (fieldSchema.getLogicalType() == LogicalType.DECIMAL) { - precision = fieldSchema.getPrecision(); - scale = fieldSchema.getScale(); + public static Pair getValidatedPrecisionAndScale(Schema fieldSchema, + Integer precision, Integer scale) + throws DirectiveParseException { //check precision and scale + Integer outputPrecision = precision; + Integer outputScale = scale; + Integer inputSchemaPrecision = null; + Integer inputSchemaScale = null; + + if (fieldSchema.getLogicalType() == LogicalType.DECIMAL) { + inputSchemaPrecision = fieldSchema.getPrecision(); + inputSchemaScale = fieldSchema.getScale(); + } + + if (scale == null && precision == null) { + outputScale = inputSchemaScale; + outputPrecision = inputSchemaPrecision; + } else if (scale == null && inputSchemaScale != null) { + if (precision - inputSchemaScale < 1) { + throw new DirectiveParseException(String.format( + "Cannot set scale as '%s' and precision as '%s' when " + + "given precision - scale is less than 1 ", inputSchemaScale, + precision)); } - return new Pair(precision, scale); + outputScale = inputSchemaScale; + outputPrecision = precision; + + } else if (precision == null && inputSchemaPrecision != null) { + if (inputSchemaPrecision - scale < 1) { + throw new DirectiveParseException(String.format( + "Cannot set scale as '%s' and precision as '%s' when " + + "given precision - scale is less than 1 ", scale, + inputSchemaPrecision)); + } + outputScale = scale; + outputPrecision = inputSchemaPrecision; } + + return new Pair(outputPrecision, outputScale); + } }