From a1a1288683b7d46a952d4885883a59d802ee09a6 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Fri, 1 Dec 2023 09:53:02 +0800 Subject: [PATCH 1/9] wip Signed-off-by: Haoyang Li --- .../advanced_configs.md | 1 + docs/supported_ops.md | 47 +++++++++++++++++++ .../src/main/python/string_test.py | 6 +++ .../nvidia/spark/rapids/GpuOverrides.scala | 6 +++ .../spark/sql/rapids/stringFunctions.scala | 31 ++++++++++++ tools/generated_files/operatorsScore.csv | 1 + tools/generated_files/supportedExprs.csv | 2 + 7 files changed, 94 insertions(+) diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 0251cacb34c..38d888d9873 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -186,6 +186,7 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysZip|`arrays_zip`|Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays.|true|None| +spark.rapids.sql.expression.Ascii|`ascii`|The numeric value of the first character of string data|true|None| spark.rapids.sql.expression.Asin|`asin`|Inverse sine|true|None| spark.rapids.sql.expression.Asinh|`asinh`|Inverse hyperbolic sine|true|None| spark.rapids.sql.expression.AtLeastNNonNulls| |Checks if number of non null/Nan values is greater than a given value|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 414a53c56ac..7f94126d98e 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -2863,6 +2863,53 @@ are limited. +Ascii +`ascii` +The numeric value of the first character of string data +None +project +input + + + + + + + + + +S + + + + + + + + + + +result + + + +S + + + + + + + + + + + + + + + + Asin `asin` Inverse sine diff --git a/integration_tests/src/main/python/string_test.py b/integration_tests/src/main/python/string_test.py index 65865d0bdc1..b88acefc231 100644 --- a/integration_tests/src/main/python/string_test.py +++ b/integration_tests/src/main/python/string_test.py @@ -650,6 +650,12 @@ def test_byte_length(): lambda spark: unary_op_df(spark, gen).selectExpr( 'BIT_LENGTH(a)', 'OCTET_LENGTH(a)')) +def test_ascii(): + gen = mk_str_gen('.{0,5}') + assert_gpu_and_cpu_are_equal_collect( + lambda spark: unary_op_df(spark, gen).select(f.col('a'), + f.ascii(f.col('a')))) + @incompat def test_initcap(): # Because we don't use the same unicode version we need to limit diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 4d45dacfd0d..ce5ddd1a1e5 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3694,6 +3694,12 @@ object GpuOverrides extends Logging { (a, conf, p, r) => new UnaryExprMeta[OctetLength](a, conf, p, r) { override def convertToGpu(child: Expression): GpuExpression = GpuOctetLength(child) }), + expr[Ascii]( + "The numeric value of the first character of string data", + ExprChecks.unaryProject(TypeSig.INT, TypeSig.INT, TypeSig.STRING, TypeSig.STRING), + (a, conf, p, r) => new UnaryExprMeta[Ascii](a, conf, p, r) { + override def convertToGpu(child: Expression): GpuExpression = GpuAscii(child) + }), expr[GetArrayStructFields]( "Extracts the `ordinal`-th fields of all array elements for the data with the type of" + " array of struct", diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index febbf75ba58..df4ea1466e8 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -92,6 +92,37 @@ case class GpuOctetLength(child: Expression) extends GpuUnaryExpression with Exp input.getBase.getByteCount } +case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitCastInputTypes + with NullIntolerant { + + override def dataType: DataType = IntegerType + override def inputTypes: Seq[AbstractDataType] = Seq(StringType) + + override def doColumnar(input: GpuColumnVector): ColumnVector = { + // replace nulls with 'n' and save the null mask + val nullMask = input.getBase.isNull + val nullsReplaced = withResource(Scalar.fromString("n")) { nullScalar => + nullMask.ifElse(nullScalar, input.getBase) + } + // replace empty strings with 'e' and save the null mask + val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => + nullsReplaced.equalTo(emptyScalar) + } + val emptyReplaced = withResource(nullsReplaced) { _ => + withResource(Scalar.fromString("e")) { eScalar => + emptyMask.ifElse(eScalar, nullsReplaced) + } + } + // substr(0,1) + val substr = emptyReplaced.substring(0, 1) + // codePoints + val codePoints = withResource(substr) { substr => + substr.codePoints() + } + codePoints + } +} + case class GpuStringLocate(substr: Expression, col: Expression, start: Expression) extends GpuTernaryExpressionArgsScalarAnyScalar with ImplicitCastInputTypes { diff --git a/tools/generated_files/operatorsScore.csv b/tools/generated_files/operatorsScore.csv index 17c80f60bfb..bdc186d126d 100644 --- a/tools/generated_files/operatorsScore.csv +++ b/tools/generated_files/operatorsScore.csv @@ -57,6 +57,7 @@ ArrayTransform,4 ArrayUnion,4 ArraysOverlap,4 ArraysZip,4 +Ascii,4 Asin,4 Asinh,4 AtLeastNNonNulls,4 diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index bff8dc7359a..89df9009625 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -60,6 +60,8 @@ ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark vers ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +Ascii,S,`ascii`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,S,`ascii`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From a57adac9c95974d6626e9235cc1d2587d3eaf3d7 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 14 Dec 2023 17:53:15 +0800 Subject: [PATCH 2/9] Support Ascii function for ascii and latin-1 Signed-off-by: Haoyang Li --- .../advanced_configs.md | 2 +- docs/supported_ops.md | 4 +- .../src/main/python/string_test.py | 6 +- .../nvidia/spark/rapids/GpuOverrides.scala | 5 +- .../spark/sql/rapids/stringFunctions.scala | 109 +++++++++++++++--- tools/generated_files/supportedExprs.csv | 4 +- 6 files changed, 104 insertions(+), 26 deletions(-) diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 4be9706a0ee..7c343c34150 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -186,7 +186,7 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysZip|`arrays_zip`|Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays.|true|None| -spark.rapids.sql.expression.Ascii|`ascii`|The numeric value of the first character of string data|true|None| +spark.rapids.sql.expression.Ascii|`ascii`|The numeric value of the first character of string data.|false|This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.| spark.rapids.sql.expression.Asin|`asin`|Inverse sine|true|None| spark.rapids.sql.expression.Asinh|`asinh`|Inverse hyperbolic sine|true|None| spark.rapids.sql.expression.AtLeastNNonNulls| |Checks if number of non null/Nan values is greater than a given value|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 987f1c655b0..30a8e654d88 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -2865,8 +2865,8 @@ are limited. Ascii `ascii` -The numeric value of the first character of string data -None +The numeric value of the first character of string data. +This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU. project input diff --git a/integration_tests/src/main/python/string_test.py b/integration_tests/src/main/python/string_test.py index f2bdfecda6b..6b440febc02 100644 --- a/integration_tests/src/main/python/string_test.py +++ b/integration_tests/src/main/python/string_test.py @@ -651,10 +651,12 @@ def test_byte_length(): 'BIT_LENGTH(a)', 'OCTET_LENGTH(a)')) def test_ascii(): + # StringGen will generate ascii and latin1 characters by default, which is the same as the supported + # range of ascii in plugin. Characters outside of this range will return mismatched results. gen = mk_str_gen('.{0,5}') assert_gpu_and_cpu_are_equal_collect( - lambda spark: unary_op_df(spark, gen).select(f.col('a'), - f.ascii(f.col('a')))) + lambda spark: unary_op_df(spark, gen).select(f.ascii(f.col('a'))), + conf={'spark.rapids.sql.expression.Ascii': True}) @incompat def test_initcap(): diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index fbf3033f448..88afcb952a0 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3718,11 +3718,12 @@ object GpuOverrides extends Logging { override def convertToGpu(child: Expression): GpuExpression = GpuOctetLength(child) }), expr[Ascii]( - "The numeric value of the first character of string data", + "The numeric value of the first character of string data.", ExprChecks.unaryProject(TypeSig.INT, TypeSig.INT, TypeSig.STRING, TypeSig.STRING), (a, conf, p, r) => new UnaryExprMeta[Ascii](a, conf, p, r) { override def convertToGpu(child: Expression): GpuExpression = GpuAscii(child) - }), + }).disabledByDefault("it only supports strings starting with ASCII or Latin-1 characters." + + " Otherwise the results will not match the CPU."), expr[GetArrayStructFields]( "Extracts the `ordinal`-th fields of all array elements for the data with the type of" + " array of struct", diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index df4ea1466e8..de159c7cad0 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -98,28 +98,103 @@ case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitC override def dataType: DataType = IntegerType override def inputTypes: Seq[AbstractDataType] = Seq(StringType) - override def doColumnar(input: GpuColumnVector): ColumnVector = { - // replace nulls with 'n' and save the null mask - val nullMask = input.getBase.isNull - val nullsReplaced = withResource(Scalar.fromString("n")) { nullScalar => - nullMask.ifElse(nullScalar, input.getBase) + private def utf8CodePointsToAscii(codePoints: ColumnVector): ColumnVector = { + // Currently we only support ASCII characters, so we need to convert the UTF8 code points + // to ASCII code points. Results for code points outside range [0, 255] are undefined. + // seg A: 0 <= codePoints < 128, already ASCII + // seg B: 49792 <= codePoints < 49856, ASCII = codePoints - 49664 + // seg C: 50048 <= codePoints < 50112, ASCII = codePoints - 49856 + + // get values of seg B + val segBL = withResource(Scalar.fromInt(49792)) { segBLeftEnd => + codePoints.greaterOrEqualTo(segBLeftEnd) + } + val segB = withResource(segBL) { _ => + val segBR = withResource(Scalar.fromInt(49856)) { segBRightEnd => + codePoints.lessThan(segBRightEnd) + } + withResource(segBR) { _ => + segBL.and(segBR) + } } - // replace empty strings with 'e' and save the null mask - val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => - nullsReplaced.equalTo(emptyScalar) + + // combine seg A and seg B + val segAB = withResource(segB) { _ => + val segBValue = withResource(Scalar.fromInt(49664)) { subValue => + codePoints.sub(subValue) + } + withResource(segBValue) { _ => + segB.ifElse(segBValue, codePoints) + } } - val emptyReplaced = withResource(nullsReplaced) { _ => - withResource(Scalar.fromString("e")) { eScalar => - emptyMask.ifElse(eScalar, nullsReplaced) + + // get values of seg C + withResource(segAB) { _ => + val segCL = withResource(Scalar.fromInt(50048)) { segCLeftEnd => + codePoints.greaterOrEqualTo(segCLeftEnd) + } + val segC = withResource(segCL) { _ => + val segCR = withResource(Scalar.fromInt(50112)) { segCRightEnd => + codePoints.lessThan(segCRightEnd) + } + withResource(segCR) { _ => + segCL.and(segCR) + } + } + + // combine seg AB and seg C, and return + withResource(segC) { _ => + val segCValue = withResource(Scalar.fromInt(49856)) { subValue => + codePoints.sub(subValue) + } + withResource(segCValue) { _ => + segC.ifElse(segCValue, segAB) + } } } - // substr(0,1) - val substr = emptyReplaced.substring(0, 1) - // codePoints - val codePoints = withResource(substr) { substr => - substr.codePoints() + } + + override def doColumnar(input: GpuColumnVector): ColumnVector = { + withResource(ArrayBuffer.empty[AutoCloseable]) { resource_array => + // replace nulls with 'n' and save the null mask + val nullMask = input.getBase.isNull + resource_array += nullMask + val nullsReplaced = withResource(Scalar.fromString("n")) { nullScalar => + nullMask.ifElse(nullScalar, input.getBase) + } + // replace empty strings with 'e' and save the null mask + val emptyMask = closeOnExcept(nullsReplaced) { _ => + withResource(Scalar.fromString("")) { emptyScalar => + nullsReplaced.equalTo(emptyScalar) + } + } + resource_array += emptyMask + val emptyReplaced = withResource(nullsReplaced) { _ => + withResource(Scalar.fromString("e")) { eScalar => + emptyMask.ifElse(eScalar, nullsReplaced) + } + } + val substr = withResource(emptyReplaced) { _ => + emptyReplaced.substring(0, 1) + } + val codePoints = withResource(substr) { substr => + substr.codePoints() + } + val segABC = withResource(codePoints) { codePoints => + utf8CodePointsToAscii(codePoints) + } + // replace nulls with null and empty strings with 0 + val segABCwithNull = withResource(segABC) { _ => + withResource(Scalar.fromNull(DType.INT32)) { nullScalar => + nullMask.ifElse(nullScalar, segABC) + } + } + withResource(segABCwithNull) { _ => + withResource(Scalar.fromInt(0)) { zeroScalar => + emptyMask.ifElse(zeroScalar, segABCwithNull) + } + } } - codePoints } } diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 0eb406f333b..50b75e3ba5e 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -60,8 +60,8 @@ ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark vers ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Ascii,S,`ascii`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Ascii,S,`ascii`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From 8b03fa11f8d4d1e57f288f00518436e5286704b3 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Tue, 19 Dec 2023 15:31:03 +0800 Subject: [PATCH 3/9] Refine to make it run faster Signed-off-by: Haoyang Li --- .../spark/sql/rapids/stringFunctions.scala | 105 +++++++----------- 1 file changed, 41 insertions(+), 64 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index de159c7cad0..dfaac4746e7 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -104,96 +104,73 @@ case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitC // seg A: 0 <= codePoints < 128, already ASCII // seg B: 49792 <= codePoints < 49856, ASCII = codePoints - 49664 // seg C: 50048 <= codePoints < 50112, ASCII = codePoints - 49856 - - // get values of seg B - val segBL = withResource(Scalar.fromInt(49792)) { segBLeftEnd => + // + // To reduce cuDF API calling, following algorithm will be performed: + // 1. For anything above 49792, we subtract 49664, now seg A and B are correct. + // 2. seg C: 50048 <= current + 49664 < 50112 => 384 <= current < 448, ASCII = current - 192 + // So for anything above 384, we subtract 192, now seg C is correct too. + val greaterThan49792 = withResource(Scalar.fromInt(49792)) { segBLeftEnd => codePoints.greaterOrEqualTo(segBLeftEnd) } - val segB = withResource(segBL) { _ => - val segBR = withResource(Scalar.fromInt(49856)) { segBRightEnd => - codePoints.lessThan(segBRightEnd) + val segAB = withResource(greaterThan49792) { _ => + val sub1 = withResource(Scalar.fromInt(49664)) { segBValue => + codePoints.sub(segBValue) } - withResource(segBR) { _ => - segBL.and(segBR) + withResource(sub1) { _ => + greaterThan49792.ifElse(sub1, codePoints) } } - - // combine seg A and seg B - val segAB = withResource(segB) { _ => - val segBValue = withResource(Scalar.fromInt(49664)) { subValue => - codePoints.sub(subValue) - } - withResource(segBValue) { _ => - segB.ifElse(segBValue, codePoints) - } - } - - // get values of seg C withResource(segAB) { _ => - val segCL = withResource(Scalar.fromInt(50048)) { segCLeftEnd => - codePoints.greaterOrEqualTo(segCLeftEnd) - } - val segC = withResource(segCL) { _ => - val segCR = withResource(Scalar.fromInt(50112)) { segCRightEnd => - codePoints.lessThan(segCRightEnd) - } - withResource(segCR) { _ => - segCL.and(segCR) - } + val geraterThan384 = withResource(Scalar.fromInt(384)) { segCLeftEnd => + segAB.greaterOrEqualTo(segCLeftEnd) } - - // combine seg AB and seg C, and return - withResource(segC) { _ => - val segCValue = withResource(Scalar.fromInt(49856)) { subValue => - codePoints.sub(subValue) + withResource(geraterThan384) { _ => + val sub2 = withResource(Scalar.fromInt(192)) { segCValue => + segAB.sub(segCValue) } - withResource(segCValue) { _ => - segC.ifElse(segCValue, segAB) + withResource(sub2) { _ => + geraterThan384.ifElse(sub2, segAB) } } } } override def doColumnar(input: GpuColumnVector): ColumnVector = { - withResource(ArrayBuffer.empty[AutoCloseable]) { resource_array => - // replace nulls with 'n' and save the null mask - val nullMask = input.getBase.isNull - resource_array += nullMask - val nullsReplaced = withResource(Scalar.fromString("n")) { nullScalar => - nullMask.ifElse(nullScalar, input.getBase) - } - // replace empty strings with 'e' and save the null mask - val emptyMask = closeOnExcept(nullsReplaced) { _ => - withResource(Scalar.fromString("")) { emptyScalar => - nullsReplaced.equalTo(emptyScalar) - } + // replace empty strings with 'NUL' (which will convert to ascii 0) + val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => + input.getBase.equalTo(emptyScalar) + } + val emptyReplaced = withResource(emptyMask) { _ => + // replace empty strings with 'NUL' (which will convert to ascii 0) + withResource(Scalar.fromString('\u0000'.toString)) { zeroScalar => + emptyMask.ifElse(zeroScalar, input.getBase) } - resource_array += emptyMask - val emptyReplaced = withResource(nullsReplaced) { _ => - withResource(Scalar.fromString("e")) { eScalar => - emptyMask.ifElse(eScalar, nullsReplaced) + } + // replace nulls with 'n' and save the null mask + val nullMask = closeOnExcept(emptyReplaced) { _ => + input.getBase.isNull + } + withResource(nullMask) { _ => + val nullsReplaced = withResource(emptyReplaced) { _ => + withResource(Scalar.fromString("n")) { nullScalar => + nullMask.ifElse(nullScalar, emptyReplaced) } } - val substr = withResource(emptyReplaced) { _ => - emptyReplaced.substring(0, 1) + val substr = withResource(nullsReplaced) { _ => + nullsReplaced.substring(0, 1) } - val codePoints = withResource(substr) { substr => + val codePoints = withResource(substr) { _ => substr.codePoints() } - val segABC = withResource(codePoints) { codePoints => + val segABC = withResource(codePoints) { _ => utf8CodePointsToAscii(codePoints) } - // replace nulls with null and empty strings with 0 - val segABCwithNull = withResource(segABC) { _ => + // replace nulls with null + withResource(segABC) { _ => withResource(Scalar.fromNull(DType.INT32)) { nullScalar => nullMask.ifElse(nullScalar, segABC) } } - withResource(segABCwithNull) { _ => - withResource(Scalar.fromInt(0)) { zeroScalar => - emptyMask.ifElse(zeroScalar, segABCwithNull) - } - } } } } From e6d3f975d6d0ba58f502c8d060b113ee55a40da7 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Wed, 27 Dec 2023 17:29:31 +0800 Subject: [PATCH 4/9] Add bytes-based solution and shims Signed-off-by: Haoyang Li --- .../advanced_configs.md | 2 +- docs/supported_ops.md | 2 +- .../nvidia/spark/rapids/GpuOverrides.scala | 6 +- .../spark/sql/rapids/stringFunctions.scala | 83 ------------ .../spark/sql/rapids/shims/GpuAscii.scala | 68 ++++++++++ .../spark/sql/rapids/shims/GpuAscii.scala | 121 ++++++++++++++++++ tools/generated_files/supportedExprs.csv | 4 +- 7 files changed, 196 insertions(+), 90 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala create mode 100644 sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 7c343c34150..04737c758a0 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -186,7 +186,7 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysZip|`arrays_zip`|Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays.|true|None| -spark.rapids.sql.expression.Ascii|`ascii`|The numeric value of the first character of string data.|false|This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.| +spark.rapids.sql.expression.Ascii|`ascii`|The numeric value of the first character of string data.|false|This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3, 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.| spark.rapids.sql.expression.Asin|`asin`|Inverse sine|true|None| spark.rapids.sql.expression.Asinh|`asinh`|Inverse hyperbolic sine|true|None| spark.rapids.sql.expression.AtLeastNNonNulls| |Checks if number of non null/Nan values is greater than a given value|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 30a8e654d88..1dc3e587cdf 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -2866,7 +2866,7 @@ are limited. Ascii `ascii` The numeric value of the first character of string data. -This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU. +This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3, 3.3.1 and 3.4.0. Otherwise the results will not match the CPU. project input diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 88afcb952a0..af6cf5daf14 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -65,7 +65,7 @@ import org.apache.spark.sql.rapids.catalyst.expressions.GpuRand import org.apache.spark.sql.rapids.execution._ import org.apache.spark.sql.rapids.execution.python._ import org.apache.spark.sql.rapids.execution.python.GpuFlatMapGroupsInPandasExecMeta -import org.apache.spark.sql.rapids.shims.GpuTimeAdd +import org.apache.spark.sql.rapids.shims.{GpuAscii, GpuTimeAdd} import org.apache.spark.sql.rapids.zorder.ZOrderRules import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} @@ -3722,8 +3722,8 @@ object GpuOverrides extends Logging { ExprChecks.unaryProject(TypeSig.INT, TypeSig.INT, TypeSig.STRING, TypeSig.STRING), (a, conf, p, r) => new UnaryExprMeta[Ascii](a, conf, p, r) { override def convertToGpu(child: Expression): GpuExpression = GpuAscii(child) - }).disabledByDefault("it only supports strings starting with ASCII or Latin-1 characters." + - " Otherwise the results will not match the CPU."), + }).disabledByDefault("it only supports strings starting with ASCII or Latin-1 characters " + + "after Spark 3.2.3, 3.3.1 and 3.4.0. Otherwise the results will not match the CPU."), expr[GetArrayStructFields]( "Extracts the `ordinal`-th fields of all array elements for the data with the type of" + " array of struct", diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index dfaac4746e7..febbf75ba58 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -92,89 +92,6 @@ case class GpuOctetLength(child: Expression) extends GpuUnaryExpression with Exp input.getBase.getByteCount } -case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitCastInputTypes - with NullIntolerant { - - override def dataType: DataType = IntegerType - override def inputTypes: Seq[AbstractDataType] = Seq(StringType) - - private def utf8CodePointsToAscii(codePoints: ColumnVector): ColumnVector = { - // Currently we only support ASCII characters, so we need to convert the UTF8 code points - // to ASCII code points. Results for code points outside range [0, 255] are undefined. - // seg A: 0 <= codePoints < 128, already ASCII - // seg B: 49792 <= codePoints < 49856, ASCII = codePoints - 49664 - // seg C: 50048 <= codePoints < 50112, ASCII = codePoints - 49856 - // - // To reduce cuDF API calling, following algorithm will be performed: - // 1. For anything above 49792, we subtract 49664, now seg A and B are correct. - // 2. seg C: 50048 <= current + 49664 < 50112 => 384 <= current < 448, ASCII = current - 192 - // So for anything above 384, we subtract 192, now seg C is correct too. - val greaterThan49792 = withResource(Scalar.fromInt(49792)) { segBLeftEnd => - codePoints.greaterOrEqualTo(segBLeftEnd) - } - val segAB = withResource(greaterThan49792) { _ => - val sub1 = withResource(Scalar.fromInt(49664)) { segBValue => - codePoints.sub(segBValue) - } - withResource(sub1) { _ => - greaterThan49792.ifElse(sub1, codePoints) - } - } - withResource(segAB) { _ => - val geraterThan384 = withResource(Scalar.fromInt(384)) { segCLeftEnd => - segAB.greaterOrEqualTo(segCLeftEnd) - } - withResource(geraterThan384) { _ => - val sub2 = withResource(Scalar.fromInt(192)) { segCValue => - segAB.sub(segCValue) - } - withResource(sub2) { _ => - geraterThan384.ifElse(sub2, segAB) - } - } - } - } - - override def doColumnar(input: GpuColumnVector): ColumnVector = { - // replace empty strings with 'NUL' (which will convert to ascii 0) - val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => - input.getBase.equalTo(emptyScalar) - } - val emptyReplaced = withResource(emptyMask) { _ => - // replace empty strings with 'NUL' (which will convert to ascii 0) - withResource(Scalar.fromString('\u0000'.toString)) { zeroScalar => - emptyMask.ifElse(zeroScalar, input.getBase) - } - } - // replace nulls with 'n' and save the null mask - val nullMask = closeOnExcept(emptyReplaced) { _ => - input.getBase.isNull - } - withResource(nullMask) { _ => - val nullsReplaced = withResource(emptyReplaced) { _ => - withResource(Scalar.fromString("n")) { nullScalar => - nullMask.ifElse(nullScalar, emptyReplaced) - } - } - val substr = withResource(nullsReplaced) { _ => - nullsReplaced.substring(0, 1) - } - val codePoints = withResource(substr) { _ => - substr.codePoints() - } - val segABC = withResource(codePoints) { _ => - utf8CodePointsToAscii(codePoints) - } - // replace nulls with null - withResource(segABC) { _ => - withResource(Scalar.fromNull(DType.INT32)) { nullScalar => - nullMask.ifElse(nullScalar, segABC) - } - } - } - } -} - case class GpuStringLocate(substr: Expression, col: Expression, start: Expression) extends GpuTernaryExpressionArgsScalarAnyScalar with ImplicitCastInputTypes { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala new file mode 100644 index 00000000000..52b75e178f4 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "321db"} +{"spark": "322"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +spark-rapids-shim-json-lines ***/ + +package org.apache.spark.sql.rapids.shims + +import ai.rapids.cudf.{ColumnVector, DType, Scalar} +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.Arm._ + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.types._ + +case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitCastInputTypes + with NullIntolerant { + + override def dataType: DataType = IntegerType + override def inputTypes: Seq[AbstractDataType] = Seq(StringType) + + override def doColumnar(input: GpuColumnVector): ColumnVector = { + // convert to byte lists + val firstBytes = withResource(input.getBase.asByteList) { bytes => + bytes.extractListElement(0) + } + val firstBytesInt = withResource(firstBytes) { _ => + firstBytes.castTo(DType.INT32) + } + withResource(firstBytesInt) { _ => + val greaterThan127 = withResource(Scalar.fromInt(127)) { scalar => + firstBytesInt.greaterThan(scalar) + } + withResource(greaterThan127) { _ => + val sub256 = withResource(Scalar.fromInt(256)) { scalar => + firstBytesInt.sub(scalar) + } + withResource(sub256) { _ => + greaterThan127.ifElse(sub256, firstBytesInt) + } + } + } + } +} diff --git a/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala new file mode 100644 index 00000000000..662ad116bf3 --- /dev/null +++ b/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "323"} +{"spark": "324"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "350"} +spark-rapids-shim-json-lines ***/ + +package org.apache.spark.sql.rapids.shims + +import ai.rapids.cudf.{ColumnVector, DType, Scalar} +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.Arm._ + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.types._ + +case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitCastInputTypes + with NullIntolerant { + + override def dataType: DataType = IntegerType + override def inputTypes: Seq[AbstractDataType] = Seq(StringType) + + private def utf8CodePointsToAscii(codePoints: ColumnVector): ColumnVector = { + // Currently we only support ASCII characters, so we need to convert the UTF8 code points + // to ASCII code points. Results for code points outside range [0, 255] are undefined. + // seg A: 0 <= codePoints < 128, already ASCII + // seg B: 49792 <= codePoints < 49856, ASCII = codePoints - 49664 + // seg C: 50048 <= codePoints < 50112, ASCII = codePoints - 49856 + // + // To reduce cuDF API calling, following algorithm will be performed: + // 1. For anything above 49792, we subtract 49664, now seg A and B are correct. + // 2. seg C: 50048 <= current + 49664 < 50112 => 384 <= current < 448, ASCII = current - 192 + // So for anything above 384, we subtract 192, now seg C is correct too. + val greaterThan49792 = withResource(Scalar.fromInt(49792)) { segBLeftEnd => + codePoints.greaterOrEqualTo(segBLeftEnd) + } + val segAB = withResource(greaterThan49792) { _ => + val sub1 = withResource(Scalar.fromInt(49664)) { segBValue => + codePoints.sub(segBValue) + } + withResource(sub1) { _ => + greaterThan49792.ifElse(sub1, codePoints) + } + } + withResource(segAB) { _ => + val geraterThan384 = withResource(Scalar.fromInt(384)) { segCLeftEnd => + segAB.greaterOrEqualTo(segCLeftEnd) + } + withResource(geraterThan384) { _ => + val sub2 = withResource(Scalar.fromInt(192)) { segCValue => + segAB.sub(segCValue) + } + withResource(sub2) { _ => + geraterThan384.ifElse(sub2, segAB) + } + } + } + } + + override def doColumnar(input: GpuColumnVector): ColumnVector = { + // replace empty strings with 'NUL' (which will convert to ascii 0) + val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => + input.getBase.equalTo(emptyScalar) + } + val emptyReplaced = withResource(emptyMask) { _ => + // replace empty strings with 'NUL' (which will convert to ascii 0) + withResource(Scalar.fromString('\u0000'.toString)) { zeroScalar => + emptyMask.ifElse(zeroScalar, input.getBase) + } + } + // replace nulls with 'n' and save the null mask + val nullMask = closeOnExcept(emptyReplaced) { _ => + input.getBase.isNull + } + withResource(nullMask) { _ => + val nullsReplaced = withResource(emptyReplaced) { _ => + withResource(Scalar.fromString("n")) { nullScalar => + nullMask.ifElse(nullScalar, emptyReplaced) + } + } + val substr = withResource(nullsReplaced) { _ => + nullsReplaced.substring(0, 1) + } + val codePoints = withResource(substr) { _ => + substr.codePoints() + } + val segABC = withResource(codePoints) { _ => + utf8CodePointsToAscii(codePoints) + } + // replace nulls with null + withResource(segABC) { _ => + withResource(Scalar.fromNull(DType.INT32)) { nullScalar => + nullMask.ifElse(nullScalar, segABC) + } + } + } + } +} diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 50b75e3ba5e..c81c913de78 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -60,8 +60,8 @@ ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark vers ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters. Otherwise the results will not match the CPU.,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Asin,S,`asin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From fa515aef13dceb63588a7d4c291650ec61b48f48 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Wed, 27 Dec 2023 23:28:32 +0800 Subject: [PATCH 5/9] empty string to 0 in lower version Signed-off-by: Haoyang Li --- .../apache/spark/sql/rapids/shims/GpuAscii.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 52b75e178f4..2c5224bfb55 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -44,8 +44,20 @@ case class GpuAscii(child: Expression) extends GpuUnaryExpression with ImplicitC override def inputTypes: Seq[AbstractDataType] = Seq(StringType) override def doColumnar(input: GpuColumnVector): ColumnVector = { + val emptyMask = withResource(Scalar.fromString("")) { emptyScalar => + input.getBase.equalTo(emptyScalar) + } + val emptyReplaced = withResource(emptyMask) { _ => + // replace empty strings with 'NUL' (which will convert to ascii 0) + withResource(Scalar.fromString('\u0000'.toString)) { zeroScalar => + emptyMask.ifElse(zeroScalar, input.getBase) + } + } // convert to byte lists - val firstBytes = withResource(input.getBase.asByteList) { bytes => + val byteLists = withResource(emptyReplaced) { _ => + emptyReplaced.asByteList() + } + val firstBytes = withResource(byteLists) { bytes => bytes.extractListElement(0) } val firstBytesInt = withResource(firstBytes) { _ => From f00ed5fd776f7c62d10d5387ac68139e44d59ba1 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 28 Dec 2023 11:02:08 +0800 Subject: [PATCH 6/9] db test Signed-off-by: Haoyang Li --- .../scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala | 2 -- .../scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 2c5224bfb55..94ed975c100 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -21,11 +21,9 @@ {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} -{"spark": "321db"} {"spark": "322"} {"spark": "330"} {"spark": "330cdh"} -{"spark": "330db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 662ad116bf3..e3e1102b1be 100644 --- a/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -15,8 +15,10 @@ */ /*** spark-rapids-shim-json-lines +{"spark": "321db"} {"spark": "323"} {"spark": "324"} +{"spark": "330db"} {"spark": "331"} {"spark": "332"} {"spark": "332cdh"} From 56df45c01a7232910870a74025ed27b8308a77a5 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 28 Dec 2023 11:28:50 +0800 Subject: [PATCH 7/9] db test Signed-off-by: Haoyang Li --- .../scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sql-plugin/src/main/{spark323 => spark321db}/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala (100%) diff --git a/sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala similarity index 100% rename from sql-plugin/src/main/spark323/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala rename to sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala From e443c1d863a39fa26383efdc098297dac1eb9c37 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 28 Dec 2023 11:41:08 +0800 Subject: [PATCH 8/9] shim 342 Signed-off-by: Haoyang Li --- .../scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index e3e1102b1be..72dc55024e4 100644 --- a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -27,6 +27,7 @@ {"spark": "340"} {"spark": "341"} {"spark": "341db"} +{"spark": "342"} {"spark": "350"} spark-rapids-shim-json-lines ***/ From 593a1fe27d92bd35338085a2bd1c959fc709d6ab Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Fri, 29 Dec 2023 08:13:15 +0800 Subject: [PATCH 9/9] Add 351 shim --- .../scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 72dc55024e4..225b8bb0e73 100644 --- a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -29,6 +29,7 @@ {"spark": "341db"} {"spark": "342"} {"spark": "350"} +{"spark": "351"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims