From 091fbb636b4a54c34bafc3b6fcc93dd4d3f7b65b Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Fri, 19 Jul 2024 12:32:45 +0200 Subject: [PATCH] Corrections related to LookupCommandIT --- .../src/main/antlr/SqlBaseLexer.g4 | 1 + .../src/main/antlr/SqlBaseParser.g4 | 52 ++++-- .../sql/planner/physical/LookupOperator.java | 3 +- .../opensearch/sql/ppl/LookupCommandIT.java | 148 ++++++++++++------ 4 files changed, 146 insertions(+), 58 deletions(-) diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index 85a4633e80..bde298c23e 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -316,6 +316,7 @@ NANOSECOND: 'NANOSECOND'; NANOSECONDS: 'NANOSECONDS'; NATURAL: 'NATURAL'; NO: 'NO'; +NONE: 'NONE'; NOT: 'NOT'; NULL: 'NULL'; NULLS: 'NULLS'; diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index 54eff14b6d..a50051715e 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -52,7 +52,7 @@ singleCompoundStatement ; beginEndCompoundBlock - : BEGIN compoundBody END + : beginLabel? BEGIN compoundBody END endLabel? ; compoundBody @@ -61,11 +61,26 @@ compoundBody compoundStatement : statement + | setStatementWithOptionalVarKeyword | beginEndCompoundBlock ; +setStatementWithOptionalVarKeyword + : SET (VARIABLE | VAR)? assignmentList #setVariableWithOptionalKeyword + | SET (VARIABLE | VAR)? LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ + LEFT_PAREN query RIGHT_PAREN #setVariableWithOptionalKeyword + ; + singleStatement - : statement SEMICOLON* EOF + : (statement|setResetStatement) SEMICOLON* EOF + ; + +beginLabel + : multipartIdentifier COLON + ; + +endLabel + : multipartIdentifier ; singleExpression @@ -174,6 +189,8 @@ statement | ALTER TABLE identifierReference (partitionSpec)? SET locationSpec #setTableLocation | ALTER TABLE identifierReference RECOVER PARTITIONS #recoverPartitions + | ALTER TABLE identifierReference + (clusterBySpec | CLUSTER BY NONE) #alterClusterBy | DROP TABLE (IF EXISTS)? identifierReference PURGE? #dropTable | DROP VIEW (IF EXISTS)? identifierReference #dropView | CREATE (OR REPLACE)? (GLOBAL? TEMPORARY)? @@ -202,7 +219,7 @@ statement identifierReference dataType? variableDefaultExpression? #createVariable | DROP TEMPORARY VARIABLE (IF EXISTS)? identifierReference #dropVariable | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? - statement #explain + (statement|setResetStatement) #explain | SHOW TABLES ((FROM | IN) identifierReference)? (LIKE? pattern=stringLit)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? @@ -241,26 +258,29 @@ statement | (MSCK)? REPAIR TABLE identifierReference (option=(ADD|DROP|SYNC) PARTITIONS)? #repairTable | op=(ADD | LIST) identifier .*? #manageResource - | SET COLLATION collationName=identifier #setCollation - | SET ROLE .*? #failNativeCommand + | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? + identifierReference (USING indexType=identifier)? + LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN + (OPTIONS options=propertyList)? #createIndex + | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex + | unsupportedHiveNativeCommands .*? #failNativeCommand + ; + +setResetStatement + : SET COLLATION collationName=identifier #setCollation + | SET ROLE .*? #failSetRole | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone | SET (VARIABLE | VAR) assignmentList #setVariable | SET (VARIABLE | VAR) LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ - LEFT_PAREN query RIGHT_PAREN #setVariable + LEFT_PAREN query RIGHT_PAREN #setVariable | SET configKey EQ configValue #setQuotedConfiguration | SET configKey (EQ .*?)? #setConfiguration | SET .*? EQ configValue #setQuotedConfiguration | SET .*? #setConfiguration | RESET configKey #resetQuotedConfiguration | RESET .*? #resetConfiguration - | CREATE INDEX (IF errorCapturingNot EXISTS)? identifier ON TABLE? - identifierReference (USING indexType=identifier)? - LEFT_PAREN columns=multipartIdentifierPropertyList RIGHT_PAREN - (OPTIONS options=propertyList)? #createIndex - | DROP INDEX (IF EXISTS)? identifier ON TABLE? identifierReference #dropIndex - | unsupportedHiveNativeCommands .*? #failNativeCommand ; executeImmediate @@ -853,13 +873,17 @@ identifierComment relationPrimary : identifierReference temporalClause? - sample? tableAlias #tableName + optionsClause? sample? tableAlias #tableName | LEFT_PAREN query RIGHT_PAREN sample? tableAlias #aliasedQuery | LEFT_PAREN relation RIGHT_PAREN sample? tableAlias #aliasedRelation | inlineTable #inlineTableDefault2 | functionTable #tableValuedFunction ; +optionsClause + : WITH options=propertyList + ; + inlineTable : VALUES expression (COMMA expression)* tableAlias ; @@ -1572,6 +1596,7 @@ ansiNonReserved | NANOSECOND | NANOSECONDS | NO + | NONE | NULLS | NUMERIC | OF @@ -1920,6 +1945,7 @@ nonReserved | NANOSECOND | NANOSECONDS | NO + | NONE | NOT | NULL | NULLS diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/LookupOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/LookupOperator.java index 7117d87f5d..c4b1ccd824 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/LookupOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/LookupOperator.java @@ -9,6 +9,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.BiFunction; @@ -108,7 +109,7 @@ public ExprValue next() { } Map tupleInputValue = ExprValueUtils.getTupleValue(inputValue); - Map resultTupleBuilder = new HashMap<>(); + Map resultTupleBuilder = new LinkedHashMap<>(); resultTupleBuilder.putAll(tupleInputValue); for (Map.Entry sourceOfAdditionalField : lookupResult.entrySet()) { String lookedUpFieldName = sourceOfAdditionalField.getKey(); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java index ade37b1241..d6a85f4687 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; +import java.math.BigDecimal; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -32,49 +33,49 @@ public void testLookup() throws IOException { verifyDataRows( result, rows( - 28.1, - "2015-01-20 15:31:32.406431", + new BigDecimal("28.1"), 255, + "2015-01-20 15:31:32.406431", "temperature-basement", "meter", 255, "VendorOne"), rows( - 27.8, - "2016-01-20 15:31:33.509334", + new BigDecimal("27.8"), 256, + "2016-01-20 15:31:33.509334", "temperature-living-room", "temperature meter", 256, "VendorTwo"), rows( - 27.4, - "2017-01-20 15:31:35.732436", + new BigDecimal("27.4"), 257, + "2017-01-20 15:31:35.732436", "temperature-bedroom", "camcorder", 257, "VendorThree"), rows( - 28.5, - "2018-01-20 15:32:32.406431", + new BigDecimal("28.5"), 255, + "2018-01-20 15:32:32.406431", "temperature-basement", "meter", 255, "VendorOne"), rows( - 27.9, - "2019-01-20 15:32:33.509334", + new BigDecimal("27.9"), 256, + "2019-01-20 15:32:33.509334", "temperature-living-room", "temperature meter", 256, "VendorTwo"), rows( - 27.4, - "2020-01-20 15:32:35.732436", + new BigDecimal("27.4"), 257, + "2020-01-20 15:32:35.732436", "temperature-bedroom", "camcorder", 257, @@ -90,12 +91,23 @@ public void testLookupSelectedAttribute() throws IOException { TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter", "VendorOne"), - rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter", "VendorTwo"), - rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder", "VendorThree"), - rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter", "VendorOne"), - rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter", "VendorTwo"), - rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder", "VendorThree")); + rows(new BigDecimal("28.1"), 255, "2015-01-20 15:31:32.406431", "meter", "VendorOne"), + rows( + new BigDecimal("27.8"), + 256, + "2016-01-20 15:31:33.509334", + "temperature meter", + "VendorTwo"), + rows(new BigDecimal("27.4"), 257, "2017-01-20 15:31:35.732436", "camcorder", "VendorThree"), + rows(new BigDecimal("28.5"), 255, "2018-01-20 15:32:32.406431", "meter", "VendorOne"), + rows( + new BigDecimal("27.9"), + 256, + "2019-01-20 15:32:33.509334", + "temperature meter", + "VendorTwo"), + rows( + new BigDecimal("27.4"), 257, "2020-01-20 15:32:35.732436", "camcorder", "VendorThree")); } @Test @@ -108,12 +120,36 @@ public void testLookupRenameSelectedAttributes() throws IOException { TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows(28.1, "2015-01-20 15:31:32.406431", 255, 255, "meter", "VendorOne"), - rows(27.8, "2016-01-20 15:31:33.509334", 256, 256, "temperature meter", "VendorTwo"), - rows(27.4, "2017-01-20 15:31:35.732436", 257, 257, "camcorder", "VendorThree"), - rows(28.5, "2018-01-20 15:32:32.406431", 255, 255, "meter", "VendorOne"), - rows(27.9, "2019-01-20 15:32:33.509334", 256, 256, "temperature meter", "VendorTwo"), - rows(27.4, "2020-01-20 15:32:35.732436", 257, 257, "camcorder", "VendorThree")); + rows(new BigDecimal("28.1"), 255, "2015-01-20 15:31:32.406431", 255, "meter", "VendorOne"), + rows( + new BigDecimal("27.8"), + 256, + "2016-01-20 15:31:33.509334", + 256, + "temperature meter", + "VendorTwo"), + rows( + new BigDecimal("27.4"), + 257, + "2017-01-20 15:31:35.732436", + 257, + "camcorder", + "VendorThree"), + rows(new BigDecimal("28.5"), 255, "2018-01-20 15:32:32.406431", 255, "meter", "VendorOne"), + rows( + new BigDecimal("27.9"), + 256, + "2019-01-20 15:32:33.509334", + 256, + "temperature meter", + "VendorTwo"), + rows( + new BigDecimal("27.4"), + 257, + "2020-01-20 15:32:35.732436", + 257, + "camcorder", + "VendorThree")); } @Test @@ -125,12 +161,12 @@ public void testLookupSelectedMultipleAttributes() throws IOException { TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter"), - rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter"), - rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder"), - rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter"), - rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter"), - rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder")); + rows(new BigDecimal("28.1"), 255, "2015-01-20 15:31:32.406431", "meter"), + rows(new BigDecimal("27.8"), 256, "2016-01-20 15:31:33.509334", "temperature meter"), + rows(new BigDecimal("27.4"), 257, "2017-01-20 15:31:35.732436", "camcorder"), + rows(new BigDecimal("28.5"), 255, "2018-01-20 15:32:32.406431", "meter"), + rows(new BigDecimal("27.9"), 256, "2019-01-20 15:32:33.509334", "temperature meter"), + rows(new BigDecimal("27.4"), 257, "2020-01-20 15:32:35.732436", "camcorder")); } @Test @@ -143,32 +179,32 @@ public void testLookupShouldAppendOnlyShouldBeFalseByDefault() throws IOExceptio TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows("2015-01-20 15:31:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255), + rows(255, "2015-01-20 15:31:32.406431", "VendorOne", "temperature-basement", "meter", 255), rows( - "2016-01-20 15:31:33.509334", 256, + "2016-01-20 15:31:33.509334", "VendorTwo", "temperature-living-room", "temperature meter", 256), rows( - "2017-01-20 15:31:35.732436", 257, + "2017-01-20 15:31:35.732436", "VendorThree", "temperature-bedroom", "camcorder", 257), - rows("2018-01-20 15:32:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255), + rows(255, "2018-01-20 15:32:32.406431", "VendorOne", "temperature-basement", "meter", 255), rows( - "2019-01-20 15:32:33.509334", 256, + "2019-01-20 15:32:33.509334", "VendorTwo", "temperature-living-room", "temperature meter", 256), rows( - "2020-01-20 15:32:35.732436", 257, + "2020-01-20 15:32:35.732436", "VendorThree", "temperature-bedroom", "camcorder", @@ -185,23 +221,47 @@ public void testLookupWithAppendOnlyFalse() throws IOException { TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows("2015-01-20 15:31:32.406431", 255, 28.1, "temperature-basement", "meter", 255), rows( - "2016-01-20 15:31:33.509334", + 255, + "2015-01-20 15:31:32.406431", + new BigDecimal("28.1"), + "temperature-basement", + "meter", + 255), + rows( 256, - 27.8, + "2016-01-20 15:31:33.509334", + new BigDecimal("27.8"), "temperature-living-room", "temperature meter", 256), - rows("2017-01-20 15:31:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257), - rows("2018-01-20 15:32:32.406431", 255, 28.5, "temperature-basement", "meter", 255), rows( - "2019-01-20 15:32:33.509334", + 257, + "2017-01-20 15:31:35.732436", + new BigDecimal("27.4"), + "temperature-bedroom", + "camcorder", + 257), + rows( + 255, + "2018-01-20 15:32:32.406431", + new BigDecimal("28.5"), + "temperature-basement", + "meter", + 255), + rows( 256, - 27.9, + "2019-01-20 15:32:33.509334", + new BigDecimal("27.9"), "temperature-living-room", "temperature meter", 256), - rows("2020-01-20 15:32:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257)); + rows( + 257, + "2020-01-20 15:32:35.732436", + new BigDecimal("27.4"), + "temperature-bedroom", + "camcorder", + 257)); } }