diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java index 7cc083cbb6..f07a27c190 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java @@ -401,7 +401,7 @@ public void testCompare() throws IOException { var result = executeQuery( String.format( - "source=%s | eval `%s` = %s | fields `%s`", + "source=%s | head 1 | eval `%s` = %s | fields `%s`", TEST_INDEX_DATATYPE_NONNUMERIC, name, functionCall, name)); verifySchema(result, schema(name, null, "boolean")); verifyDataRows(result, rows(expectedResult)); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index c1356ce838..bb4e1dd3d1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -29,7 +29,7 @@ public void typeof_sql_types() throws IOException { JSONObject response = executeQuery( String.format( - "source=%s | eval `str` = typeof('pewpew')," + "source=%s | head 1 | eval `str` = typeof('pewpew')," + " `double` = typeof(1.0)," + "`int` = typeof(12345)," + " `long` = typeof(1234567891011)," @@ -42,7 +42,7 @@ public void typeof_sql_types() throws IOException { response = executeQuery( String.format( - "source=%s | eval " + "source=%s | head 1 | eval " + "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP))," + "`time` = typeof(CAST('09:07:00' AS TIME))," + "`date` = typeof(CAST('1961-04-12' AS DATE))" @@ -56,7 +56,7 @@ public void typeof_opensearch_types() throws IOException { JSONObject response = executeQuery( String.format( - "source=%s | eval `double` = typeof(double_number), `long` =" + "source=%s | head 1 | eval `double` = typeof(double_number), `long` =" + " typeof(long_number),`integer` = typeof(integer_number), `byte` =" + " typeof(byte_number),`short` = typeof(short_number), `float` =" + " typeof(float_number),`half_float` = typeof(half_float_number)," @@ -69,11 +69,11 @@ public void typeof_opensearch_types() throws IOException { response = executeQuery( String.format( - "source=%s | eval `text` = typeof(text_value), `date` = typeof(date_value)," - + " `date_nanos` = typeof(date_nanos_value),`boolean` = typeof(boolean_value)," - + " `object` = typeof(object_value),`keyword` = typeof(keyword_value), `ip` =" - + " typeof(ip_value),`binary` = typeof(binary_value), `geo_point` =" - + " typeof(geo_point_value)" + "source=%s | head 1 | eval `text` = typeof(text_value), `date` =" + + " typeof(date_value), `date_nanos` = typeof(date_nanos_value),`boolean` =" + + " typeof(boolean_value), `object` = typeof(object_value),`keyword` =" + + " typeof(keyword_value), `ip` = typeof(ip_value),`binary` =" + + " typeof(binary_value), `geo_point` = typeof(geo_point_value)" // TODO activate this test once `ARRAY` type supported, see // ExpressionAnalyzer::isTypeNotSupported // + ", `nested` = typeof(nested_value)" diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DataTypeIT.java new file mode 100644 index 0000000000..6c7c9ab237 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DataTypeIT.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class DataTypeIT extends SQLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(DATA_TYPE_NUMERIC); + loadIndex(DATA_TYPE_NONNUMERIC); + } + + @Test + public void testReadingData() throws IOException { + String query = + String.format( + "SELECT" + + " long_number,integer_number,short_number,byte_number,double_number,float_number,half_float_number,scaled_float_number" + + " FROM %s LIMIT 5", + Index.DATA_TYPE_NUMERIC.getName()); + JSONObject result = executeJdbcRequest(query); + verifySchema( + result, + schema("long_number", "long"), + schema("integer_number", "integer"), + schema("short_number", "short"), + schema("byte_number", "byte"), + schema("float_number", "float"), + schema("double_number", "double"), + schema("half_float_number", "float"), + schema("scaled_float_number", "double")); + verifyDataRows( + result, + rows(1, 2, 3, 4, 5.1, 6.2, 7.3, 8.4), + rows(1, 2, 3, 4, 5.1, 6.2, 7.3, 8.4), + rows(1, 2, 3, 4, 5.1, 6.2, 7.3, 8.4)); + + query = + String.format( + "SELECT boolean_value,keyword_value FROM %s LIMIT 5", + Index.DATA_TYPE_NONNUMERIC.getName()); + result = executeJdbcRequest(query); + verifySchema(result, schema("boolean_value", "boolean"), schema("keyword_value", "keyword")); + verifyDataRows(result, rows(true, "123"), rows(true, "123"), rows(true, "123")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 13c2eecd56..2298b9d584 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -223,7 +223,8 @@ public void testDateNanosGroupBy() { @SneakyThrows public void testDateNanosWithNanos() { String query = - String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); + String.format( + "SELECT date_nanos_value" + " FROM %s limit 1", TEST_INDEX_DATATYPE_NONNUMERIC); JSONObject result = executeQuery(query); verifySchema(result, schema("date_nanos_value", null, "timestamp")); verifyDataRows(result, rows("2019-03-24 01:34:46.123456789")); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 7129d058c0..e634cd347d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -46,7 +46,7 @@ public void typeof_opensearch_types() { String.format( "SELECT typeof(double_number),typeof(long_number), typeof(integer_number)," + " typeof(byte_number), typeof(short_number),typeof(float_number)," - + " typeof(half_float_number), typeof(scaled_float_number) from %s;", + + " typeof(half_float_number), typeof(scaled_float_number) from %s limit 1;", TEST_INDEX_DATATYPE_NUMERIC)); verifyDataRows( response, rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); @@ -61,7 +61,7 @@ public void typeof_opensearch_types() { // TODO activate this test once `ARRAY` type supported, see // ExpressionAnalyzer::isTypeNotSupported // + ", typeof(nested_value)" - + " from %s;", + + " from %s limit 1;", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json index 70ddd28763..c052190c03 100644 --- a/integ-test/src/test/resources/datatypes.json +++ b/integ-test/src/test/resources/datatypes.json @@ -1,2 +1,6 @@ {"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"boolean_value": true, "keyword_value": "123", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"index":{"_id":"2"}} +{"boolean_value": "true", "keyword_value": 123, "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"index":{"_id":"3"}} +{"boolean_value": ["true"], "keyword_value": [123], "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} diff --git a/integ-test/src/test/resources/datatypes_numeric.json b/integ-test/src/test/resources/datatypes_numeric.json index 5aa3b00511..a07874584d 100644 --- a/integ-test/src/test/resources/datatypes_numeric.json +++ b/integ-test/src/test/resources/datatypes_numeric.json @@ -1,2 +1,6 @@ {"index":{"_id":"1"}} {"long_number": 1, "integer_number": 2, "short_number": 3, "byte_number": 4, "double_number": 5.1, "float_number": 6.2, "half_float_number": 7.3, "scaled_float_number": 8.4} +{"index":{"_id":"2"}} +{"long_number": "1", "integer_number": "2", "short_number": "3", "byte_number": "4", "double_number": "5.1", "float_number": "6.2", "half_float_number": "7.3", "scaled_float_number": "8.4"} +{"index":{"_id":"3"}} +{"long_number": ["1"], "integer_number": ["2"], "short_number": ["3"], "byte_number": ["4"], "double_number": ["5.1"], "float_number": ["6.2"], "half_float_number": ["7.3"], "scaled_float_number": ["8.4"]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java index 4446c1f979..80430827fe 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java @@ -14,6 +14,7 @@ import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.OpenSearchParseException; +import org.opensearch.common.Numbers; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.common.xcontent.json.JsonXContentParser; @@ -29,32 +30,32 @@ public class OpenSearchJsonContent implements Content { @Override public Integer intValue() { - return value().intValue(); + return (int) extractDoubleValue(value()); } @Override public Long longValue() { - return value().longValue(); + return extractLongValue(value()); } @Override public Short shortValue() { - return value().shortValue(); + return (short) extractDoubleValue(value()); } @Override public Byte byteValue() { - return (byte) value().shortValue(); + return (byte) extractDoubleValue(value()); } @Override public Float floatValue() { - return value().floatValue(); + return (float) extractDoubleValue(value()); } @Override public Double doubleValue() { - return value().doubleValue(); + return extractDoubleValue(value()); } @Override @@ -64,7 +65,7 @@ public String stringValue() { @Override public Boolean booleanValue() { - return value().booleanValue(); + return extractBooleanValue(value()); } @Override @@ -148,4 +149,40 @@ public Pair geoValue() { private JsonNode value() { return value; } + + /** Get double value from JsonNode if possible. */ + private double extractDoubleValue(JsonNode node) { + if (node.isTextual()) { + return Double.parseDouble(node.textValue()); + } + if (node.isNumber()) { + return node.doubleValue(); + } else { + throw new OpenSearchParseException("node must be a number"); + } + } + + /** Get long value from JsonNode if possible. */ + private long extractLongValue(JsonNode node) { + if (node.isTextual()) { + return Numbers.toLong(node.textValue(), true); + } + if (node.isNumber()) { + return node.longValue(); + } else { + throw new OpenSearchParseException("node must be a number"); + } + } + + /** Get boolean value from JsonNode if possible. */ + private boolean extractBooleanValue(JsonNode node) { + if (node.isTextual()) { + return Boolean.parseBoolean(node.textValue()); + } + if (node.isBoolean()) { + return node.booleanValue(); + } else { + throw new OpenSearchParseException("node must be a boolean"); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 417aaddaee..c00ff3d781 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -60,10 +60,8 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; -import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -346,9 +344,8 @@ private ExprValue parseArray( if (content.objectValue() instanceof ObjectNode) { result.add(parseStruct(content, prefix, supportArrays)); // non-object type arrays are only supported when parsing inner_hits of OS response. - } else if (!(type instanceof OpenSearchDataType - && ((OpenSearchDataType) type).getExprType().equals(ARRAY)) - && !supportArrays) { + } else if (((OpenSearchDataType) type).getExprType().equals(ARRAY) == false + && supportArrays == false) { return parseInnerArrayValue(content.array().next(), prefix, type, supportArrays); } else { content @@ -415,10 +412,10 @@ private ExprValue parseGeoPoint(Content content, boolean supportArrays) { */ private ExprValue parseInnerArrayValue( Content content, String prefix, ExprType type, boolean supportArrays) { - if (type instanceof OpenSearchIpType - || type instanceof OpenSearchBinaryType - || type instanceof OpenSearchDateType) { - return parse(content, prefix, Optional.of(type), supportArrays); + if (content.isArray()) { + return parseArray(content, prefix, type, supportArrays); + } else if (typeActionMap.containsKey(type)) { + return typeActionMap.get(type).apply(content, type); } else if (content.isString()) { return parse(content, prefix, Optional.of(OpenSearchDataType.of(STRING)), supportArrays); } else if (content.isLong()) { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 6b4d825ab1..22ef531a9c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -156,6 +156,7 @@ public void constructNullArrayValue() { public void constructByte() { assertAll( () -> assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")), + () -> assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":\"1\"}").get("byteV")), () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", 1)), () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", "1.0"))); } @@ -164,6 +165,7 @@ public void constructByte() { public void constructShort() { assertAll( () -> assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")), + () -> assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":\"1\"}").get("shortV")), () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", 1)), () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", "1.0"))); } @@ -172,19 +174,16 @@ public void constructShort() { public void constructInteger() { assertAll( () -> assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")), + () -> assertEquals(integerValue(1), tupleValue("{\"intV\":\"1\"}").get("intV")), () -> assertEquals(integerValue(1), constructFromObject("intV", 1)), () -> assertEquals(integerValue(1), constructFromObject("intV", "1.0"))); } - @Test - public void constructIntegerValueInStringValue() { - assertEquals(integerValue(1), constructFromObject("intV", "1")); - } - @Test public void constructLong() { assertAll( () -> assertEquals(longValue(1L), tupleValue("{\"longV\":1}").get("longV")), + () -> assertEquals(longValue(1L), tupleValue("{\"longV\":\"1\"}").get("longV")), () -> assertEquals(longValue(1L), constructFromObject("longV", 1L)), () -> assertEquals(longValue(1L), constructFromObject("longV", "1.0"))); } @@ -193,6 +192,7 @@ public void constructLong() { public void constructFloat() { assertAll( () -> assertEquals(floatValue(1f), tupleValue("{\"floatV\":1.0}").get("floatV")), + () -> assertEquals(floatValue(1f), tupleValue("{\"floatV\":\"1.0\"}").get("floatV")), () -> assertEquals(floatValue(1f), constructFromObject("floatV", 1f))); } @@ -200,6 +200,7 @@ public void constructFloat() { public void constructDouble() { assertAll( () -> assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":1.0}").get("doubleV")), + () -> assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":\"1.0\"}").get("doubleV")), () -> assertEquals(doubleValue(1d), constructFromObject("doubleV", 1d))); } @@ -215,6 +216,7 @@ public void constructString() { public void constructBoolean() { assertAll( () -> assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")), + () -> assertEquals(booleanValue(true), tupleValue("{\"boolV\":\"true\"}").get("boolV")), () -> assertEquals(booleanValue(true), constructFromObject("boolV", true)), () -> assertEquals(booleanValue(true), constructFromObject("boolV", "true")), () -> assertEquals(booleanValue(true), constructFromObject("boolV", 1)), @@ -755,6 +757,27 @@ public void constructGeoPointFromUnsupportedFormatShouldThrowException() { assertEquals("lat must be a number", exception.getMessage()); } + @Test + public void constructNumberFromUnsupportedFormatShouldThrowException() { + OpenSearchParseException exception = + assertThrows( + OpenSearchParseException.class, () -> tupleValue("{\"intV\": false}").get("intV")); + assertEquals("node must be a number", exception.getMessage()); + + exception = + assertThrows( + OpenSearchParseException.class, () -> tupleValue("{\"longV\": false}").get("intV")); + assertEquals("node must be a number", exception.getMessage()); + } + + @Test + public void constructBooleanFromUnsupportedFormatShouldThrowException() { + OpenSearchParseException exception = + assertThrows( + OpenSearchParseException.class, () -> tupleValue("{\"boolV\": 1}").get("boolV")); + assertEquals("node must be a boolean", exception.getMessage()); + } + @Test public void constructBinary() { assertEquals( @@ -769,6 +792,7 @@ public void constructBinary() { @Test public void constructFromOpenSearchArrayReturnFirstElement() { assertEquals(integerValue(1), tupleValue("{\"intV\":[1, 2, 3]}").get("intV")); + assertEquals(integerValue(1), tupleValue("{\"intV\":[\"1\", 2, 3]}").get("intV")); assertEquals( new ExprTupleValue( new LinkedHashMap() {