Skip to content

Commit

Permalink
[AD-980] Handle in-driver date add calculation for literals. (#428)
Browse files Browse the repository at this point in the history 
* [AD-980] Handle in-driver date add calculation for literals.

* Commit Code Coverage Badge

* [AD-980] Add support for CURRENT_TIMESTAMP and remove support for handling unused data types in the literal conversion.

* [AD-980] Fix file size.

* Commit Code Coverage Badge

* [AD-980] Test TIMESTAMPADD on left and DATE literal.

* Commit Code Coverage Badge

Co-authored-by: birschick-bq <[email protected]>
  • Loading branch information
@birschick-bq
Bruce Irschick and birschick-bq authored Oct 28, 2022
1 parent 724fdda commit 3b3c2ad
Show file tree
Hide file tree
Showing 5 changed files with 318 additions and 7 deletions.
2 changes: 1 addition & 1 deletion .github/badges/branches.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
59 changes: 54 additions & 5 deletions src/main/java/software/amazon/documentdb/jdbc/calcite/adapter/DocumentDbRules.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@
import org.apache.calcite.util.TimeString;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.trace.CalciteTrace;
import org.bson.BsonDocument;
import org.bson.BsonType;
import org.bson.BsonValue;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.slf4j.Logger;
import software.amazon.documentdb.jdbc.common.utilities.SqlError;
Expand Down Expand Up @@ -204,7 +206,7 @@ private static boolean needsQuote(final String s) {
* @param fieldName The non-normalized string
* @return The input string with '$' replaced by '_'
*/
protected static String getNormalizedIdentifier(final String fieldName) {
static String getNormalizedIdentifier(final String fieldName) {
return fieldName.startsWith("$") ? "_" + fieldName.substring(1) : fieldName;
}

Expand Down Expand Up @@ -757,6 +759,8 @@ private static Operand reformatObjectIdLiteral(

private static class DateFunctionTranslator {

private static final String CURRENT_DATE = "CURRENT_DATE";
private static final String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
private static final Map<TimeUnitRange, String> DATE_PART_OPERATORS =
new HashMap<>();
private static final Instant FIRST_DAY_OF_WEEK_AFTER_EPOCH =
Expand Down Expand Up @@ -786,9 +790,54 @@ private static Operand translateCurrentTimestamp(final Instant currentTime) {
@SneakyThrows
private static Operand translateDateAdd(final RexCall call, final List<Operand> strings) {
verifySupportedDateAddType(call.getOperands().get(1));

// Is date addition between literals (including CURRENT_DATE)?
final boolean isLiteralCandidate = isDateLiteralCandidate(call, strings);
if (isLiteralCandidate) {
// Perform in-memory calculation before sending to server.
return getDateAddLiteralOperand(strings);
}
// Otherwise, perform addition on server.
return new Operand("{ \"$add\":" + "[" + Util.commaList(strings) + "]}");
}

private static boolean isDateLiteralCandidate(final RexCall call, final List<Operand> strings) {
final boolean allLiterals = call.getOperands().stream()
.allMatch(op -> {
final SqlKind opKind = op.getKind();
final String opName = op.toString();
return opKind == SqlKind.LITERAL
|| opName.equalsIgnoreCase(CURRENT_DATE)
|| opName.equalsIgnoreCase(CURRENT_TIMESTAMP);
});
final boolean allHaveQueryValue = strings.stream().allMatch(op -> op.getQueryValue() != null);
return allLiterals && allHaveQueryValue;
}

private static Operand getDateAddLiteralOperand(final List<Operand> strings) {
final String queryValue0 = strings.get(0).getQueryValue();
final String queryValue1 = strings.get(1).getQueryValue();
final BsonDocument document0 = BsonDocument.parse("{field: " + queryValue0 + "}");
final BsonDocument document1 = BsonDocument.parse("{field: " + queryValue1 + "}");

long sum = 0L;
for (BsonValue v : new BsonValue[]{document0.get("field"), document1.get("field")}) {
switch (v.getBsonType()) {
case DATE_TIME:
sum += v.asDateTime().getValue();
break;
case INT64:
sum += v.asInt64().getValue();
break;
default:
throw new UnsupportedOperationException(
"Unsupported data type '" + v.getBsonType().name() + "'");
}
}
final String query = "{\"$date\": {\"$numberLong\": \"" + sum + "\"}}";
return new Operand(query, query, true);
}

private static void verifySupportedDateAddType(final RexNode node)
throws SQLFeatureNotSupportedException {
if (node.getType().getSqlTypeName() == SqlTypeName.INTERVAL_MONTH
Expand Down Expand Up @@ -1169,7 +1218,7 @@ private static class StringFunctionTranslator {
new HashMap<>();

static {
STRING_OPERATORS.put(SqlStdOperatorTable.CONCAT, "$concat");;
STRING_OPERATORS.put(SqlStdOperatorTable.CONCAT, "$concat");
STRING_OPERATORS.put(SqlStdOperatorTable.LOWER, "$toLower");
STRING_OPERATORS.put(SqlStdOperatorTable.UPPER, "$toUpper");
STRING_OPERATORS.put(SqlStdOperatorTable.CHAR_LENGTH, "$strLenCP");
Expand Down Expand Up @@ -1215,15 +1264,15 @@ private static Operand getMongoAggregateForPositionStringOperator(
args.add("{" + STRING_OPERATORS.get(SqlStdOperatorTable.LOWER) + ":" + strings.get(1) + "}");
args.add("{" + STRING_OPERATORS.get(SqlStdOperatorTable.LOWER) + ":" + strings.get(0) + "}");
// Check if either string is null.
operand.append("{\"$cond\": [" + RexToMongoTranslator.getNullCheckExpr(strings) + ", ");
operand.append("{\"$cond\": [").append(RexToMongoTranslator.getNullCheckExpr(strings)).append(", ");
// Add starting index if any.
if (strings.size() == 3) {
args.add("{\"$subtract\": [" + strings.get(2) + ", 1]}"); // Convert to 0-based.
operand.append("{\"$cond\": [{\"$lte\": [" + strings.get(2) + ", 0]}, 0, "); // Check if 1-based index > 0.
operand.append("{\"$cond\": [{\"$lte\": [").append(strings.get(2)).append(", 0]}, 0, "); // Check if 1-based index > 0.
finish.append("]}");
}
// Convert 0-based index to 1-based.
operand.append("{\"$add\": [{" + STRING_OPERATORS.get(SqlStdOperatorTable.POSITION) + ": [" + Util.commaList(args) + "]}, 1]}");
operand.append("{\"$add\": [{").append(STRING_OPERATORS.get(SqlStdOperatorTable.POSITION)).append(": [").append(Util.commaList(args)).append("]}, 1]}");
operand.append(finish);
operand.append(", null ]}");
// Return 1-based index when string is found.
Expand Down
77 changes: 76 additions & 1 deletion src/markdown/support/troubleshooting-guide.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,4 +262,79 @@ For example:
- In Tableau, a parameter must be used in the command line or terminal:
- In Windows: `start "" "c:\program files\Tableau\Tableau [version]\bin\tableau.exe" -DLogLevel=DEBUG`
- In MacOS: `/Applications/Tableau\ Desktop\[version].app/Contents/MacOS/Tableau -DLogLevel=DEBUG`
- Tableau logs are located at: `{user.home}/Documents/My Tableau Repository/Logs`
- Tableau logs are located at: `{user.home}/Documents/My Tableau Repository/Logs`

## Permanently Setting Environment Variables

### Windows

- From the start menu (or press the `Windows` key), type '***Edit environment variables for your account***' and launch
the settings application.
- If an environment variable is already listed, click the '***Edit...***' button. Otherwise, click the
"***New...***" button.
- Enter the name of the variable (e.g., `JAVA_TOOL_OPTIONS`) in the '***Variable name***' field and then enter the value
in the '***Variable value***' field (e.g., `-Ddocumentdb.jdbc.log.level=DEBUG`. Click the '***Ok***' button to save the value.
- Restart the application or command window for the change to take effect.

### MacOS

- Create the file `~/Library/LaunchAgents/environment.plist` if it doesn't exist
- Edit the file `~/Library/LaunchAgents/environment.plist`
- If the contents do not exist, enter the following template:
```xml
?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>my.startup</string>
<key>ProgramArguments</key>
<array>
<string>sh</string>
<string>-c</string>
<string>
<!-- Add more 'launchctl setenv ...' commands here -->
</string>
</array>
<key>RunAtLoad</key>
<true/>
</dict>
</plist>
```

- Add environment variables in the path `/dict/array/string` after the `<string>-c</string>` node.
- In the example below, we add two environment variables `JAVA_TOOL_OPTIONS` and `DOCUMENTDB_CUSTOM_OPTIONS`

```xml
?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>my.startup</string>
<key>ProgramArguments</key>
<array>
<string>sh</string>
<string>-c</string>
<string>
<!-- Add more 'launchctl setenv ...' commands here -->
launchctl setenv JAVA_TOOL_OPTIONS -Ddocumentdb.jdbc.log.level=DEBUG
launchctl setenv DOCUMENTDB_CUSTOM_OPTIONS allowDiskUse=enable
</string>
</array>
<key>RunAtLoad</key>
<true/>
</dict>
</plist>
```

- You can reboot your machine or in a terminal windows type the following commands to load your changes.

```shell
launchctl stop ~/Library/LaunchAgents/environment.plist
launchctl unload ~/Library/LaunchAgents/environment.plist
launchctl load ~/Library/LaunchAgents/environment.plist
launchctl start ~/Library/LaunchAgents/environment.plist
```

- Restart your application for the changes to take effect.
69 changes: 69 additions & 0 deletions src/test/java/software/amazon/documentdb/jdbc/DocumentDbStatementFilterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.bson.BsonDouble;
import org.bson.BsonInt64;
import org.bson.BsonMinKey;
import org.bson.BsonNull;
import org.bson.BsonObjectId;
import org.bson.BsonString;
import org.bson.types.ObjectId;
Expand All @@ -37,6 +38,7 @@
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.temporal.ChronoUnit;

public class DocumentDbStatementFilterTest extends DocumentDbStatementTest {

Expand Down Expand Up @@ -602,6 +604,73 @@ void testQueryWhereTimestampAdd(final DocumentDbTestEnvironment testEnvironment)
}
}

/**
* Tests where condition of field compared to CURRENT_DATE.
*
* @throws SQLException occurs if query fails.
*/
@DisplayName("Tests where condition of field compared to CURRENT_DATE.")
@ParameterizedTest(name = "testWhereFieldComparedToCurrentDate - [{index}] - {arguments}")
@MethodSource({"getTestEnvironments"})
void testWhereFieldComparedToCurrentDate(final DocumentDbTestEnvironment testEnvironment) throws SQLException {
setTestEnvironment(testEnvironment);
final String tableName = "testWhereFieldComparedToCurrentDate";
final long dateTimePast = Instant.now().minus(2, ChronoUnit.DAYS).toEpochMilli();
final long dateTimeFuture = Instant.now().plus(2, ChronoUnit.DAYS).toEpochMilli();
final BsonDocument doc1 = BsonDocument.parse("{\"_id\": 101}");
doc1.append("field", new BsonDateTime(dateTimePast));
final BsonDocument doc2 = BsonDocument.parse("{\"_id\": 102}");
doc2.append("field", new BsonDateTime(dateTimeFuture));
final BsonDocument doc3 = BsonDocument.parse("{\"_id\": 103}");
doc3.append("field", new BsonNull());
insertBsonDocuments(tableName, new BsonDocument[]{doc1, doc2, doc3});

try (Connection connection = getConnection()) {
final Statement statement = getDocumentDbStatement(connection);

for (final String currentFunc : new String[]{"CURRENT_DATE", "CURRENT_TIMESTAMP"}) {
// Find condition that does exist.
final ResultSet resultSet1 = statement.executeQuery(
String.format(
"SELECT \"field\"%n" +
" FROM \"%s\".\"%s\"%n" +
" WHERE \"field\" < TIMESTAMPADD(DAY, 1, %s)",
getDatabaseName(), tableName, currentFunc));
Assertions.assertNotNull(resultSet1);
Assertions.assertTrue(resultSet1.next());
Assertions.assertFalse(resultSet1.next());
// Find condition that does exist.
final ResultSet resultSet2 = statement.executeQuery(
String.format(
"SELECT \"field\"%n" +
" FROM \"%s\".\"%s\"%n" +
" WHERE \"field\" > TIMESTAMPADD(DAY, 1, %s)",
getDatabaseName(), tableName, currentFunc));
Assertions.assertNotNull(resultSet2);
Assertions.assertTrue(resultSet2.next());
Assertions.assertFalse(resultSet2.next());
// Find condition that does NOT exist.
final ResultSet resultSet3 = statement.executeQuery(
String.format(
"SELECT \"field\"%n" +
" FROM \"%s\".\"%s\"%n" +
" WHERE \"field\" > TIMESTAMPADD(DAY, 10, %s)",
getDatabaseName(), tableName, currentFunc));
Assertions.assertNotNull(resultSet3);
Assertions.assertFalse(resultSet3.next());
// Find condition that does NOT exist.
final ResultSet resultSet4 = statement.executeQuery(
String.format(
"SELECT \"field\"%n" +
" FROM \"%s\".\"%s\"%n" +
" WHERE \"field\" < TIMESTAMPADD(DAY, -10, %s)",
getDatabaseName(), tableName, currentFunc));
Assertions.assertNotNull(resultSet4);
Assertions.assertFalse(resultSet4.next());
}
}
}

/**
* Tests for queries filtering by IS NULL.
*
Expand Down
Loading

0 comments on commit 3b3c2ad

Please sign in to comment.