From bae42dc90a1dcd888c8247f3566b978b6cdd12fd Mon Sep 17 00:00:00 2001 From: Eugene R Date: Sun, 26 May 2024 12:54:02 +0300 Subject: [PATCH] FMWK-444 Allow record metadata columns inclusion (#69) --- .github/workflows/snyk-scan.yml | 6 +- docs/params.md | 1 + .../aerospike/jdbc/model/DriverPolicy.java | 6 + .../java/com/aerospike/jdbc/model/Pair.java | 4 + .../jdbc/schema/AerospikeSchemaBuilder.java | 51 +++++-- .../jdbc/sql/AerospikeRecordResultSet.java | 78 ++++++----- .../com/aerospike/jdbc/util/Constants.java | 4 + .../aerospike/jdbc/PreparedQueriesTest.java | 3 + .../aerospike/jdbc/RecordMetadataTest.java | 130 ++++++++++++++++++ .../com/aerospike/jdbc/SimpleQueriesTest.java | 3 + 10 files changed, 234 insertions(+), 52 deletions(-) create mode 100644 src/test/java/com/aerospike/jdbc/RecordMetadataTest.java diff --git a/.github/workflows/snyk-scan.yml b/.github/workflows/snyk-scan.yml index e3e88d3..10d501a 100644 --- a/.github/workflows/snyk-scan.yml +++ b/.github/workflows/snyk-scan.yml @@ -22,6 +22,10 @@ jobs: with: args: --all-projects --sarif-file-output=snyk.sarif + - name: Handle undefined security-severity + run: | + sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif + - name: Check output file id: out-file run: | @@ -32,6 +36,6 @@ jobs: - name: Upload result to GitHub Code Scanning if: steps.out-file.outputs.exists == 'true' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: snyk.sarif \ No newline at end of file diff --git a/docs/params.md b/docs/params.md index 648590f..ad88701 100644 --- a/docs/params.md +++ b/docs/params.md @@ -39,3 +39,4 @@ Consider setting a custom value if really necessary. | recordSetTimeoutMs | 1000 | Timeout for the asynchronous queue write operation in milliseconds | | metadataCacheTtlSeconds | 3600 | Database metadata cache TTL in seconds | | schemaBuilderMaxRecords | 1000 | The number of records to be used to build the table schema | +| showRecordMetadata | false | Add record metadata columns (__digest, __ttl, __gen) | diff --git a/src/main/java/com/aerospike/jdbc/model/DriverPolicy.java b/src/main/java/com/aerospike/jdbc/model/DriverPolicy.java index 5725c44..3b9eccd 100644 --- a/src/main/java/com/aerospike/jdbc/model/DriverPolicy.java +++ b/src/main/java/com/aerospike/jdbc/model/DriverPolicy.java @@ -13,6 +13,7 @@ public class DriverPolicy { private final int recordSetTimeoutMs; private final int metadataCacheTtlSeconds; private final int schemaBuilderMaxRecords; + private final boolean showRecordMetadata; public DriverPolicy(Properties properties) { recordSetQueueCapacity = parseInt(properties.getProperty("recordSetQueueCapacity"), @@ -23,6 +24,7 @@ public DriverPolicy(Properties properties) { DEFAULT_METADATA_CACHE_TTL_SECONDS); schemaBuilderMaxRecords = parseInt(properties.getProperty("schemaBuilderMaxRecords"), DEFAULT_SCHEMA_BUILDER_MAX_RECORDS); + showRecordMetadata = Boolean.parseBoolean(properties.getProperty("showRecordMetadata")); } public int getRecordSetQueueCapacity() { @@ -41,6 +43,10 @@ public int getSchemaBuilderMaxRecords() { return schemaBuilderMaxRecords; } + public boolean getShowRecordMetadata() { + return showRecordMetadata; + } + private int parseInt(String value, int defaultValue) { if (value != null) { return Integer.parseInt(value); diff --git a/src/main/java/com/aerospike/jdbc/model/Pair.java b/src/main/java/com/aerospike/jdbc/model/Pair.java index 2799613..736a53a 100644 --- a/src/main/java/com/aerospike/jdbc/model/Pair.java +++ b/src/main/java/com/aerospike/jdbc/model/Pair.java @@ -10,6 +10,10 @@ public Pair(L left, R right) { this.right = right; } + public static Pair of(L left, R right) { + return new Pair<>(left, right); + } + public L getLeft() { return left; } diff --git a/src/main/java/com/aerospike/jdbc/schema/AerospikeSchemaBuilder.java b/src/main/java/com/aerospike/jdbc/schema/AerospikeSchemaBuilder.java index 69a2232..53db7ca 100644 --- a/src/main/java/com/aerospike/jdbc/schema/AerospikeSchemaBuilder.java +++ b/src/main/java/com/aerospike/jdbc/schema/AerospikeSchemaBuilder.java @@ -6,16 +6,21 @@ import com.aerospike.jdbc.model.CatalogTableName; import com.aerospike.jdbc.model.DataColumn; import com.aerospike.jdbc.model.DriverPolicy; +import com.aerospike.jdbc.model.Pair; import java.sql.Types; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; import static com.aerospike.jdbc.util.Constants.DEFAULT_SCHEMA_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_DIGEST_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_GEN_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_TTL_COLUMN_NAME; import static com.aerospike.jdbc.util.Constants.PRIMARY_KEY_COLUMN_NAME; public final class AerospikeSchemaBuilder { @@ -23,30 +28,22 @@ public final class AerospikeSchemaBuilder { private static final Logger logger = Logger.getLogger(AerospikeSchemaBuilder.class.getName()); private final IAerospikeClient client; + private final DriverPolicy driverPolicy; private final AerospikeSchemaCache schemaCache; - private final int scanMaxRecords; public AerospikeSchemaBuilder(IAerospikeClient client, DriverPolicy driverPolicy) { this.client = client; + this.driverPolicy = driverPolicy; schemaCache = new AerospikeSchemaCache(Duration.ofSeconds(driverPolicy.getMetadataCacheTtlSeconds())); - scanMaxRecords = driverPolicy.getSchemaBuilderMaxRecords(); } public List getSchema(CatalogTableName catalogTableName) { return schemaCache.get(catalogTableName).orElseGet(() -> { logger.info(() -> "Fetching CatalogTableName: " + catalogTableName); - final Map columnHandles = new TreeMap<>(String::compareToIgnoreCase); - ScanPolicy policy = new ScanPolicy(client.getScanPolicyDefault()); - policy.maxRecords = scanMaxRecords; + final Map columnHandles = initColumnHandles(catalogTableName); - // add record key column handler - columnHandles.put(PRIMARY_KEY_COLUMN_NAME, - new DataColumn( - catalogTableName.getCatalogName(), - catalogTableName.getTableName(), - Types.VARCHAR, - PRIMARY_KEY_COLUMN_NAME, - PRIMARY_KEY_COLUMN_NAME)); + ScanPolicy policy = new ScanPolicy(client.getScanPolicyDefault()); + policy.maxRecords = driverPolicy.getSchemaBuilderMaxRecords(); client.scanAll(policy, catalogTableName.getCatalogName(), toSet(catalogTableName.getTableName()), (key, rec) -> { @@ -69,6 +66,34 @@ public List getSchema(CatalogTableName catalogTableName) { }); } + private Map initColumnHandles(CatalogTableName catalogTableName) { + final Map columnHandles = new TreeMap<>(String::compareToIgnoreCase); + final List> metadataColumns = new ArrayList<>(); + + // add record key column handler + metadataColumns.add(Pair.of(PRIMARY_KEY_COLUMN_NAME, Types.VARCHAR)); + + // add record metadata column handlers + if (driverPolicy.getShowRecordMetadata()) { + metadataColumns.addAll(Arrays.asList( + Pair.of(METADATA_DIGEST_COLUMN_NAME, Types.VARCHAR), + Pair.of(METADATA_GEN_COLUMN_NAME, Types.INTEGER), + Pair.of(METADATA_TTL_COLUMN_NAME, Types.INTEGER) + )); + } + + for (Pair md : metadataColumns) { + columnHandles.put(md.getLeft(), + new DataColumn( + catalogTableName.getCatalogName(), + catalogTableName.getTableName(), + md.getRight(), + md.getLeft(), + md.getLeft())); + } + return columnHandles; + } + private String toSet(String tableName) { if (tableName.equals(DEFAULT_SCHEMA_NAME)) { return null; diff --git a/src/main/java/com/aerospike/jdbc/sql/AerospikeRecordResultSet.java b/src/main/java/com/aerospike/jdbc/sql/AerospikeRecordResultSet.java index 0ef3337..ecb725b 100644 --- a/src/main/java/com/aerospike/jdbc/sql/AerospikeRecordResultSet.java +++ b/src/main/java/com/aerospike/jdbc/sql/AerospikeRecordResultSet.java @@ -4,13 +4,19 @@ import com.aerospike.client.Value; import com.aerospike.jdbc.async.RecordSet; import com.aerospike.jdbc.model.DataColumn; +import com.google.common.io.BaseEncoding; import java.math.BigDecimal; import java.sql.Statement; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; +import static com.aerospike.jdbc.util.Constants.METADATA_DIGEST_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_GEN_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_TTL_COLUMN_NAME; import static com.aerospike.jdbc.util.Constants.PRIMARY_KEY_COLUMN_NAME; public class AerospikeRecordResultSet extends BaseResultSet { @@ -18,6 +24,7 @@ public class AerospikeRecordResultSet extends BaseResultSet { private static final Logger logger = Logger.getLogger(AerospikeRecordResultSet.class.getName()); private final RecordSet recordSet; + private final Set columnNames; public AerospikeRecordResultSet( RecordSet recordSet, @@ -28,6 +35,7 @@ public AerospikeRecordResultSet( ) { super(statement, catalog, table, columns); this.recordSet = recordSet; + this.columnNames = columns.stream().map(DataColumn::getName).collect(Collectors.toSet()); } @Override @@ -43,12 +51,7 @@ protected boolean moveToNext() { @Override public Object getObject(String columnLabel) { logger.fine(() -> "getObject: " + columnLabel); - Object obj; - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - obj = getUserKey().map(Value::getObject).orElse(null); - } else { - obj = getBin(columnLabel).orElse(null); - } + Object obj = getValue(columnLabel).map(Value::getObject).orElse(null); wasNull = obj == null; return obj; } @@ -56,12 +59,7 @@ public Object getObject(String columnLabel) { @Override public String getString(String columnLabel) { logger.fine(() -> "getString: " + columnLabel); - String str; - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - str = getUserKey().map(Value::toString).orElse(null); - } else { - str = getBin(columnLabel).map(Object::toString).orElse(null); - } + String str = getValue(columnLabel).map(Value::toString).orElse(null); wasNull = str == null; return str; } @@ -69,10 +67,7 @@ public String getString(String columnLabel) { @Override public boolean getBoolean(String columnLabel) { logger.fine(() -> "getBoolean: " + columnLabel); - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - return getUserKey().map(Value::toString).map(Boolean::parseBoolean).orElse(false); - } - return getBin(columnLabel).map(Object::toString).map(Boolean::parseBoolean).orElse(false); + return getValue(columnLabel).map(Value::toString).map(Boolean::parseBoolean).orElse(false); } @Override @@ -90,19 +85,13 @@ public short getShort(String columnLabel) { @Override public int getInt(String columnLabel) { logger.fine(() -> "getInt: " + columnLabel); - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - return getUserKey().map(Value::toInteger).orElse(0); - } - return getBin(columnLabel).map(Object::toString).map(Integer::parseInt).orElse(0); + return getValue(columnLabel).map(Value::toString).map(Integer::parseInt).orElse(0); } @Override public long getLong(String columnLabel) { logger.fine(() -> "getLong: " + columnLabel); - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - return getUserKey().map(Value::toLong).orElse(0L); - } - return getBin(columnLabel).map(Object::toString).map(Long::parseLong).orElse(0L); + return getValue(columnLabel).map(Value::toString).map(Long::parseLong).orElse(0L); } @Override @@ -114,10 +103,7 @@ public float getFloat(String columnLabel) { @Override public double getDouble(String columnLabel) { logger.fine(() -> "getDouble: " + columnLabel); - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - return getUserKey().map(Value::toString).map(Double::parseDouble).orElse(0.0d); - } - return getBin(columnLabel).map(Object::toString).map(Double::parseDouble).orElse(0.0d); + return getValue(columnLabel).map(Value::toString).map(Double::parseDouble).orElse(0.0d); } @Override @@ -129,17 +115,33 @@ public BigDecimal getBigDecimal(String columnLabel, int scale) { @Override public byte[] getBytes(String columnLabel) { logger.fine(() -> "getBytes: " + columnLabel); - if (columnLabel.equals(PRIMARY_KEY_COLUMN_NAME)) { - return getUserKey().map(Value::toString).map(String::getBytes).orElse(null); - } - return getBin(columnLabel).map(byte[].class::cast).orElse(null); - } - - private Optional getUserKey() { - return Optional.ofNullable(recordSet.getKey().userKey); + return getValue(columnLabel).map(Value::getObject).map(byte[].class::cast).orElse(null); } - private Optional getBin(String columnLabel) { - return Optional.ofNullable(recordSet.getRecord().bins.get(columnLabel)); + private Optional getValue(String columnLabel) { + if (!columnNames.contains(columnLabel)) { + return Optional.empty(); + } + switch (columnLabel) { + case PRIMARY_KEY_COLUMN_NAME: + return Optional.ofNullable(recordSet.getKey()) + .map(key -> key.userKey); + case METADATA_DIGEST_COLUMN_NAME: + return Optional.ofNullable(recordSet.getKey()) + .map(key -> BaseEncoding.base16().lowerCase().encode(key.digest)) + .map(Value::get); + case METADATA_TTL_COLUMN_NAME: + return Optional.ofNullable(recordSet.getRecord()) + .map(rec -> rec.expiration) + .map(Value::get); + case METADATA_GEN_COLUMN_NAME: + return Optional.ofNullable(recordSet.getRecord()) + .map(rec -> rec.generation) + .map(Value::get); + default: // regular bin value + return Optional.ofNullable(recordSet.getRecord()) + .map(rec -> rec.bins.get(columnLabel)) + .map(Value::get); + } } } diff --git a/src/main/java/com/aerospike/jdbc/util/Constants.java b/src/main/java/com/aerospike/jdbc/util/Constants.java index 1ff8738..ef9b680 100644 --- a/src/main/java/com/aerospike/jdbc/util/Constants.java +++ b/src/main/java/com/aerospike/jdbc/util/Constants.java @@ -5,6 +5,10 @@ public final class Constants { public static final String PRIMARY_KEY_COLUMN_NAME = "__key"; public static final String DEFAULT_SCHEMA_NAME = "__default"; + public static final String METADATA_DIGEST_COLUMN_NAME = "__digest"; + public static final String METADATA_TTL_COLUMN_NAME = "__ttl"; + public static final String METADATA_GEN_COLUMN_NAME = "__gen"; + public static final String UNSUPPORTED_QUERY_TYPE_MESSAGE = "Unsupported query type"; // Driver version diff --git a/src/test/java/com/aerospike/jdbc/PreparedQueriesTest.java b/src/test/java/com/aerospike/jdbc/PreparedQueriesTest.java index 086e1b8..20fdab8 100644 --- a/src/test/java/com/aerospike/jdbc/PreparedQueriesTest.java +++ b/src/test/java/com/aerospike/jdbc/PreparedQueriesTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.Executors; import java.util.logging.Logger; +import static com.aerospike.jdbc.util.Constants.METADATA_DIGEST_COLUMN_NAME; import static com.aerospike.jdbc.util.Constants.PRIMARY_KEY_COLUMN_NAME; import static com.aerospike.jdbc.util.TestConfig.HOSTNAME; import static com.aerospike.jdbc.util.TestConfig.NAMESPACE; @@ -25,6 +26,7 @@ import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; public class PreparedQueriesTest { @@ -96,6 +98,7 @@ public void testSelectQuery() throws SQLException { statement = connection.prepareStatement(query); resultSet = statement.executeQuery(); while (resultSet.next()) { + assertNull(resultSet.getObject(METADATA_DIGEST_COLUMN_NAME)); testRecord.assertPreparedResultSet(resultSet); total++; diff --git a/src/test/java/com/aerospike/jdbc/RecordMetadataTest.java b/src/test/java/com/aerospike/jdbc/RecordMetadataTest.java new file mode 100644 index 0000000..ecc9868 --- /dev/null +++ b/src/test/java/com/aerospike/jdbc/RecordMetadataTest.java @@ -0,0 +1,130 @@ +package com.aerospike.jdbc; + +import com.aerospike.jdbc.util.TestRecord; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +import static com.aerospike.jdbc.util.Constants.METADATA_DIGEST_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_GEN_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.METADATA_TTL_COLUMN_NAME; +import static com.aerospike.jdbc.util.Constants.PRIMARY_KEY_COLUMN_NAME; +import static com.aerospike.jdbc.util.TestConfig.HOSTNAME; +import static com.aerospike.jdbc.util.TestConfig.NAMESPACE; +import static com.aerospike.jdbc.util.TestConfig.PORT; +import static com.aerospike.jdbc.util.TestConfig.TABLE_NAME; +import static com.aerospike.jdbc.util.TestUtil.closeQuietly; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +public class RecordMetadataTest { + + private static final Logger logger = Logger.getLogger(RecordMetadataTest.class.getName()); + private static Connection connection; + + private final TestRecord testRecord; + + RecordMetadataTest() { + testRecord = new TestRecord("key1", true, 11100, 1, "bar"); + } + + @BeforeClass + public static void connectionInit() throws Exception { + logger.info("connectionInit"); + Class.forName("com.aerospike.jdbc.AerospikeDriver").newInstance(); + String url = String.format("jdbc:aerospike:%s:%d/%s?sendKey=true&showRecordMetadata=true", + HOSTNAME, PORT, NAMESPACE); + connection = DriverManager.getConnection(url); + connection.setNetworkTimeout(Executors.newSingleThreadExecutor(), 5000); + } + + @AfterClass + public static void connectionClose() throws SQLException { + logger.info("connectionClose"); + connection.close(); + } + + @BeforeMethod + public void setUp() throws SQLException { + Objects.requireNonNull(connection, "connection is null"); + Statement statement = null; + int count; + String query = testRecord.toInsertQuery(); + try { + statement = connection.createStatement(); + count = statement.executeUpdate(query); + } finally { + closeQuietly(statement); + } + assertEquals(count, 1); + } + + @AfterMethod + public void tearDown() throws SQLException { + Objects.requireNonNull(connection, "connection is null"); + Statement statement = null; + String query = format("DELETE FROM %s", TABLE_NAME); + try { + statement = connection.createStatement(); + boolean result = statement.execute(query); + assertFalse(result); + } finally { + closeQuietly(statement); + } + assertTrue(statement.getUpdateCount() > 0); + } + + @Test + public void testSelectAllColumns() throws SQLException { + Statement statement = null; + ResultSet resultSet = null; + String query = format("SELECT * FROM %s LIMIT 10", TABLE_NAME); + try { + statement = connection.createStatement(); + resultSet = statement.executeQuery(query); + assertTrue(resultSet.next()); + assertEquals(resultSet.getString(METADATA_DIGEST_COLUMN_NAME), "212ddf97ff3fe0f6dec5e1626d92a635a55171c2"); + assertEquals(resultSet.getInt(METADATA_GEN_COLUMN_NAME), 1); + assertTrue(resultSet.getInt(METADATA_TTL_COLUMN_NAME) > 0); + assertFalse(resultSet.next()); + } finally { + closeQuietly(statement); + closeQuietly(resultSet); + } + } + + @Test + public void testSelectMetadataColumns() throws SQLException { + Statement statement = null; + ResultSet resultSet = null; + String query = format("SELECT %s, %s, int1 FROM %s WHERE %s='key1'", METADATA_GEN_COLUMN_NAME, + METADATA_TTL_COLUMN_NAME, TABLE_NAME, PRIMARY_KEY_COLUMN_NAME); + try { + statement = connection.createStatement(); + resultSet = statement.executeQuery(query); + assertTrue(resultSet.next()); + assertNull(resultSet.getObject(METADATA_DIGEST_COLUMN_NAME)); + assertEquals(resultSet.getInt(METADATA_GEN_COLUMN_NAME), 1); + assertTrue(resultSet.getInt(METADATA_TTL_COLUMN_NAME) > 0); + assertEquals(resultSet.getInt("int1"), 11100); + assertFalse(resultSet.next()); + } finally { + closeQuietly(statement); + closeQuietly(resultSet); + } + } +} diff --git a/src/test/java/com/aerospike/jdbc/SimpleQueriesTest.java b/src/test/java/com/aerospike/jdbc/SimpleQueriesTest.java index 4e5ff78..e9969e0 100644 --- a/src/test/java/com/aerospike/jdbc/SimpleQueriesTest.java +++ b/src/test/java/com/aerospike/jdbc/SimpleQueriesTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.Executors; import java.util.logging.Logger; +import static com.aerospike.jdbc.util.Constants.METADATA_DIGEST_COLUMN_NAME; import static com.aerospike.jdbc.util.Constants.PRIMARY_KEY_COLUMN_NAME; import static com.aerospike.jdbc.util.TestConfig.HOSTNAME; import static com.aerospike.jdbc.util.TestConfig.NAMESPACE; @@ -25,6 +26,7 @@ import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; public class SimpleQueriesTest { @@ -93,6 +95,7 @@ public void testSelectQuery() throws SQLException { statement = connection.createStatement(); resultSet = statement.executeQuery(query); while (resultSet.next()) { + assertNull(resultSet.getObject(METADATA_DIGEST_COLUMN_NAME)); testRecord.assertResultSet(resultSet); total++;