forked from apache/cassandra-java-driver
-
Notifications
You must be signed in to change notification settings - Fork 38
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve DefaultMetadata's TabletMap tablet invalidation logic (#388)
Previously the tablet map would be completely emptied on node removal, necessitating full rebuild of it. With this change the TabletMap will be scanned through on node removals (`RemoveNodeRefresh`) in order to delete only those tablets, that contain removed node as one of the replicas. Additionally we introduce `TabletMapSchemaChangeListener` which will be automatically registered on context initialization as long as schema metadata is enabled. It won't be added if the schema metadata is enabled later at runtime. This listener ensures that relevant tablets will be removed on removals and updates of both keyspaces and tables. The `TabletMapSchemaChangesIT` tests its behaviour. Addresses #377.
- Loading branch information
Showing
6 changed files
with
273 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
.../com/datastax/oss/driver/internal/core/metadata/schema/TabletMapSchemaChangeListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package com.datastax.oss.driver.internal.core.metadata.schema; | ||
|
|
||
| import com.datastax.oss.driver.api.core.metadata.TabletMap; | ||
| import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; | ||
| import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListenerBase; | ||
| import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; | ||
| import edu.umd.cs.findbugs.annotations.NonNull; | ||
|
|
||
| public class TabletMapSchemaChangeListener extends SchemaChangeListenerBase { | ||
| private final TabletMap tabletMap; | ||
|
|
||
| public TabletMapSchemaChangeListener(TabletMap tabletMap) { | ||
| this.tabletMap = tabletMap; | ||
| } | ||
|
|
||
| @Override | ||
| public void onKeyspaceDropped(@NonNull KeyspaceMetadata keyspace) { | ||
| tabletMap.removeByKeyspace(keyspace.getName()); | ||
| } | ||
|
|
||
| @Override | ||
| public void onKeyspaceUpdated( | ||
| @NonNull KeyspaceMetadata current, @NonNull KeyspaceMetadata previous) { | ||
| tabletMap.removeByKeyspace(previous.getName()); | ||
| } | ||
|
|
||
| @Override | ||
| public void onTableDropped(@NonNull TableMetadata table) { | ||
| tabletMap.removeByTable(table.getName()); | ||
| } | ||
|
|
||
| @Override | ||
| public void onTableUpdated(@NonNull TableMetadata current, @NonNull TableMetadata previous) { | ||
| tabletMap.removeByTable(previous.getName()); | ||
| } | ||
| } |
178 changes: 178 additions & 0 deletions
178
...n-tests/src/test/java/com/datastax/oss/driver/core/metadata/TabletMapSchemaChangesIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
| package com.datastax.oss.driver.core.metadata; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.awaitility.Awaitility.await; | ||
|
|
||
| import com.datastax.oss.driver.api.core.CqlIdentifier; | ||
| import com.datastax.oss.driver.api.core.CqlSession; | ||
| import com.datastax.oss.driver.api.core.config.DefaultDriverOption; | ||
| import com.datastax.oss.driver.api.core.cql.BoundStatement; | ||
| import com.datastax.oss.driver.api.core.cql.PreparedStatement; | ||
| import com.datastax.oss.driver.api.core.metadata.KeyspaceTableNamePair; | ||
| import com.datastax.oss.driver.api.core.metadata.Node; | ||
| import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; | ||
| import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; | ||
| import com.datastax.oss.driver.api.testinfra.CassandraSkip; | ||
| import com.datastax.oss.driver.api.testinfra.ScyllaRequirement; | ||
| import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; | ||
| import com.datastax.oss.driver.api.testinfra.session.SessionRule; | ||
| import com.datastax.oss.driver.api.testinfra.session.SessionUtils; | ||
| import com.datastax.oss.driver.internal.core.loadbalancing.BasicLoadBalancingPolicy; | ||
| import com.datastax.oss.driver.internal.core.metadata.schema.TabletMapSchemaChangeListener; | ||
| import java.time.Duration; | ||
| import java.util.concurrent.TimeUnit; | ||
| import org.junit.Before; | ||
| import org.junit.ClassRule; | ||
| import org.junit.Test; | ||
| import org.junit.rules.RuleChain; | ||
| import org.junit.rules.TestRule; | ||
| import org.mockito.ArgumentCaptor; | ||
| import org.mockito.Mockito; | ||
|
|
||
| @ScyllaRequirement( | ||
| minOSS = "6.0.0", | ||
| minEnterprise = "2024.2", | ||
| description = "Needs to support tablets") | ||
| @CassandraSkip(description = "Tablets are ScyllaDB-only extension") | ||
| // Ensures that TabletMap used by MetadataManager behaves as desired on certain events | ||
| public class TabletMapSchemaChangesIT { | ||
|
|
||
| // Same listener as the one registered on initialization by | ||
| // DefaultDriverContext#buildSchemaChangeListener | ||
| // for TabletMap updates. Note that this mock only verifies that it reacts to ".onXhappening()" | ||
| // calls and the | ||
| // actual working listener updates the TabletMap. | ||
| private static final TabletMapSchemaChangeListener listener = | ||
| Mockito.mock(TabletMapSchemaChangeListener.class); | ||
| private static final CustomCcmRule CCM_RULE = | ||
| CustomCcmRule.builder() | ||
| .withNodes(2) | ||
| .withCassandraConfiguration( | ||
| "experimental_features", "['consistent-topology-changes','tablets']") | ||
| .build(); | ||
| private static final SessionRule<CqlSession> SESSION_RULE = | ||
| SessionRule.builder(CCM_RULE) | ||
| .withConfigLoader( | ||
| SessionUtils.configLoaderBuilder() | ||
| .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(15)) | ||
| .withClass( | ||
| DefaultDriverOption.LOAD_BALANCING_POLICY_CLASS, | ||
| BasicLoadBalancingPolicy.class) | ||
| .withBoolean(DefaultDriverOption.METADATA_SCHEMA_ENABLED, true) | ||
| .withDuration(DefaultDriverOption.METADATA_SCHEMA_WINDOW, Duration.ofSeconds(0)) | ||
| .build()) | ||
| .withSchemaChangeListener(listener) | ||
| .build(); | ||
|
|
||
| @ClassRule | ||
| public static final TestRule CHAIN = RuleChain.outerRule(CCM_RULE).around(SESSION_RULE); | ||
|
|
||
| private static final int INITIAL_TABLETS = 32; | ||
| private static final int REPLICATION_FACTOR = 1; | ||
| private static final String KEYSPACE_NAME = "TabletMapSchemaChangesIT"; | ||
| private static final String TABLE_NAME = "testTable"; | ||
| private static final KeyspaceTableNamePair TABLET_MAP_KEY = | ||
| new KeyspaceTableNamePair( | ||
| CqlIdentifier.fromCql(KEYSPACE_NAME), CqlIdentifier.fromCql(TABLE_NAME)); | ||
| private static final String CREATE_KEYSPACE_QUERY = | ||
| "CREATE KEYSPACE IF NOT EXISTS " | ||
| + KEYSPACE_NAME | ||
| + " WITH replication = {'class': " | ||
| + "'NetworkTopologyStrategy', " | ||
| + "'replication_factor': '" | ||
| + REPLICATION_FACTOR | ||
| + "'} AND durable_writes = true AND tablets = " | ||
| + "{'initial': " | ||
| + INITIAL_TABLETS | ||
| + "};"; | ||
| private static final String CREATE_TABLE_QUERY = | ||
| "CREATE TABLE IF NOT EXISTS " | ||
| + KEYSPACE_NAME | ||
| + "." | ||
| + TABLE_NAME | ||
| + " (pk int, ck int, PRIMARY KEY(pk, ck));"; | ||
| private static final String DROP_KEYSPACE = "DROP KEYSPACE IF EXISTS " + KEYSPACE_NAME; | ||
|
|
||
| private static final String INSERT_QUERY_TEMPLATE = | ||
| "INSERT INTO " + KEYSPACE_NAME + "." + TABLE_NAME + " (pk, ck) VALUES (%s, %s)"; | ||
| private static final String SELECT_QUERY_TEMPLATE = | ||
| "SELECT pk, ck FROM " + KEYSPACE_NAME + "." + TABLE_NAME + " WHERE pk = ?"; | ||
|
|
||
| private static final long NOTIF_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1); | ||
|
|
||
| @Before | ||
| public void setup() { | ||
| SESSION_RULE.session().execute(DROP_KEYSPACE); | ||
| SESSION_RULE.session().execute(CREATE_KEYSPACE_QUERY); | ||
| SESSION_RULE.session().execute(CREATE_TABLE_QUERY); | ||
| SESSION_RULE.session().execute(String.format(INSERT_QUERY_TEMPLATE, "1", "2")); | ||
| SESSION_RULE.session().execute(String.format(INSERT_QUERY_TEMPLATE, "3", "4")); | ||
| PreparedStatement ps = SESSION_RULE.session().prepare(SELECT_QUERY_TEMPLATE); | ||
| BoundStatement bs = ps.bind(1); | ||
| // This ensures we hit the node that is not tablet replica | ||
| for (Node node : SESSION_RULE.session().getMetadata().getNodes().values()) { | ||
| SESSION_RULE.session().execute(bs.setNode(node)); | ||
| } | ||
| // Make sure the tablet information is present | ||
| await() | ||
| .atMost(30, TimeUnit.SECONDS) | ||
| .until( | ||
| () -> | ||
| SESSION_RULE | ||
| .session() | ||
| .getMetadata() | ||
| .getTabletMap() | ||
| .getMapping() | ||
| .containsKey(TABLET_MAP_KEY)); | ||
| // Reset invocations for the next test method | ||
| Mockito.clearInvocations(listener); | ||
| } | ||
|
|
||
| @Test | ||
| public void should_remove_tablets_on_keyspace_update() { | ||
| SESSION_RULE | ||
| .session() | ||
| .execute("ALTER KEYSPACE " + KEYSPACE_NAME + " WITH durable_writes = false"); | ||
| ArgumentCaptor<KeyspaceMetadata> previous = ArgumentCaptor.forClass(KeyspaceMetadata.class); | ||
| Mockito.verify(listener, Mockito.timeout(NOTIF_TIMEOUT_MS).times(1)) | ||
| .onKeyspaceUpdated(Mockito.any(), previous.capture()); | ||
| assertThat(previous.getValue().getName()).isEqualTo(CqlIdentifier.fromCql(KEYSPACE_NAME)); | ||
| assertThat(SESSION_RULE.session().getMetadata().getTabletMap().getMapping().keySet()) | ||
| .doesNotContain(TABLET_MAP_KEY); | ||
| } | ||
|
|
||
| @Test | ||
| public void should_remove_tablets_on_keyspace_drop() { | ||
| SESSION_RULE.session().execute(DROP_KEYSPACE); | ||
| ArgumentCaptor<KeyspaceMetadata> keyspace = ArgumentCaptor.forClass(KeyspaceMetadata.class); | ||
| Mockito.verify(listener, Mockito.timeout(NOTIF_TIMEOUT_MS).times(1)) | ||
| .onKeyspaceDropped(keyspace.capture()); | ||
| assertThat(keyspace.getValue().getName()).isEqualTo(CqlIdentifier.fromCql(KEYSPACE_NAME)); | ||
| assertThat(SESSION_RULE.session().getMetadata().getTabletMap().getMapping().keySet()) | ||
| .doesNotContain(TABLET_MAP_KEY); | ||
| } | ||
|
|
||
| @Test | ||
| public void should_remove_tablets_on_table_update() { | ||
| SESSION_RULE | ||
| .session() | ||
| .execute("ALTER TABLE " + KEYSPACE_NAME + "." + TABLE_NAME + " ADD anotherCol int"); | ||
| ArgumentCaptor<TableMetadata> previous = ArgumentCaptor.forClass(TableMetadata.class); | ||
| Mockito.verify(listener, Mockito.timeout(NOTIF_TIMEOUT_MS).times(1)) | ||
| .onTableUpdated(Mockito.any(), previous.capture()); | ||
| assertThat(previous.getValue().getName()).isEqualTo(CqlIdentifier.fromCql(TABLE_NAME)); | ||
| assertThat(SESSION_RULE.session().getMetadata().getTabletMap().getMapping().keySet()) | ||
| .doesNotContain(TABLET_MAP_KEY); | ||
| } | ||
|
|
||
| @Test | ||
| public void should_remove_tablets_on_table_drop() { | ||
| SESSION_RULE.session().execute("DROP TABLE " + KEYSPACE_NAME + "." + TABLE_NAME); | ||
| ArgumentCaptor<TableMetadata> table = ArgumentCaptor.forClass(TableMetadata.class); | ||
| Mockito.verify(listener, Mockito.timeout(NOTIF_TIMEOUT_MS).times(1)) | ||
| .onTableDropped(table.capture()); | ||
| assertThat(table.getValue().getName()).isEqualTo(CqlIdentifier.fromCql(TABLE_NAME)); | ||
| assertThat(SESSION_RULE.session().getMetadata().getTabletMap().getMapping().keySet()) | ||
| .doesNotContain(TABLET_MAP_KEY); | ||
| } | ||
| } |