[WIP] [DNM] SnapshotReader Rebased

.palantir/revapi.yml

@@ -1,12 +1,65 @@
 acceptedBreaks:
   "0.1040.0":
+    com.palantir.atlasdb:atlasdb-api:
+    - code: "java.method.removed"
+      old: "method java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, com.palantir.atlasdb.keyvalue.api.Value>\
+        \ com.palantir.atlasdb.cell.api.AutoDelegate_TransactionKeyValueService::get(com.palantir.atlasdb.keyvalue.api.TableReference,\
+        \ java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, java.lang.Long>)"
+      justification: "TransactionKeyValueService API not widely used yet"
+    - code: "java.method.removed"
+      old: "method java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, com.palantir.atlasdb.keyvalue.api.Value>\
+        \ com.palantir.atlasdb.cell.api.TransactionKeyValueService::get(com.palantir.atlasdb.keyvalue.api.TableReference,\
+        \ java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, java.lang.Long>)"
+      justification: "TransactionKeyValueService API not widely used yet"
+  "0.1041.0-rc3":
     com.palantir.atlasdb:atlasdb-api:
     - code: "java.class.removed"
       old: "interface com.palantir.atlasdb.transaction.api.AutoDelegate_TransactionKeyValueServiceManager"
       justification: "Still iterating on the APIs"
     - code: "java.class.removed"
       old: "interface com.palantir.atlasdb.transaction.api.TransactionKeyValueServiceManager"
       justification: "Still iterating on the APIs"
+    - code: "java.method.addedToInterface"
+      new: "method java.util.NavigableMap<byte[], com.palantir.atlasdb.keyvalue.api.RowResult<byte[]>>\
+        \ com.palantir.atlasdb.transaction.api.KeyValueSnapshotReader::getRows(com.palantir.atlasdb.keyvalue.api.TableReference,\
+        \ java.lang.Iterable<byte[]>, com.palantir.atlasdb.keyvalue.api.ColumnSelection,\
+        \ com.google.common.collect.ImmutableMap.Builder<com.palantir.atlasdb.keyvalue.api.Cell,\
+        \ byte[]>)"
+      justification: "SnapshotReader and TimestampLoader are new internal interfaces"
+    - code: "java.method.addedToInterface"
+      new: "method void com.palantir.atlasdb.transaction.api.KeyValueSnapshotEventRecorder::recordEmptyValueRead(com.palantir.atlasdb.keyvalue.api.TableReference)"
+      justification: "SnapshotReader and TimestampLoader are new internal interfaces"
+    - code: "java.method.numberOfParametersChanged"
+      old: "method com.google.common.util.concurrent.ListenableFuture<org.eclipse.collections.api.map.primitive.LongLongMap>\
+        \ com.palantir.atlasdb.transaction.api.CommitTimestampLoader::getCommitTimestamps(com.palantir.atlasdb.keyvalue.api.TableReference,\
+        \ org.eclipse.collections.api.LongIterable, boolean, com.palantir.atlasdb.transaction.service.AsyncTransactionService)"
+      new: "method com.google.common.util.concurrent.ListenableFuture<org.eclipse.collections.api.map.primitive.LongLongMap>\
+        \ com.palantir.atlasdb.transaction.api.CommitTimestampLoader::getCommitTimestamps(com.palantir.atlasdb.keyvalue.api.TableReference,\
+        \ org.eclipse.collections.api.LongIterable, boolean)"
+      justification: "SnapshotReader and TimestampLoader are new internal interfaces"
+    - code: "java.method.numberOfParametersChanged"
+      old: "method com.palantir.atlasdb.transaction.api.TransactionKeyValueServiceManager\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceManagerFactory<T>::create(com.palantir.atlasdb.coordination.CoordinationService<T>,\
+        \ com.palantir.atlasdb.spi.KeyValueServiceManager, com.palantir.atlasdb.spi.TransactionKeyValueServiceConfig,\
+        \ com.palantir.refreshable.Refreshable<com.palantir.atlasdb.spi.TransactionKeyValueServiceRuntimeConfig>,\
+        \ boolean)"
+      new: "method com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceManagerFactory<T>::create(java.lang.String,\
+        \ com.palantir.dialogue.clients.DialogueClients.ReloadingFactory, com.palantir.atlasdb.util.MetricsManager,\
+        \ com.palantir.atlasdb.coordination.CoordinationService<T>, com.palantir.atlasdb.spi.KeyValueServiceManager,\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceConfig, com.palantir.refreshable.Refreshable<com.palantir.atlasdb.spi.TransactionKeyValueServiceRuntimeConfig>,\
+        \ boolean)"
+      justification: "updating internal APIs"
+    - code: "java.method.parameterTypeChanged"
+      old: "parameter com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManager\
+        \ com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManagerFactory::createKeyValueSnapshotReaderManager(===com.palantir.atlasdb.transaction.api.TransactionKeyValueServiceManager===,\
+        \ com.palantir.atlasdb.transaction.service.TransactionService, boolean, com.palantir.atlasdb.transaction.api.OrphanedSentinelDeleter,\
+        \ com.palantir.atlasdb.transaction.api.DeleteExecutor)"
+      new: "parameter com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManager\
+        \ com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManagerFactory::createKeyValueSnapshotReaderManager(===com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager===,\
+        \ com.palantir.atlasdb.transaction.service.TransactionService, boolean, com.palantir.atlasdb.transaction.api.OrphanedSentinelDeleter,\
+        \ com.palantir.atlasdb.transaction.api.DeleteExecutor)"
+      justification: "updating internal APIs"
     - code: "java.method.returnTypeChanged"
       old: "method com.palantir.atlasdb.transaction.api.TransactionKeyValueServiceManager\
         \ com.palantir.atlasdb.spi.TransactionKeyValueServiceManagerFactory<T>::create(com.palantir.atlasdb.coordination.CoordinationService<T>,\
@@ -19,6 +72,19 @@ acceptedBreaks:
         \ com.palantir.refreshable.Refreshable<com.palantir.atlasdb.spi.TransactionKeyValueServiceRuntimeConfig>,\
         \ boolean)"
       justification: "Still iterating on the APIs"
+    - code: "java.method.returnTypeChanged"
+      old: "method com.palantir.atlasdb.transaction.api.TransactionKeyValueServiceManager\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceManagerFactory<T>::create(com.palantir.atlasdb.coordination.CoordinationService<T>,\
+        \ com.palantir.atlasdb.spi.KeyValueServiceManager, com.palantir.atlasdb.spi.TransactionKeyValueServiceConfig,\
+        \ com.palantir.refreshable.Refreshable<com.palantir.atlasdb.spi.TransactionKeyValueServiceRuntimeConfig>,\
+        \ boolean)"
+      new: "method com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceManagerFactory<T>::create(java.lang.String,\
+        \ com.palantir.dialogue.clients.DialogueClients.ReloadingFactory, com.palantir.atlasdb.util.MetricsManager,\
+        \ com.palantir.atlasdb.coordination.CoordinationService<T>, com.palantir.atlasdb.spi.KeyValueServiceManager,\
+        \ com.palantir.atlasdb.spi.TransactionKeyValueServiceConfig, com.palantir.refreshable.Refreshable<com.palantir.atlasdb.spi.TransactionKeyValueServiceRuntimeConfig>,\
+        \ boolean)"
+      justification: "updating internal APIs"
     com.palantir.atlasdb:atlasdb-config:
     - code: "java.method.numberOfParametersChanged"
       old: "method com.palantir.atlasdb.factory.TransactionManagersInitializer com.palantir.atlasdb.factory.TransactionManagersInitializer::createInitialTables(com.palantir.atlasdb.keyvalue.api.KeyValueService,\
@@ -47,18 +113,18 @@ acceptedBreaks:
     - code: "java.method.addedToInterface"
       new: "method boolean com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager::isInitialized()"
       justification: "Adding async init to TransactionKeyValueServiceManager"
-  "0.1050.0":
+  "0.1051.0-rc1":
     com.palantir.atlasdb:atlasdb-api:
-    - code: "java.method.removed"
-      old: "method java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, com.palantir.atlasdb.keyvalue.api.Value>\
-        \ com.palantir.atlasdb.cell.api.AutoDelegate_TransactionKeyValueService::get(com.palantir.atlasdb.keyvalue.api.TableReference,\
-        \ java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, java.lang.Long>)"
-      justification: "TransactionKeyValueService is largely internal"
-    - code: "java.method.removed"
-      old: "method java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, com.palantir.atlasdb.keyvalue.api.Value>\
-        \ com.palantir.atlasdb.cell.api.TransactionKeyValueService::get(com.palantir.atlasdb.keyvalue.api.TableReference,\
-        \ java.util.Map<com.palantir.atlasdb.keyvalue.api.Cell, java.lang.Long>)"
-      justification: "TransactionKeyValueService is largely internal"
+    - code: "java.method.numberOfParametersChanged"
+      old: "method com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManager\
+        \ com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManagerFactory::createKeyValueSnapshotReaderManager(com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager,\
+        \ com.palantir.atlasdb.transaction.service.TransactionService, boolean, com.palantir.atlasdb.transaction.api.OrphanedSentinelDeleter,\
+        \ com.palantir.atlasdb.transaction.api.DeleteExecutor)"
+      new: "method com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManager\
+        \ com.palantir.atlasdb.transaction.api.KeyValueSnapshotReaderManagerFactory::createKeyValueSnapshotReaderManager(com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager,\
+        \ com.palantir.atlasdb.transaction.service.TransactionService, boolean, com.palantir.atlasdb.transaction.api.OrphanedSentinelDeleter,\
+        \ com.palantir.atlasdb.transaction.api.DeleteExecutor, com.palantir.atlasdb.util.MetricsManager)"
+      justification: "internal API"
   "0.770.0":
     com.palantir.atlasdb:atlasdb-api:
     - code: "java.class.removed"

atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/transaction/impl/DeleteExecutor.java → atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/DeleteExecutor.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.palantir.atlasdb.transaction.impl;
+package com.palantir.atlasdb.transaction.api;
 
 import com.palantir.atlasdb.keyvalue.api.Cell;
 import com.palantir.atlasdb.keyvalue.api.TableReference;

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/KeyValueSnapshotEventRecorder.java

@@ -0,0 +1,37 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import com.palantir.atlasdb.keyvalue.api.TableReference;
+
+public interface KeyValueSnapshotEventRecorder {
+    void recordCellsRead(TableReference tableReference, long cellsRead);
+
+    void recordCellsReturned(TableReference tableReference, long cellsReturned);
+
+    void recordManyBytesReadForTable(TableReference tableReference, long bytesRead);
+
+    void recordFilteredSweepSentinel(TableReference tableReference);
+
+    void recordFilteredUncommittedTransaction(TableReference tableReference);
+
+    void recordFilteredTransactionCommittingAfterOurStart(TableReference tableReference);
+
+    void recordRolledBackOtherTransaction();
+
+    void recordEmptyValueRead(TableReference tableReference);
+}

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/KeyValueSnapshotReader.java

@@ -0,0 +1,52 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.palantir.atlasdb.keyvalue.api.Cell;
+import com.palantir.atlasdb.keyvalue.api.ColumnSelection;
+import com.palantir.atlasdb.keyvalue.api.RowResult;
+import com.palantir.atlasdb.keyvalue.api.TableReference;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+/**
+ * Reads state of a {@link com.palantir.atlasdb.keyvalue.api.KeyValueService} in accordance with the provided
+ * AtlasDB timestamp, following the AtlasDB read protocol. This includes reading the most recent committed value for
+ * each cell that would be visible at the provided timestamp(s) and filtering out versions that have been aborted or
+ * not committed yet.
+ * <p>
+ * If used in the context of a transaction, users are responsible for validating that snapshots read are still
+ * guaranteed to be consistent (for example, transactions may need to validate their pre-commit conditions or check
+ * that sweep has not progressed). Some methods may have partial, intermediate validation required as part of servicing
+ * a read; this class will carry out this intermediate validation.
+ * <p>
+ * Although this interface performs user-level reads, internal writes may be performed (for example, as part of the
+ * read protocol, to abort a long-running transaction).
+ */
+public interface KeyValueSnapshotReader {
+    ListenableFuture<Map<Cell, byte[]>> getAsync(TableReference tableReference, Set<Cell> cells);
+
+    // TODO (jkong): This one is kind of awful, but it's used to avoid allocations and I'm not keen to add too many.
+    NavigableMap<byte[], RowResult<byte[]>> getRows(
+            TableReference tableReference,
+            Iterable<byte[]> rows,
+            ColumnSelection columnSelection,
+            ImmutableMap.Builder<Cell, byte[]> resultCollector);
+}

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/KeyValueSnapshotReaderManager.java

@@ -0,0 +1,37 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import java.util.function.LongSupplier;
+import org.immutables.value.Value;
+
+public interface KeyValueSnapshotReaderManager {
+    KeyValueSnapshotReader createKeyValueSnapshotReader(TransactionContext transactionContext);
+
+    @Value.Immutable
+    interface TransactionContext {
+        LongSupplier startTimestampSupplier();
+
+        TransactionReadSentinelBehavior transactionReadSentinelBehavior();
+
+        CommitTimestampLoader commitTimestampLoader();
+
+        PreCommitRequirementValidator preCommitRequirementValidator();
+
+        KeyValueSnapshotEventRecorder keyValueSnapshotEventRecorder();
+    }
+}

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/KeyValueSnapshotReaderManagerFactory.java

@@ -0,0 +1,33 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import com.palantir.atlasdb.cell.api.TransactionKeyValueServiceManager;
+import com.palantir.atlasdb.transaction.service.TransactionService;
+import com.palantir.atlasdb.util.MetricsManager;
+
+public interface KeyValueSnapshotReaderManagerFactory {
+    String getType();
+
+    KeyValueSnapshotReaderManager createKeyValueSnapshotReaderManager(
+            TransactionKeyValueServiceManager transactionKeyValueServiceManager,
+            TransactionService transactionService,
+            boolean allowHiddenTableAccess,
+            OrphanedSentinelDeleter orphanedSentinelDeleter,
+            DeleteExecutor deleteExecutor,
+            MetricsManager metricsManager);
+}

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/OrphanedSentinelDeleter.java

@@ -0,0 +1,25 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import com.palantir.atlasdb.keyvalue.api.Cell;
+import com.palantir.atlasdb.keyvalue.api.TableReference;
+import java.util.Set;
+
+public interface OrphanedSentinelDeleter {
+    void scheduleSentinelsForDeletion(TableReference tableReference, Set<Cell> orphanedSentinels);
+}

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/PreCommitRequirementValidator.java

@@ -0,0 +1,38 @@
+/*
+ * (c) Copyright 2024 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.transaction.api;
+
+import com.palantir.atlasdb.keyvalue.api.Cell;
+import com.palantir.atlasdb.keyvalue.api.TableReference;
+import com.palantir.lock.v2.LockToken;
+import java.util.Map;
+import java.util.Optional;
+
+public interface PreCommitRequirementValidator {
+    // TODO (jkong): The boolean means "are there unvalidated reads"?
+    boolean throwIfPreCommitRequirementsNotMetOnRead(
+            TableReference tableRef, long timestamp, boolean allPossibleCellsReadAndPresent);
+
+    void throwIfPreCommitConditionInvalid(long timestamp);
+
+    void throwIfPreCommitConditionInvalidAtCommitOnWriteTransaction(
+            Map<TableReference, ? extends Map<Cell, byte[]>> mutations, long timestamp);
+
+    void throwIfPreCommitRequirementsNotMet(Optional<LockToken> commitLocksToken, long timestamp);
+
+    void throwIfImmutableTsOrCommitLocksExpired(Optional<LockToken> commitLocksToken);
+}