feat: add the read concern header (#408)

* feat: add the read concern header

Add the read concern option to the cache Configuration and thread it
through to the gRPC internals of the client.

Add an argument to the UserHeaderInterceptor constructor representing a
list of extra headers to add to every request. The stubs manager uses
this to add the read concern header if it isn't set to balanced.

Make the base cache test class create a client with either balanced or
consistent read concern based on whether the CONSISTENT_READS
env var is defined.

Add tasks to run integration tests for the different clients.

Add make targets to use the new tasks.

Organize integration tests into auth/cache/storage/topics packages.

.github/workflows/ci.yml

@@ -58,9 +58,7 @@ jobs:
         arguments: clean build
 
     - name: Run integration tests
-      uses: gradle/gradle-build-action@v2
-      with:
-        arguments: integrationTest
+      run: make prod-test
 
   build-examples:
     runs-on: ubuntu-latest

Makefile

@@ -1,45 +1,66 @@
-.PHONY: all
+.PHONY: all clean build test prod-test test-unit test-integration test-auth-service test-cache-service \
+    test-leaderboard-service test-storage-service test-topics-service test-http-service format lint precommit help
+
 all: precommit
 
-.PHONY: clean
 ## Clean the project
 clean:
 	./gradlew clean
 
-.PHONY: build
 ## Build the project
 build:
 	./gradlew build
 
-.PHONY: test
 ## Run all the tests
 test: test-unit test-integration
 
-.PHONY: test-unit
 ## Run the unit tests
 test-unit:
 	./gradlew test
 
-.PHONY: test-integration
-## Run the integration tests
+## Run all integration tests
 test-integration:
-	./gradlew intTest
+	./gradlew integrationTest
+
+## Run all integration tests with consistent reads enabled
+prod-test:
+	@CONSISTENT_READS=1 $(MAKE) test-integration
+
+## Run the auth service tests
+test-auth-service:
+	@CONSISTENT_READS=1 ./gradlew test-auth-service
+
+## Run the cache service tests
+test-cache-service:
+	@CONSISTENT_READS=1 ./gradlew test-cache-service
+
+## Run the leaderboard service tests
+test-leaderboard-service:
+	@echo "Leaderboard client not implemented yet."
+
+## Run the storage service tests
+test-storage-service:
+	@CONSISTENT_READS=1 ./gradlew test-storage-service
+
+## Run the topics service tests
+test-topics-service:
+	@CONSISTENT_READS=1 ./gradlew test-topics-service
+
+## Run the http service tests
+test-http-service:
+	@echo "No tests for http service."
 
-.PHONY: format
 ## Format the code
 format:
 	./gradlew spotlessApply
 
-.PHONY: lint
 ## Lint the code
 lint:
 	./gradlew spotlessCheck
 
-.PHONY: precommit
 ## Run the precommit checks
 precommit: format lint build test
 
-.PHONY: help
 # See <https://gist.github.com/klmr/575726c7e05d8780505a> for explanation.
 help:
 	@echo "$$(tput bold)Available rules:$$(tput sgr0)";echo;sed -ne"/^## /{h;s/.*//;:d" -e"H;n;s/^## //;td" -e"s/:.*//;G;s/\\n## /---/;s/\\n/ /g;p;}" ${MAKEFILE_LIST}|LC_ALL='C' sort -f|awk -F --- -v n=$$(tput cols) -v i=19 -v a="$$(tput setaf 6)" -v z="$$(tput sgr0)" '{printf"%s%*s%s ",a,-i,$$1,z;m=split($$2,w," ");l=n-i;for(j=1;j<=m;j++){l-=length(w[j])+1;if(l<= 0){l=n-i-length(w[j])-1;printf"\n%*s ",-i," ";}printf"%s ",w[j];}printf"\n";}'|more $(shell test $(shell uname) == Darwin && echo '-Xr')

momento-sdk/build.gradle.kts

@@ -1,3 +1,5 @@
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+
 plugins {
     id("momento.publishable-java-lib")
     id("momento.junit-tests")
@@ -53,3 +55,47 @@ tasks.named("analyzeIntTestClassesDependencies").configure {
 tasks.named("analyzeTestClassesDependencies").configure {
     enabled = false
 }
+
+fun registerIntegrationTestTask(name: String, testClasses: List<String>) {
+    tasks.register<Test>(name) {
+        description = "Runs $name integration tests"
+        group = "verification"
+
+        val integrationTestTask = tasks.named<Test>("integrationTest").get()
+        classpath = integrationTestTask.classpath
+        testClassesDirs = integrationTestTask.testClassesDirs
+
+        filter {
+            testClasses.forEach { testClass ->
+                includeTestsMatching(testClass)
+            }
+        }
+
+        useJUnitPlatform()
+        testLogging {
+            exceptionFormat = TestExceptionFormat.FULL
+            events("passed", "skipped", "failed")
+        }
+        outputs.upToDateWhen { false }
+    }
+}
+
+registerIntegrationTestTask(
+    "test-auth-service",
+    listOf("momento.sdk.auth.*")
+)
+
+registerIntegrationTestTask(
+    "test-cache-service",
+    listOf("momento.sdk.cache.*")
+)
+
+registerIntegrationTestTask(
+    "test-storage-service",
+    listOf("momento.sdk.storage.*")
+)
+
+registerIntegrationTestTask(
+    "test-topics-service",
+    listOf("momento.sdk.topics.*")
+)

momento-sdk/src/intTest/java/momento/sdk/MomentoBatchUtilsIntegrationTest.java

@@ -12,6 +12,7 @@
 import momento.sdk.batchutils.MomentoBatchUtils;
 import momento.sdk.batchutils.request.BatchGetRequest;
 import momento.sdk.batchutils.response.BatchGetResponse;
+import momento.sdk.cache.BaseCacheTestClass;
 import momento.sdk.responses.cache.GetResponse;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;

momento-sdk/src/intTest/java/momento/sdk/AuthClientCacheTests.java → momento-sdk/src/intTest/java/momento/sdk/auth/AuthClientCacheTests.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.auth;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -9,7 +9,8 @@
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
-import momento.sdk.auth.StringCredentialProvider;
+import momento.sdk.AuthClient;
+import momento.sdk.CacheClient;
 import momento.sdk.auth.accessControl.CacheItemSelector;
 import momento.sdk.auth.accessControl.CacheRole;
 import momento.sdk.auth.accessControl.CacheSelector;
@@ -18,6 +19,7 @@
 import momento.sdk.auth.accessControl.DisposableTokenScope;
 import momento.sdk.auth.accessControl.DisposableTokenScopes;
 import momento.sdk.auth.accessControl.ExpiresIn;
+import momento.sdk.cache.BaseCacheTestClass;
 import momento.sdk.config.Configurations;
 import momento.sdk.exceptions.MomentoErrorCode;
 import momento.sdk.responses.auth.GenerateDisposableTokenResponse;

momento-sdk/src/intTest/java/momento/sdk/AuthClientTopicTests.java → momento-sdk/src/intTest/java/momento/sdk/auth/AuthClientTopicTests.java

@@ -1,13 +1,15 @@
-package momento.sdk;
+package momento.sdk.auth;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
+import momento.sdk.AuthClient;
 import momento.sdk.auth.accessControl.CacheSelector;
 import momento.sdk.auth.accessControl.DisposableTokenScopes;
 import momento.sdk.auth.accessControl.ExpiresIn;
 import momento.sdk.auth.accessControl.TopicSelector;
+import momento.sdk.cache.BaseCacheTestClass;
 import momento.sdk.exceptions.MomentoErrorCode;
 import momento.sdk.responses.auth.GenerateDisposableTokenResponse;
 import org.junit.jupiter.api.BeforeAll;

momento-sdk/src/intTest/java/momento/sdk/BaseCacheTestClass.java → momento-sdk/src/intTest/java/momento/sdk/cache/BaseCacheTestClass.java

@@ -1,9 +1,12 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import java.time.Duration;
 import java.util.UUID;
+import momento.sdk.CacheClient;
 import momento.sdk.auth.CredentialProvider;
+import momento.sdk.config.Configuration;
 import momento.sdk.config.Configurations;
+import momento.sdk.config.ReadConcern;
 import momento.sdk.responses.cache.control.CacheCreateResponse;
 import momento.sdk.responses.cache.control.CacheDeleteResponse;
 import org.junit.jupiter.api.AfterAll;
@@ -19,10 +22,18 @@ public class BaseCacheTestClass {
 
   @BeforeAll
   static void beforeAll() {
+    final boolean consistentReads = System.getenv("CONSISTENT_READS") != null;
+
     credentialProvider = CredentialProvider.fromEnvVar("MOMENTO_API_KEY");
+
+    final Configuration config = Configurations.Laptop.latest();
+    final Configuration consistentConfig = config.withReadConcern(ReadConcern.CONSISTENT);
+
     cacheClient =
-        CacheClient.builder(credentialProvider, Configurations.Laptop.latest(), DEFAULT_TTL_SECONDS)
-            .build();
+        consistentReads
+            ? CacheClient.builder(credentialProvider, consistentConfig, DEFAULT_TTL_SECONDS).build()
+            : CacheClient.builder(credentialProvider, config, DEFAULT_TTL_SECONDS).build();
+
     cacheName = testCacheName();
     ensureTestCacheExists(cacheName);
   }
@@ -52,8 +63,4 @@ public static void cleanupTestCache(String cacheName) {
   public static String testCacheName() {
     return "java-integration-test-default-" + UUID.randomUUID();
   }
-
-  public static String testStoreName() {
-    return "java-integration-test-default-" + UUID.randomUUID();
-  }
 }

momento-sdk/src/intTest/java/momento/sdk/CacheClientTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/CacheClientTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomBytes;
 import static momento.sdk.TestUtils.randomString;
@@ -12,6 +12,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import momento.sdk.CacheClient;
 import momento.sdk.auth.CredentialProvider;
 import momento.sdk.auth.StringCredentialProvider;
 import momento.sdk.config.Configuration;

momento-sdk/src/intTest/java/momento/sdk/CacheControlPlaneTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/CacheControlPlaneTest.java

@@ -1,10 +1,11 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 import java.time.Duration;
+import momento.sdk.CacheClient;
 import momento.sdk.auth.CredentialProvider;
 import momento.sdk.config.Configurations;
 import momento.sdk.exceptions.AuthenticationException;
@@ -199,17 +200,4 @@ public void throwsInvalidArgumentForNegativeRequestTimeout() {
                         DEFAULT_TTL_SECONDS)
                     .build());
   }
-
-  @Test
-  public void throwsInvalidArgumentForNullRequestTimeout() {
-    //noinspection resource
-    assertThatExceptionOfType(InvalidArgumentException.class)
-        .isThrownBy(
-            () ->
-                CacheClient.builder(
-                        credentialProvider,
-                        Configurations.Laptop.latest().withTimeout(null),
-                        DEFAULT_TTL_SECONDS)
-                    .build());
-  }
 }

momento-sdk/src/intTest/java/momento/sdk/CacheDataPlaneClientSideTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/CacheDataPlaneClientSideTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static org.assertj.core.api.Assertions.assertThat;
 

momento-sdk/src/intTest/java/momento/sdk/CacheDataPlaneEagerConnectionTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/CacheDataPlaneEagerConnectionTest.java

@@ -1,9 +1,10 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.time.Duration;
+import momento.sdk.CacheClient;
 import momento.sdk.config.Configurations;
 import momento.sdk.responses.cache.GetResponse;
 import momento.sdk.responses.cache.SetResponse;

momento-sdk/src/intTest/java/momento/sdk/CacheDataPlaneTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/CacheDataPlaneTest.java

@@ -1,9 +1,10 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.time.Duration;
+import momento.sdk.CacheClient;
 import momento.sdk.auth.CredentialProvider;
 import momento.sdk.auth.StringCredentialProvider;
 import momento.sdk.config.Configurations;

momento-sdk/src/intTest/java/momento/sdk/DictionaryTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/DictionaryTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;

momento-sdk/src/intTest/java/momento/sdk/ListTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/ListTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomBytes;
 import static momento.sdk.TestUtils.randomString;

momento-sdk/src/intTest/java/momento/sdk/SetTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/SetTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;

momento-sdk/src/intTest/java/momento/sdk/SortedSetTest.java → momento-sdk/src/intTest/java/momento/sdk/cache/SortedSetTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.cache;
 
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;

momento-sdk/src/intTest/java/momento/sdk/BaseStorageTestClass.java → momento-sdk/src/intTest/java/momento/sdk/storage/BaseStorageTestClass.java

@@ -1,7 +1,8 @@
-package momento.sdk;
+package momento.sdk.storage;
 
 import java.time.Duration;
 import java.util.UUID;
+import momento.sdk.PreviewStorageClient;
 import momento.sdk.auth.CredentialProvider;
 import momento.sdk.config.StorageConfigurations;
 import momento.sdk.responses.storage.CreateStoreResponse;
@@ -19,10 +20,7 @@ public class BaseStorageTestClass {
   static void beforeAll() {
     credentialProvider = CredentialProvider.fromEnvVar("MOMENTO_API_KEY");
     storageClient =
-        new PreviewStorageClientBuilder()
-            .withCredentialProvider(credentialProvider)
-            .withConfiguration(StorageConfigurations.Laptop.latest())
-            .build();
+        new PreviewStorageClient(credentialProvider, StorageConfigurations.Laptop.latest());
     storeName = testStoreName();
     ensureTestStoreExists(storeName);
   }

momento-sdk/src/intTest/java/momento/sdk/storage/ControlTests.java

@@ -3,7 +3,6 @@
 import static momento.sdk.TestUtils.randomString;
 import static org.assertj.core.api.Assertions.assertThat;
 
-import momento.sdk.BaseStorageTestClass;
 import momento.sdk.PreviewStorageClient;
 import momento.sdk.auth.CredentialProvider;
 import momento.sdk.config.StorageConfigurations;

momento-sdk/src/intTest/java/momento/sdk/storage/DataTests.java

@@ -4,7 +4,6 @@
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import momento.sdk.BaseStorageTestClass;
 import momento.sdk.exceptions.ClientSdkException;
 import momento.sdk.exceptions.StoreNotFoundException;
 import momento.sdk.responses.storage.DeleteResponse;

momento-sdk/src/intTest/java/momento/sdk/TopicClientTest.java → momento-sdk/src/intTest/java/momento/sdk/topics/TopicClientTest.java

@@ -1,4 +1,4 @@
-package momento.sdk;
+package momento.sdk.topics;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -7,6 +7,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import momento.sdk.ISubscriptionCallbacks;
+import momento.sdk.TopicClient;
+import momento.sdk.cache.BaseCacheTestClass;
 import momento.sdk.config.TopicConfigurations;
 import momento.sdk.exceptions.MomentoErrorCode;
 import momento.sdk.responses.topic.TopicDiscontinuity;
@@ -23,7 +26,7 @@ public class TopicClientTest extends BaseCacheTestClass {
   private static TopicClient topicClient;
 
   private final String topicName = "test-topic";
-  private final Logger logger = LoggerFactory.getLogger(SubscriptionWrapper.class);
+  private final Logger logger = LoggerFactory.getLogger(TopicClientTest.class);
 
   private final List<String> receivedStringValues = new ArrayList<>();
   private final List<byte[]> receivedByteArrayValues = new ArrayList<>();