Merge pull request #4434 from cfpb/master-after-filing

Master after filing
cfpb · Apr 27, 2022 · d0dd987 · d0dd987
2 parents 1b2b76e + a0fda34
commit d0dd987
Show file tree

Hide file tree

Showing 26 changed files with 875 additions and 181 deletions.
diff --git a/UPGRADES.md b/UPGRADES.md
@@ -0,0 +1,82 @@
+# Upgrades
+
+## Akka Cassandra Persistence Plugin Upgrade
+Migration Documentation: https://doc.akka.io/docs/akka-persistence-cassandra/current/migrations.html
+
+##### new table needed for upgrading to 1.0+
+```sql
+CREATE TABLE IF NOT EXISTS akka.all_persistence_ids(
+  persistence_id text PRIMARY KEY);
+```
+
+new environment variable to put into `cassandra-configmap`: `cassandra-cluster-dc` (e.g.: `dc` / `dc2`)
+
+### Migration
+newly added table needs to be populated, refer to [Migration Documentation](https://doc.akka.io/docs/akka-persistence-cassandra/current/migrations.html) in [plugin upgrade](#akka-cassandra-persistence-plugin-upgrade) section,
+code snippet for migration tool:
+```scala
+import scala.util.Failure
+import scala.util.Success
+
+import akka.actor.ActorSystem
+import akka.persistence.cassandra.reconciler.Reconciliation
+
+// System should have the same Cassandra plugin configuration as your application
+// but be careful to remove seed nodes so this doesn't join the cluster
+val system = ActorSystem()
+import system.dispatcher
+
+val rec = new Reconciliation(system)
+val result = rec.rebuildAllPersistenceIds()
+
+result.onComplete {
+  case Success(_) =>
+    system.log.info("All persistenceIds migrated.")
+    system.terminate()
+  case Failure(e) =>
+    system.log.error(e, "All persistenceIds migration failed.")
+    system.terminate()
+}
+```
+when running this migration code, make sure to adjust the datastax driver configurations to allow logging,
+so it is visible if errors occur, or if the process stalled,
+and to increase request timeout.
+```HOCON
+datastax-java-driver {
+  basic {
+    contact-points = ["localhost:9042"]
+    contact-points = [${?CASSANDRA_CLUSTER_HOSTS}":9042"]
+    load-balancing-policy.local-datacenter = "datacenter1"
+    load-balancing-policy.local-datacenter = ${?CASSANDRA_CLUSTER_DC}
+    request {
+      timeout = 30 seconds
+    }
+  }
+  advanced {
+    connection {
+      init-query-timeout = 30 seconds
+    }
+    auth-provider {
+      class = PlainTextAuthProvider
+      username = ""
+      username = ${?CASSANDRA_CLUSTER_USERNAME}
+      password = ""
+      password = ${?CASSANDRA_CLUSTER_PASSWORD}
+    }
+    request-tracker {
+      class = RequestLogger
+      logs {
+        success.enabled = true
+        error.enabled = true
+        show-values = false
+        slow {
+          threshold = 1 second
+          enabled = true
+        }
+      }
+    }
+  }
+}
+```
+Lightbend do not have performance figures, but they have reported the longest customer reported migration took days.
+Our scenario should take 10 ~ 20 minutes.
diff --git a/common/src/test/scala/hmda/serialization/projection/ProjectionProtobufConverterSpec.scala b/common/src/test/scala/hmda/serialization/projection/ProjectionProtobufConverterSpec.scala
@@ -5,14 +5,10 @@ import akka.actor.testkit.typed.scaladsl.TestProbe
 import akka.actor.typed.ActorRefResolver
 import akka.actor.typed.scaladsl.adapter._
 import akka.persistence.query.TimeBasedUUID
-import com.datastax.driver.core.utils.UUIDs
-import hmda.messages.projection.CommonProjectionMessages.{
-  GetOffset,
-  OffsetSaved,
-  SaveOffset
-}
+import com.datastax.oss.driver.api.core.uuid.Uuids
+import hmda.messages.projection.CommonProjectionMessages.{ GetOffset, OffsetSaved, SaveOffset }
 import hmda.serialization.projection.ProjectionProtobufConverter._
-import org.scalatest.{BeforeAndAfterAll, MustMatchers, WordSpec}
+import org.scalatest.{ BeforeAndAfterAll, MustMatchers, WordSpec }
 
 class ProjectionProtobufConverterSpec
     extends WordSpec
@@ -28,7 +24,7 @@ class ProjectionProtobufConverterSpec
   private val resolver = ActorRefResolver(systemTyped)
 
   "Projection Protobuf Converter" must {
-    val uuid = UUIDs.timeBased()
+    val uuid = Uuids.timeBased()
     val offset = TimeBasedUUID(uuid)
     val probe = TestProbe[OffsetSaved](name = "projection-command")
     "convert SaveOffset to and from protobuf" in {
@@ -44,7 +40,7 @@ class ProjectionProtobufConverterSpec
     }
 
     "convert OffsetSaved to and from protobuf" in {
-      val uuid = UUIDs.timeBased()
+      val uuid = Uuids.timeBased()
       val offset = TimeBasedUUID(uuid)
       val saved = OffsetSaved(offset)
       val protobuf = offsetSavedToProtobuf(saved)

diff --git a/hmda-dashboard/src/main/resources/application.conf b/hmda-dashboard/src/main/resources/application.conf
@@ -24,6 +24,12 @@ hmda {
   runtime.mode = "dev"
   runtime.mode = ${?HMDA_RUNTIME_MODE}
   runtime.mode = "kubernetes"
+  sources {
+    yearly = "2018,2019,2020,2021"
+    yearly = ${?AVAILABLE_YEARLY}
+    quarterly = "2020-Q1,2020-Q2,2020-Q3,2021-Q1,2021-Q2,2021-Q3,2022-Q1,2022-Q2,2022-Q3"
+    quarterly = ${?AVAILABLE_QUARTERLY}
+  }
 }
 
 dashboard {