[Issue #2976] Run analytics migrations like the API does them

.github/workflows/ci-analytics.yml

@@ -39,8 +39,8 @@ jobs:
       - name: Run linting
         run: make lint
 
-      - name: Run database initialization
-        run: docker compose down --volumes && make init-db
+      - name: Run database migrations
+        run: docker compose down --volumes && make db-migrate
 
       - name: Run tests
         run: make test-audit

analytics/Makefile

@@ -144,10 +144,10 @@ lint: ## runs code quality checks
 # Data Commands #
 #################
 
-init-db:
-	@echo "=> Initializing the database schema"
+db-migrate:
+	@echo "=> Migrating the database schema"
 	@echo "====================================================="
-	$(POETRY) analytics etl initialize_database
+	$(POETRY) analytics etl db_migrate
 	@echo "====================================================="
 
 gh-transform-and-load:

analytics/pyproject.toml

@@ -8,11 +8,12 @@ version = "0.1.0"
 
 [tool.poetry.scripts]
 analytics = "analytics.cli:app"
+db-migrate = "src.analytics.cli:db_migrate"
 
 [tool.poetry.dependencies]
 dynaconf = "^3.2.4"
 kaleido = "0.2.1"
-notebook = "^7.0.0"                              # Goal is to replace this with another method of presenting charts
+notebook = "^7.0.0"                               # Goal is to replace this with another method of presenting charts
 pandas = "^2.0.3"
 pandas-stubs = "^2.0.2.230605"
 plotly = "^5.15.0"

analytics/src/analytics/cli.py

@@ -264,12 +264,12 @@ def export_json_to_database(delivery_file: Annotated[str, ISSUE_FILE_ARG]) -> No
 # ===========================================================
 
 
-@etl_app.command(name="initialize_database")
-@ecs_background_task("initialize_database")
-def initialize_database() -> None:
+@etl_app.command(name="db_migrate")
+@ecs_background_task("db_migrate")
+def migrate_database() -> None:
     """Initialize etl database."""
     logger.info("initializing database")
-    etldb.initialize_database()
+    etldb.migrate_database()
     logger.info("done")
 
 

analytics/src/analytics/integrations/etldb/__init__.py

@@ -1,11 +1,11 @@
 """Read and write data from/to delivery metrics database."""
 
 __all__ = [
-    "initialize_database",
+    "migrate_database",
     "sync_data",
 ]
 
 from analytics.integrations.etldb.main import (
-    initialize_database,
+    migrate_database,
     sync_data,
 )

analytics/src/analytics/integrations/etldb/main.py

@@ -18,7 +18,7 @@
 VERBOSE = False
 
 
-def initialize_database() -> None:
+def migrate_database() -> None:
     """
     Create and/or update an etl database by applying a sequential set of migration scripts.
 

analytics/tests/test_cli.py

@@ -285,7 +285,7 @@ def test_init_db(self):
         # setup - create command
         command = [
             "etl",
-            "initialize_database",
+            "db_migrate",
         ]
         # execution
         result = runner.invoke(app, command)

infra/analytics/app-config/env-config/scheduled_jobs.tf

@@ -11,10 +11,5 @@ locals {
       schedule_expression = "rate(1 days)"
       state               = "ENABLED"
     }
-    init-etldb = {
-      task_command        = ["make", "init-db"]
-      schedule_expression = "rate(1 days)"
-      state               = "ENABLED"
-    }
   }
 }

infra/analytics/app-config/outputs.tf

@@ -18,10 +18,6 @@ output "environment_configs" {
   value = local.environment_configs
 }
 
-# This variable is slightly misnamed. It should really be called "has_migrations".
-# It controls whether or not the `run-database-migrations.sh` script tries to run database
-# migrations. The entire analytics application is going to have its schema controlled
-# via ETL jobs, so we don't need to run migrations in the same way as the API.
 output "has_database" {
-  value = false
+  value = true
 }