[Task] Update analytics db to use local.env and Remove Dynaconf from Analytics

analytics/.gitignore

@@ -1,4 +1,2 @@
 data
 
-# Ignore dynaconf secret files
-.secrets.*

analytics/config.py

@@ -1,35 +1,19 @@
-"""Loads configuration variables from settings files and settings files
+"""Loads configuration variables from settings files
 
-Dynaconf provides a few valuable features for configuration management:
-- Load variables from env vars and files with predictable overrides
-- Validate the existence and format of required configs
-- Connect with secrets managers like HashiCorp's Vault server
-- Load different configs based on environment (e.g. DEV, PROD, STAGING)
-
-For more information visit: https://www.dynaconf.com/
 """
+import os 
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import Field
 
-from dynaconf import Dynaconf, Validator, ValidationError
-
-settings = Dynaconf(
-    # set env vars with `export ANALYTICS_FOO=bar`
-    envvar_prefix="ANALYTICS",
-    # looks for config vars in the following files
-    # with vars in .secrets.toml overriding vars in settings.toml
-    settings_files=["settings.toml", ".secrets.toml"],
-    # merge the settings found in all files
-    merge_enabled= True,
-    # add validators for our required config vars
-    validators=[
-        Validator("SLACK_BOT_TOKEN", must_exist=True),
-        Validator("REPORTING_CHANNEL_ID", must_exist=True),
-    ],
-)
+# reads environment variables from .env files defaulting to "local.env"
+class PydanticBaseEnvConfig(BaseSettings):
+    model_config = SettingsConfigDict(env_file="%s.env" % os.getenv("ENVIRONMENT", "local"), extra="ignore") # set extra to ignore so that it ignores variables irrelevant to the database config (e.g. metabase settings)
 
-# raises after all possible errors are evaluated
-try:
-    settings.validators.validate_all()
-except ValidationError as error:
-    list_of_all_errors = error.details
-    print(list_of_all_errors)
-    raise
+class DBSettings(PydanticBaseEnvConfig):
+     db_host: str = Field(alias="DB_HOST")
+     port: int = Field(5432,alias="DB_PORT")
+     user: str = Field (alias="DB_USER")
+     password: str = Field(alias="DB_PASSWORD")
+     ssl_mode: str = Field(alias="DB_SSL_MODE")
+     slack_bot_token: str = Field(alias="SLACK_BOT_TOKEN")
+     reporting_channel_id: str = Field(alias="REPORTING_CHANNEL_ID")

analytics/pyproject.toml

@@ -22,6 +22,7 @@ slack-sdk = "^3.23.0"
 typer = { extras = ["all"], version = "^0.9.0" }
 sqlalchemy = "^2.0.30"
 psycopg = ">=3.0.7"
+pydantic-settings = "^2.3.4"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"

analytics/src/analytics/cli.py

@@ -241,17 +241,17 @@ def show_and_or_post_results(
     """Optionally show the results of a metric and/or post them to slack."""
     # defer load of settings until this command is called
     # this prevents an error if ANALYTICS_SLACK_BOT_TOKEN env var is unset
-    from config import settings
+    from config import DBSettings
 
     # optionally display the burndown chart in the browser
     if show_results:
         metric.show_chart()
         print("Slack message:\n")
         print(metric.format_slack_message())
     if post_results:
-        slackbot = slack.SlackBot(client=WebClient(token=settings.slack_bot_token))
+        slackbot = slack.SlackBot(client=WebClient(token=DBSettings.slack_bot_token))
         metric.post_results_to_slack(
             slackbot=slackbot,
-            channel_id=settings.reporting_channel_id,
+            channel_id=DBSettings.reporting_channel_id,
             output_dir=Path(output_dir),
         )

analytics/src/analytics/integrations/db.py

@@ -3,13 +3,12 @@
 
 from sqlalchemy import Engine, create_engine
 
-from config import settings
+from config import DBSettings
+
+# The variables used in the connection url are pulled from local.env
+# and configured in the DBSettings class found in config.py
 
 
-# The variables used in the connection url are set in settings.toml and
-# .secrets.toml. These can be overridden with the custom prefix defined in config.py: "ANALYTICS".
-# e.g. `export ANALYTICS_POSTGRES_USER=new_usr`.
-# Docs: https://www.dynaconf.com/envvars/
 def get_db() -> Engine:
     """
     Get a connection to the database using a SQLAlchemy engine object.
@@ -22,8 +21,10 @@ def get_db() -> Engine:
     sqlalchemy.engine.Engine
     A SQLAlchemy engine object representing the connection to the database.
     """
+    db = DBSettings
+    print(f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}")
     return create_engine(
-        f"postgresql+psycopg://{settings.postgres_user}:{settings.postgres_password}@{settings.postgres_host}:{settings.postgres_port}",
+        f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}",
         pool_pre_ping=True,
         hide_parameters=True,
     )

analytics/tests/integrations/test_slack.py

@@ -5,9 +5,9 @@
 from slack_sdk import WebClient
 
 from analytics.integrations.slack import FileMapping, SlackBot
-from config import settings
+from config import DBSettings
 
-client = WebClient(token=settings.slack_bot_token)
+client = WebClient(token=DBSettings.slack_bot_token)
 
 
 @pytest.fixture(name="slackbot")
@@ -19,7 +19,9 @@ def mock_slackbot() -> SlackBot:
 @pytest.mark.skip(reason="requires Slack token")
 def test_fetch_slack_channels(slackbot: SlackBot):
     """The fetch_slack_channels() function should execute correctly."""
-    result = slackbot.fetch_slack_channel_info(channel_id=settings.reporting_channel_id)
+    result = slackbot.fetch_slack_channel_info(
+        channel_id=DBSettings.reporting_channel_id,
+    )
     assert result["ok"] is True
     assert result["channel"]["name"] == "z_bot-analytics-ci-test"
 
@@ -47,7 +49,7 @@ def test_upload_files_to_slack_channel(slackbot: SlackBot):
 """
     result = slackbot.upload_files_to_slack_channel(
         files=files,
-        channel_id=settings.reporting_channel_id,
+        channel_id=DBSettings.reporting_channel_id,
         message=markdown,
     )
     assert result["ok"] is True