From 79aa2cd75673a6928ad64f9e0ec2b701d0052f25 Mon Sep 17 00:00:00 2001 From: Alsia Plybeah Date: Fri, 26 Jul 2024 16:05:54 -0400 Subject: [PATCH] [Task] Update analytics db to use local.env and Remove Dynaconf from Analytics (#136) ## Summary Fixes #107 Fixes #115 ### Time to review: __5 mins__ ## Changes proposed * removed the `*.toml` files related to dynaconf * removed references to Dynaconf (e.g. in Docstrings, gitignore) * use Pydantic for loading ## Context for reviewers > After getting feedback for https://github.com/navapbc/simpler-grants-gov/issues/107, the consensus was to reevaluate the way the database loader works for more uniformity. Changes will need to be made primarily in db.py and cli.py > > With the PR https://github.com/navapbc/simpler-grants-gov/pull/84 , the env settings for the db are stored in settings.toml. The config settings should be updated to use the existing local.env file ## Additional information > Screenshots, GIF demos, code examples or output to help show the changes working as expected. --- analytics/.gitignore | 2 - analytics/config.py | 48 ++++++++-------------- analytics/local.env | 10 +++++ analytics/poetry.lock | 41 ++++++++++++++++-- analytics/pyproject.toml | 1 + analytics/settings.toml | 4 -- analytics/src/analytics/cli.py | 4 +- analytics/src/analytics/integrations/db.py | 13 +++--- analytics/tests/integrations/test_slack.py | 7 +++- 9 files changed, 81 insertions(+), 49 deletions(-) delete mode 100644 analytics/settings.toml diff --git a/analytics/.gitignore b/analytics/.gitignore index e2a66772c..9e67bd47d 100644 --- a/analytics/.gitignore +++ b/analytics/.gitignore @@ -1,4 +1,2 @@ data -# Ignore dynaconf secret files -.secrets.* diff --git a/analytics/config.py b/analytics/config.py index 60a7786b6..7bc91adc3 100644 --- a/analytics/config.py +++ b/analytics/config.py @@ -1,35 +1,23 @@ -"""Loads configuration variables from settings files and settings files +"""Loads configuration variables from settings files -Dynaconf provides a few valuable features for configuration management: -- Load variables from env vars and files with predictable overrides -- Validate the existence and format of required configs -- Connect with secrets managers like HashiCorp's Vault server -- Load different configs based on environment (e.g. DEV, PROD, STAGING) - -For more information visit: https://www.dynaconf.com/ """ +import os +from typing import Optional +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import Field -from dynaconf import Dynaconf, Validator, ValidationError +# reads environment variables from .env files defaulting to "local.env" +class PydanticBaseEnvConfig(BaseSettings): + model_config = SettingsConfigDict(env_file="%s.env" % os.getenv("ENVIRONMENT", "local"), extra="ignore") # set extra to ignore so that it ignores variables irrelevant to the database config (e.g. metabase settings) -settings = Dynaconf( - # set env vars with `export ANALYTICS_FOO=bar` - envvar_prefix="ANALYTICS", - # looks for config vars in the following files - # with vars in .secrets.toml overriding vars in settings.toml - settings_files=["settings.toml", ".secrets.toml"], - # merge the settings found in all files - merge_enabled= True, - # add validators for our required config vars - validators=[ - Validator("SLACK_BOT_TOKEN", must_exist=True), - Validator("REPORTING_CHANNEL_ID", must_exist=True), - ], -) +class DBSettings(PydanticBaseEnvConfig): + db_host: str = Field(alias="DB_HOST") + port: int = Field(5432,alias="DB_PORT") + user: str = Field (alias="DB_USER") + password: str = Field(alias="DB_PASSWORD") + ssl_mode: str = Field(alias="DB_SSL_MODE") + slack_bot_token: str = Field(alias="ANALYTICS_SLACK_BOT_TOKEN") + reporting_channel_id: str = Field(alias="ANALYTICS_REPORTING_CHANNEL_ID") -# raises after all possible errors are evaluated -try: - settings.validators.validate_all() -except ValidationError as error: - list_of_all_errors = error.details - print(list_of_all_errors) - raise +def get_db_settings() -> DBSettings: + return DBSettings() \ No newline at end of file diff --git a/analytics/local.env b/analytics/local.env index ed6697f5c..f8e658c37 100644 --- a/analytics/local.env +++ b/analytics/local.env @@ -28,3 +28,13 @@ MB_DB_PORT=5432 MB_DB_USER=app MB_DB_PASS=secret123 MB_DB_HOST=grants-analytics-db + +########################### +# Slack Configuration # +########################### +# Do not add these values to this file +# to avoid mistakenly committing them. +# Set these in your shell +# by doing `export ANALYTICS_REPORTING_CHANNEL_ID=whatever` +ANALYTICS_REPORTING_CHANNEL_ID=DO_NOT_SET_HERE +ANALYTICS_SLACK_BOT_TOKEN=DO_NOT_SET_HERE \ No newline at end of file diff --git a/analytics/poetry.lock b/analytics/poetry.lock index 3029d81fe..3b24238a9 100644 --- a/analytics/poetry.lock +++ b/analytics/poetry.lock @@ -276,13 +276,13 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -2104,6 +2104,25 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pydantic-settings" +version = "2.3.4" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.3.4-py3-none-any.whl", hash = "sha256:11ad8bacb68a045f00e4f862c7a718c8a9ec766aa8fd4c32e39a0594b207b53a"}, + {file = "pydantic_settings-2.3.4.tar.gz", hash = "sha256:c5802e3d62b78e82522319bbc9b8f8ffb28ad1c988a99311d04f2a6051fca0a7"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" + +[package.extras] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "pygments" version = "2.18.0" @@ -2197,6 +2216,20 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-json-logger" version = "2.0.7" @@ -3183,4 +3216,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "5cfdff4542a0784685c9464acce097c7fb579ddea7bfe49946b62c7c512144ee" +content-hash = "0a82827fbe80ff77cf44f3b02808a828b311bae027b6955afbd774e426e15063" diff --git a/analytics/pyproject.toml b/analytics/pyproject.toml index 0972263d4..85f0edbec 100644 --- a/analytics/pyproject.toml +++ b/analytics/pyproject.toml @@ -22,6 +22,7 @@ slack-sdk = "^3.23.0" typer = { extras = ["all"], version = "^0.9.0" } sqlalchemy = "^2.0.30" psycopg = ">=3.0.7" +pydantic-settings = "^2.3.4" [tool.poetry.group.dev.dependencies] black = "^23.7.0" diff --git a/analytics/settings.toml b/analytics/settings.toml deleted file mode 100644 index 7ff7221cc..000000000 --- a/analytics/settings.toml +++ /dev/null @@ -1,4 +0,0 @@ -POSTGRES_NAME = "app" -POSTGRES_HOST = "grants-analytics-db" -POSTGRES_USER = "app" -POSTGRES_PORT = 5432 \ No newline at end of file diff --git a/analytics/src/analytics/cli.py b/analytics/src/analytics/cli.py index 60b482160..68d49d48e 100644 --- a/analytics/src/analytics/cli.py +++ b/analytics/src/analytics/cli.py @@ -241,7 +241,9 @@ def show_and_or_post_results( """Optionally show the results of a metric and/or post them to slack.""" # defer load of settings until this command is called # this prevents an error if ANALYTICS_SLACK_BOT_TOKEN env var is unset - from config import settings + from config import get_db_settings + + settings = get_db_settings() # optionally display the burndown chart in the browser if show_results: diff --git a/analytics/src/analytics/integrations/db.py b/analytics/src/analytics/integrations/db.py index 6f6ef1155..89bdeaa09 100644 --- a/analytics/src/analytics/integrations/db.py +++ b/analytics/src/analytics/integrations/db.py @@ -3,13 +3,12 @@ from sqlalchemy import Engine, create_engine -from config import settings +from config import get_db_settings + +# The variables used in the connection url are pulled from local.env +# and configured in the DBSettings class found in config.py -# The variables used in the connection url are set in settings.toml and -# .secrets.toml. These can be overridden with the custom prefix defined in config.py: "ANALYTICS". -# e.g. `export ANALYTICS_POSTGRES_USER=new_usr`. -# Docs: https://www.dynaconf.com/envvars/ def get_db() -> Engine: """ Get a connection to the database using a SQLAlchemy engine object. @@ -22,8 +21,10 @@ def get_db() -> Engine: sqlalchemy.engine.Engine A SQLAlchemy engine object representing the connection to the database. """ + db = get_db_settings() + print(f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}") return create_engine( - f"postgresql+psycopg://{settings.postgres_user}:{settings.postgres_password}@{settings.postgres_host}:{settings.postgres_port}", + f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}", pool_pre_ping=True, hide_parameters=True, ) diff --git a/analytics/tests/integrations/test_slack.py b/analytics/tests/integrations/test_slack.py index 6caef5c7a..32ec6b6bc 100644 --- a/analytics/tests/integrations/test_slack.py +++ b/analytics/tests/integrations/test_slack.py @@ -5,8 +5,9 @@ from slack_sdk import WebClient from analytics.integrations.slack import FileMapping, SlackBot -from config import settings +from config import get_db_settings +settings = get_db_settings() client = WebClient(token=settings.slack_bot_token) @@ -19,7 +20,9 @@ def mock_slackbot() -> SlackBot: @pytest.mark.skip(reason="requires Slack token") def test_fetch_slack_channels(slackbot: SlackBot): """The fetch_slack_channels() function should execute correctly.""" - result = slackbot.fetch_slack_channel_info(channel_id=settings.reporting_channel_id) + result = slackbot.fetch_slack_channel_info( + channel_id=settings.reporting_channel_id, + ) assert result["ok"] is True assert result["channel"]["name"] == "z_bot-analytics-ci-test"