[Issue #2050] Setup agency tables for transformation (navapbc#127)

Fixes #2050

Added agency tables

Added tgroups tables to the staging & foreign tables

Added factories / data setup for these tables

https://docs.google.com/document/d/1EPZJyqTQruq-BkQoojtrkLqgVl8GrpfsZAIY9T1dEg8/edit
provides a lot of rough details on how this data works in the existing
system

The next ticket will be to transform the data stored in tgroups into the
new agency table(s).

The agency table I created contains most of the fields from the legacy
system (in a normalized structure) including a few that I don't think we
quite need, but just in case they have a separate purpose than what I
can understand, I'm preferring to keep them.

---------

Co-authored-by: nava-platform-bot <[email protected]>

api/bin/create_erds.py

@@ -12,7 +12,7 @@
 import src.db.models.staging.opportunity as staging_opportunity_models
 import src.db.models.staging.synopsis as staging_synopsis_models
 import src.logging
-from src.db.models import opportunity_models
+from src.db.models import agency_models, opportunity_models
 from src.db.models.transfer import topportunity_models
 
 logger = logging.getLogger(__name__)
@@ -23,7 +23,10 @@
 ERD_FOLDER = pathlib.Path(__file__).parent.resolve()
 
 # If we want to generate separate files for more specific groups, we can set that up here
-API_MODULES = (opportunity_models,)
+API_MODULES = (
+    opportunity_models,
+    agency_models,
+)
 STAGING_TABLE_MODULES = (
     staging_opportunity_models,
     staging_forecast_models,

api/src/constants/lookup_constants.py

@@ -105,3 +105,14 @@ class FundingInstrument(StrEnum):
     GRANT = "grant"  # G
     PROCUREMENT_CONTRACT = "procurement_contract"  # PC
     OTHER = "other"  # O
+
+
+class AgencyDownloadFileType(StrEnum):
+    XML = "xml"
+    PDF = "pdf"
+
+
+class AgencySubmissionNotificationSetting(StrEnum):
+    NEVER = "never"
+    FIRST_APPLICATION_ONLY = "first_application_only"
+    ALWAYS = "always"

api/src/db/foreign/__init__.py

@@ -2,8 +2,8 @@
 # SQLAlchemy models for foreign tables.
 #
 
-from . import forecast, foreignbase, opportunity, synopsis
+from . import forecast, foreignbase, opportunity, synopsis, tgroups
 
 metadata = foreignbase.metadata
 
-__all__ = ["metadata", "forecast", "opportunity", "synopsis"]
+__all__ = ["metadata", "forecast", "opportunity", "synopsis", "tgroups"]

api/src/db/foreign/tgroups.py

@@ -0,0 +1,14 @@
+#
+# SQLAlchemy models for foreign tables.
+#
+# The order of the columns must match the remote Oracle database. The names are not required to
+# match by oracle_fdw, but we are matching them for maintainability.
+#
+
+from src.db.legacy_mixin import tgroups_mixin
+
+from . import foreignbase
+
+
+class Tgroups(foreignbase.ForeignBase, tgroups_mixin.TGroupsMixin):
+    __tablename__ = "tgroups"

api/src/db/legacy_mixin/tgroups_mixin.py

@@ -0,0 +1,20 @@
+#
+# SQLAlchemy models for foreign tables.
+#
+# The order of the columns must match the remote Oracle database. The names are not required to
+# match by oracle_fdw, but we are matching them for maintainability.
+#
+
+import datetime
+
+from sqlalchemy.orm import Mapped, declarative_mixin, mapped_column
+
+
+@declarative_mixin
+class TGroupsMixin:
+    keyfield: Mapped[str] = mapped_column(primary_key=True)
+    value: Mapped[str | None]
+    created_date: Mapped[datetime.datetime | None]
+    last_upd_date: Mapped[datetime.datetime | None]
+    creator_id: Mapped[str | None]
+    last_upd_id: Mapped[str | None]

api/src/db/migrations/versions/2024_07_08_add_agency_related_tables.py

@@ -0,0 +1,223 @@
+"""Add agency related tables
+
+Revision ID: 4f7acbb61548
+Revises: 61c58638e56b
+Create Date: 2024-07-08 12:43:45.240782
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4f7acbb61548"
+down_revision = "61c58638e56b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "agency_contact_info",
+        sa.Column("agency_contact_info_id", sa.BigInteger(), nullable=False),
+        sa.Column("contact_name", sa.Text(), nullable=False),
+        sa.Column("address_line_1", sa.Text(), nullable=False),
+        sa.Column("address_line_2", sa.Text(), nullable=True),
+        sa.Column("city", sa.Text(), nullable=False),
+        sa.Column("state", sa.Text(), nullable=False),
+        sa.Column("zip_code", sa.Text(), nullable=False),
+        sa.Column("phone_number", sa.Text(), nullable=False),
+        sa.Column("primary_email", sa.Text(), nullable=False),
+        sa.Column("secondary_email", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("agency_contact_info_id", name=op.f("agency_contact_info_pkey")),
+        schema="api",
+    )
+    op.create_table(
+        "lk_agency_download_file_type",
+        sa.Column("agency_download_file_type_id", sa.Integer(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint(
+            "agency_download_file_type_id", name=op.f("lk_agency_download_file_type_pkey")
+        ),
+        schema="api",
+    )
+    op.create_table(
+        "lk_agency_submission_notification_setting",
+        sa.Column("agency_submission_notification_setting_id", sa.Integer(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint(
+            "agency_submission_notification_setting_id",
+            name=op.f("lk_agency_submission_notification_setting_pkey"),
+        ),
+        schema="api",
+    )
+    op.create_table(
+        "agency",
+        sa.Column("agency_id", sa.BigInteger(), nullable=False),
+        sa.Column("agency_name", sa.Text(), nullable=False),
+        sa.Column("agency_code", sa.Text(), nullable=False),
+        sa.Column("sub_agency_code", sa.Text(), nullable=True),
+        sa.Column("assistance_listing_number", sa.Text(), nullable=False),
+        sa.Column("agency_submission_notification_setting_id", sa.Integer(), nullable=False),
+        sa.Column("agency_contact_info_id", sa.BigInteger(), nullable=True),
+        sa.Column("is_test_agency", sa.Boolean(), nullable=False),
+        sa.Column("ldap_group", sa.Text(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=False),
+        sa.Column("label", sa.Text(), nullable=False),
+        sa.Column("is_multilevel_agency", sa.Boolean(), nullable=False),
+        sa.Column("is_multiproject", sa.Boolean(), nullable=False),
+        sa.Column("has_system_to_system_certificate", sa.Boolean(), nullable=False),
+        sa.Column("can_view_packages_in_grace_period", sa.Boolean(), nullable=False),
+        sa.Column("is_image_workspace_enabled", sa.Boolean(), nullable=False),
+        sa.Column("is_validation_workspace_enabled", sa.Boolean(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["agency_contact_info_id"],
+            ["api.agency_contact_info.agency_contact_info_id"],
+            name=op.f("agency_agency_contact_info_id_agency_contact_info_fkey"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["agency_submission_notification_setting_id"],
+            [
+                "api.lk_agency_submission_notification_setting.agency_submission_notification_setting_id"
+            ],
+            name=op.f(
+                "agency_agency_submission_notification_setting_id_lk_agency_submission_notification_setting_fkey"
+            ),
+        ),
+        sa.PrimaryKeyConstraint("agency_id", name=op.f("agency_pkey")),
+        schema="api",
+    )
+    op.create_index(
+        op.f("agency_agency_code_idx"), "agency", ["agency_code"], unique=True, schema="api"
+    )
+    op.create_table(
+        "link_agency_download_file_type",
+        sa.Column("agency_id", sa.BigInteger(), nullable=False),
+        sa.Column("agency_download_file_type_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["agency_download_file_type_id"],
+            ["api.lk_agency_download_file_type.agency_download_file_type_id"],
+            name=op.f(
+                "link_agency_download_file_type_agency_download_file_type_id_lk_agency_download_file_type_fkey"
+            ),
+        ),
+        sa.ForeignKeyConstraint(
+            ["agency_id"],
+            ["api.agency.agency_id"],
+            name=op.f("link_agency_download_file_type_agency_id_agency_fkey"),
+        ),
+        sa.PrimaryKeyConstraint(
+            "agency_id",
+            "agency_download_file_type_id",
+            name=op.f("link_agency_download_file_type_pkey"),
+        ),
+        schema="api",
+    )
+    op.create_table(
+        "tgroups",
+        sa.Column("keyfield", sa.Text(), nullable=False),
+        sa.Column("value", sa.Text(), nullable=True),
+        sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("creator_id", sa.Text(), nullable=True),
+        sa.Column("last_upd_id", sa.Text(), nullable=True),
+        sa.Column("is_deleted", sa.Boolean(), nullable=False),
+        sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column("deleted_at", sa.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("transformation_notes", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("keyfield", name=op.f("tgroups_pkey")),
+        schema="staging",
+    )
+    op.create_index(
+        op.f("tgroups_transformed_at_idx"),
+        "tgroups",
+        ["transformed_at"],
+        unique=False,
+        schema="staging",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("tgroups_transformed_at_idx"), table_name="tgroups", schema="staging")
+    op.drop_table("tgroups", schema="staging")
+    op.drop_table("link_agency_download_file_type", schema="api")
+    op.drop_index(op.f("agency_agency_code_idx"), table_name="agency", schema="api")
+    op.drop_table("agency", schema="api")
+    op.drop_table("lk_agency_submission_notification_setting", schema="api")
+    op.drop_table("lk_agency_download_file_type", schema="api")
+    op.drop_table("agency_contact_info", schema="api")
+    # ### end Alembic commands ###

api/src/db/models/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-from . import base, lookup_models, opportunity_models
+from . import agency_models, base, lookup_models, opportunity_models
 from .transfer import topportunity_models
 
 logger = logging.getLogger(__name__)
@@ -9,4 +9,10 @@
 # This is used by tests to create the test database.
 metadata = base.metadata
 
-__all__ = ["metadata", "opportunity_models", "lookup_models", "topportunity_models"]
+__all__ = [
+    "metadata",
+    "opportunity_models",
+    "lookup_models",
+    "topportunity_models",
+    "agency_models",
+]