Add make sprint-data-import and issue-data-import to import github sprint and issue data to database

.grype.yml

@@ -16,11 +16,11 @@ ignore:
   - fix-state: not-fixed
   - fix-state: wont-fix
   - fix-state: unknown
-  # Golang vulnerability inside of a python docker image. It's basically impossible to find
-  # out where the impacted golang vulnerability is coming from. The python image in question
-  # does not even have golang installed, yet somehow still there's a golang vulnerability.
-  # We are ignoring the finding because it would take undue effort to track down the source.
+  # Golang vulnerabilities inside of a python docker image. Both originate from lower level packages within the GitHub CLI:
+  # https://github.com/cli/cli/blob/trunk/go.mod#L101 
+  # https://github.com/cli/cli/blob/trunk/go.mod#L161
   - vulnerability: GHSA-4v7x-pqxf-cx7m
+  - vulnerability: GHSA-v6v8-xj6m-xwqh
   # https://github.com/anchore/grype/issues/1172
   - vulnerability: GHSA-xqr8-7jwr-rhp7
   - vulnerability: GHSA-7fh5-64p2-3v2j

analytics/Makefile

@@ -146,6 +146,15 @@ sprint-data-export:
 	--project $(SPRINT_PROJECT) \
 	--output-file $(SPRINT_FILE)
 
+# Import file data to database
+# Takes sprint and issue and puts it in database
+sprint-db-data-import:
+	@echo "=> Importing project data to the database"
+	@echo "====================================================="
+	$(POETRY) analytics export db_export \
+	--sprint-file $(SPRINT_FILE) \
+	--issue-file $(ISSUE_FILE)
+
 roadmap-data-export:
 	@echo "=> Exporting project data from the product roadmap"
 	@echo "====================================================="

analytics/pyproject.toml

@@ -21,6 +21,7 @@ python = "^3.11"
 slack-sdk = "^3.23.0"
 typer = { extras = ["all"], version = "^0.9.0" }
 sqlalchemy = "^2.0.30"
+psycopg = ">=3.0.7"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"

analytics/settings.toml

@@ -1,4 +1,4 @@
 POSTGRES_NAME = "app"
-POSTGRES_HOST = "0.0.0.0"
+POSTGRES_HOST = "grants-analytics-db"
 POSTGRES_USER = "app"
 POSTGRES_PORT = 5432

analytics/src/analytics/cli.py

@@ -149,6 +149,28 @@ def test_connection() -> None:
     result.close()
 
 
+@export_app.command(name="db_export")
+def export_json_to_database(
+    sprint_file: Annotated[str, SPRINT_FILE_ARG],
+    issue_file: Annotated[str, ISSUE_FILE_ARG],
+) -> None:
+    """Export JSON data to the database."""
+    # Get the database engine and establish a connection
+    engine = db.get_db()
+
+    # get data and load from JSON
+    deliverable_data = DeliverableTasks.load_from_json_files(
+        sprint_file=sprint_file,
+        issue_file=issue_file,
+    )
+
+    deliverable_data.to_sql(
+        output_table="github_project_data",
+        engine=engine,
+        replace_table=True,
+    )  # replace_table=True is the default
+
+
 @metrics_app.command(name="deliverable_percent_complete")
 def calculate_deliverable_percent_complete(
     sprint_file: Annotated[str, SPRINT_FILE_ARG],

analytics/tests/integrations/test_slack.py

@@ -16,13 +16,15 @@ def mock_slackbot() -> SlackBot:
     return SlackBot(client=client)
 
 
+@pytest.mark.skip(reason="requires Slack token")
 def test_fetch_slack_channels(slackbot: SlackBot):
     """The fetch_slack_channels() function should execute correctly."""
     result = slackbot.fetch_slack_channel_info(channel_id=settings.reporting_channel_id)
     assert result["ok"] is True
     assert result["channel"]["name"] == "z_bot-analytics-ci-test"
 
 
+@pytest.mark.skip(reason="requires Slack token")
 def test_upload_files_to_slack_channel(slackbot: SlackBot):
     """The upload_files_to_slack_channel() function should execute correctly."""
     # setup - create test files to upload