diff --git a/analytics/src/analytics/datasets/etl_dataset.py b/analytics/src/analytics/datasets/etl_dataset.py index c452cb2c8..fda83079e 100644 --- a/analytics/src/analytics/datasets/etl_dataset.py +++ b/analytics/src/analytics/datasets/etl_dataset.py @@ -21,8 +21,9 @@ class EtlEntityType(Enum): DELIVERABLE = "deliverable" EPIC = "epic" ISSUE = "issue" - SPRINT = "sprint" + PROJECT = "project" QUAD = "quad" + SPRINT = "sprint" class EtlDataset(BaseDataset): @@ -32,6 +33,7 @@ class EtlDataset(BaseDataset): "deliverable_url": "deliverable_ghid", "deliverable_title": "deliverable_title", "deliverable_pillar": "deliverable_pillar", + "deliverable_status": "deliverable_status", "epic_url": "epic_ghid", "epic_title": "epic_title", "issue_url": "issue_ghid", @@ -43,6 +45,8 @@ class EtlDataset(BaseDataset): "issue_closed_at": "issue_closed_at", "issue_points": "issue_points", "issue_status": "issue_status", + "project_owner": "project_name", + "project_number": "project_ghid", "sprint_id": "sprint_ghid", "sprint_name": "sprint_name", "sprint_start": "sprint_start", @@ -144,3 +148,15 @@ def get_issue_ghids(self) -> NDArray[Any]: """Fetch an array of unique non-null issue ghids.""" df = self.df[self.df.issue_ghid.notna()] return df.issue_ghid.unique() + + # PROJECT getters + + def get_project(self, project_ghid: int) -> pd.Series: + """Fetch data about a given project.""" + query_string = f"project_ghid == {project_ghid}" + return self.df.query(query_string).iloc[0] + + def get_project_ghids(self) -> NDArray[Any]: + """Fetch an array of unique non-null project ghids.""" + df = self.df[self.df.project_ghid.notna()] + return df.project_ghid.unique() diff --git a/analytics/src/analytics/integrations/etldb/deliverable_model.py b/analytics/src/analytics/integrations/etldb/deliverable_model.py index 0f0d8cd35..1916e86db 100644 --- a/analytics/src/analytics/integrations/etldb/deliverable_model.py +++ b/analytics/src/analytics/integrations/etldb/deliverable_model.py @@ -1,7 +1,9 @@ """Define EtlDeliverableModel class to encapsulate db CRUD operations.""" from pandas import Series +from psycopg.errors import InsufficientPrivilege from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.datasets.etl_dataset import EtlEntityType from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb @@ -21,20 +23,30 @@ def sync_deliverable( ) -> tuple[int | None, EtlChangeType]: """Write deliverable data to etl database.""" # initialize return value + deliverable_id = None change_type = EtlChangeType.NONE - # insert dimensions - deliverable_id = self._insert_dimensions(deliverable_df) - if deliverable_id is not None: - change_type = EtlChangeType.INSERT - - # if insert failed, select and update - if deliverable_id is None: - deliverable_id, change_type = self._update_dimensions(deliverable_df) - - # insert facts - if deliverable_id is not None: - self._insert_facts(deliverable_id, deliverable_df, ghid_map) + try: + # insert dimensions + deliverable_id = self._insert_dimensions(deliverable_df) + if deliverable_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, select and update + if deliverable_id is None: + deliverable_id, change_type = self._update_dimensions(deliverable_df) + + # insert facts + if deliverable_id is not None: + _ = self._insert_facts(deliverable_id, deliverable_df, ghid_map) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync deliverable data: {e}" + raise RuntimeError(message) from e return deliverable_id, change_type @@ -69,10 +81,10 @@ def _insert_facts( deliverable_id: int, deliverable_df: Series, ghid_map: dict, - ) -> int | None: + ) -> tuple[int | None, int | None]: """Write deliverable fact data to etl database.""" # insert into fact table: deliverable_quad_map - new_row_id = None + map_id = None cursor = self.dbh.connection() result = cursor.execute( text( @@ -91,12 +103,31 @@ def _insert_facts( ) row = result.fetchone() if row: - new_row_id = row[0] + map_id = row[0] + + # insert into fact table: deliverable_history + history_id = None + result = cursor.execute( + text( + "insert into gh_deliverable_history(deliverable_id, status, d_effective) " + "values (:deliverable_id, :status, :effective) " + "on conflict(deliverable_id, d_effective) do update " + "set (status, t_modified) = (:status, current_timestamp) returning id", + ), + { + "deliverable_id": deliverable_id, + "status": deliverable_df["deliverable_status"], + "effective": self.dbh.effective_date, + }, + ) + row = result.fetchone() + if row: + history_id = row[0] # commit self.dbh.commit(cursor) - return new_row_id + return history_id, map_id def _update_dimensions( self, diff --git a/analytics/src/analytics/integrations/etldb/epic_model.py b/analytics/src/analytics/integrations/etldb/epic_model.py index af0fdf45d..13bdb4dc7 100644 --- a/analytics/src/analytics/integrations/etldb/epic_model.py +++ b/analytics/src/analytics/integrations/etldb/epic_model.py @@ -1,7 +1,9 @@ """Defines EtlEpicModel class to encapsulate db CRUD operations.""" from pandas import Series +from psycopg.errors import InsufficientPrivilege from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.datasets.etl_dataset import EtlEntityType from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb @@ -21,20 +23,30 @@ def sync_epic( ) -> tuple[int | None, EtlChangeType]: """Write epic data to etl database.""" # initialize return value + epic_id = None change_type = EtlChangeType.NONE - # insert dimensions - epic_id = self._insert_dimensions(epic_df) - if epic_id is not None: - change_type = EtlChangeType.INSERT - - # if insert failed, select and update - if epic_id is None: - epic_id, change_type = self._update_dimensions(epic_df) - - # insert facts - if epic_id is not None: - self._insert_facts(epic_id, epic_df, ghid_map) + try: + # insert dimensions + epic_id = self._insert_dimensions(epic_df) + if epic_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, select and update + if epic_id is None: + epic_id, change_type = self._update_dimensions(epic_df) + + # insert facts + if epic_id is not None: + self._insert_facts(epic_id, epic_df, ghid_map) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync epic data: {e}" + raise RuntimeError(message) from e return epic_id, change_type diff --git a/analytics/src/analytics/integrations/etldb/issue_model.py b/analytics/src/analytics/integrations/etldb/issue_model.py index 36740438d..ec2f6be21 100644 --- a/analytics/src/analytics/integrations/etldb/issue_model.py +++ b/analytics/src/analytics/integrations/etldb/issue_model.py @@ -3,7 +3,9 @@ from datetime import datetime from pandas import Series +from psycopg.errors import InsufficientPrivilege from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.datasets.etl_dataset import EtlEntityType from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb @@ -23,20 +25,30 @@ def sync_issue( ) -> tuple[int | None, EtlChangeType]: """Write issue data to etl database.""" # initialize return value + issue_id = None change_type = EtlChangeType.NONE - # insert dimensions - issue_id = self._insert_dimensions(issue_df, ghid_map) - if issue_id is not None: - change_type = EtlChangeType.INSERT - - # if insert failed, select and update - if issue_id is None: - issue_id, change_type = self._update_dimensions(issue_df, ghid_map) - - # insert facts - if issue_id is not None: - self._insert_facts(issue_id, issue_df, ghid_map) + try: + # insert dimensions + issue_id = self._insert_dimensions(issue_df, ghid_map) + if issue_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, select and update + if issue_id is None: + issue_id, change_type = self._update_dimensions(issue_df, ghid_map) + + # insert facts + if issue_id is not None: + self._insert_facts(issue_id, issue_df, ghid_map) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync issue data: {e}" + raise RuntimeError(message) from e return issue_id, change_type diff --git a/analytics/src/analytics/integrations/etldb/main.py b/analytics/src/analytics/integrations/etldb/main.py index 81cc549b4..ea9854d8d 100644 --- a/analytics/src/analytics/integrations/etldb/main.py +++ b/analytics/src/analytics/integrations/etldb/main.py @@ -4,15 +4,14 @@ import re from pathlib import Path -from psycopg.errors import InsufficientPrivilege from sqlalchemy import text -from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.datasets.etl_dataset import EtlDataset, EtlEntityType from analytics.integrations.etldb.deliverable_model import EtlDeliverableModel from analytics.integrations.etldb.epic_model import EtlEpicModel from analytics.integrations.etldb.etldb import EtlDb from analytics.integrations.etldb.issue_model import EtlIssueModel +from analytics.integrations.etldb.project_model import EtlProjectModel from analytics.integrations.etldb.quad_model import EtlQuadModel from analytics.integrations.etldb.sprint_model import EtlSprintModel @@ -73,83 +72,44 @@ def sync_data(dataset: EtlDataset, effective: str) -> None: ghid_map: dict[EtlEntityType, dict[str, int]] = { EtlEntityType.DELIVERABLE: {}, EtlEntityType.EPIC: {}, - EtlEntityType.SPRINT: {}, + EtlEntityType.PROJECT: {}, EtlEntityType.QUAD: {}, + EtlEntityType.SPRINT: {}, } # initialize db connection db = EtlDb(effective) + # note: the following code assumes SCHEMA VERSION >= 4 + # sync project data to db resulting in row id for each project + ghid_map[EtlEntityType.PROJECT] = sync_projects(db, dataset) + print(f"project row(s) processed: {len(ghid_map[EtlEntityType.PROJECT])}") + # sync quad data to db resulting in row id for each quad - try: - ghid_map[EtlEntityType.QUAD] = sync_quads(db, dataset) - print(f"quad row(s) processed: {len(ghid_map[EtlEntityType.QUAD])}") - except ( - InsufficientPrivilege, - OperationalError, - ProgrammingError, - RuntimeError, - ) as e: - message = f"FATAL: Failed to sync quad data: {e}" - raise RuntimeError(message) from e + ghid_map[EtlEntityType.QUAD] = sync_quads(db, dataset) + print(f"quad row(s) processed: {len(ghid_map[EtlEntityType.QUAD])}") # sync deliverable data to db resulting in row id for each deliverable - try: - ghid_map[EtlEntityType.DELIVERABLE] = sync_deliverables( - db, - dataset, - ghid_map, - ) - print( - f"deliverable row(s) processed: {len(ghid_map[EtlEntityType.DELIVERABLE])}", - ) - except ( - InsufficientPrivilege, - OperationalError, - ProgrammingError, - RuntimeError, - ) as e: - message = f"FATAL: Failed to sync deliverable data: {e}" - raise RuntimeError(message) from e + ghid_map[EtlEntityType.DELIVERABLE] = sync_deliverables( + db, + dataset, + ghid_map, + ) + print( + f"deliverable row(s) processed: {len(ghid_map[EtlEntityType.DELIVERABLE])}", + ) # sync sprint data to db resulting in row id for each sprint - try: - ghid_map[EtlEntityType.SPRINT] = sync_sprints(db, dataset, ghid_map) - print(f"sprint row(s) processed: {len(ghid_map[EtlEntityType.SPRINT])}") - except ( - InsufficientPrivilege, - OperationalError, - ProgrammingError, - RuntimeError, - ) as e: - message = f"FATAL: Failed to sync sprint data: {e}" - raise RuntimeError(message) from e + ghid_map[EtlEntityType.SPRINT] = sync_sprints(db, dataset, ghid_map) + print(f"sprint row(s) processed: {len(ghid_map[EtlEntityType.SPRINT])}") # sync epic data to db resulting in row id for each epic - try: - ghid_map[EtlEntityType.EPIC] = sync_epics(db, dataset, ghid_map) - print(f"epic row(s) processed: {len(ghid_map[EtlEntityType.EPIC])}") - except ( - InsufficientPrivilege, - OperationalError, - ProgrammingError, - RuntimeError, - ) as e: - message = f"FATAL: Failed to sync epic data: {e}" - raise RuntimeError(message) from e + ghid_map[EtlEntityType.EPIC] = sync_epics(db, dataset, ghid_map) + print(f"epic row(s) processed: {len(ghid_map[EtlEntityType.EPIC])}") # sync issue data to db resulting in row id for each issue - try: - issue_map = sync_issues(db, dataset, ghid_map) - print(f"issue row(s) processed: {len(issue_map)}") - except ( - InsufficientPrivilege, - OperationalError, - ProgrammingError, - RuntimeError, - ) as e: - message = f"FATAL: Failed to sync issue data: {e}" - raise RuntimeError(message) from e + issue_map = sync_issues(db, dataset, ghid_map) + print(f"issue row(s) processed: {len(issue_map)}") def sync_deliverables(db: EtlDb, dataset: EtlDataset, ghid_map: dict) -> dict: @@ -188,6 +148,20 @@ def sync_issues(db: EtlDb, dataset: EtlDataset, ghid_map: dict) -> dict: return result +def sync_projects(db: EtlDb, dataset: EtlDataset) -> dict: + """Insert or update (if necessary) a row for each project and return a map of row ids.""" + result = {} + model = EtlProjectModel(db) + for ghid in dataset.get_project_ghids(): + project_df = dataset.get_project(ghid) + result[ghid], _ = model.sync_project(project_df) + if VERBOSE: + print( + f"PROJECT '{ghid}' title = '{project_df['project_name']}', row_id = {result[ghid]}", + ) + return result + + def sync_sprints(db: EtlDb, dataset: EtlDataset, ghid_map: dict) -> dict: """Insert or update (if necessary) a row for each sprint and return a map of row ids.""" result = {} diff --git a/analytics/src/analytics/integrations/etldb/migrations/versions/0004_create_tables_deliv_hist_and_project.sql b/analytics/src/analytics/integrations/etldb/migrations/versions/0004_create_tables_deliv_hist_and_project.sql new file mode 100644 index 000000000..709ac8a4b --- /dev/null +++ b/analytics/src/analytics/integrations/etldb/migrations/versions/0004_create_tables_deliv_hist_and_project.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS gh_project ( + id SERIAL PRIMARY KEY, + ghid INTEGER UNIQUE NOT NULL, + name TEXT NOT NULL, + t_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + t_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +ALTER TABLE IF EXISTS gh_sprint ADD COLUMN IF NOT EXISTS project_id INTEGER; + +CREATE TABLE IF NOT EXISTS gh_deliverable_history ( + id SERIAL PRIMARY KEY, + deliverable_id INTEGER NOT NULL, + status TEXT, + d_effective DATE NOT NULL, + t_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + t_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(deliverable_id, d_effective) +); +CREATE INDEX IF NOT EXISTS gh_dh_i1 on gh_deliverable_history(deliverable_id, d_effective); + diff --git a/analytics/src/analytics/integrations/etldb/project_model.py b/analytics/src/analytics/integrations/etldb/project_model.py new file mode 100644 index 000000000..2def66da1 --- /dev/null +++ b/analytics/src/analytics/integrations/etldb/project_model.py @@ -0,0 +1,115 @@ +"""Defines EtlProjectModel class to encapsulate db CRUD operations.""" + +from pandas import Series +from psycopg.errors import InsufficientPrivilege +from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError + +from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb + + +class EtlProjectModel: + """Encapsulates CRUD operations for project entity.""" + + def __init__(self, dbh: EtlDb) -> None: + """Instantiate a class instance.""" + self.dbh = dbh + + def sync_project(self, project_df: Series) -> tuple[int | None, EtlChangeType]: + """Write project data to etl database.""" + # initialize return value + project_id = None + change_type = EtlChangeType.NONE + + try: + # insert dimensions + project_id = self._insert_dimensions(project_df) + if project_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, then select and update + if project_id is None: + project_id, change_type = self._update_dimensions(project_df) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync project data: {e}" + raise RuntimeError(message) from e + + return project_id, change_type + + def _insert_dimensions(self, project_df: Series) -> int | None: + """Write project dimension data to etl database.""" + # insert into dimension table: project + new_row_id = None + cursor = self.dbh.connection() + result = cursor.execute( + text( + "insert into gh_project(ghid, name) values (:ghid, :name) " + "on conflict(ghid) do nothing returning id", + ), + {"ghid": project_df["project_ghid"], "name": project_df["project_name"]}, + ) + row = result.fetchone() + if row: + new_row_id = row[0] + + # commit + self.dbh.commit(cursor) + + return new_row_id + + def _update_dimensions( + self, + project_df: Series, + ) -> tuple[int | None, EtlChangeType]: + """Update project dimension data in etl database.""" + # initialize return value + change_type = EtlChangeType.NONE + + # get new values + new_name = project_df["project_name"] + + # select old values + project_id, old_name = self._select( + project_df["project_ghid"], + ) + + # compare + if project_id is not None and new_name != old_name: + change_type = EtlChangeType.UPDATE + cursor = self.dbh.connection() + cursor.execute( + text( + "update gh_project set name = :new_name, " + "t_modified = current_timestamp where id = :project_id", + ), + { + "new_name": new_name, + "project_id": project_id, + }, + ) + self.dbh.commit(cursor) + + return project_id, change_type + + def _select(self, ghid: str) -> tuple[ + int | None, + str | None, + ]: + """Select epic data from etl database.""" + cursor = self.dbh.connection() + result = cursor.execute( + text( + "select id, name from gh_project where ghid = :ghid", + ), + {"ghid": ghid}, + ) + row = result.fetchone() + if row: + return row[0], row[1] + + return None, None diff --git a/analytics/src/analytics/integrations/etldb/quad_model.py b/analytics/src/analytics/integrations/etldb/quad_model.py index 6324710ec..a312c6c87 100644 --- a/analytics/src/analytics/integrations/etldb/quad_model.py +++ b/analytics/src/analytics/integrations/etldb/quad_model.py @@ -3,7 +3,9 @@ from datetime import datetime from pandas import Series +from psycopg.errors import InsufficientPrivilege from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb @@ -18,16 +20,26 @@ def __init__(self, dbh: EtlDb) -> None: def sync_quad(self, quad_df: Series) -> tuple[int | None, EtlChangeType]: """Write quad data to etl database.""" # initialize return value + quad_id = None change_type = EtlChangeType.NONE - # insert dimensions - quad_id = self._insert_dimensions(quad_df) - if quad_id is not None: - change_type = EtlChangeType.INSERT - - # if insert failed, then select and update - if quad_id is None: - quad_id, change_type = self._update_dimensions(quad_df) + try: + # insert dimensions + quad_id = self._insert_dimensions(quad_df) + if quad_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, then select and update + if quad_id is None: + quad_id, change_type = self._update_dimensions(quad_df) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync quad data: {e}" + raise RuntimeError(message) from e return quad_id, change_type diff --git a/analytics/src/analytics/integrations/etldb/sprint_model.py b/analytics/src/analytics/integrations/etldb/sprint_model.py index f14dc14cb..31951e55c 100644 --- a/analytics/src/analytics/integrations/etldb/sprint_model.py +++ b/analytics/src/analytics/integrations/etldb/sprint_model.py @@ -1,7 +1,9 @@ """Define EtlSprintModel class to encapsulate db CRUD operations.""" from pandas import Series +from psycopg.errors import InsufficientPrivilege from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from analytics.datasets.etl_dataset import EtlEntityType from analytics.integrations.etldb.etldb import EtlChangeType, EtlDb @@ -20,16 +22,26 @@ def sync_sprint(self, sprint_df: Series, ghid_map: dict) -> tuple[ ]: """Write sprint data to etl database.""" # initialize return value + sprint_id = None change_type = EtlChangeType.NONE - # insert dimensions - sprint_id = self._insert_dimensions(sprint_df, ghid_map) - if sprint_id is not None: - change_type = EtlChangeType.INSERT - - # if insert failed, select and update - if sprint_id is None: - sprint_id, change_type = self._update_dimensions(sprint_df, ghid_map) + try: + # insert dimensions + sprint_id = self._insert_dimensions(sprint_df, ghid_map) + if sprint_id is not None: + change_type = EtlChangeType.INSERT + + # if insert failed, select and update + if sprint_id is None: + sprint_id, change_type = self._update_dimensions(sprint_df, ghid_map) + except ( + InsufficientPrivilege, + OperationalError, + ProgrammingError, + RuntimeError, + ) as e: + message = f"FATAL: Failed to sync sprint data: {e}" + raise RuntimeError(message) from e return sprint_id, change_type @@ -40,8 +52,9 @@ def _insert_dimensions(self, sprint_df: Series, ghid_map: dict) -> int | None: cursor = self.dbh.connection() result = cursor.execute( text( - "insert into gh_sprint(ghid, name, start_date, end_date, duration, quad_id) " - "values (:ghid, :name, :start, :end, :duration, :quad_id) " + "insert into gh_sprint " + "(ghid, name, start_date, end_date, duration, quad_id, project_id) " + "values (:ghid, :name, :start, :end, :duration, :quad_id, :project_id) " "on conflict(ghid) do nothing returning id", ), { @@ -51,6 +64,9 @@ def _insert_dimensions(self, sprint_df: Series, ghid_map: dict) -> int | None: "end": sprint_df["sprint_end"], "duration": sprint_df["sprint_length"], "quad_id": ghid_map[EtlEntityType.QUAD].get(sprint_df["quad_ghid"]), + "project_id": ghid_map[EtlEntityType.PROJECT].get( + sprint_df["project_ghid"], + ), }, ) row = result.fetchone() @@ -77,13 +93,14 @@ def _update_dimensions(self, sprint_df: Series, ghid_map: dict) -> tuple[ sprint_df["sprint_end"], sprint_df["sprint_length"], ghid_map[EtlEntityType.QUAD].get(sprint_df["quad_ghid"]), + ghid_map[EtlEntityType.PROJECT].get(sprint_df["project_ghid"]), ) # select old values - sprint_id, old_name, old_start, old_end, old_duration, old_quad_id = ( + sprint_id, o_name, o_start, o_end, o_duration, o_quad_id, o_project_id = ( self._select(sprint_df["sprint_ghid"]) ) - old_values = (old_name, old_start, old_end, old_duration, old_quad_id) + old_values = (o_name, o_start, o_end, o_duration, o_quad_id, o_project_id) # compare if sprint_id is not None and new_values != old_values: @@ -93,7 +110,8 @@ def _update_dimensions(self, sprint_df: Series, ghid_map: dict) -> tuple[ text( "update gh_sprint set name = :new_name, start_date = :new_start, " "end_date = :new_end, duration = :new_duration, quad_id = :quad_id, " - "t_modified = current_timestamp where id = :sprint_id", + "project_id = :project_id, t_modified = current_timestamp " + "where id = :sprint_id", ), { "new_name": new_values[0], @@ -101,6 +119,7 @@ def _update_dimensions(self, sprint_df: Series, ghid_map: dict) -> tuple[ "new_end": new_values[2], "new_duration": new_values[3], "quad_id": new_values[4], + "project_id": new_values[5], "sprint_id": sprint_id, }, ) @@ -115,18 +134,19 @@ def _select(self, ghid: str) -> tuple[ str | None, int | None, int | None, + int | None, ]: """Select epic data from etl database.""" cursor = self.dbh.connection() result = cursor.execute( text( - "select id, name, start_date, end_date, duration, quad_id " + "select id, name, start_date, end_date, duration, quad_id, project_id " "from gh_sprint where ghid = :ghid", ), {"ghid": ghid}, ) row = result.fetchone() if row: - return row[0], row[1], row[2], row[3], row[4], row[5] + return row[0], row[1], row[2], row[3], row[4], row[5], row[6] - return None, None, None, None, None, None + return None, None, None, None, None, None, None diff --git a/analytics/tests/datasets/test_etldb.py b/analytics/tests/datasets/test_etldb.py index 042022a89..ee93f83d3 100644 --- a/analytics/tests/datasets/test_etldb.py +++ b/analytics/tests/datasets/test_etldb.py @@ -15,33 +15,33 @@ def test_load_from_json_files(self): row_count = dataset.df.shape[0] col_count = dataset.df.shape[1] assert row_count == 22 - assert col_count == 24 + assert col_count == 27 def test_deliverable_fetchers(self): """Deliverable fetchers should return expected values.""" dataset = EtlDataset.load_from_json_file(self.TEST_FILE_1) unique_ghids = dataset.get_deliverable_ghids() - assert len(unique_ghids) == 2 + assert len(unique_ghids) == 4 ghid = unique_ghids[0] - assert ghid == "agilesix/simpler-grants-sandbox/issues/2" + assert ghid == "HHS/simpler-grants-gov/issues/2200" deliverable = dataset.get_deliverable(ghid) - assert deliverable["deliverable_title"] == "Opportunity listing page" + assert deliverable["deliverable_title"] == "Search *" def test_epic_fetchers(self): """Epic fetchers should return expected values.""" dataset = EtlDataset.load_from_json_file(self.TEST_FILE_1) unique_ghids = dataset.get_epic_ghids() - assert len(unique_ghids) == 4 + assert len(unique_ghids) == 6 ghid = unique_ghids[0] - assert ghid == "agilesix/simpler-grants-sandbox/issues/8" + assert ghid == "HHS/simpler-grants-gov/issues/2719" epic = dataset.get_epic(ghid) - assert epic["epic_title"] == "Deploy opportunity listing behind a feature flag" + assert epic["epic_title"] == "Search API Engagement" def test_issue_fetchers(self): """Issue fetchers should return expected values.""" @@ -51,23 +51,23 @@ def test_issue_fetchers(self): assert len(unique_ghids) == 22 ghid = unique_ghids[0] - assert ghid == "agilesix/simpler-grants-sandbox/issues/46" + assert ghid == "HHS/simpler-grants-gov/issues/2763" issue = dataset.get_issue(ghid) - assert issue["issue_opened_at"] == "2024-09-27T15:29:37Z" + assert issue["issue_opened_at"] == "2024-11-07" def test_sprint_fetchers(self): """Deliverable fetchers should return expected values.""" dataset = EtlDataset.load_from_json_file(self.TEST_FILE_1) unique_ghids = dataset.get_sprint_ghids() - assert len(unique_ghids) == 5 + assert len(unique_ghids) == 4 ghid = unique_ghids[0] - assert ghid == "74402b12" + assert ghid == "b8f1831e" sprint = dataset.get_sprint(ghid) - assert sprint["sprint_name"] == "Sprint 2" + assert sprint["sprint_name"] == "Sprint 1.3" def test_quad_fetchers(self): """Quad fetchers should return expected values.""" @@ -77,7 +77,20 @@ def test_quad_fetchers(self): assert len(unique_ghids) == 1 ghid = unique_ghids[0] - assert ghid == "de5f962b" + assert ghid == "93412e1c" quad = dataset.get_quad(ghid) - assert quad["quad_name"] == "BY1 Quad 1" + assert quad["quad_name"] == "Quad 1.1" + + def test_project_fetchers(self): + """Project fetchers should return expected values.""" + dataset = EtlDataset.load_from_json_file(self.TEST_FILE_1) + + unique_ghids = dataset.get_project_ghids() + assert len(unique_ghids) == 2 + + ghid = unique_ghids[0] + assert ghid == 13 + + project = dataset.get_project(ghid) + assert project["project_name"] == "HHS" diff --git a/analytics/tests/etldb_test_01.json b/analytics/tests/etldb_test_01.json index 2c3801b44..d1ad03588 100644 --- a/analytics/tests/etldb_test_01.json +++ b/analytics/tests/etldb_test_01.json @@ -1,97 +1,51 @@ [ { - "issue_title": "exampel that doesn't ask type when created", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/46", - "issue_parent": null, - "issue_type": null, - "issue_is_closed": false, - "issue_opened_at": "2024-09-27T15:29:37Z", - "issue_closed_at": null, - "issue_points": null, - "issue_status": "In Progress", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": null, - "quad_name": null, - "quad_start": null, - "quad_length": null, - "quad_end": null, - "deliverable_pillar": null, - "deliverable_url": null, - "deliverable_title": null, - "epic_url": null, - "epic_title": null - }, - { - "issue_title": "Implement opportunity listing UI", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/11", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:49:17Z", - "issue_closed_at": null, - "issue_points": 5, - "issue_status": "In Progress", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", - "quad_start": "2024-09-09", - "quad_length": 122, - "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "epic_title": "Deploy opportunity listing behind a feature flag" - }, - { - "issue_title": "Implement opportunity listing API", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/10", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", + "project_owner": "HHS", + "project_number": 13, + "issue_title": "Draft email content for participants", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2763", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2719", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:49:03Z", - "issue_closed_at": null, - "issue_points": 5, - "issue_status": "In Progress", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "issue_is_closed": true, + "issue_opened_at": "2024-11-07", + "issue_closed_at": "2024-11-12", + "issue_points": 2.0, + "issue_status": "Done", + "sprint_id": "b8f1831e", + "sprint_name": "Sprint 1.3", + "sprint_start": "2024-10-30", + "sprint_length": 14.0, + "sprint_end": "2024-11-13", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "epic_title": "Deploy opportunity listing behind a feature flag" + "deliverable_pillar": "🔎 SimplerFind", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2200", + "deliverable_title": "Search *", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2719", + "epic_title": "Search API Engagement", + "issue_state": "closed" }, { - "issue_title": "exampel creating from project interface", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/45", + "project_owner": "HHS", + "project_number": 13, + "issue_title": "[DRAFT] Save and Subscribe to a Search - Not Ready for Feedback", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2820", "issue_parent": null, - "issue_type": null, + "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-26T23:23:31Z", + "issue_opened_at": "2024-11-13", "issue_closed_at": null, "issue_points": null, - "issue_status": "In Progress", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", + "issue_status": null, + "sprint_id": null, + "sprint_name": null, + "sprint_start": null, + "sprint_length": null, + "sprint_end": null, "quad_id": null, "quad_name": null, "quad_start": null, @@ -100,206 +54,208 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "Implement search API", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/5", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", + "project_owner": "HHS", + "project_number": 13, + "issue_title": "Content design: post Search launch beta alert", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2634", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2368", "issue_type": "Task", "issue_is_closed": true, - "issue_opened_at": "2024-09-18T15:41:49Z", - "issue_closed_at": "2024-09-18T19:40:40Z", - "issue_points": 3, + "issue_opened_at": "2024-10-29", + "issue_closed_at": "2024-11-18", + "issue_points": 1.0, "issue_status": "Done", - "sprint_id": "26a4c39d", - "sprint_name": "Sprint 1", - "sprint_start": "2024-09-09", - "sprint_length": 14, - "sprint_end": "2024-09-23", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "sprint_id": "3fd64daa", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "\ud83d\udd0e SimplerFind", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/1", - "deliverable_title": "Search", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", - "epic_title": "Deploy search behind a feature flag" + "deliverable_pillar": "🔎 SimplerFind", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2200", + "deliverable_title": "Search *", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2368", + "epic_title": "Search launch", + "issue_state": "closed" }, { - "issue_title": "Enable feature flag for 1000 users", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/15", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Finalize Agenda", + "issue_url": "https://github.com/HHS/grants-product-and-delivery/issues/330", + "issue_parent": "https://github.com/HHS/grants-product-and-delivery/issues/297", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:50:41Z", - "issue_closed_at": null, - "issue_points": 3, - "issue_status": "Todo", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "issue_is_closed": true, + "issue_opened_at": "2024-10-16", + "issue_closed_at": "2024-10-29", + "issue_points": 0.0, + "issue_status": "Done", + "sprint_id": "20188ca5", + "sprint_name": "Sprint 1.2", + "sprint_start": "2024-10-16", + "sprint_length": 14.0, + "sprint_end": "2024-10-30", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", - "epic_title": "Release to opportunity listing to 10k users" + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2357", + "deliverable_title": "Cross-program team health", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2362", + "epic_title": "Team forming", + "issue_state": "closed" }, { - "issue_title": "Load test for 10k active users", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/14", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Order lunch catering", + "issue_url": "https://github.com/HHS/grants-product-and-delivery/issues/331", + "issue_parent": "https://github.com/HHS/grants-product-and-delivery/issues/297", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:50:20Z", - "issue_closed_at": null, + "issue_is_closed": true, + "issue_opened_at": "2024-10-16", + "issue_closed_at": "2024-10-29", "issue_points": null, - "issue_status": "Todo", - "sprint_id": "8a6d26a4", - "sprint_name": "Sprint 4", - "sprint_start": "2024-10-21", - "sprint_length": 14, - "sprint_end": "2024-11-04", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "issue_status": "Done", + "sprint_id": "20188ca5", + "sprint_name": "Sprint 1.2", + "sprint_start": "2024-10-16", + "sprint_length": 14.0, + "sprint_end": "2024-10-30", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", - "epic_title": "Release to opportunity listing to 10k users" + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2357", + "deliverable_title": "Cross-program team health", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2362", + "epic_title": "Team forming", + "issue_state": "closed" }, { - "issue_title": "Enable feature flag for first 100 users", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/13", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Send a reminder email", + "issue_url": "https://github.com/HHS/grants-product-and-delivery/issues/332", + "issue_parent": "https://github.com/HHS/grants-product-and-delivery/issues/297", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:50:02Z", - "issue_closed_at": null, + "issue_is_closed": true, + "issue_opened_at": "2024-10-16", + "issue_closed_at": "2024-10-29", "issue_points": null, - "issue_status": "Todo", - "sprint_id": "11878b69", - "sprint_name": "Sprint 5", - "sprint_start": "2024-11-04", - "sprint_length": 14, - "sprint_end": "2024-11-18", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "issue_status": "Done", + "sprint_id": "20188ca5", + "sprint_name": "Sprint 1.2", + "sprint_start": "2024-10-16", + "sprint_length": 14.0, + "sprint_end": "2024-10-30", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/9", - "epic_title": "Release to opportunity listing to 10k users" + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2357", + "deliverable_title": "Cross-program team health", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2362", + "epic_title": "Team forming", + "issue_state": "closed" }, { - "issue_title": "Conduct first usability test for opportunity listing", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/12", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Create guide for tracking SimplerNOFOs deliverables in GitHub", + "issue_url": "https://github.com/HHS/grants-product-and-delivery/issues/337", + "issue_parent": "https://github.com/HHS/grants-product-and-delivery/issues/291", "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:49:32Z", + "issue_opened_at": "2024-11-13", "issue_closed_at": null, - "issue_points": 3, + "issue_points": 3.0, "issue_status": "Todo", - "sprint_id": "0a9ff409", - "sprint_name": "Sprint 3", - "sprint_start": "2024-10-07", - "sprint_length": 14, - "sprint_end": "2024-10-21", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "epic_title": "Deploy opportunity listing behind a feature flag" + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2356", + "deliverable_title": "Cross-program product strategy (Base Year 1)", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2358", + "epic_title": "Roadmap and deliverables for Base Year 1", + "issue_state": "open" }, { - "issue_title": "Implement Search UI", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/6", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Analytics - Add logging and lint for `print` statements", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/918", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2560", "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:41:58Z", + "issue_opened_at": "2023-12-14", "issue_closed_at": null, - "issue_points": 8, + "issue_points": 2.0, "issue_status": "Todo", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "\ud83d\udd0e SimplerFind", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/1", - "deliverable_title": "Search", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", - "epic_title": "Deploy search behind a feature flag" + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "open" }, { - "issue_title": "Host first usability test for search", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/7", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Set assignee on issue close", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2561", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2830", "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-18T15:42:24Z", + "issue_opened_at": "2024-10-24", "issue_closed_at": null, - "issue_points": null, + "issue_points": 3.0, "issue_status": "Todo", - "sprint_id": "8a6d26a4", - "sprint_name": "Sprint 4", - "sprint_start": "2024-10-21", - "sprint_length": 14, - "sprint_end": "2024-11-04", - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", - "quad_start": "2024-09-09", - "quad_length": 122, - "quad_end": "2025-01-09", - "deliverable_pillar": "\ud83d\udd0e SimplerFind", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/1", - "deliverable_title": "Search", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/3", - "epic_title": "Deploy search behind a feature flag" - }, - { - "issue_title": "[Bug] DD test 01", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/25", - "issue_parent": null, - "issue_type": "Bug", - "issue_is_closed": true, - "issue_opened_at": "2024-09-21T01:09:08Z", - "issue_closed_at": "2024-09-21T01:26:21Z", - "issue_points": 2, - "issue_status": "Done", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -308,76 +264,88 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "[Bug] DD test 02", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/26", - "issue_parent": null, + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Analytics package: `make gh-data-db-import` fails", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2569", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2355", "issue_type": "Bug", "issue_is_closed": true, - "issue_opened_at": "2024-09-21T01:14:39Z", - "issue_closed_at": "2024-09-21T01:26:39Z", - "issue_points": 1, + "issue_opened_at": "2024-10-25", + "issue_closed_at": "2024-10-28", + "issue_points": 2.0, "issue_status": "Done", - "sprint_id": "74402b12", - "sprint_name": "Sprint 2", - "sprint_start": "2024-09-23", - "sprint_length": 14, - "sprint_end": "2024-10-07", - "quad_id": null, - "quad_name": null, - "quad_start": null, - "quad_length": null, - "quad_end": null, - "deliverable_pillar": null, - "deliverable_url": null, - "deliverable_title": null, - "epic_url": null, - "epic_title": null + "sprint_id": "20188ca5", + "sprint_name": "Sprint 1.2", + "sprint_start": "2024-10-16", + "sprint_length": 14.0, + "sprint_end": "2024-10-30", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "closed" }, { - "issue_title": "[BUG] DD test 03", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/27", - "issue_parent": null, - "issue_type": "Bug", - "issue_is_closed": true, - "issue_opened_at": "2024-09-21T01:22:52Z", - "issue_closed_at": "2024-09-21T01:26:45Z", - "issue_points": 5, - "issue_status": "Done", - "sprint_id": "26a4c39d", - "sprint_name": "Sprint 1", - "sprint_start": "2024-09-09", - "sprint_length": 14, - "sprint_end": "2024-09-23", - "quad_id": null, - "quad_name": null, - "quad_start": null, - "quad_length": null, - "quad_end": null, - "deliverable_pillar": null, - "deliverable_url": null, - "deliverable_title": null, - "epic_url": null, - "epic_title": null + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Call GitHub graphql API directly from python", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2590", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2560", + "issue_type": "Task", + "issue_is_closed": false, + "issue_opened_at": "2024-10-25", + "issue_closed_at": null, + "issue_points": 5.0, + "issue_status": "Todo", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "open" }, { - "issue_title": "[BUG] DD test 04 with screenshot", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/28", - "issue_parent": null, - "issue_type": "Bug", - "issue_is_closed": true, - "issue_opened_at": "2024-09-21T01:24:39Z", - "issue_closed_at": "2024-09-21T01:26:52Z", - "issue_points": 2, - "issue_status": "Done", - "sprint_id": "26a4c39d", - "sprint_name": "Sprint 1", - "sprint_start": "2024-09-09", - "sprint_length": 14, - "sprint_end": "2024-09-23", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Limit automatic linting to issues with linked PR", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2834", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2830", + "issue_type": "Task", + "issue_is_closed": false, + "issue_opened_at": "2024-11-13", + "issue_closed_at": null, + "issue_points": 3.0, + "issue_status": "In Progress", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -386,42 +354,50 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "Sub-issue 1", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/32", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/10", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "1099-K Handling for Participant Receiving >$600 in a Calendar Year", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2350", + "issue_parent": null, "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-24T17:06:03Z", + "issue_opened_at": "2024-10-03", "issue_closed_at": null, "issue_points": null, - "issue_status": "Todo", + "issue_status": "In Progress", "sprint_id": null, "sprint_name": null, "sprint_start": null, "sprint_length": null, "sprint_end": null, - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", - "quad_start": "2024-09-09", - "quad_length": 122, - "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "epic_title": "Deploy opportunity listing behind a feature flag" + "quad_id": null, + "quad_name": null, + "quad_start": null, + "quad_length": null, + "quad_end": null, + "deliverable_pillar": null, + "deliverable_url": null, + "deliverable_title": null, + "deliverable_status": null, + "epic_url": null, + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "Sub issue 2", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/33", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/10", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Cross-Program: Streamline Meeting Structures", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2672", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2658", "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-24T17:06:18Z", + "issue_opened_at": "2024-10-30", "issue_closed_at": null, "issue_points": null, "issue_status": "Todo", @@ -430,32 +406,36 @@ "sprint_start": null, "sprint_length": null, "sprint_end": null, - "quad_id": "de5f962b", - "quad_name": "BY1 Quad 1", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", "quad_start": "2024-09-09", - "quad_length": 122, + "quad_length": 122.0, "quad_end": "2025-01-09", - "deliverable_pillar": "SimplerApply", - "deliverable_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/2", - "deliverable_title": "Opportunity listing page", - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/8", - "epic_title": "Deploy opportunity listing behind a feature flag" + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2357", + "deliverable_title": "Cross-program team health", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2658", + "epic_title": "Nava Team Retro Feedback", + "issue_state": "open" }, { - "issue_title": "[Bug] Sample bug created with issue template", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/35", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "ADR for how to add slack users", + "issue_url": "https://github.com/HHS/grants-product-and-delivery/issues/333", "issue_parent": null, - "issue_type": "Bug", + "issue_type": null, "issue_is_closed": false, - "issue_opened_at": "2024-09-25T17:58:41Z", + "issue_opened_at": "2024-11-05", "issue_closed_at": null, "issue_points": null, "issue_status": "Todo", - "sprint_id": null, - "sprint_name": null, - "sprint_start": null, - "sprint_length": null, - "sprint_end": null, + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -464,50 +444,58 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "[Feature] Sample feature request created with issue template", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/36", - "issue_parent": null, - "issue_type": "Enhancement", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Cross-Program: Clarify Communication Channels", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2675", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2658", + "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-25T17:59:29Z", + "issue_opened_at": "2024-10-30", "issue_closed_at": null, "issue_points": null, - "issue_status": "Todo", - "sprint_id": null, - "sprint_name": null, - "sprint_start": null, - "sprint_length": null, - "sprint_end": null, - "quad_id": null, - "quad_name": null, - "quad_start": null, - "quad_length": null, - "quad_end": null, - "deliverable_pillar": null, - "deliverable_url": null, - "deliverable_title": null, - "epic_url": null, - "epic_title": null + "issue_status": "In Progress", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "SimplerDelivery", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2357", + "deliverable_title": "Cross-program team health", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2658", + "epic_title": "Nava Team Retro Feedback", + "issue_state": "open" }, { - "issue_title": "[ADR] Sample decision created with issue template", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/37", - "issue_parent": null, + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Improve sprint reporting linter", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2830", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2829", "issue_type": "Task", "issue_is_closed": false, - "issue_opened_at": "2024-09-25T18:00:34Z", + "issue_opened_at": "2024-11-13", "issue_closed_at": null, "issue_points": null, "issue_status": "Todo", - "sprint_id": null, - "sprint_name": null, - "sprint_start": null, - "sprint_length": null, - "sprint_end": null, + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -516,24 +504,28 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "open" }, { - "issue_title": "[Task] Sample task created with issue template", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/40", - "issue_parent": null, + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Turn off comment for sprint reporting linter", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2833", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2830", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-25T18:02:47Z", - "issue_closed_at": null, - "issue_points": null, - "issue_status": "Todo", - "sprint_id": null, - "sprint_name": null, - "sprint_start": null, - "sprint_length": null, - "sprint_end": null, + "issue_is_closed": true, + "issue_opened_at": "2024-11-13", + "issue_closed_at": "2024-11-14", + "issue_points": 1.0, + "issue_status": "Done", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -542,24 +534,28 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, + "deliverable_status": null, "epic_url": null, - "epic_title": null + "epic_title": null, + "issue_state": "closed" }, { - "issue_title": "[Task] Bar 1", - "issue_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/42", - "issue_parent": "https://github.com/agilesix/simpler-grants-sandbox/issues/39", + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Turn off auto-close for issues in done column", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2835", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2830", "issue_type": "Task", - "issue_is_closed": false, - "issue_opened_at": "2024-09-25T19:14:49Z", - "issue_closed_at": null, - "issue_points": null, - "issue_status": "Todo", - "sprint_id": null, - "sprint_name": null, - "sprint_start": null, - "sprint_length": null, - "sprint_end": null, + "issue_is_closed": true, + "issue_opened_at": "2024-11-13", + "issue_closed_at": "2024-11-14", + "issue_points": 1.0, + "issue_status": "Done", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", "quad_id": null, "quad_name": null, "quad_start": null, @@ -568,7 +564,99 @@ "deliverable_pillar": null, "deliverable_url": null, "deliverable_title": null, - "epic_url": "https://github.com/agilesix/simpler-grants-sandbox/issues/39", - "epic_title": "[Epic] Sample epic created with issue template" + "deliverable_status": null, + "epic_url": null, + "epic_title": null, + "issue_state": "closed" + }, + { + "project_owner": "HHS", + "project_number": 17, + "issue_title": "[SPIKE] Connect Metabase to Postgres", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2845", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2560", + "issue_type": "Task", + "issue_is_closed": true, + "issue_opened_at": "2024-11-13", + "issue_closed_at": "2024-11-15", + "issue_points": 1.0, + "issue_status": "Done", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "closed" + }, + { + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Automate analytics DB creation in staging and prod environments", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2857", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2560", + "issue_type": "Task", + "issue_is_closed": false, + "issue_opened_at": "2024-11-14", + "issue_closed_at": null, + "issue_points": 3.0, + "issue_status": "In Progress", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "open" + }, + { + "project_owner": "HHS", + "project_number": 17, + "issue_title": "Add new columns to analytics database to enable sprint burndown charts", + "issue_url": "https://github.com/HHS/simpler-grants-gov/issues/2887", + "issue_parent": "https://github.com/HHS/simpler-grants-gov/issues/2666", + "issue_type": "Task", + "issue_is_closed": false, + "issue_opened_at": "2024-11-15", + "issue_closed_at": null, + "issue_points": 3.0, + "issue_status": "In Progress", + "sprint_id": "dfa48c43", + "sprint_name": "Sprint 1.4", + "sprint_start": "2024-11-13", + "sprint_length": 14.0, + "sprint_end": "2024-11-27", + "quad_id": "93412e1c", + "quad_name": "Quad 1.1", + "quad_start": "2024-09-09", + "quad_length": 122.0, + "quad_end": "2025-01-09", + "deliverable_pillar": "📊 SimplerReporting", + "deliverable_url": "https://github.com/HHS/simpler-grants-gov/issues/2347", + "deliverable_title": "Cross-program delivery metrics", + "deliverable_status": "In Progress", + "epic_url": "https://github.com/HHS/simpler-grants-gov/issues/2353", + "epic_title": "Public dashboard with at least one static chart", + "issue_state": "open" } -] \ No newline at end of file +] diff --git a/analytics/tests/test_cli.py b/analytics/tests/test_cli.py index dec9ed29d..05e5ada08 100644 --- a/analytics/tests/test_cli.py +++ b/analytics/tests/test_cli.py @@ -315,10 +315,11 @@ def test_transform_and_load_with_valid_parameters(self): f"running transform and load with effective date {self.EFFECTIVE_DATE}" in result.stdout ) + assert "project row(s) processed: 2" in result.stdout assert "quad row(s) processed: 1" in result.stdout - assert "deliverable row(s) processed: 2" in result.stdout - assert "sprint row(s) processed: 5" in result.stdout - assert "epic row(s) processed: 4" in result.stdout + assert "deliverable row(s) processed: 4" in result.stdout + assert "sprint row(s) processed: 4" in result.stdout + assert "epic row(s) processed: 6" in result.stdout assert "issue row(s) processed: 22" in result.stdout assert "transform and load is done" in result.stdout