diff --git a/daily_read/__init__.py b/daily_read/__init__.py index 5e26bdc..b7a88be 100644 --- a/daily_read/__init__.py +++ b/daily_read/__init__.py @@ -1,3 +1,4 @@ -import daily_read.config +""" DailyRead version +""" -config_values = daily_read.config.Config() +__version__ = "0.0.1" diff --git a/daily_read/__main__.py b/daily_read/__main__.py index 647959e..04d797e 100644 --- a/daily_read/__main__.py +++ b/daily_read/__main__.py @@ -21,43 +21,46 @@ import daily_read.utils -config_values = daily_read.config.Config() +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +@click.option("--env_file_path", type=click.Path()) +@click.pass_context +def daily_read_cli(ctx, env_file_path): + config_values = daily_read.config.Config(env_file_path=env_file_path) + if ctx.obj is None: + ctx.obj = dict() + ctx.obj["config_values"] = config_values -rich_handler = RichHandler() -rich_handler.setFormatter(logging.Formatter("%(message)s")) -LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - Commit: %(commit)s - %(message)s" + rich_handler = RichHandler() + rich_handler.setFormatter(logging.Formatter("%(message)s")) + LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - Commit: %(commit)s - %(message)s" -if not os.path.isabs(config_values.LOG_LOCATION): - raise ValueError(f"Log location is not an absolute path: {config_values.LOG_LOCATION}") + if not os.path.isabs(config_values.LOG_LOCATION): + raise ValueError(f"Log location is not an absolute path: {config_values.LOG_LOCATION}") -if os.path.exists(config_values.LOG_LOCATION) and not os.path.isdir(config_values.LOG_LOCATION): - raise ValueError(f"Log Location exists but is not a directory: {config_values.LOG_LOCATION}") + if os.path.exists(config_values.LOG_LOCATION) and not os.path.isdir(config_values.LOG_LOCATION): + raise ValueError(f"Log Location exists but is not a directory: {config_values.LOG_LOCATION}") -log_file = os.path.join(config_values.LOG_LOCATION, "DailyRead.log") + log_file = os.path.join(config_values.LOG_LOCATION, "DailyRead.log") -rotating_file_handler = logging.handlers.RotatingFileHandler( - log_file, maxBytes=1024 * 1024 * 100, backupCount=5 -) # 5 files of 100MB -rotating_file_handler.addFilter(daily_read.utils.ContextFilter()) + rotating_file_handler = logging.handlers.RotatingFileHandler( + log_file, maxBytes=1024 * 1024 * 100, backupCount=5 + ) # 5 files of 100MB + rotating_file_handler.addFilter(daily_read.utils.ContextFilter()) + logging.basicConfig( + level="INFO", + format=LOG_FORMAT, + handlers=[rich_handler, rotating_file_handler], + ) -logging.basicConfig( - level="INFO", - format=LOG_FORMAT, - handlers=[rich_handler, rotating_file_handler], -) log = logging.getLogger(__name__) -@click.group(context_settings=dict(help_option_names=["-h", "--help"])) -def daily_read_cli(): - pass - - ### GENERATE ### @daily_read_cli.group() -def generate(): +@click.pass_context +def generate(ctx): """Generate reports and save in a local git repository""" pass @@ -65,8 +68,10 @@ def generate(): @generate.command(name="all") @click.option("-u", "--upload", is_flag=True, help="Trigger upload of reports.") @click.option("--develop", is_flag=True, help="Only generate max 5 reports, for dev purposes.") -def generate_all(upload=False, develop=False): +@click.pass_context +def generate_all(ctx, upload=False, develop=False): # Fetch data from all sources (configurable) + config_values = ctx.obj["config_values"] projects_data = daily_read.ngi_data.ProjectDataMaster(config_values) log.info(f"Fetching data for {projects_data.source_names}") @@ -137,7 +142,9 @@ def generate_all(upload=False, develop=False): help="Include projects that are older than 6 months.", is_flag=True, ) -def generate_single(project, include_older=False): +@click.pass_context +def generate_single(ctx, project, include_older=False): + config_values = ctx.obj["config_values"] projects_data = daily_read.ngi_data.ProjectDataMaster(config_values) # Fetch all projects so that the report will look the same log.info("Fetching data from NGI sources") diff --git a/daily_read/config.py b/daily_read/config.py index 2bc77c8..9ddd49d 100644 --- a/daily_read/config.py +++ b/daily_read/config.py @@ -4,8 +4,8 @@ class Config(object): - def __init__(self): - dotenv.load_dotenv() + def __init__(self, env_file_path): + dotenv.load_dotenv(dotenv_path=env_file_path) self.ORDER_PORTAL_URL = os.getenv("DAILY_READ_ORDER_PORTAL_URL") self.ORDER_PORTAL_API_KEY = os.getenv("DAILY_READ_ORDER_PORTAL_API_KEY") self.REPORTS_LOCATION = os.getenv("DAILY_READ_REPORTS_LOCATION") diff --git a/daily_read/templates/daily_report.html.j2 b/daily_read/templates/daily_report.html.j2 index 5f8432c..2cf1c4a 100644 --- a/daily_read/templates/daily_report.html.j2 +++ b/daily_read/templates/daily_report.html.j2 @@ -1,6 +1,8 @@ - - - \ No newline at end of file + ' + width="120%" + height="2000" + frameborder="0" + sandbox="allow-scripts allow-top-navigation" + > + diff --git a/setup.py b/setup.py index a786006..749c14c 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = "0.0.1" +from daily_read import __version__ with open("README.md") as f: readme = f.read() @@ -10,7 +10,7 @@ setup( name="Daily Read", - version=version, + version=__version__, description="A utility to generate and upload automatic progress reports for NGI Sweden.", long_description=readme, long_description_content_type="text/markdown", diff --git a/tests/conftest.py b/tests/conftest.py index e6719d6..22822c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -323,7 +323,7 @@ def data_repo_full( @pytest.fixture def mock_project_data_record(): def _method(status): - config_values = config.Config() + config_values = config.Config(env_file_path=".test.env") if status == "open": mock_record = ngi_data.ProjectDataRecord( "NGIS/2023/NGI123456.json", @@ -441,3 +441,10 @@ def mocked_statusdb_conn_rows(): }, ) return [row1, row2, row3] + + +@pytest.fixture(autouse=True) +def get_env_file_path(): + """returns the test env file path""" + + return ".test.env" diff --git a/tests/test_daily_report.py b/tests/test_daily_report.py index f98585a..0c741a1 100644 --- a/tests/test_daily_report.py +++ b/tests/test_daily_report.py @@ -6,11 +6,11 @@ from daily_read import daily_report, config, ngi_data, order_portal -def test_write_report_to_out_dir(data_repo_full, mock_project_data_record, create_report_path): +def test_write_report_to_out_dir(data_repo_full, mock_project_data_record, create_report_path, get_env_file_path): """Test existence of html report when provided with out_dir""" orderer = "dummy@dummy.se" order_id = "NGI123456" - config_values = config.Config() + config_values = config.Config(env_file_path=get_env_file_path) daily_rep = daily_report.DailyReport() with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) diff --git a/tests/test_ngi_data.py b/tests/test_ngi_data.py index 636b6b8..a65d86e 100644 --- a/tests/test_ngi_data.py +++ b/tests/test_ngi_data.py @@ -9,9 +9,9 @@ ####################################################### TESTS ######################################################### -def test_create_project_data_master(data_repo_full): +def test_create_project_data_master(data_repo_full, get_env_file_path): """With existing git repo, test creation of a ProjectDataMaster class""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -21,9 +21,9 @@ def test_create_project_data_master(data_repo_full): assert data_master._data_fetched == False -def test_create_project_data_master_no_commit(data_repo_no_commit): +def test_create_project_data_master_no_commit(data_repo_no_commit, get_env_file_path): """With existing git repo but without commits, test creation of a ProjectDataMaster class""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -35,8 +35,8 @@ def test_create_project_data_master_no_commit(data_repo_no_commit): assert data_master.data_repo.commit().message == "Empty file as a first commit" -def test_modified_or_new_no_commit(data_repo_no_commit): - config_values = config.Config() +def test_modified_or_new_no_commit(data_repo_no_commit, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -47,8 +47,8 @@ def test_modified_or_new_no_commit(data_repo_no_commit): assert len(set(file_names)) == 0 -def test_modified_or_new(data_repo_full): - config_values = config.Config() +def test_modified_or_new(data_repo_full, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -59,8 +59,8 @@ def test_modified_or_new(data_repo_full): assert len(set(file_names)) == 12 -def test_modified_or_new_untracked(data_repo_untracked): - config_values = config.Config() +def test_modified_or_new_untracked(data_repo_untracked, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -73,8 +73,8 @@ def test_modified_or_new_untracked(data_repo_untracked): assert "untracked_file" in file_names[0] -def test_modified_or_new_staged(data_repo_new_staged): - config_values = config.Config() +def test_modified_or_new_staged(data_repo_new_staged, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -86,8 +86,8 @@ def test_modified_or_new_staged(data_repo_new_staged): assert any("staged_file" in s for s in file_names) -def test_modified_or_new_modified_not_staged(data_repo_modified_not_staged): - config_values = config.Config() +def test_modified_or_new_modified_not_staged(data_repo_modified_not_staged, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -100,8 +100,8 @@ def test_modified_or_new_modified_not_staged(data_repo_modified_not_staged): assert "modified_file" in file_names[0] -def test_modified_or_new_modified_staged(data_repo_modified_staged): - config_values = config.Config() +def test_modified_or_new_modified_staged(data_repo_modified_staged, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -114,8 +114,8 @@ def test_modified_or_new_modified_staged(data_repo_modified_staged): assert "modified_staged_file" in file_names[0] -def test_modified_or_new_tracked(data_repo_tracked): - config_values = config.Config() +def test_modified_or_new_tracked(data_repo_tracked, get_env_file_path): + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -127,9 +127,9 @@ def test_modified_or_new_tracked(data_repo_tracked): assert len(set(file_names)) == 0 -def test_get_unique_orderers(data_repo_full): +def test_get_unique_orderers(data_repo_full, get_env_file_path): """Test getting unique orders in the project data from statusdb""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.get_data() @@ -137,9 +137,9 @@ def test_get_unique_orderers(data_repo_full): assert orderers == set(["dummy@dummy.se"]) -def test_user_list(data_repo_full, tmp_path, mocked_statusdb_conn_rows): +def test_user_list(data_repo_full, tmp_path, mocked_statusdb_conn_rows, get_env_file_path): """Test getting and reading users from the user list url""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) temp_file = tmp_path / "test_file.txt" temp_file.write_text("dummy@dummy.se\ntest@dummy.se") config_values.USERS_LIST_LOCATION = temp_file @@ -154,9 +154,9 @@ def test_user_list(data_repo_full, tmp_path, mocked_statusdb_conn_rows): assert orderers == set(["dummy@dummy.se", "test@dummy.se"]) -def test_save_data_to_disk(data_repo_full, mocked_statusdb_conn_rows): +def test_save_data_to_disk(data_repo_full, mocked_statusdb_conn_rows, get_env_file_path): """Test saving in git repo the data gotten from statusdb""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.sources[0].statusdb_session.rows.return_value = mocked_statusdb_conn_rows @@ -166,9 +166,9 @@ def test_save_data_to_disk(data_repo_full, mocked_statusdb_conn_rows): assert os.path.exists(os.path.join(config_values.DATA_LOCATION, "NGIS/2023/NGI123458.json")) -def test_get_data_with_project(data_repo_full, mocked_statusdb_conn_rows): +def test_get_data_with_project(data_repo_full, mocked_statusdb_conn_rows, get_env_file_path): """Test getting data for a specific order""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) order_id = "NGI123457" with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -179,9 +179,9 @@ def test_get_data_with_project(data_repo_full, mocked_statusdb_conn_rows): assert data_master.data[order_id].internal_id_or_portal_id == "P123457" -def test_get_data_with_project_unknown(data_repo_full, mocked_statusdb_conn_rows): +def test_get_data_with_project_unknown(data_repo_full, mocked_statusdb_conn_rows, get_env_file_path): """Test error thrown when the order specified is not found in statusdb""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.sources[0].statusdb_session.rows.return_value = mocked_statusdb_conn_rows @@ -190,9 +190,9 @@ def test_get_data_with_project_unknown(data_repo_full, mocked_statusdb_conn_rows @mock.patch("daily_read.statusdb.StatusDBSession") -def test_data_loc_not_abs(mock_status): +def test_data_loc_not_abs(mock_status, get_env_file_path): """Test error thrown when given data location is not an absolute path""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) config_values.DATA_LOCATION = "tests/test_data_location" with pytest.raises( ValueError, match=f"Data location is not an absolute path: {config_values.DATA_LOCATION}" @@ -201,9 +201,9 @@ def test_data_loc_not_abs(mock_status): @mock.patch("daily_read.statusdb.StatusDBSession") -def test_data_loc_not_dir(mock_status, tmp_path): +def test_data_loc_not_dir(mock_status, tmp_path, get_env_file_path): """Test error thrown when data location is not a directory""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) temp_file = tmp_path / "test_file.txt" temp_file.write_text("test") config_values.DATA_LOCATION = temp_file @@ -213,11 +213,11 @@ def test_data_loc_not_dir(mock_status, tmp_path): ngi_data.ProjectDataMaster(config_values) -def test_get_data_with_no_project_dates(data_repo_full, mocked_statusdb_conn_rows, caplog): +def test_get_data_with_no_project_dates(data_repo_full, mocked_statusdb_conn_rows, caplog, get_env_file_path): """Test log output when no project dates are found in statusdb for a specific project""" from copy import deepcopy - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.sources[0].statusdb_session.rows.return_value = mocked_statusdb_conn_rows @@ -232,10 +232,10 @@ def test_get_data_with_no_project_dates(data_repo_full, mocked_statusdb_conn_row assert "No project dates found for NGI123459" in caplog.text -def test_skip_order_with_no_year(data_repo_full, mocked_statusdb_conn_rows, caplog): +def test_skip_order_with_no_year(data_repo_full, mocked_statusdb_conn_rows, caplog, get_env_file_path): """Test that orders with no order year (i.e. with no contract signed) are skipped""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.sources[0].statusdb_session.rows.return_value = mocked_statusdb_conn_rows @@ -247,10 +247,10 @@ def test_skip_order_with_no_year(data_repo_full, mocked_statusdb_conn_rows, capl @mock.patch("daily_read.statusdb.StatusDBSession") -def test_no_source_specified(mock_status): +def test_no_source_specified(mock_status, get_env_file_path): """Test error thrown when no sources are specified""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) config_values.FETCH_FROM_NGIS = "" config_values.FETCH_FROM_SNPSEQ = "" config_values.FETCH_FROM_UGC = "" @@ -258,9 +258,9 @@ def test_no_source_specified(mock_status): ngi_data.ProjectDataMaster(config_values) -def test_commit_staged_data(data_repo_full, mocked_statusdb_conn_rows): +def test_commit_staged_data(data_repo_full, mocked_statusdb_conn_rows, get_env_file_path): """Test file is staged for commit and committed and then try adding it for staging again""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) data_master.sources[0].statusdb_session.rows.return_value = mocked_statusdb_conn_rows diff --git a/tests/test_order_portal.py b/tests/test_order_portal.py index f8d3581..5a24687 100644 --- a/tests/test_order_portal.py +++ b/tests/test_order_portal.py @@ -7,11 +7,11 @@ from daily_read import order_portal, config, ngi_data -def test_get_and_process_orders_open_upload_fail(data_repo_full, mock_project_data_record, caplog): +def test_get_and_process_orders_open_upload_fail(data_repo_full, mock_project_data_record, caplog, get_env_file_path): """Test getting and processing an open order and upload to Order portal failing""" orderer = "dummy@dummy.se" order_id = "NGI123456" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -32,11 +32,11 @@ def test_get_and_process_orders_open_upload_fail(data_repo_full, mock_project_da assert f"Report not uploaded for order with project id: {order_id}\nReason: 404" in caplog.text -def test_get_and_process_orders_open_and_upload(data_repo_full, mock_project_data_record): +def test_get_and_process_orders_open_and_upload(data_repo_full, mock_project_data_record, get_env_file_path): """Test getting and processing an open order and uploading its Project progress report and uploading the report to the Order portal""" orderer = "dummy@dummy.se" order_id = "NGI123456" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -69,11 +69,13 @@ def test_get_and_process_orders_open_and_upload(data_repo_full, mock_project_dat ) -def test_get_and_process_orders_open_with_report_and_upload(data_repo_full, mock_project_data_record, caplog): +def test_get_and_process_orders_open_with_report_and_upload( + data_repo_full, mock_project_data_record, caplog, get_env_file_path +): """Test getting, processing an open order with an existing Project progress report and uploading the report to the Order portal""" orderer = "dummy@dummy.se" order_id = "NGI123453" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -109,11 +111,13 @@ def test_get_and_process_orders_open_with_report_and_upload(data_repo_full, mock assert f"Updated report for order with project id: {order_id}" in caplog.text -def test_get_and_process_orders_open_to_aborted_with_report_and_upload(data_repo_full, mock_project_data_record): +def test_get_and_process_orders_open_to_aborted_with_report_and_upload( + data_repo_full, mock_project_data_record, get_env_file_path +): """Test getting, processing an open order with an existing Project progress report and uploading the report to the Order portal""" orderer = "dummy@dummy.se" order_id = "NGI123461" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -142,11 +146,11 @@ def test_get_and_process_orders_open_to_aborted_with_report_and_upload(data_repo ) -def test_get_and_process_orders_closed(data_repo_full, mock_project_data_record): +def test_get_and_process_orders_closed(data_repo_full, mock_project_data_record, get_env_file_path): """Test getting and processing an order closed within the timeframe of Project progress report deletion""" orderer = "dummy@dummy.se" order_id = "NGI123455" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -160,11 +164,11 @@ def test_get_and_process_orders_closed(data_repo_full, mock_project_data_record) assert modified_orders[orderer]["delete_report_for"]["All Raw data Delivered"][0] == data_master.data[order_id] -def test_get_and_process_orders_mult_reports(data_repo_full, mock_project_data_record): +def test_get_and_process_orders_mult_reports(data_repo_full, mock_project_data_record, get_env_file_path): """Test getting and processing orders with multiple Project progress reports""" orderer = "dummy@dummy.se" order_id = "NGI123454" - config_values = config.Config() + config_values = config.Config(get_env_file_path) with mock.patch("daily_read.statusdb.StatusDBSession"): data_master = ngi_data.ProjectDataMaster(config_values) @@ -180,10 +184,10 @@ def test_get_and_process_orders_mult_reports(data_repo_full, mock_project_data_r op.process_orders(config_values.STATUS_PRIORITY_REV) -def test_base_url_and_api_key_not_set(data_repo_full, mock_project_data_record): +def test_base_url_and_api_key_not_set(data_repo_full, mock_project_data_record, get_env_file_path): """Test the conditions when environment variables ORDER_PORTAL_URL and ORDER_PORTAL_API_KEY are not set""" order_id = "NGI123456" - config_values = config.Config() + config_values = config.Config(get_env_file_path) order_portal_url = config_values.ORDER_PORTAL_URL api_key = config_values.ORDER_PORTAL_API_KEY diff --git a/tests/test_statusdb.py b/tests/test_statusdb.py index ae20bbe..72173ac 100644 --- a/tests/test_statusdb.py +++ b/tests/test_statusdb.py @@ -3,9 +3,9 @@ from daily_read import config, statusdb -def test_no_statusdb_conn(data_repo_full): +def test_no_statusdb_conn(data_repo_full, get_env_file_path): """Test error thrown when statusdb conn fails""" - config_values = config.Config() + config_values = config.Config(get_env_file_path) # Escape special chars because match uses regex display_url_string = ( rf"https:\/\/{config_values.STHLM_STATUSDB_USERNAME}:\*\*\*\*\*\*\*\*\*@{config_values.STHLM_STATUSDB_URL}"