From 803b0370fd25b81edfce559dbb7e2f986b0901d7 Mon Sep 17 00:00:00 2001 From: John A Stevenson Date: Fri, 6 Dec 2019 11:41:20 +0000 Subject: [PATCH 1/4] Update BGS integration test instructions --- CONTRIBUTING.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61d4618..bd5f978 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,15 +78,15 @@ A Dockerised PostGIS version can be started with: ```bash export TEST_PG_PORT=5432 +export TEST_PG_PASSWORD=etlhelper_pw docker run -e POSTGRES_USER=etlhelper_user -e POSTGRES_DB=etlhelper \ - -e POSTGRES_PASSWORD=etlhelper_pw --name etlhelper_postgis \ + -e POSTGRES_PASSWORD=$TEST_PG_PASSWORD --name etlhelper_postgis \ -d --rm -p $TEST_PG_PORT:5432 mdillon/postgis:11-alpine ``` Tests are run with: ```bash -export TEST_PG_PASSWORD=etlhelper_pw bash bin/run_tests_for_developer.sh ``` @@ -94,24 +94,27 @@ The test-runner script will run tests within a dedicated container and provide HTML coverage output. It can be viewed with `firefox htmlcov/index.html`. -#### Additional integration tests +#### Running additional BGS integration tests Additional integration tests can be run against internal BGS Oracle and SQL Server databases. The DbParams for these databases are defined by environment variables stored within the Continuous Integration system. -The required environment variables to run the integration test suite can be -seen in the `bin/run_tests_for_developers.sh` file. - -Internal BGS developers: `source` the `test_env_vars` Snippet from GitLab to -configure local variables. +To run these: ++ Go to Settings > CI / CD > Variables in GitLab ++ Use "Reveal values" and copy the contents of TEST_ENV_VARS ++ Paste into the terminal to set environment variables ++ Run tests as before ## Creating a new release -Releases are created manually from the master branch with the following -commands. -The full integration test suite should be run before creating a release. +Releases are created manually from the master branch via tags. +This should be done via the GitHub web interface. +The GitLab CI system will automatically perform the release following the next +repository mirror synchronisation. + +The instructions below explain how to do a manual release. #### Tagging From ce09526434735a784dbb2bc87f1fef8e817640ed Mon Sep 17 00:00:00 2001 From: John A Stevenson Date: Fri, 6 Dec 2019 11:44:17 +0000 Subject: [PATCH 2/4] Pin Dockerfile version to python:3.6.9-slim --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b197a7c..015a00b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim +FROM python:3.6.9-slim # Install package dependencies RUN apt-get update -y && \ From f7379990fd6bceaeb166f7cd9a03aeba16a0cf88 Mon Sep 17 00:00:00 2001 From: John A Stevenson Date: Fri, 6 Dec 2019 13:07:39 +0000 Subject: [PATCH 3/4] Use named arguments for pytest fixtures Passing the fixture scope as a position argument has been deprecated in more recent versions of pytest. --- test/conftest.py | 4 ++-- test/integration/db/test_oracle.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 921bafe..a350b5a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,7 +21,7 @@ user='etlhelper_user') -@pytest.fixture('module') +@pytest.fixture(scope='module') def pgtestdb_insert_sql(): """Return SQL command used to populate test database.""" insert_sql = dedent(""" @@ -33,7 +33,7 @@ def pgtestdb_insert_sql(): return insert_sql -@pytest.fixture('function') +@pytest.fixture(scope='function') def pgtestdb_test_tables(test_table_data, pgtestdb_conn, pgtestdb_insert_sql): """ Create a table and fill with test data. Teardown after the yield drops it diff --git a/test/integration/db/test_oracle.py b/test/integration/db/test_oracle.py index e30f86d..7761529 100644 --- a/test/integration/db/test_oracle.py +++ b/test/integration/db/test_oracle.py @@ -69,7 +69,7 @@ def testdb_conn(): yield conn -@pytest.fixture('function') +@pytest.fixture(scope='function') def test_tables(test_table_data, testdb_conn): """ Create a table and fill with test data. Teardown after the yield drops it From 0d323385707f1dbf146097c1eda58a61e7345ac3 Mon Sep 17 00:00:00 2001 From: John A Stevenson Date: Fri, 6 Dec 2019 13:18:16 +0000 Subject: [PATCH 4/4] Make DbParams raise AttributeError not KeyError Python specifies that the getattr function should raise an AttributeError if an attribute does not exist. https://docs.python.org/3/library/functions.html#getattr This issue was noticed when pytest.parametrize began crashing on an update to pytest 5.3.1. See this (non-)bug report for details: https://github.com/pytest-dev/pytest/issues/6321 --- etlhelper/db_params.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etlhelper/db_params.py b/etlhelper/db_params.py index df0df3a..6cbfb4e 100644 --- a/etlhelper/db_params.py +++ b/etlhelper/db_params.py @@ -15,14 +15,18 @@ class DbParams(dict): subclasses dict, to give dynamic attributes, following the pattern described here: https://amir.rachum.com/blog/2016/10/05/python-dynamic-attributes/ """ - def __init__(self, dbtype='dbtype not set', **kwargs): kwargs.update(dbtype=dbtype.upper()) super().__init__(kwargs) self.validate_params() def __getattr__(self, item): - return self[item] + try: + return self[item] + except KeyError: + # getattr should raise AttributeError, not KeyError + # https://docs.python.org/3/library/functions.html#getattr + raise AttributeError(f'No such attribute: {item}') def __dir__(self): return super().__dir__() + [str(k) for k in self.keys()]