From 05d3e067096918b1968bc3186ea31fd6e166d662 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 12:13:42 +0200 Subject: [PATCH 1/7] #700 django specific api template --- .../django-app/api/test_st.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 application-templates/django-app/api/test_st.py diff --git a/application-templates/django-app/api/test_st.py b/application-templates/django-app/api/test_st.py new file mode 100644 index 00000000..fb852433 --- /dev/null +++ b/application-templates/django-app/api/test_st.py @@ -0,0 +1,27 @@ +import os +from pprint import pprint +import schemathesis as st +from schemathesis.checks import response_schema_conformance, not_a_server_error + +from cloudharness_test import apitest_init # include to perform default authorization + +app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") + +try: + schema = st.from_uri(app_url + "/openapi.json") +except: + # support alternative schema location + schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json") + + +@schema.parametrize(endpoint="/ping") +def test_ping(case): + response = case.call() + pprint(response.__dict__) + assert response.status_code == 200, "this api errors on purpose" + +def test_state_machine(): + schema.as_state_machine().run() +# APIWorkflow = schema.as_state_machine() +# APIWorkflow.run() +# TestAPI = APIWorkflow.TestCase \ No newline at end of file From 8a6d67c282d15985e62d244ca5a58456426645fc Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 12:14:15 +0200 Subject: [PATCH 2/7] #700 white list alt openapi path --- deployment-configuration/value-template.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment-configuration/value-template.yaml b/deployment-configuration/value-template.yaml index 16c00b3a..0e39e395 100644 --- a/deployment-configuration/value-template.yaml +++ b/deployment-configuration/value-template.yaml @@ -22,6 +22,8 @@ harness: - administrator - uri: /api/openapi.json white-listed: true + - uri: /openapi.json + white-listed: true # -- Defines reference deployment parameters. Values maps to k8s spec deployment: # -- When true, enables automatic deployment From ac7c6469badebbf53fcadfae6582a128eeb4005a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 12:19:42 +0200 Subject: [PATCH 3/7] #700 #703 Improve local testing + alt schema url --- .../cloudharness_utils/testing/util.py | 4 +- .../cloudharness_test/apitest_init.py | 54 ++++++++++++------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py index b0e98624..58e20fa0 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py +++ b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py @@ -11,7 +11,9 @@ def get_user_password(main_user: ApplicationUser): def get_app_environment(app_config: ApplicationHarnessConfig, app_domain, use_local_env=True): my_env = os.environ.copy() if use_local_env else {} my_env["APP_URL"] = app_domain - + schema_file = f"applications/{app_config.name}/api/openapi.yaml" + if os.path.exists(schema_file): + my_env["APP_SCHEMA_FILE"] = schema_file if app_config.accounts and app_config.accounts.users: main_user: ApplicationUser = app_config.accounts.users[0] diff --git a/tools/cloudharness-test/cloudharness_test/apitest_init.py b/tools/cloudharness-test/cloudharness_test/apitest_init.py index 4ba65f7e..52d20f53 100644 --- a/tools/cloudharness-test/cloudharness_test/apitest_init.py +++ b/tools/cloudharness-test/cloudharness_test/apitest_init.py @@ -1,44 +1,62 @@ import os import logging +import requests import schemathesis as st from cloudharness.auth import get_token -if "APP_URL" in os.environ: - - app_url = os.environ["APP_URL"] - - try: - openapi_uri = app_url + "/openapi.json" - logging.info("Using openapi spec at %s", openapi_uri) - schema = st.from_uri(openapi_uri) - except Exception as e: - raise Exception(f"Cannot setup api tests: {openapi_uri} not reachable. Check your deployment is up and configuration") from e +if "APP_URL" or "APP_SCHEMA_FILE" in os.environ: + app_schema = os.environ.get("APP_SCHEMA_FILE", None) + app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") + logging.info("Start schemathesis tests on %s", app_url) + if app_schema: + openapi_uri = app_schema + schema = st.from_file(openapi_uri) + else: + try: + openapi_uri = openapi_uri = app_url + "/openapi.json" + schema = st.from_file(openapi_uri) + except st.exceptions.SchemaLoadingError as e: + # Use alternative configuration + try: + openapi_uri = app_url.replace("/api", "") + "/openapi.json" + print(requests.get(openapi_uri)) + schema = st.from_uri(openapi_uri) + except st.exceptions.SchemaLoadingError as e: + raise Exception( + f"Cannot setup api tests: {openapi_uri} not valid. Check your deployment is up and configuration") from e + + except Exception as e: + raise Exception( + f"Cannot setup api tests: {openapi_uri}: {e}") from e + + logging.info("Using openapi spec at %s", openapi_uri) if "USERNAME" in os.environ and "PASSWORD" in os.environ: logging.info("Setting token from username and password") + @st.auth.register() class TokenAuth: def get(self, context): - + username = os.environ["USERNAME"] - password = os.environ["PASSWORD"] - + password = os.environ["PASSWORD"] + return get_token(username, password) def set(self, case, data, context): case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" + case.headers["Authorization"] = f"Bearer {data}" + case.headers["Cookie"] = f"kc-access={data}" else: @st.auth.register() class TokenAuth: def get(self, context): - + return "" def set(self, case, data, context): case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" \ No newline at end of file + case.headers["Authorization"] = f"Bearer {data}" + case.headers["Cookie"] = f"kc-access={data}" From 82696dc3cd6270552d59d4fd85acccddc4cea189 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 12:21:37 +0200 Subject: [PATCH 4/7] #700 small fix --- tools/cloudharness-test/cloudharness_test/apitest_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cloudharness-test/cloudharness_test/apitest_init.py b/tools/cloudharness-test/cloudharness_test/apitest_init.py index 52d20f53..32772abd 100644 --- a/tools/cloudharness-test/cloudharness_test/apitest_init.py +++ b/tools/cloudharness-test/cloudharness_test/apitest_init.py @@ -16,7 +16,7 @@ else: try: openapi_uri = openapi_uri = app_url + "/openapi.json" - schema = st.from_file(openapi_uri) + schema = st.from_uri(openapi_uri) except st.exceptions.SchemaLoadingError as e: # Use alternative configuration try: From 6c5298544e0c80dcd2fc0d237bf89153607e89fc Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 12:34:57 +0200 Subject: [PATCH 5/7] #703 local schema file only for harness-test --- .../cloudharness-utils/cloudharness_utils/testing/util.py | 4 +--- tools/cloudharness-test/cloudharness_test/api.py | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py index 58e20fa0..b0e98624 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py +++ b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py @@ -11,9 +11,7 @@ def get_user_password(main_user: ApplicationUser): def get_app_environment(app_config: ApplicationHarnessConfig, app_domain, use_local_env=True): my_env = os.environ.copy() if use_local_env else {} my_env["APP_URL"] = app_domain - schema_file = f"applications/{app_config.name}/api/openapi.yaml" - if os.path.exists(schema_file): - my_env["APP_SCHEMA_FILE"] = schema_file + if app_config.accounts and app_config.accounts.users: main_user: ApplicationUser = app_config.accounts.users[0] diff --git a/tools/cloudharness-test/cloudharness_test/api.py b/tools/cloudharness-test/cloudharness_test/api.py index d7cc87bd..e3c93ea5 100644 --- a/tools/cloudharness-test/cloudharness_test/api.py +++ b/tools/cloudharness-test/cloudharness_test/api.py @@ -60,6 +60,12 @@ def run_api_tests(root_paths, helm_values: HarnessMainConfig, base_domain, inclu app_env = get_app_environment(app_config, app_domain) + schema_file = f"applications/{app_config.name}/api/openapi.yaml" + + for path in root_paths: + if os.path.exists(os.path.join(path, schema_file)): + app_env["APP_SCHEMA_FILE"] = schema_file + if api_config.autotest: logging.info("Running auto api tests") cmd = get_schemathesis_command(api_filename, app_config, app_domain) From 86d3c86aad2a27f638592f97931b73dfe71f274b Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 23 Aug 2023 17:42:46 +0200 Subject: [PATCH 6/7] Chore: improve docs --- docs/testing.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 11fd8349..ef37128a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -108,16 +108,7 @@ The test can use environmental variables: Examples: - [Sample api test](../applications/samples/test/api/test_st.py) -### Common smoke tests - -Once a test is created for your application, generic smoke tests are also -executed, checking for: - -- Main page is reachable -- No errors in the console -- No error responses from network resources and fetch requests (code < 400) -The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec.ts). @@ -125,6 +116,22 @@ The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec End to end tests run in a headless browser ([Puppeteer](https://github.com/puppeteer/puppeteer)) against the full deployment on Kubernetes. +Custom configuration: + +```yaml +harness: + ... + test: + e2e: + # -- enable/disable e2e tests + enabled: true + # -- ignore errors on console by default + ignoreConsoleErrors: false + # -- ignore fetched resources errors by default + ignoreRequestErrors: false + # -- enable common smoke tests + smoketest: true +``` ### Write tests with Jest and Puppeteer @@ -159,7 +166,7 @@ executed, checking for: - No errors in the console - No error responses from network resources and fetch requests (code < 400) -The smoke tests is defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts). +The smoke tests are defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts). ## Run API and E2E tests in the CI/CD pipeline From 042189b73ddf2bf71f9ddc2ed7c18ae52994856d Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 24 Aug 2023 09:43:19 +0200 Subject: [PATCH 7/7] #703 improve docs and code clarity --- docs/testing.md | 21 ++++++++++++++++++- .../cloudharness_test/api.py | 1 + .../cloudharness_test/apitest_init.py | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index ef37128a..27356fea 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -189,7 +189,7 @@ deployment. In order to use `harness-test` install the library with ``` -pip install -r requirements-test.txt +pip install -e tools/cloudharness-test ``` In order to run tests against an existing deployment based on a domain (say, my.domain), run: @@ -205,6 +205,25 @@ If you want to run the deployment locally and then run the tests, can use skaffo 1. Wait the deployment to settle 1. Run `harness-test PATHS` +### Tests development +The `harness-test` client is useful while developing and tweaking the tests. +In that case it's better to target the application under development and +the kind of tests we are working on. + + +To target a specific application for end-to-end tests, use: +``` +harness-test . -i [APPNAME] -e +``` + +To target a specific application for api tests, use: +``` +harness-test . -i [APPNAME] -a +``` + +Note that the local version of the openapi.yaml file located at applications/[APPNAME]/api/openapi.yaml is used if available. That's useful to tweak examples and responses +used by schemathesis to generate the test hypotheses. + ## Create test users for your application To create test users: diff --git a/tools/cloudharness-test/cloudharness_test/api.py b/tools/cloudharness-test/cloudharness_test/api.py index e3c93ea5..5193902d 100644 --- a/tools/cloudharness-test/cloudharness_test/api.py +++ b/tools/cloudharness-test/cloudharness_test/api.py @@ -63,6 +63,7 @@ def run_api_tests(root_paths, helm_values: HarnessMainConfig, base_domain, inclu schema_file = f"applications/{app_config.name}/api/openapi.yaml" for path in root_paths: + # use local schema if available to simplify test development if os.path.exists(os.path.join(path, schema_file)): app_env["APP_SCHEMA_FILE"] = schema_file diff --git a/tools/cloudharness-test/cloudharness_test/apitest_init.py b/tools/cloudharness-test/cloudharness_test/apitest_init.py index 32772abd..1f83eabd 100644 --- a/tools/cloudharness-test/cloudharness_test/apitest_init.py +++ b/tools/cloudharness-test/cloudharness_test/apitest_init.py @@ -11,9 +11,11 @@ app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") logging.info("Start schemathesis tests on %s", app_url) if app_schema: + # Test locally with harness-test -- use local schema for convenience during test development openapi_uri = app_schema schema = st.from_file(openapi_uri) else: + # remote testing: might be /api/openapi.json or /openapi.json try: openapi_uri = openapi_uri = app_url + "/openapi.json" schema = st.from_uri(openapi_uri)