diff --git a/application-templates/django-app/api/test_st.py b/application-templates/django-app/api/test_st.py new file mode 100644 index 00000000..fb852433 --- /dev/null +++ b/application-templates/django-app/api/test_st.py @@ -0,0 +1,27 @@ +import os +from pprint import pprint +import schemathesis as st +from schemathesis.checks import response_schema_conformance, not_a_server_error + +from cloudharness_test import apitest_init # include to perform default authorization + +app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") + +try: + schema = st.from_uri(app_url + "/openapi.json") +except: + # support alternative schema location + schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json") + + +@schema.parametrize(endpoint="/ping") +def test_ping(case): + response = case.call() + pprint(response.__dict__) + assert response.status_code == 200, "this api errors on purpose" + +def test_state_machine(): + schema.as_state_machine().run() +# APIWorkflow = schema.as_state_machine() +# APIWorkflow.run() +# TestAPI = APIWorkflow.TestCase \ No newline at end of file diff --git a/deployment-configuration/value-template.yaml b/deployment-configuration/value-template.yaml index 98810629..8702e6f6 100644 --- a/deployment-configuration/value-template.yaml +++ b/deployment-configuration/value-template.yaml @@ -22,6 +22,8 @@ harness: - administrator - uri: /api/openapi.json white-listed: true + - uri: /openapi.json + white-listed: true # -- Defines reference deployment parameters. Values maps to k8s spec deployment: # -- When true, enables automatic deployment diff --git a/docs/testing.md b/docs/testing.md index 11fd8349..27356fea 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -108,16 +108,7 @@ The test can use environmental variables: Examples: - [Sample api test](../applications/samples/test/api/test_st.py) -### Common smoke tests - -Once a test is created for your application, generic smoke tests are also -executed, checking for: -- Main page is reachable -- No errors in the console -- No error responses from network resources and fetch requests (code < 400) - -The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec.ts). @@ -125,6 +116,22 @@ The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec End to end tests run in a headless browser ([Puppeteer](https://github.com/puppeteer/puppeteer)) against the full deployment on Kubernetes. +Custom configuration: + +```yaml +harness: + ... + test: + e2e: + # -- enable/disable e2e tests + enabled: true + # -- ignore errors on console by default + ignoreConsoleErrors: false + # -- ignore fetched resources errors by default + ignoreRequestErrors: false + # -- enable common smoke tests + smoketest: true +``` ### Write tests with Jest and Puppeteer @@ -159,7 +166,7 @@ executed, checking for: - No errors in the console - No error responses from network resources and fetch requests (code < 400) -The smoke tests is defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts). +The smoke tests are defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts). ## Run API and E2E tests in the CI/CD pipeline @@ -182,7 +189,7 @@ deployment. In order to use `harness-test` install the library with ``` -pip install -r requirements-test.txt +pip install -e tools/cloudharness-test ``` In order to run tests against an existing deployment based on a domain (say, my.domain), run: @@ -198,6 +205,25 @@ If you want to run the deployment locally and then run the tests, can use skaffo 1. Wait the deployment to settle 1. Run `harness-test PATHS` +### Tests development +The `harness-test` client is useful while developing and tweaking the tests. +In that case it's better to target the application under development and +the kind of tests we are working on. + + +To target a specific application for end-to-end tests, use: +``` +harness-test . -i [APPNAME] -e +``` + +To target a specific application for api tests, use: +``` +harness-test . -i [APPNAME] -a +``` + +Note that the local version of the openapi.yaml file located at applications/[APPNAME]/api/openapi.yaml is used if available. That's useful to tweak examples and responses +used by schemathesis to generate the test hypotheses. + ## Create test users for your application To create test users: diff --git a/tools/cloudharness-test/cloudharness_test/api.py b/tools/cloudharness-test/cloudharness_test/api.py index d7cc87bd..5193902d 100644 --- a/tools/cloudharness-test/cloudharness_test/api.py +++ b/tools/cloudharness-test/cloudharness_test/api.py @@ -60,6 +60,13 @@ def run_api_tests(root_paths, helm_values: HarnessMainConfig, base_domain, inclu app_env = get_app_environment(app_config, app_domain) + schema_file = f"applications/{app_config.name}/api/openapi.yaml" + + for path in root_paths: + # use local schema if available to simplify test development + if os.path.exists(os.path.join(path, schema_file)): + app_env["APP_SCHEMA_FILE"] = schema_file + if api_config.autotest: logging.info("Running auto api tests") cmd = get_schemathesis_command(api_filename, app_config, app_domain) diff --git a/tools/cloudharness-test/cloudharness_test/apitest_init.py b/tools/cloudharness-test/cloudharness_test/apitest_init.py index 4ba65f7e..1f83eabd 100644 --- a/tools/cloudharness-test/cloudharness_test/apitest_init.py +++ b/tools/cloudharness-test/cloudharness_test/apitest_init.py @@ -1,44 +1,64 @@ import os import logging +import requests import schemathesis as st from cloudharness.auth import get_token -if "APP_URL" in os.environ: +if "APP_URL" or "APP_SCHEMA_FILE" in os.environ: + app_schema = os.environ.get("APP_SCHEMA_FILE", None) + app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") + logging.info("Start schemathesis tests on %s", app_url) + if app_schema: + # Test locally with harness-test -- use local schema for convenience during test development + openapi_uri = app_schema + schema = st.from_file(openapi_uri) + else: + # remote testing: might be /api/openapi.json or /openapi.json + try: + openapi_uri = openapi_uri = app_url + "/openapi.json" + schema = st.from_uri(openapi_uri) + except st.exceptions.SchemaLoadingError as e: + # Use alternative configuration + try: + openapi_uri = app_url.replace("/api", "") + "/openapi.json" + print(requests.get(openapi_uri)) + schema = st.from_uri(openapi_uri) + except st.exceptions.SchemaLoadingError as e: + raise Exception( + f"Cannot setup api tests: {openapi_uri} not valid. Check your deployment is up and configuration") from e - app_url = os.environ["APP_URL"] + except Exception as e: + raise Exception( + f"Cannot setup api tests: {openapi_uri}: {e}") from e - try: - openapi_uri = app_url + "/openapi.json" - logging.info("Using openapi spec at %s", openapi_uri) - schema = st.from_uri(openapi_uri) - except Exception as e: - raise Exception(f"Cannot setup api tests: {openapi_uri} not reachable. Check your deployment is up and configuration") from e + logging.info("Using openapi spec at %s", openapi_uri) if "USERNAME" in os.environ and "PASSWORD" in os.environ: logging.info("Setting token from username and password") + @st.auth.register() class TokenAuth: def get(self, context): - + username = os.environ["USERNAME"] - password = os.environ["PASSWORD"] - + password = os.environ["PASSWORD"] + return get_token(username, password) def set(self, case, data, context): case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" + case.headers["Authorization"] = f"Bearer {data}" + case.headers["Cookie"] = f"kc-access={data}" else: @st.auth.register() class TokenAuth: def get(self, context): - + return "" def set(self, case, data, context): case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" \ No newline at end of file + case.headers["Authorization"] = f"Bearer {data}" + case.headers["Cookie"] = f"kc-access={data}"