Merge pull request #712 from MetaCell/release/2.2.0

Release/2.2.0

application-templates/django-app/api/test_st.py

@@ -0,0 +1,27 @@
+import os
+from pprint import pprint
+import schemathesis as st
+from schemathesis.checks import response_schema_conformance, not_a_server_error
+
+from cloudharness_test import apitest_init # include to perform default authorization
+
+app_url = os.environ.get("APP_URL", "http://samples.ch.local/api")
+
+try:
+    schema = st.from_uri(app_url + "/openapi.json")
+except:
+    # support alternative schema location
+    schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json")
+
+
+@schema.parametrize(endpoint="/ping")
+def test_ping(case):
+    response = case.call()
+    pprint(response.__dict__)
+    assert response.status_code == 200, "this api errors on purpose"
+
+def test_state_machine():
+    schema.as_state_machine().run()
+# APIWorkflow = schema.as_state_machine()
+# APIWorkflow.run()
+# TestAPI = APIWorkflow.TestCase

deployment-configuration/helm/templates/auto-database.yaml

@@ -74,11 +74,6 @@ spec:
         volumeMounts:
           - name: {{ .app.harness.database.name | quote }}
             mountPath: /data/db
-          {{- if .root.Values.backup.active }}
-          - name: "db-backups"
-            mountPath: {{ (printf "%s/%s/%s" .root.Values.backup.dir .app.harness.database.type .app.harness.database.name) | quote }}
-            readOnly: true
-          {{- end }}
           {{- if eq .app.harness.database.type "postgres" }}
           - mountPath: /dev/shm
             name: dshm  
@@ -92,11 +87,6 @@ spec:
           medium: Memory
         name: dshm     
       {{- end }}     
-      {{- if .root.Values.backup.active }}
-      - name: "db-backups"
-        persistentVolumeClaim:
-          claimName: "db-backups"
-      {{- end }}
 ---
 {{- if .root.Values.backup.active }}
 {{- include (print "deploy_utils.database." .app.harness.database.type ".backup") . }}

deployment-configuration/helm/templates/auto-gatekeepers.yaml

@@ -18,8 +18,12 @@ data:
     enable-default-deny:  {{ eq (.app.harness.secured | toString) "true" }}
     listen: 0.0.0.0:8080
     enable-refresh-tokens: true
-    server-write-timeout: 180s
-    upstream-response-header-timeout: 180s
+    server-write-timeout: {{ .app.harness.proxy.timeout.send | default .root.Values.proxy.timeout.send | default 180 }}s
+    upstream-timeout: {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-response-header-timeout:   {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-expect-continue-timeout:  {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    server-read-timeout:  {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-keepalive-timeout:  {{ .app.harness.proxy.timeout.keepalive | default .root.Values.proxy.timeout.keepalive | default 180 }}s
     http-only-cookie: false
     tls-cert:
     tls-private-key:
@@ -65,7 +69,7 @@ data:
               <h2 class="message">403 Permission Denied</h2>
               <div class="error-details">
                 Sorry, you do not have access to this page, please contact your administrator.
-                If you have been assigned new authorizations try to <a href="/oauth/logout?redirect=/">login again</a>.
+                If you have been assigned new authorizations, try to refresh the page or to <a href="/oauth/logout?redirect=/">login again</a>.
               </div>
             </div>
           </div>

deployment-configuration/helm/templates/ingress.yaml

@@ -38,10 +38,13 @@ metadata:
     cert-manager.io/issuer: {{ printf "%s-%s" "letsencrypt" .Values.namespace }}
     {{- end }}
     nginx.ingress.kubernetes.io/ssl-redirect: {{ (and $tls .Values.ingress.ssl_redirect) | quote }}
-    nginx.ingress.kubernetes.io/proxy-body-size: '250m'
+    nginx.ingress.kubernetes.io/proxy-body-size: '{{ .Values.proxy.payload.max }}m'
     nginx.ingress.kubernetes.io/proxy-buffer-size: '128k'
     nginx.ingress.kubernetes.io/from-to-www-redirect: 'true'
     nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/auth-keepalive-timeout: {{ .Values.proxy.timeout.keepalive | quote }}
+    nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.proxy.timeout.read | quote }}
+    nginx.ingress.kubernetes.io/proxy-send-timeout: {{ .Values.proxy.timeout.send | quote }}
 spec:
   rules:
   {{- range $app := .Values.apps }}

deployment-configuration/helm/values.yaml

@@ -67,3 +67,13 @@ backup:
       memory: "64Mi"
       # -- K8s cpu resource definition.
       cpu: "50m"
+proxy:
+  timeout:
+    # -- Timeout for proxy connections in seconds.
+    send: 60
+    # -- Timeout for proxy responses in seconds.
+    read: 60
+    keepalive: 60
+  payload:
+    # -- Maximum size of payload in MB
+    max: 250

deployment-configuration/value-template.yaml

@@ -22,6 +22,8 @@ harness:
         - administrator
     - uri: /api/openapi.json
       white-listed: true
+    - uri: /openapi.json
+      white-listed: true
   # -- Defines reference deployment parameters. Values maps to k8s spec
   deployment:
     # -- When true, enables automatic deployment
@@ -125,3 +127,13 @@ harness:
       smoketest: true
       ignoreConsoleErrors: false
       ignoreRequestErrors: false
+  proxy:
+    timeout:
+      # -- Timeout for proxy connections in seconds.
+      send:
+      # -- Timeout for proxy responses in seconds.
+      read:
+      keepalive:
+    payload:
+      # -- Maximum size of payload in MB
+      max: 

docs/accounts.md

@@ -62,7 +62,22 @@ harness:
   secured: open
 ```
 
+#### Proxy specific configurations
+Proxy configurations can be personalized in the application in the case that we want to have more restrictive values than the global ones (see [here](./ingress-domains-proxies.md#proxy-configurations) for more )
 
+```yaml
+harness:
+  proxy:
+    timeout:
+      # -- Timeout for proxy connections in seconds.
+      send:
+      # -- Timeout for proxy responses in seconds.
+      read:
+      keepalive:
+    payload:
+      # -- Maximum size of payload in MB
+      max: 
+```
 ### Secure an enpoint with OpenAPI
 
 In every api endpoint that you want to secure, add the bearerAuth security as in the example:

docs/applications/databases.md

@@ -111,14 +111,49 @@ Per default, database backups are disabled. However, you can overwrite backups b
 
 ```yaml
 backup:
-    active: true
+  active: true
 ```
 
+See all the default values [here](../../deployment-configuration/helm/values.yaml).
 You can find additional configuration fields for backups to overwrite in the generated `deployment/helm/values.yaml` once you deploy your applications.
 
 Backups are defined for `mongo` and `postgres` database in form of a [K8s CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) that creates a dump of the database and stores it in a different persistent volume on the same cluster.
 
-This is done periodically according to a configurable schedule, per default once a day.
+This is done periodically according to a configurable schedule, per default every 5 minutes.
+
+A smart retention strategy is used for backups, by default:
+- all current days backups
+- one per day, last 7 days
+- one per week, last 4 weeks
+- one per month, last 6 months
+
+Implementation of backups and retention is based on https://github.com/prodrigestivill/docker-postgres-backup-local.
+
+#### How to monitor and restore backups
+
+Backups are stored in a Kubernetes volume named `db-backups`.
+
+Can mount the volume to your database pod by adding the following to your db deployment:
+
+```yaml
+...
+spec:
+  template:
+    spec:
+      containers:
+      - ...
+        volumeMounts:
+        - name: "db-backups"
+          mountPath: /backups
+          readOnly: true
+      ...
+      volumes:
+      ...
+      - name: "db-backups"
+        persistentVolumeClaim:
+          claimName: "db-backups"
+```
+
 
 
 ### MongoDB

docs/ingress-domains-proxies.md

@@ -0,0 +1,69 @@
+# Ingress, domains and proxies
+
+## Default configurations for domain and subdomains
+Cloud Harness makes it very easy to configure domains and proxies, by making
+an underlying assumption:
+
+- Applications share a main base domain (say ch.org)
+- Applications can define a subdomain (say myapp)
+
+The main domain is configured in the [root values file](../deployment-configuration/values-template.yaml) and
+it is usually overridden by the `harness-deployment` command, e.g.
+
+```
+harness-deployment ... -d ch.org
+```
+
+The subdomain is defined in the application's values.yaml file in 
+harness.subdomain (see for instance the [samples application configuration](../applications/samples/deploy/values.yaml))
+
+For instance on applications/myapp/deploy/values.yaml:
+
+```yaml
+harness:
+  subdomain: myapp
+```
+
+The above configurations put together create an ingress configuration for https://myapp.ch.org and automatically configure letsencrypt to create and renew certificates.
+
+Note:
+that the tls and letsencrypt configurations are enabled by default but should usually be disabled locally with
+
+```
+harness-deployment ... -dtls -l
+```
+
+## Main application
+
+The "main" application is deployed on the base domain.
+In order to specify a main application, override the value in your `/deployment-configuration/values-template.yaml` file.
+
+Example
+```yaml
+mainapp: myapp
+```
+This creates a reverse proxy to https://ch.org pointing to myapp
+
+## Proxy configurations
+
+Ingress is a reverse proxy and as such has some configurations to take into account.
+The most common configurations are connection timeouts and payload size.
+
+To configure it, override the following values in your `deployment-configuration/values-template.yaml` file.
+
+```yaml
+proxy:
+  timeout:
+    # -- Timeout for proxy connections in seconds.
+    send: 60
+    # -- Timeout for proxy responses in seconds.
+    read: 60
+    keepalive: 60
+  payload:
+    # -- Maximum size of payload in MB
+    max: 250
+```
+
+Note that in the case that gatekeepers are enabled, the same configurations are applied
+to the gatekeepers, unless the application override them on `harness.proxy.*`.
+See also the [gatekeepers documentation](./accounts.md#secure-and-enpoint-with-the-gatekeeper).

docs/model/ApplicationHarnessConfig.md

@@ -30,6 +30,7 @@ Key | Input Type | Accessed Type | Description | Notes
 **jupyterhub** | [**JupyterHubConfig**](JupyterHubConfig.md) | [**JupyterHubConfig**](JupyterHubConfig.md) |  | [optional] 
 **accounts** | [**ApplicationAccountsConfig**](ApplicationAccountsConfig.md) | [**ApplicationAccountsConfig**](ApplicationAccountsConfig.md) |  | [optional] 
 **test** | [**ApplicationTestConfig**](ApplicationTestConfig.md) | [**ApplicationTestConfig**](ApplicationTestConfig.md) |  | [optional] 
+**quotas** | [**Quota**](Quota.md) | [**Quota**](Quota.md) |  | [optional] 
 **any_string_name** | dict, frozendict.frozendict, str, date, datetime, uuid.UUID, int, float, decimal.Decimal, bool, None, list, tuple, bytes, io.FileIO, io.BufferedReader,  | frozendict.frozendict, str, decimal.Decimal, BoolClass, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional]
 
 # aliases

docs/model/HarnessMainConfig.md

@@ -21,7 +21,7 @@ Key | Input Type | Accessed Type | Description | Notes
 **backup** | [**BackupConfig**](BackupConfig.md) | [**BackupConfig**](BackupConfig.md) |  | [optional] 
 **name** | str,  | str,  | Base name | [optional] 
 **task-images** | [**SimpleMap**](SimpleMap.md) | [**SimpleMap**](SimpleMap.md) |  | [optional] 
-**any_string_name** | dict, frozendict.frozendict, str, date, datetime, int, float, bool, decimal.Decimal, None, list, tuple, bytes, io.FileIO, io.BufferedReader | frozendict.frozendict, str, BoolClass, decimal.Decimal, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional]
+**any_string_name** | dict, frozendict.frozendict, str, date, datetime, uuid.UUID, int, float, decimal.Decimal, bool, None, list, tuple, bytes, io.FileIO, io.BufferedReader,  | frozendict.frozendict, str, decimal.Decimal, BoolClass, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional]
 
 # env
 

docs/model/Quota.md

@@ -8,7 +8,7 @@ dict, frozendict.frozendict,  | frozendict.frozendict,  |  |
 ### Dictionary Keys
 Key | Input Type | Accessed Type | Description | Notes
 ------------ | ------------- | ------------- | ------------- | -------------
-**any_string_name** | str,  | str,  | any string name can be used but the value must be the correct type | [optional] 
+**any_string_name** | dict, frozendict.frozendict, str, date, datetime, uuid.UUID, int, float, decimal.Decimal, bool, None, list, tuple, bytes, io.FileIO, io.BufferedReader,  | frozendict.frozendict, str, decimal.Decimal, BoolClass, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional]
 
 [[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md)
 

docs/model/SimpleMap.md

@@ -8,7 +8,7 @@ dict, frozendict.frozendict,  | frozendict.frozendict,  |  |
 ### Dictionary Keys
 Key | Input Type | Accessed Type | Description | Notes
 ------------ | ------------- | ------------- | ------------- | -------------
-**any_string_name** | str,  | str,  | any string name can be used but the value must be the correct type | [optional] 
+**any_string_name** | dict, frozendict.frozendict, str, date, datetime, uuid.UUID, int, float, decimal.Decimal, bool, None, list, tuple, bytes, io.FileIO, io.BufferedReader,  | frozendict.frozendict, str, decimal.Decimal, BoolClass, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional]
 
 [[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md)
 

docs/testing.md

@@ -108,23 +108,30 @@ The test can use environmental variables:
 Examples:
 - [Sample api test](../applications/samples/test/api/test_st.py)
 
-### Common smoke tests
-
-Once a test is created for your application, generic smoke tests are also
-executed, checking for:
 
-- Main page is reachable
-- No errors in the console
-- No error responses from network resources and fetch requests (code < 400)
-
-The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec.ts).
 
 
 
 ## End to end (E2E) tests
 
 End to end tests run in a headless browser ([Puppeteer](https://github.com/puppeteer/puppeteer)) against the full deployment on Kubernetes.
 
+Custom configuration:
+
+```yaml
+harness:
+  ...
+  test:
+    e2e:
+      # -- enable/disable e2e tests
+      enabled: true
+      # -- ignore errors on console by default
+      ignoreConsoleErrors: false
+      # -- ignore fetched resources errors by default
+      ignoreRequestErrors: false
+      # -- enable common smoke tests
+      smoketest: true
+```
 
 ### Write tests with Jest and Puppeteer
 
@@ -159,7 +166,7 @@ executed, checking for:
 - No errors in the console
 - No error responses from network resources and fetch requests (code < 400)
 
-The smoke tests is defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts).
+The smoke tests are defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts).
 
 
 ## Run API and E2E tests in the CI/CD pipeline
@@ -182,7 +189,7 @@ deployment.
 In order to use `harness-test` install the library with
 
 ```
-pip install -r requirements-test.txt
+pip install -e tools/cloudharness-test
 ```
 
 In order to run tests against an existing deployment based on a domain (say, my.domain), run:
@@ -198,6 +205,25 @@ If you want to run the deployment locally and then run the tests, can use skaffo
 1. Wait the deployment to settle
 1. Run `harness-test PATHS`
 
+### Tests development
+The `harness-test` client is useful while developing and tweaking the tests.
+In that case it's better to target the application under development and 
+the kind of tests we are working on.
+
+
+To target a specific application for end-to-end tests, use:
+```
+harness-test . -i [APPNAME] -e
+```
+
+To target a specific application for api tests, use:
+```
+harness-test . -i [APPNAME] -a
+```
+
+Note that the local version of the openapi.yaml file located at applications/[APPNAME]/api/openapi.yaml is used if available. That's useful to tweak examples and responses
+used by schemathesis to generate the test hypotheses.
+
 ## Create test users for your application
 
 To create test users: