Merge pull request #603 from MetaCell/release/2.0.0

Release/2.0.0

.dockerignore

@@ -1,10 +1,12 @@
 **/node_modules
 .tox
 docs
-applications
-infrastructure
-blueprint
+/applications
+/infrastructure
+/blueprint
 test
 .github
 .git
-.vscode
+.vscode
+/deployment
+skaffold.yaml

.gitignore

@@ -6,11 +6,14 @@ node_modules
 .vscode
 .coverage
 *.DS_Store
-deployment
+deployment/helm
 *.egg-info
 *.idea
 /build
 skaffold.yaml
 .tox
 .pytest_cache
-.overrides
+.overrides
+deployment.yaml
+.hypothesis
+__pycache__

README.md

@@ -17,7 +17,6 @@ What building your cloud solution with CloudHarness gives to you:
     - access gatekeepers configuration
     - secrets
     - templated config maps from files
-    - secrets
   * Automatic build and push of images
   * REST-API scaffolding building based on OpenApi
   * Continuous deployment script generation
@@ -28,7 +27,12 @@ What building your cloud solution with CloudHarness gives to you:
   * Submit batch and asynchronous workflows - based on Argo
   * Orchestrate microservices - based on Kafka
   * Assign compute workspaces to users - based Jupyterhub
-
+* Testing framework to help you write and run tests
+  * Unit tests
+  * API integration tests
+  * End to End tests (with Puppeteer)
+* CI/CD pipelines generation
+
 # Why CloudHarness?
 
 The microservice architecture is a great to get code separation and flexible development, but may not be of easy implementation, especially for small development teams/projects.
@@ -52,7 +56,7 @@ CloudHarness provides the following command line tools to help application scaff
 * `harness-deployment` - generate the helm chart to deploy on Kubernetes. 
 * `harness-application` - create a new CloudHarness REST application.
 * `harness-generate` - generates server and client code for all CloudHarness REST applications.
-
+* `harness-test` - run end to end tests
 # Get started
 
 ## Prerequisites
@@ -66,14 +70,14 @@ Cloudharness can be used on all major operative systems.
 - Windows native: mostly working, unsupported  
 
 ### Python
-Python 3.7-3.9 must be installed.
+Python 3.9 must be installed.
 
 It is recommended to setup a virtual environment.
 With conda: 
- ```bash
- conda create --name ch python=3.7
- conda activate ch
- ```
+```bash
+conda create --name ch python=3.9
+conda activate ch
+```
 
 ### Docker
 [Docker](https://www.docker.com) is required to build locally.
@@ -90,11 +94,26 @@ With conda:
 
 [Skaffold](https://skaffold.dev/docs/install/) is the way to go to build and debug your application in your local development environment.
 
+### Node environment
+
+A node environment with npm is required for developing web applications and to run end to end tests.
+
+Recommended:
+- node >= v14.0.0
+- npm >= 8.0.0
+
+### Java Runtime Environment
+
+A JRE is needed to run the code generators based on openapi-generator.
+
+For more info, see [here](https://openapi-generator.tech/docs/installation).
+
+
 ## CloudHarness command line tools
 To use the cli tools, install requirements first:
 
 ```bash
-source install.sh
+bash install.sh
 ```
 ### Generate deployment
 
@@ -141,7 +160,7 @@ harness-deployment cloud-harness . [PARAMS]
 ```
 
 to create the build and deployment artifacts for your solution.
-See the dedicated [Build and deploy](./docs/build-deploy-howto.md) document for more details and examples.
+See the dedicated [Build and deploy](./docs/build-deploy/README.md) document for more details and examples.
 
 # Add and manage applications
 

application-templates/base/deploy/values-test.yaml

@@ -0,0 +1,6 @@
+harness:
+    users:
+    - username: __APP_NAME__
+      email: [email protected]
+      clientRoles: []
+      realmRoles: []

tools/cloudharness_utilities/application-templates/base/deploy/values.yaml → application-templates/base/deploy/values.yaml

@@ -14,4 +14,10 @@ harness:
   dependencies:
     build:
       - cloudharness-base
-      - cloudharness-flask
+      - cloudharness-flask
+  test:
+    api:
+      enabled: true
+      autotest: true
+      checks:
+        - all

application-templates/base/test/api/test_st.py

@@ -0,0 +1,18 @@
+import os
+from pprint import pprint
+import schemathesis as st
+from schemathesis.checks import response_schema_conformance, not_a_server_error
+
+from cloudharness_test import apitest_init # include to perform default authorization
+
+app_url = os.environ.get("APP_URL", "http://samples.ch.local/api")
+
+schema = st.from_uri(app_url + "/openapi.json")
+
+
+@schema.parametrize(endpoint="/ping")
+def test_ping(case):
+    response = case.call()
+    pprint(response.__dict__)
+    assert response.status_code == 200, "this api errors on purpose"
+

application-templates/base/test/e2e/landing.spec.ts

@@ -0,0 +1,35 @@
+import * as puppeteer from "puppeteer";
+
+let page: any;
+let browser: any;
+
+describe("End to end test", () => {
+  beforeAll(async () => {
+    browser = await puppeteer.launch({args: ['--no-sandbox'],});
+    page = await browser.newPage();
+    console.log(process.env.APP_URL)
+    await page
+      .goto(process.env.APP_URL, {
+        waitUntil: "networkidle0",
+      })
+      // tslint:disable-next-line:no-empty
+      .catch(() => {});
+  });
+
+  afterAll(() => {
+    if (!page.isClosed()) {
+      browser.close();
+    }
+  });
+
+  // TODO CHANGEME
+  test("should see the landing page", async () => {
+    await page.waitForSelector("h1");
+    const title = await page.$eval("h1", (el: { textContent: any }) => {
+      return el.textContent;
+    });
+
+    expect(await page.title()).toEqual("Samples");
+    expect(title).toEqual("Sample React application is working!");
+  });
+});

application-templates/django-app/Dockerfile

@@ -0,0 +1,34 @@
+ARG CLOUDHARNESS_FRONTEND_BUILD
+ARG CLOUDHARNESS_DJANGO
+
+FROM $CLOUDHARNESS_FRONTEND_BUILD as frontend
+
+ENV APP_DIR=/app
+
+WORKDIR ${APP_DIR}
+COPY frontend/package.json ${APP_DIR}
+COPY frontend/package-lock.json ${APP_DIR}
+RUN npm ci
+
+COPY frontend ${APP_DIR}
+RUN npm run build
+
+#####
+
+FROM $CLOUDHARNESS_DJANGO
+
+WORKDIR ${APP_DIR}
+RUN mkdir -p ${APP_DIR}/static/www
+
+COPY backend/requirements.txt ${APP_DIR}
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY backend/requirements.txt backend/setup.py ${APP_DIR}
+RUN python3 -m pip install -e .
+
+COPY backend ${APP_DIR}
+RUN python3 manage.py collectstatic --noinput
+
+COPY --from=frontend /app/dist ${APP_DIR}/static/www
+
+ENTRYPOINT uvicorn --workers ${WORKERS} --host 0.0.0.0 --port ${PORT} main:app

application-templates/django-app/README.md

@@ -0,0 +1,108 @@
+# __APP_NAME__
+
+FastAPI/Django/React-based web application.
+This application is constructed to be deployed inside a cloud-harness Kubernetes.
+It can be also run locally for development and test purpose.
+
+The code is generated with the script `harness-application` and is in part automatically generated 
+from [openapi definition](./api/openapi.yaml).
+
+## Configuration
+
+### Accounts
+
+The CloudHarness Django application template comes with a configuration that can retrieve user account updates from Keycloak (accounts)
+To enable this feature:
+* log in into the accounts admin interface
+* select in the left sidebar Events
+* select the `Config` tab
+* enable "metacell-admin-event-listener" under the `Events Config` - `Event Listeners`
+
+An other option is to enable the "metacell-admin-event-listener" through customizing the Keycloak realm.json from the CloudHarness repository.
+
+## Develop
+
+This application is composed of a FastAPI Django backend and a React frontend.
+
+### Backend
+
+Backend code is inside the *backend* directory.
+See [backend/README.md#Develop]
+
+### Frontend
+
+Backend code is inside the *frontend* directory.
+
+Frontend is by default generated as a React web application, but no constraint about this specific technology.
+
+#### Call the backend apis
+All the api stubs are automatically generated in the [frontend/rest](frontend/rest) directory by `harness-application`
+and `harness-generate`.
+
+#### Update the backend apis from openapi.yaml
+THe backend openapi models and main.py can be updated using the `genapi.sh` from the api folder.
+
+## Local build & run
+
+### Install dependencies 
+1 - Clone cloud-harness into your project root folder 
+
+2 - Install cloud-harness requirements
+```
+cd cloud-harness
+bash install.sh
+```
+
+3 - Install cloud-harness common library
+```
+cd libraries/cloudharness-common
+pip install -e .
+```
+
+4 - Install cloud-harness django library
+```
+cd ../../infrastructure/common-images/cloudharness-django/libraries/cloudharness-django
+pip install -e .
+```
+
+5 - Install cloud-harness fastapi requirements
+```
+cd ../fastapi
+pip install -r requirements.txt
+```
+
+### Prepare backend
+
+Create a Django local superuser account, this you only need to do on initial setup
+```bash
+cd backend
+python3 manage.py migrate # to sync the database with the Django models
+python3 manage.py collectstatic --noinput # to copy all assets to the static folder
+python3 manage.py createsuperuser
+# link the frontend dist to the django static folder, this is only needed once, frontend updates will automatically be applied
+cd static/www
+ln -s ../../../frontend/dist dist
+```
+
+### Build frontend
+
+Compile the frontend
+```bash
+cd frontend
+npm install
+npm run build
+```
+
+### Run backend application
+
+start the FastAPI server
+```bash
+uvicorn --workers 2 --host 0.0.0.0 --port 8000 main:app
+```
+
+
+### Running local with port forwardings to a kubernetes cluster
+When you create port forwards to microservices in your k8s cluster you want to forced your local backend server to initialize
+the AuthService and EventService services.
+This can be done by setting the `KUBERNETES_SERVICE_HOST` environment variable to a dummy or correct k8s service host.
+The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak client and client roles of this microservice.

application-templates/django-app/api/genapi.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+fastapi-codegen --input openapi.yaml --output app -t templates && mv app/main.py ../backend/ && mv app/models.py ../backend/openapi/
+rm -rf app
+
+echo Generated new models and main.py