From 066b139060ab90ac31b07fac083e769dea695270 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 30 Jun 2023 15:52:48 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20Run=20CI=20against=20actual=20se?= =?UTF-8?q?rver=20(#10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .firecrest-demo-config.json | 8 +++ .github/workflows/server-tests.yml | 92 ++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 1 + .gitignore | 1 - README.md | 16 ++++-- firecrest_demo.py | 1 - 6 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 .firecrest-demo-config.json create mode 100644 .github/workflows/server-tests.yml diff --git a/.firecrest-demo-config.json b/.firecrest-demo-config.json new file mode 100644 index 0000000..9479e1d --- /dev/null +++ b/.firecrest-demo-config.json @@ -0,0 +1,8 @@ +{ + "url": "http://localhost:8000/", + "token_uri": "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token", + "client_id": "firecrest-sample", + "client_secret": "b391e177-fa50-4987-beaf-e6d33ca93571", + "machine": "cluster", + "scratch_path": "/home/service-account-firecrest-sample" +} diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml new file mode 100644 index 0000000..859e230 --- /dev/null +++ b/.github/workflows/server-tests.yml @@ -0,0 +1,92 @@ +# Run pytest against an actual FirecREST server, +# rather than just a mock server. + +name: Server + +on: + push: + branches: [main] + tags: + - 'v*' + pull_request: # TODO delete + +jobs: + + tests: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9"] + firecrest-version: ["v1.13.0"] + + services: + rabbitmq: + image: rabbitmq:3.8.14-management + ports: + - 5672:5672 + - 15672:15672 + + steps: + - uses: actions/checkout@v3 + + - name: checkout the firecrest repository + uses: actions/checkout@v3 + with: + repository: eth-cscs/firecrest + ref: ${{ matrix.firecrest-version }} + path: .demo-server + + - name: Cache Docker images + uses: jpribyl/action-docker-layer-caching@v0.1.1 + continue-on-error: true + with: + key: ${{ runner.os }}-docker-${{ matrix.firecrest-version }} + + # note, for some reason, the certificator image fails to build + # if you build them in order, so here we build everything except that first + # and then it seems to work + - name: Build the FirecREST images + run: | + docker-compose build f7t-base + docker-compose build compute + docker-compose build status + docker-compose build storage + docker-compose build tasks + docker-compose build utilities + docker-compose build reservations + docker-compose build client + docker-compose build cluster + docker-compose build keycloak + docker-compose build kong + docker-compose build minio + docker-compose build taskpersistence + docker-compose build opa + docker-compose build openapi + docker-compose build jaeger + + docker-compose build certificator + working-directory: .demo-server/deploy/demo + + - name: Ensure permissions of SSH Keys + run: | + chmod 400 .demo-server/deploy/test-build/environment/keys/ca-key + chmod 400 .demo-server/deploy/test-build/environment/keys/user-key + + - name: Start the FirecREST server + run: docker-compose up --detach + working-directory: .demo-server/deploy/demo + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + - name: Test with pytest + run: pytest -vv --cov=aiida_firecrest --firecrest-config .firecrest-demo-config.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b9156fa..2c50215 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.gitignore b/.gitignore index 3e2ed24..bbf1cd9 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,3 @@ dmypy.json .vscode/ .demo-server/ _archive/ -.firecrest-config.json diff --git a/README.md b/README.md index 41e45f3..e58374d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # aiida-firecrest [IN-DEVELOPMENT] -AiiDA Transport/Scheduler plugins for interfacing with [FirecREST](https://products.cscs.ch/firecrest/) (currently based on [v1.12.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.12.0)), +AiiDA Transport/Scheduler plugins for interfacing with [FirecREST](https://products.cscs.ch/firecrest/) (currently based on [v1.13.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.13.0)), via [pyfirecrest](https://github.com/eth-cscs/pyfirecrest). ## Usage @@ -125,14 +125,18 @@ It is recommended to run the tests via [tox](https://tox.readthedocs.io/en/lates tox ``` -By default, the tests are run using a mock FirecREST server (in a temporary folder). +By default, the tests are run using a mock FirecREST server, in a temporary folder +(see [aiida_fircrest.utils_test.FirecrestConfig](aiida_firecrest/utils_test.py)). +This allows for quick testing and debugging of the plugin, without needing to connect to a real server, +but is obviously not guaranteed to be fully representative of the real behaviour. + You can also provide connections details to a real FirecREST server: ```bash -tox -- --firecrest-config=".firecrest-config.json" +tox -- --firecrest-config=".firecrest-demo-config.json" ``` -The format of the `.firecrest-config.json` file is: +The format of the `.firecrest-demo-config.json` file is: ```json { @@ -148,8 +152,8 @@ The format of the `.firecrest-config.json` file is: In this mode, if you want to inspect the generated files, after a failure, you can use: ```bash -tox -- --firecrest-config=".firecrest-config.json" --firecrest-no-clean +tox -- --firecrest-config=".firecrest-demo-config.json" --firecrest-no-clean ``` See [firecrest_demo.py](firecrest_demo.py) for how to start up a demo server. -(note the issue with OSX abd turning off the AirPlay port) +(note the issue with OSX and turning off the AirPlay port) diff --git a/firecrest_demo.py b/firecrest_demo.py index 13c9484..743d752 100644 --- a/firecrest_demo.py +++ b/firecrest_demo.py @@ -66,7 +66,6 @@ def main(args: list[str] | None = None): # ensure permissions of SSH keys (chmod 400) print("Ensuring permissions of SSH keys") - # tester/deploy/test-build/environment/keys/ca-key folder.joinpath("deploy", "test-build", "environment", "keys", "ca-key").chmod( 0o400 )