From 0e9ab68530ff4f813b49f0f9a910a41532d61752 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 12 Dec 2023 20:56:46 +0100 Subject: [PATCH 01/38] 1 connector Signed-off-by: Dominik --- config/config-sil-ocpp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config-sil-ocpp.yaml b/config/config-sil-ocpp.yaml index bf3d22fe5..c748f46a7 100644 --- a/config/config-sil-ocpp.yaml +++ b/config/config-sil-ocpp.yaml @@ -137,8 +137,8 @@ active_modules: evse_manager: - module_id: evse_manager_1 implementation_id: evse - - module_id: evse_manager_2 - implementation_id: evse + # - module_id: evse_manager_2 + # implementation_id: evse reservation: - module_id: auth implementation_id: reservation From 95bdf3e3cfa37cca9128c37ee0c5a3364b8b5693 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 18 Dec 2023 16:50:31 +0100 Subject: [PATCH 02/38] CI: define workflow as reusable Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 934c32cef..9bcdddfea 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -1,6 +1,7 @@ name: Build and test on: pull_request: {} + workflow_call: # defines it as reusable workflow workflow_dispatch: inputs: runner: From fa82ebe1b2e87dcc30b7fbaef435da863c2ac467 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 18 Dec 2023 17:39:55 +0100 Subject: [PATCH 03/38] checkout caller repo Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 38 ++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9bcdddfea..19e972df2 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -1,7 +1,20 @@ name: Build and test on: - pull_request: {} + pull_request: {} # TODO set `repository` and `repository_ref` to still be used in `everest-core` workflow_call: # defines it as reusable workflow + inputs: + repository: + description: The repo to checkout and run the workflow steps in + type: string + required: true + repository_ref: + description: The ref (branch, tag, commit-hash) of the `inputs.repository` + type: string + required: true + run_integration_tests: + description: Wether to run integration-tests or not + type: boolean + default: false workflow_dispatch: inputs: runner: @@ -32,20 +45,30 @@ jobs: restore-keys: | compile-${{ env.branch_name_for_cache }}- compile- - - name: Checkout everest-core - uses: actions/checkout@v3 + - name: Checkout from `everest-core` the `.ci/` directory + uses: actions/checkout@v4 + with: + sparse-checkout: | + .ci + # for reusable workflow + repository: EVerest/everest-core + ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 + - name: Checkout repo {{ inputs.repository }} to directory `source/` + uses: actions/checkout@v4 with: - path: source + path: source/ + repository: "{{ inputs.repository }}" + ref: "{{ inputs.repository_ref }}" - name: Run clang-format uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 with: source-dir: source extensions: hpp,cpp exclude: cache - - name: Setup run scripts + - name: Setup run scripts # FIXME update `build-kit` to not use the `scripts/`-dir implicitly run: | mkdir scripts - rsync -a source/.ci/build-kit/ scripts + rsync -a .ci/build-kit/ scripts - name: Pull docker container run: | docker pull --quiet ghcr.io/everest/build-kit-alpine:latest @@ -64,7 +87,8 @@ jobs: --name test-container \ build-image run-script test_and_install - name: Run integration tests + if: ${{ inputs.run_integration_tests }} run: | docker commit test-container integration-image - pushd source/.ci/e2e + pushd .ci/e2e docker-compose run e2e-test-server run-script tests From 55c303ecd846f8da2642424072e3cfec8d4950ed Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 18 Dec 2023 17:47:00 +0100 Subject: [PATCH 04/38] fix usage of `{{ inputs. }}` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 19e972df2..76db600f8 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -57,8 +57,8 @@ jobs: uses: actions/checkout@v4 with: path: source/ - repository: "{{ inputs.repository }}" - ref: "{{ inputs.repository_ref }}" + repository: ${{ inputs.repository }} + ref: ${{ inputs.repository_ref }} - name: Run clang-format uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 with: From b0ea8330701339b786aec309129511f50763af58 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 18 Dec 2023 18:17:22 +0100 Subject: [PATCH 05/38] `install` and `everestpy` to integration-tests Signed-off-by: Dominik --- .ci/build-kit/test_and_install.sh | 10 +--------- .ci/e2e/scripts/tests.sh | 9 +++++++++ .github/workflows/build_and_test.yaml | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.ci/build-kit/test_and_install.sh b/.ci/build-kit/test_and_install.sh index 7cc838a1b..2a89effa6 100755 --- a/.ci/build-kit/test_and_install.sh +++ b/.ci/build-kit/test_and_install.sh @@ -1,11 +1,3 @@ #!/bin/sh -# ninja -j$(nproc) -C build tests -ninja -j$(nproc) -C build install - -# install everestpy via cmake target from everest-framework -ninja -C build everestpy_pip_install_dist - -rsync -a "$EXT_MOUNT/source/tests" ./ - -rm -rf build +ninja -j$(nproc) -C build tests diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index 9ddf2cf6f..5aad23e44 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -1,4 +1,13 @@ #!/bin/sh +ninja -j$(nproc) -C build install + +# install everestpy via cmake target from everest-framework +ninja -C build everestpy_pip_install_dist + +rsync -a "$EXT_MOUNT/source/tests" ./ + +rm -rf build + cd tests pytest --everest-prefix ../dist core_tests/*.py framework_tests/*.py diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 76db600f8..b667118c4 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -53,7 +53,7 @@ jobs: # for reusable workflow repository: EVerest/everest-core ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - - name: Checkout repo {{ inputs.repository }} to directory `source/` + - name: Checkout repo ${{ inputs.repository }} to directory `source/` uses: actions/checkout@v4 with: path: source/ @@ -89,6 +89,7 @@ jobs: - name: Run integration tests if: ${{ inputs.run_integration_tests }} run: | + docker commit test-container integration-image pushd .ci/e2e docker-compose run e2e-test-server run-script tests From ce0da9e9a975529a243385c3d063a78fe247d3f5 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 12:04:10 +0100 Subject: [PATCH 06/38] undo unrelated change Signed-off-by: Dominik --- config/config-sil-ocpp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config-sil-ocpp.yaml b/config/config-sil-ocpp.yaml index c748f46a7..bf3d22fe5 100644 --- a/config/config-sil-ocpp.yaml +++ b/config/config-sil-ocpp.yaml @@ -137,8 +137,8 @@ active_modules: evse_manager: - module_id: evse_manager_1 implementation_id: evse - # - module_id: evse_manager_2 - # implementation_id: evse + - module_id: evse_manager_2 + implementation_id: evse reservation: - module_id: auth implementation_id: reservation From ef701ab7a6c1089838a2c7f3ccd945601ef56e10 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 12:10:16 +0100 Subject: [PATCH 07/38] comment 1st partial checkout of `everest-core` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index b667118c4..b8f3e0970 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -45,12 +45,11 @@ jobs: restore-keys: | compile-${{ env.branch_name_for_cache }}- compile- - - name: Checkout from `everest-core` the `.ci/` directory + - name: Checkout from `everest-core` the `.ci/` directory (reusable workflow mode, see https://stackoverflow.com/a/74123003/1168315) uses: actions/checkout@v4 with: sparse-checkout: | .ci - # for reusable workflow repository: EVerest/everest-core ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - name: Checkout repo ${{ inputs.repository }} to directory `source/` @@ -89,7 +88,6 @@ jobs: - name: Run integration tests if: ${{ inputs.run_integration_tests }} run: | - docker commit test-container integration-image pushd .ci/e2e docker-compose run e2e-test-server run-script tests From b7dcc8b35782dcdf7bc426799baa60b497ae2072 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 12:45:24 +0100 Subject: [PATCH 08/38] `run_unit_tests`; output shell cmds, correct & portable shebang https://en.wikipedia.org/wiki/Shebang_(Unix)#Program_location Signed-off-by: Dominik --- .ci/build-kit/compile.sh | 3 ++- .ci/build-kit/{test_and_install.sh => unit_tests.sh} | 3 ++- .ci/e2e/scripts/tests.sh | 3 ++- .github/workflows/build_and_test.yaml | 9 +++++++-- 4 files changed, 13 insertions(+), 5 deletions(-) rename .ci/build-kit/{test_and_install.sh => unit_tests.sh} (56%) diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh index cedc73115..08bf014fb 100755 --- a/.ci/build-kit/compile.sh +++ b/.ci/build-kit/compile.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/usr/bin/env sh +set -x cmake \ -B build \ diff --git a/.ci/build-kit/test_and_install.sh b/.ci/build-kit/unit_tests.sh similarity index 56% rename from .ci/build-kit/test_and_install.sh rename to .ci/build-kit/unit_tests.sh index 2a89effa6..50d2a003e 100755 --- a/.ci/build-kit/test_and_install.sh +++ b/.ci/build-kit/unit_tests.sh @@ -1,3 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh +set -x ninja -j$(nproc) -C build tests diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index 5aad23e44..18786d293 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/usr/bin/env sh +set -x ninja -j$(nproc) -C build install diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index b8f3e0970..ce7a17539 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -11,10 +11,14 @@ on: description: The ref (branch, tag, commit-hash) of the `inputs.repository` type: string required: true + run_unit_tests: + default: false + description: Whether to run unit-tests or not + type: boolean run_integration_tests: + default: false description: Wether to run integration-tests or not type: boolean - default: false workflow_dispatch: inputs: runner: @@ -79,12 +83,13 @@ jobs: --name compile-container \ build-kit run-script compile - name: Unit tests and install + if: ${{ inputs.run_unit_tests }} run: | docker commit compile-container build-image docker run \ --volume "$(pwd):/ext" \ --name test-container \ - build-image run-script test_and_install + build-image run-script unit_tests - name: Run integration tests if: ${{ inputs.run_integration_tests }} run: | From 5371770c48c19e43d6cae691ac722e8c18ec0f7d Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 12:46:05 +0100 Subject: [PATCH 09/38] correct unit-tests invocation Signed-off-by: Dominik --- .ci/build-kit/unit_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build-kit/unit_tests.sh b/.ci/build-kit/unit_tests.sh index 50d2a003e..d98026c51 100755 --- a/.ci/build-kit/unit_tests.sh +++ b/.ci/build-kit/unit_tests.sh @@ -1,4 +1,4 @@ #!/usr/bin/env sh set -x -ninja -j$(nproc) -C build tests +ninja -j$(nproc) -C build tests/test From df793dd8d95516f940e46a545258d2a5383c71c7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 13:12:02 +0100 Subject: [PATCH 10/38] store cache even on failing build Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index ce7a17539..b084351a0 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -41,10 +41,11 @@ jobs: run: | BRANCH_NAME_FOR_CACHE="${GITHUB_REF_NAME//-/_}" echo "branch_name_for_cache=${BRANCH_NAME_FOR_CACHE}" >> "$GITHUB_ENV" - - name: Setup cache - uses: actions/cache@v3 + - name: Restore cache + uses: actions/cache/restore@v3 + id: restore-cache with: - path: cache + path: cache # keep in sync with "Store cache" key: compile-${{ env.branch_name_for_cache }}-${{ github.sha }} restore-keys: | compile-${{ env.branch_name_for_cache }}- @@ -95,4 +96,10 @@ jobs: run: | docker commit test-container integration-image pushd .ci/e2e - docker-compose run e2e-test-server run-script tests + docker-compose run e2e-test-server run-script e2e-test-server + - name: Store cache + uses: actions/cache/save@v3 + if: always() + with: + path: cache # keep in sync with "Restore cache" + key: ${{ steps.restore-cache.outputs.cache-primary-key }} From b00bb1ed184227c96356142d9a51479ca094b0e1 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 13:33:07 +0100 Subject: [PATCH 11/38] defaults for `inputs` if `on.pull_request` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index b084351a0..383a23c59 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -1,6 +1,6 @@ name: Build and test on: - pull_request: {} # TODO set `repository` and `repository_ref` to still be used in `everest-core` + pull_request: {} workflow_call: # defines it as reusable workflow inputs: repository: @@ -12,13 +12,13 @@ on: type: string required: true run_unit_tests: - default: false description: Whether to run unit-tests or not type: boolean + required: true run_integration_tests: - default: false description: Wether to run integration-tests or not type: boolean + required: true workflow_dispatch: inputs: runner: @@ -57,12 +57,12 @@ jobs: .ci repository: EVerest/everest-core ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - - name: Checkout repo ${{ inputs.repository }} to directory `source/` + - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} to directory `source/` uses: actions/checkout@v4 with: path: source/ - repository: ${{ inputs.repository }} - ref: ${{ inputs.repository_ref }} + repository: ${{ inputs.repository || 'EVerest/everest-core' }} + ref: ${{ inputs.repository_ref || env.branch_name_for_cache }} - name: Run clang-format uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 with: @@ -84,7 +84,7 @@ jobs: --name compile-container \ build-kit run-script compile - name: Unit tests and install - if: ${{ inputs.run_unit_tests }} + if: ${{ inputs.run_unit_tests || false }} run: | docker commit compile-container build-image docker run \ @@ -92,7 +92,7 @@ jobs: --name test-container \ build-image run-script unit_tests - name: Run integration tests - if: ${{ inputs.run_integration_tests }} + if: ${{ inputs.run_integration_tests || true }} run: | docker commit test-container integration-image pushd .ci/e2e From 3ffae7b2741e63dfd91d4d7bd5ef8785e5340db6 Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 19 Dec 2023 13:36:16 +0100 Subject: [PATCH 12/38] no defaults needed for `actions/checkout` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 383a23c59..5b79ece0f 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -61,8 +61,8 @@ jobs: uses: actions/checkout@v4 with: path: source/ - repository: ${{ inputs.repository || 'EVerest/everest-core' }} - ref: ${{ inputs.repository_ref || env.branch_name_for_cache }} + repository: ${{ inputs.repository }} + ref: ${{ inputs.repository_ref }} - name: Run clang-format uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 with: From a62c1cc2e0646b631f87dacedf0f167545de26e3 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 21 Dec 2023 10:41:41 +0100 Subject: [PATCH 13/38] `.gitignore`: ignore all `/build*/` Signed-off-by: Dominik --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 60c93294b..a137094c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ -*build -*build-cross +/build*/ .cache/ workspace.yaml .vscode/ From f65b397536b2e1d5a729b38174d85eec936df5d9 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 21 Dec 2023 10:30:08 +0100 Subject: [PATCH 14/38] `integration-tests-image` relies on `compile-container` Signed-off-by: Dominik --- .ci/e2e/docker-compose.yaml | 2 +- .github/workflows/build_and_test.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/e2e/docker-compose.yaml b/.ci/e2e/docker-compose.yaml index b6a5391e8..38d6985e6 100644 --- a/.ci/e2e/docker-compose.yaml +++ b/.ci/e2e/docker-compose.yaml @@ -11,7 +11,7 @@ services: driver: none e2e-test-server: - image: integration-image + image: integration-tests-image depends_on: - mqtt-server environment: diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 5b79ece0f..63e5a8996 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -83,18 +83,18 @@ jobs: --volume "$(pwd):/ext" \ --name compile-container \ build-kit run-script compile - - name: Unit tests and install + - name: Unit tests if: ${{ inputs.run_unit_tests || false }} run: | - docker commit compile-container build-image + docker commit compile-container unit-tests-image docker run \ --volume "$(pwd):/ext" \ --name test-container \ - build-image run-script unit_tests - - name: Run integration tests + unit-tests-image run-script unit_tests + - name: Integration tests if: ${{ inputs.run_integration_tests || true }} run: | - docker commit test-container integration-image + docker commit compile-container integration-tests-image pushd .ci/e2e docker-compose run e2e-test-server run-script e2e-test-server - name: Store cache From 1b93218fd4bc61db8ad6f2c30fe6906733e7faa2 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 21 Dec 2023 10:35:14 +0100 Subject: [PATCH 15/38] compile: common set of cmake flags Signed-off-by: Dominik --- .ci/build-kit/compile.sh | 5 ++--- .github/workflows/build_and_test.yaml | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh index 08bf014fb..3811c11ee 100755 --- a/.ci/build-kit/compile.sh +++ b/.ci/build-kit/compile.sh @@ -5,9 +5,8 @@ cmake \ -B build \ -S "$EXT_MOUNT/source" \ -G Ninja \ - -DEVC_ENABLE_CCACHE=1 \ - -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF \ -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ - -DBUILD_TESTING=ON + -DBUILD_TESTING=ON \ + "${CMAKE_FLAGS_EXTRA}" ninja -j$(nproc) -C build diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 63e5a8996..9d807618a 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -11,6 +11,9 @@ on: description: The ref (branch, tag, commit-hash) of the `inputs.repository` type: string required: true + cmake_flags_extra: + description: additional flags to add to the common set of flags (see `.ci/build-kit/compile.sh`) + type: string run_unit_tests: description: Whether to run unit-tests or not type: boolean @@ -78,6 +81,8 @@ jobs: docker pull --quiet ghcr.io/everest/build-kit-alpine:latest docker image tag ghcr.io/everest/build-kit-alpine:latest build-kit - name: Compile + env: + CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | docker run \ --volume "$(pwd):/ext" \ From f6eb6dbb4f6fe99bc85e115a80bb6f79e4ea2b7a Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 21 Dec 2023 10:55:06 +0100 Subject: [PATCH 16/38] shorter API Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9d807618a..d07da5828 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -14,12 +14,12 @@ on: cmake_flags_extra: description: additional flags to add to the common set of flags (see `.ci/build-kit/compile.sh`) type: string - run_unit_tests: + unit_tests: description: Whether to run unit-tests or not type: boolean required: true - run_integration_tests: - description: Wether to run integration-tests or not + integration_tests: + description: Whether to run integration-tests or not type: boolean required: true workflow_dispatch: @@ -89,7 +89,7 @@ jobs: --name compile-container \ build-kit run-script compile - name: Unit tests - if: ${{ inputs.run_unit_tests || false }} + if: ${{ inputs.unit_tests || false }} run: | docker commit compile-container unit-tests-image docker run \ @@ -97,7 +97,7 @@ jobs: --name test-container \ unit-tests-image run-script unit_tests - name: Integration tests - if: ${{ inputs.run_integration_tests || true }} + if: ${{ inputs.integration_tests || true }} run: | docker commit compile-container integration-tests-image pushd .ci/e2e From 2b3bf15c20077c6616c0cf3fa9e7ee91d15d0c9b Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 21 Dec 2023 11:28:54 +0100 Subject: [PATCH 17/38] fix `cache/` upload Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index d07da5828..0d1458a6f 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -85,23 +85,28 @@ jobs: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | docker run \ - --volume "$(pwd):/ext" \ - --name compile-container \ - build-kit run-script compile + --volume "$(pwd):/ext" \ + --name compile-container \ + build-kit run-script compile - name: Unit tests if: ${{ inputs.unit_tests || false }} run: | docker commit compile-container unit-tests-image docker run \ - --volume "$(pwd):/ext" \ - --name test-container \ - unit-tests-image run-script unit_tests + --volume "$(pwd):/ext" \ + --name test-container \ + unit-tests-image run-script unit_tests - name: Integration tests if: ${{ inputs.integration_tests || true }} run: | docker commit compile-container integration-tests-image pushd .ci/e2e - docker-compose run e2e-test-server run-script e2e-test-server + docker-compose run e2e-test-server run-script tests + - name: FIXME make all files in `cache/` readable + # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 + if: always() + run: | + chmod --recursive +r cache/ - name: Store cache uses: actions/cache/save@v3 if: always() From 8e3cf21c76a1ce412429af500705a32f1a5b1d8d Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 21:29:49 +0100 Subject: [PATCH 18/38] `sudo chown cache/josev/` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 0d1458a6f..38e919757 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -106,7 +106,7 @@ jobs: # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() run: | - chmod --recursive +r cache/ + sudo chown --recursive $(id --user --name):$(id --user --name) cache/josev/ - name: Store cache uses: actions/cache/save@v3 if: always() From 9d0b89d0a8787e44292e93d1c2b46c8b805f04b1 Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 21:40:12 +0100 Subject: [PATCH 19/38] set `--workdir` in `docker run` Signed-off-by: Dominik --- .ci/build-kit/compile.sh | 1 - .ci/e2e/scripts/tests.sh | 2 +- .github/workflows/build_and_test.yaml | 7 ++++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh index 3811c11ee..787d38ca2 100755 --- a/.ci/build-kit/compile.sh +++ b/.ci/build-kit/compile.sh @@ -3,7 +3,6 @@ set -x cmake \ -B build \ - -S "$EXT_MOUNT/source" \ -G Ninja \ -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ -DBUILD_TESTING=ON \ diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index 18786d293..41cd2229d 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -6,7 +6,7 @@ ninja -j$(nproc) -C build install # install everestpy via cmake target from everest-framework ninja -C build everestpy_pip_install_dist -rsync -a "$EXT_MOUNT/source/tests" ./ +rsync -a "tests" ./ rm -rf build diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 38e919757..205458d54 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -86,6 +86,7 @@ jobs: run: | docker run \ --volume "$(pwd):/ext" \ + --workdir /ext \ --name compile-container \ build-kit run-script compile - name: Unit tests @@ -94,14 +95,18 @@ jobs: docker commit compile-container unit-tests-image docker run \ --volume "$(pwd):/ext" \ + --workdir /ext \ --name test-container \ unit-tests-image run-script unit_tests - name: Integration tests if: ${{ inputs.integration_tests || true }} + working-directory: source/ run: | docker commit compile-container integration-tests-image pushd .ci/e2e - docker-compose run e2e-test-server run-script tests + docker-compose run \ + --workdir /ext \ + e2e-test-server run-script tests - name: FIXME make all files in `cache/` readable # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() From 9a4da904d45c880b507d2fe80818553f83c1dfd3 Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 22:03:41 +0100 Subject: [PATCH 20/38] upload test-results as artifact Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 205458d54..9dedce212 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -93,6 +93,7 @@ jobs: if: ${{ inputs.unit_tests || false }} run: | docker commit compile-container unit-tests-image + trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ --volume "$(pwd):/ext" \ --workdir /ext \ @@ -118,3 +119,9 @@ jobs: with: path: cache # keep in sync with "Restore cache" key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: Archive test results + if: always() + uses: actions/upload-artifact@v3 + with: + name: ctest-report + path: /tmp/ctest-report From c8f65471ca7527fa3a98bbeb2f4804e95320b3a3 Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 22:43:04 +0100 Subject: [PATCH 21/38] checkout directly to cwd Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9602a278a..10edf0d71 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -60,16 +60,15 @@ jobs: .ci repository: EVerest/everest-core ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} to directory `source/` + - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} uses: actions/checkout@v4 with: - path: source/ repository: ${{ inputs.repository }} ref: ${{ inputs.repository_ref }} - name: Run clang-format uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 with: - source-dir: source + source-dir: . # FIXME Make sure `.` is the default to remove this line. extensions: hpp,cpp exclude: cache - name: Setup run scripts # FIXME update `build-kit` to not use the `scripts/`-dir implicitly @@ -101,7 +100,6 @@ jobs: unit-tests-image run-script unit_tests - name: Integration tests if: ${{ inputs.integration_tests || true }} - working-directory: source/ run: | docker commit compile-container integration-tests-image pushd .ci/e2e From e1c8c55f6918e7e6a01c68e085cc241f5a29c559 Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 22:43:50 +0100 Subject: [PATCH 22/38] stop execution in scripts on failure Signed-off-by: Dominik --- .ci/build-kit/compile.sh | 2 +- .ci/build-kit/unit_tests.sh | 2 +- .ci/e2e/scripts/tests.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh index 787d38ca2..af70570af 100755 --- a/.ci/build-kit/compile.sh +++ b/.ci/build-kit/compile.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -x +set -ex cmake \ -B build \ diff --git a/.ci/build-kit/unit_tests.sh b/.ci/build-kit/unit_tests.sh index d98026c51..d00a01bc1 100755 --- a/.ci/build-kit/unit_tests.sh +++ b/.ci/build-kit/unit_tests.sh @@ -1,4 +1,4 @@ #!/usr/bin/env sh -set -x +set -ex ninja -j$(nproc) -C build tests/test diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index 5adbcd18f..f5af98cd3 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -x +set -ex ninja -j$(nproc) -C build install From 483526ace87c90d6b830b756ef874658e3ecc0ac Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 22:44:35 +0100 Subject: [PATCH 23/38] tests.sh: remove obsolete cmds Signed-off-by: Dominik --- .ci/e2e/scripts/tests.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index f5af98cd3..a56e99f01 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -6,9 +6,5 @@ ninja -j$(nproc) -C build install # install everest testing by cmake target to make sure using the version defined in dependencies.yaml ninja -C build install_everest_testing -rsync -a "tests" ./ - -rm -rf build - cd tests pytest --everest-prefix ../dist core_tests/*.py framework_tests/*.py From 653707ff4df063cb330aea9784f996928f4dfaec Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 23:30:28 +0100 Subject: [PATCH 24/38] explicitly run scripts in docker containers Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 10edf0d71..d4fccdd0c 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -71,10 +71,6 @@ jobs: source-dir: . # FIXME Make sure `.` is the default to remove this line. extensions: hpp,cpp exclude: cache - - name: Setup run scripts # FIXME update `build-kit` to not use the `scripts/`-dir implicitly - run: | - mkdir scripts - rsync -a .ci/build-kit/ scripts - name: Pull build-kit image run: | docker pull --quiet ghcr.io/everest/build-kit-alpine:latest @@ -87,7 +83,7 @@ jobs: --volume "$(pwd):/ext" \ --workdir /ext \ --name compile-container \ - build-kit run-script compile + build-kit .ci/build-kit/compile.sh - name: Unit tests if: ${{ inputs.unit_tests || false }} run: | @@ -97,15 +93,15 @@ jobs: --volume "$(pwd):/ext" \ --workdir /ext \ --name test-container \ - unit-tests-image run-script unit_tests + unit-tests-image .ci/build-kit/unit_tests.sh - name: Integration tests if: ${{ inputs.integration_tests || true }} + working-directory: .ci/e2e/ run: | docker commit compile-container integration-tests-image - pushd .ci/e2e docker-compose run \ --workdir /ext \ - e2e-test-server run-script tests + e2e-test-server scripts/tests.sh - name: FIXME make all files in `cache/` readable # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() From 5ddc7289867639e9d53a39ad6cfbfa9f9d3202ad Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 22 Dec 2023 23:51:30 +0100 Subject: [PATCH 25/38] forward `CMAKE_FLAGS_EXTRA` to docker-container Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index d4fccdd0c..edf614f5d 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -82,6 +82,7 @@ jobs: docker run \ --volume "$(pwd):/ext" \ --workdir /ext \ + --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name compile-container \ build-kit .ci/build-kit/compile.sh - name: Unit tests From 58f05e18d9f754bd7e70be959595335b774c3baf Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:01:38 +0100 Subject: [PATCH 26/38] change volumes & workdirs Signed-off-by: Dominik --- .ci/build-kit/build_install.sh | 14 ++++++++++++++ .ci/build-kit/compile.sh | 11 ----------- .ci/e2e/docker-compose.yaml | 1 + .ci/e2e/scripts/tests.sh | 5 ----- .github/workflows/build_and_test.yaml | 20 +++++++++----------- 5 files changed, 24 insertions(+), 27 deletions(-) create mode 100755 .ci/build-kit/build_install.sh delete mode 100755 .ci/build-kit/compile.sh diff --git a/.ci/build-kit/build_install.sh b/.ci/build-kit/build_install.sh new file mode 100755 index 000000000..19ed8c80b --- /dev/null +++ b/.ci/build-kit/build_install.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +set -ex + +cmake \ + -B build \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ + -DBUILD_TESTING=ON \ + "${CMAKE_FLAGS_EXTRA}" + +ninja -j$(nproc) -C build install + +# install everest testing by cmake target to make sure using the version defined in dependencies.yaml +ninja -C build install_everest_testing diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh deleted file mode 100755 index af70570af..000000000 --- a/.ci/build-kit/compile.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env sh -set -ex - -cmake \ - -B build \ - -G Ninja \ - -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ - -DBUILD_TESTING=ON \ - "${CMAKE_FLAGS_EXTRA}" - -ninja -j$(nproc) -C build diff --git a/.ci/e2e/docker-compose.yaml b/.ci/e2e/docker-compose.yaml index 38d6985e6..c828790bc 100644 --- a/.ci/e2e/docker-compose.yaml +++ b/.ci/e2e/docker-compose.yaml @@ -20,5 +20,6 @@ services: - type: bind source: ./scripts target: /ext/scripts + working_dir: /ext/ sysctls: - net.ipv6.conf.all.disable_ipv6=0 diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index a56e99f01..0472c3b29 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -1,10 +1,5 @@ #!/usr/bin/env sh set -ex -ninja -j$(nproc) -C build install - -# install everest testing by cmake target to make sure using the version defined in dependencies.yaml -ninja -C build install_everest_testing - cd tests pytest --everest-prefix ../dist core_tests/*.py framework_tests/*.py diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index edf614f5d..f435f8998 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -80,29 +80,27 @@ jobs: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | docker run \ - --volume "$(pwd):/ext" \ - --workdir /ext \ + --volume "$(pwd):$(pwd)" \ + --workdir "$(pwd)" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ - --name compile-container \ - build-kit .ci/build-kit/compile.sh + --name build-container \ + build-kit .ci/build-kit/build_install.sh - name: Unit tests if: ${{ inputs.unit_tests || false }} run: | - docker commit compile-container unit-tests-image + docker commit build-container unit-tests-image trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ - --volume "$(pwd):/ext" \ - --workdir /ext \ + --volume "$(pwd):$(pwd)" \ + --workdir "$(pwd)" \ --name test-container \ unit-tests-image .ci/build-kit/unit_tests.sh - name: Integration tests if: ${{ inputs.integration_tests || true }} working-directory: .ci/e2e/ run: | - docker commit compile-container integration-tests-image - docker-compose run \ - --workdir /ext \ - e2e-test-server scripts/tests.sh + docker commit build-container integration-tests-image + docker-compose run e2e-test-server scripts/tests.sh - name: FIXME make all files in `cache/` readable # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() From 24e20d257bcf4804d976bca17d27e833c3fcd7ec Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:30:51 +0100 Subject: [PATCH 27/38] cache-path external to workspace Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index f435f8998..92a2a6082 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -39,6 +39,8 @@ jobs: build: name: Build, Lint and Test runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} + env: + CACHE_PATH: /tmp/cache steps: - name: Format branch name for cache key run: | @@ -48,7 +50,7 @@ jobs: uses: actions/cache/restore@v3 id: restore-cache with: - path: cache # keep in sync with "Store cache" + path: ${{ env.CACHE_PATH }} key: compile-${{ env.branch_name_for_cache }}-${{ github.sha }} restore-keys: | compile-${{ env.branch_name_for_cache }}- @@ -80,6 +82,7 @@ jobs: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | docker run \ + --volume "${CACHE_PATH:?}:/ext/cache" \ --volume "$(pwd):$(pwd)" \ --workdir "$(pwd)" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ @@ -110,7 +113,7 @@ jobs: uses: actions/cache/save@v3 if: always() with: - path: cache # keep in sync with "Restore cache" + path: ${{ env.CACHE_PATH }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Archive test results if: always() From a84d5d9140f0151c98ada051ae602abddaf09766 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:44:21 +0100 Subject: [PATCH 28/38] bughunt: upload CMake logs Signed-off-by: Dominik --- .ci/build-kit/build_install.sh | 4 ++-- .github/workflows/build_and_test.yaml | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/build-kit/build_install.sh b/.ci/build-kit/build_install.sh index 19ed8c80b..cd673a8c2 100755 --- a/.ci/build-kit/build_install.sh +++ b/.ci/build-kit/build_install.sh @@ -2,9 +2,9 @@ set -ex cmake \ - -B build \ + -B build/ \ -G Ninja \ - -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ + -DCMAKE_INSTALL_PREFIX="build/dist/" \ -DBUILD_TESTING=ON \ "${CMAKE_FLAGS_EXTRA}" diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 92a2a6082..a5e6adc76 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -77,7 +77,7 @@ jobs: run: | docker pull --quiet ghcr.io/everest/build-kit-alpine:latest docker image tag ghcr.io/everest/build-kit-alpine:latest build-kit - - name: Compile + - name: Build and install env: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | @@ -88,6 +88,12 @@ jobs: --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name build-container \ build-kit .ci/build-kit/build_install.sh + - name: Archive CMake output & error logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: build-install-logs + path: build/CMakeFiles/ - name: Unit tests if: ${{ inputs.unit_tests || false }} run: | From 29b8d1badfc7422d14bd9dd678e9a02d6cde30d9 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:48:02 +0100 Subject: [PATCH 29/38] `lint` as extra job Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index a5e6adc76..a4bc6e41b 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -37,7 +37,7 @@ on: jobs: build: - name: Build, Lint and Test + name: Build and Test runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} env: CACHE_PATH: /tmp/cache @@ -67,12 +67,7 @@ jobs: with: repository: ${{ inputs.repository }} ref: ${{ inputs.repository_ref }} - - name: Run clang-format - uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 - with: - source-dir: . # FIXME Make sure `.` is the default to remove this line. - extensions: hpp,cpp - exclude: cache + - name: Pull build-kit image run: | docker pull --quiet ghcr.io/everest/build-kit-alpine:latest @@ -88,7 +83,7 @@ jobs: --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name build-container \ build-kit .ci/build-kit/build_install.sh - - name: Archive CMake output & error logs + - name: upload CMake output & error logs if: always() uses: actions/upload-artifact@v3 with: @@ -121,9 +116,22 @@ jobs: with: path: ${{ env.CACHE_PATH }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - - name: Archive test results + - name: upload test results if: always() uses: actions/upload-artifact@v3 with: name: ctest-report path: /tmp/ctest-report + lint: + name: Lint C++/C files + steps: + - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.repository_ref }} + - name: Run clang-format + uses: everest/everest-ci/github-actions/run-clang-format@v1.0.0 + with: + source-dir: . # FIXME Make sure `.` is the default to remove this line. + extensions: hpp,cpp From 5928cd8f33f1b5e7644af5d97823389a8ab57e32 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:50:10 +0100 Subject: [PATCH 30/38] bughunt: list workspace root Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index a4bc6e41b..7415a95c8 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -76,13 +76,14 @@ jobs: env: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} run: | + ls -hal . docker run \ --volume "${CACHE_PATH:?}:/ext/cache" \ --volume "$(pwd):$(pwd)" \ --workdir "$(pwd)" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name build-container \ - build-kit .ci/build-kit/build_install.sh + build-kit sh -c "ls -hal . && .ci/build-kit/build_install.sh' - name: upload CMake output & error logs if: always() uses: actions/upload-artifact@v3 @@ -124,6 +125,7 @@ jobs: path: /tmp/ctest-report lint: name: Lint C++/C files + runs-on: ubuntu-latest steps: - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} uses: actions/checkout@v4 From 90f4781cc2b341fa23c663d206dac67ea0d3e6c0 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 01:59:16 +0100 Subject: [PATCH 31/38] move `.ci` out of `everest-core` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 7415a95c8..07278f965 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -62,12 +62,19 @@ jobs: .ci repository: EVerest/everest-core ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 + - name: Move `.ci/` to `/tmp` and remove `everest-core` + run: | + mv .ci /tmp + pwd && ls -hal . + cd .. + pwd && ls -hal . + rm -rf everest-core + pwd && ls -hal . - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} uses: actions/checkout@v4 with: repository: ${{ inputs.repository }} ref: ${{ inputs.repository_ref }} - - name: Pull build-kit image run: | docker pull --quiet ghcr.io/everest/build-kit-alpine:latest @@ -79,11 +86,12 @@ jobs: ls -hal . docker run \ --volume "${CACHE_PATH:?}:/ext/cache" \ + --volume "/tmp/.ci:/.ci" \ --volume "$(pwd):$(pwd)" \ --workdir "$(pwd)" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name build-container \ - build-kit sh -c "ls -hal . && .ci/build-kit/build_install.sh' + build-kit /.ci/build-kit/build_install.sh - name: upload CMake output & error logs if: always() uses: actions/upload-artifact@v3 @@ -97,12 +105,13 @@ jobs: trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ --volume "$(pwd):$(pwd)" \ + --volume "/tmp/.ci:/.ci" \ --workdir "$(pwd)" \ --name test-container \ - unit-tests-image .ci/build-kit/unit_tests.sh + unit-tests-image /.ci/build-kit/unit_tests.sh - name: Integration tests if: ${{ inputs.integration_tests || true }} - working-directory: .ci/e2e/ + working-directory: /tmp/.ci/e2e/ run: | docker commit build-container integration-tests-image docker-compose run e2e-test-server scripts/tests.sh From 6d4a2e1f68ec8c2ecef77ec5a6e937b8f803d55f Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 02:10:59 +0100 Subject: [PATCH 32/38] avoid doubled checkout Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 29 ++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 07278f965..d179d5ed9 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -41,6 +41,7 @@ jobs: runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} env: CACHE_PATH: /tmp/cache + CI_REPO: EVerest/everest-core steps: - name: Format branch name for cache key run: | @@ -55,24 +56,17 @@ jobs: restore-keys: | compile-${{ env.branch_name_for_cache }}- compile- - - name: Checkout from `everest-core` the `.ci/` directory (reusable workflow mode, see https://stackoverflow.com/a/74123003/1168315) + - name: Checkout `${{ env.CI_REPO }}` (to only get the `.ci/` directory if reusable workflow mode, see https://stackoverflow.com/a/74123003/1168315) uses: actions/checkout@v4 with: - sparse-checkout: | - .ci - repository: EVerest/everest-core + path: ci-repo + repository: ${{ env.CI_REPO }} ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - - name: Move `.ci/` to `/tmp` and remove `everest-core` - run: | - mv .ci /tmp - pwd && ls -hal . - cd .. - pwd && ls -hal . - rm -rf everest-core - pwd && ls -hal . - - name: Checkout repo ${{ inputs.repository || 'EVerest/everest-core' }} + - name: Checkout repo ${{ inputs.repository }} + if: ${{ github.repository != env.CI_REPO }} uses: actions/checkout@v4 with: + path: source repository: ${{ inputs.repository }} ref: ${{ inputs.repository_ref }} - name: Pull build-kit image @@ -82,11 +76,12 @@ jobs: - name: Build and install env: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} + working-directory: source run: | ls -hal . docker run \ --volume "${CACHE_PATH:?}:/ext/cache" \ - --volume "/tmp/.ci:/.ci" \ + --volume "ci-repo/.ci:/.ci" \ --volume "$(pwd):$(pwd)" \ --workdir "$(pwd)" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ @@ -100,20 +95,22 @@ jobs: path: build/CMakeFiles/ - name: Unit tests if: ${{ inputs.unit_tests || false }} + working-directory: source run: | docker commit build-container unit-tests-image trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ + --volume "ci-repo/.ci:/.ci" \ --volume "$(pwd):$(pwd)" \ - --volume "/tmp/.ci:/.ci" \ --workdir "$(pwd)" \ --name test-container \ unit-tests-image /.ci/build-kit/unit_tests.sh - name: Integration tests if: ${{ inputs.integration_tests || true }} - working-directory: /tmp/.ci/e2e/ + working-directory: source run: | docker commit build-container integration-tests-image + pushd ci-repo/.ci/e2e/ docker-compose run e2e-test-server scripts/tests.sh - name: FIXME make all files in `cache/` readable # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 From 2147cb83200570d87d8d3791073d7815e6ec153e Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 02:24:18 +0100 Subject: [PATCH 33/38] checkout `ci-repo/` and `source/` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 33 ++++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index d179d5ed9..69be6a69c 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -42,6 +42,8 @@ jobs: env: CACHE_PATH: /tmp/cache CI_REPO: EVerest/everest-core + CI_REPO_DIR: ci-repo + SOURCE_DIR: source steps: - name: Format branch name for cache key run: | @@ -56,17 +58,16 @@ jobs: restore-keys: | compile-${{ env.branch_name_for_cache }}- compile- - - name: Checkout `${{ env.CI_REPO }}` (to only get the `.ci/` directory if reusable workflow mode, see https://stackoverflow.com/a/74123003/1168315) + - name: Checkout `${{ env.CI_REPO }}` to `${{ env.CI_REPO_DIR }}/` (for reusable workflow mode, see https://stackoverflow.com/a/74123003/1168315) uses: actions/checkout@v4 with: - path: ci-repo + path: ${{ env.CI_REPO_DIR }} repository: ${{ env.CI_REPO }} ref: CI/reusable-workflow # TODO get reference in general, see https://stackoverflow.com/questions/74784735 resp. https://github.com/actions/toolkit/issues/1264 - - name: Checkout repo ${{ inputs.repository }} - if: ${{ github.repository != env.CI_REPO }} + - name: Checkout repo ${{ inputs.repository }} to `${{ env.SOURCE_DIR }}/` uses: actions/checkout@v4 with: - path: source + path: ${{ env.SOURCE_DIR }} repository: ${{ inputs.repository }} ref: ${{ inputs.repository_ref }} - name: Pull build-kit image @@ -76,14 +77,13 @@ jobs: - name: Build and install env: CMAKE_FLAGS_EXTRA: ${{ inputs.cmake_flags_extra || '-DEVC_ENABLE_CCACHE=1 -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF' }} - working-directory: source run: | ls -hal . docker run \ --volume "${CACHE_PATH:?}:/ext/cache" \ - --volume "ci-repo/.ci:/.ci" \ - --volume "$(pwd):$(pwd)" \ - --workdir "$(pwd)" \ + --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ + --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ + --workdir "/source" \ --env CMAKE_FLAGS_EXTRA="${CMAKE_FLAGS_EXTRA:?}" \ --name build-container \ build-kit /.ci/build-kit/build_install.sh @@ -95,23 +95,24 @@ jobs: path: build/CMakeFiles/ - name: Unit tests if: ${{ inputs.unit_tests || false }} - working-directory: source run: | docker commit build-container unit-tests-image trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ - --volume "ci-repo/.ci:/.ci" \ - --volume "$(pwd):$(pwd)" \ - --workdir "$(pwd)" \ + --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ + --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ + --workdir "/source" \ --name test-container \ unit-tests-image /.ci/build-kit/unit_tests.sh - name: Integration tests if: ${{ inputs.integration_tests || true }} - working-directory: source run: | docker commit build-container integration-tests-image - pushd ci-repo/.ci/e2e/ - docker-compose run e2e-test-server scripts/tests.sh + docker-compose run \ + --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ + --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ + --workdir "/source" \ + e2e-test-server /.ci/e2e/scripts/tests.sh - name: FIXME make all files in `cache/` readable # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() From 8701afa01af4dcf643d0ed72273e86b5fbb30547 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 02:52:47 +0100 Subject: [PATCH 34/38] CACHE_PATH=cache Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 69be6a69c..dd9039b27 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -40,7 +40,7 @@ jobs: name: Build and Test runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} env: - CACHE_PATH: /tmp/cache + CACHE_PATH: cache CI_REPO: EVerest/everest-core CI_REPO_DIR: ci-repo SOURCE_DIR: source From 07768926a9936a1bd7e07277ee01d1e9a06bb5fc Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 03:13:58 +0100 Subject: [PATCH 35/38] fixes Signed-off-by: Dominik --- .ci/build-kit/build_install.sh | 3 --- .ci/e2e/scripts/tests.sh | 5 ++++- .github/workflows/build_and_test.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/build-kit/build_install.sh b/.ci/build-kit/build_install.sh index cd673a8c2..05f14a353 100755 --- a/.ci/build-kit/build_install.sh +++ b/.ci/build-kit/build_install.sh @@ -9,6 +9,3 @@ cmake \ "${CMAKE_FLAGS_EXTRA}" ninja -j$(nproc) -C build install - -# install everest testing by cmake target to make sure using the version defined in dependencies.yaml -ninja -C build install_everest_testing diff --git a/.ci/e2e/scripts/tests.sh b/.ci/e2e/scripts/tests.sh index 0472c3b29..62fdc7c17 100755 --- a/.ci/e2e/scripts/tests.sh +++ b/.ci/e2e/scripts/tests.sh @@ -1,5 +1,8 @@ #!/usr/bin/env sh set -ex +# install everest testing by cmake target to make sure using the version defined in dependencies.yaml +ninja -C build install_everest_testing + cd tests -pytest --everest-prefix ../dist core_tests/*.py framework_tests/*.py +pytest --everest-prefix ../build/dist core_tests/*.py framework_tests/*.py diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index dd9039b27..97050f3c7 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -113,11 +113,11 @@ jobs: --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ --workdir "/source" \ e2e-test-server /.ci/e2e/scripts/tests.sh - - name: FIXME make all files in `cache/` readable + - name: FIXME own all files in cache by current user and group # FIXME Workaround for non-readable certs in `cache/josev/[ID]/Josev/iso15118/shared/pki/iso15118_2/certs . See https://github.com/EVerest/everest-core/actions/runs/7286554315/job/19855498928#step:13:10 if: always() run: | - sudo chown --recursive $(id --user --name):$(id --user --name) cache/josev/ + sudo chown --recursive $(id --user --name):$(id --user --name) ${CACHE_PATH} - name: Store cache uses: actions/cache/save@v3 if: always() From e1ef485f800cf5a964705707fe6fb0087f3a61e7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 03:22:19 +0100 Subject: [PATCH 36/38] fix integration-test run Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 97050f3c7..65791f143 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -109,6 +109,7 @@ jobs: run: | docker commit build-container integration-tests-image docker-compose run \ + --file ${CI_REPO_DIR:?}/.ci/e2e/docker-compose.yaml \ --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ --workdir "/source" \ From 6c21b2c050d72532a5b5820f716f412c29c75302 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 03:28:55 +0100 Subject: [PATCH 37/38] fix unit-test report upload Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 65791f143..1a57f92b1 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -97,7 +97,6 @@ jobs: if: ${{ inputs.unit_tests || false }} run: | docker commit build-container unit-tests-image - trap "cp build/Testing/Temporary/LastTest.log /tmp/ctest-report" EXIT docker run \ --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ @@ -130,7 +129,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: ctest-report - path: /tmp/ctest-report + path: ${{ env.SOURCE_DIR }}/build/Testing/Temporary/ lint: name: Lint C++/C files runs-on: ubuntu-latest From e3d60cda0e4b1df4a57ee894f03724dc0f58b8e7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 23 Dec 2023 03:59:42 +0100 Subject: [PATCH 38/38] fix `docker-compose` Signed-off-by: Dominik --- .github/workflows/build_and_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 1a57f92b1..e9a9f2c24 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -107,8 +107,8 @@ jobs: if: ${{ inputs.integration_tests || true }} run: | docker commit build-container integration-tests-image - docker-compose run \ - --file ${CI_REPO_DIR:?}/.ci/e2e/docker-compose.yaml \ + docker-compose --file ${CI_REPO_DIR:?}/.ci/e2e/docker-compose.yaml \ + run \ --volume "$(pwd)/${CI_REPO_DIR:?}/.ci:/.ci" \ --volume "$(pwd)/${SOURCE_DIR:?}:/source" \ --workdir "/source" \