diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 774cd7f49..f6d17153b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,17 +8,14 @@ env: CCACHE_DIR: ${{ github.workspace }}/.ccache jobs: coverage: - name: Coverage on Ubuntu Bionic + name: Build and Test on Debian Bullseye runs-on: ubuntu-latest env: - DOCKER_TAG: docker.pkg.github.com/uptane/aktualizr/aktualizr-ci:bionic-master - DOCKERFILE: docker/Dockerfile.ubuntu.bionic - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DOCKER_TAG: docker.pkg.github.com/uptane/aktualizr/aktualizr-ci:bullseye-master + DOCKERFILE: docker/Dockerfile.debian.bullseye DARGS: >- -eCCACHE_DIR - -eCODECOV_TOKEN -eTEST_CMAKE_BUILD_TYPE=Valgrind - -eTEST_WITH_COVERAGE=1 -eTEST_WITH_P11=1 -eTEST_WITH_FAULT_INJECTION=1 -eTEST_TESTSUITE_EXCLUDE=credentials @@ -35,18 +32,25 @@ jobs: run: | docker pull "$DOCKER_TAG" || true docker build --cache-from "$DOCKER_TAG" --pull -t "$DOCKER_TAG" -f "$DOCKERFILE" . - - uses: actions/cache@v1.1.0 + - uses: actions/cache@v4 with: - path: ${{ github.workspace }}/.ccache - key: ubuntu-bionic-${{ github.run_id }} + path: ${{ github.workspace }}/cache + key: debian-bullseye-${{ github.run_id }} restore-keys: | - ubuntu-bionic-${{ github.run_id }} - ubuntu-bionic- + debian-bullseye-${{ github.run_id }} + debian-bullseye- + - name: Unpack ccache + run: docker run --mount=type=volume,source=ccache,destination=/home/testuser/.cache -i "$DOCKER_TAG" tar -xf- < ${{github.workspace}}/cache/cache.tar || echo "No cache found" - name: Test - run: docker run -v "$PWD:$PWD" -w "$PWD" $DARGS -t "$DOCKER_TAG" ./scripts/test.sh + run: | + docker run --mount=type=volume,source=ccache,destination=/home/testuser/.cache -t "$DOCKER_TAG" source/scripts/build-and-test.sh + - name: Stash ccache + run: | + mkdir -p ${{github.workspace}}/cache + docker run --mount=type=volume,source=ccache,destination=/home/testuser/.cache -i "$DOCKER_TAG" tar -cf- .cache > ${{github.workspace}}/cache/cache.tar nop11: - name: Tests without p11 support on Ubuntu Bionic + name: Tests with PKCS#11 support on Ubuntu Bionic runs-on: ubuntu-latest env: DOCKER_TAG: docker.pkg.github.com/uptane/aktualizr/aktualizr-ci:bionic-master @@ -54,7 +58,7 @@ jobs: DARGS: >- -eCCACHE_DIR -eTEST_CMAKE_BUILD_TYPE=Debug - -eTEST_WITH_P11=0 + -eTEST_WITH_P11=1 -eTEST_WITH_TESTSUITE=0 steps: - uses: actions/checkout@main diff --git a/ci/gitlab/.gitlab-ci.yml b/ci/gitlab/.gitlab-ci.yml deleted file mode 100644 index 6217bf434..000000000 --- a/ci/gitlab/.gitlab-ci.yml +++ /dev/null @@ -1,457 +0,0 @@ -stages: - - docker - - test - - static scans - - pkg-test - - oe-checkout - - oe-test - - deploy - - trigger - -variables: - UBUNTU_BIONIC_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-UBUNTU_BIONIC - UBUNTU_XENIAL_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-UBUNTU_XENIAL - UBUNTU_FOCAL_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-UBUNTU_FOCAL - UBUNTU_BIONIC_MASTER_INSTALLIMAGE: ${CI_REGISTRY_IMAGE}:ci-install-master-UBUNTU_BIONIC - UBUNTU_XENIAL_MASTER_INSTALLIMAGE: ${CI_REGISTRY_IMAGE}:ci-install-master-UBUNTU_XENIAL - UBUNTU_BIONIC_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-UBUNTU_BIONIC - UBUNTU_XENIAL_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-UBUNTU_XENIAL - UBUNTU_FOCAL_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-UBUNTU_FOCAL - UBUNTU_BIONIC_PR_INSTALLIMAGE: ${CI_REGISTRY_IMAGE}:ci-install-${CI_COMMIT_REF_SLUG}-UBUNTU_BIONIC - UBUNTU_XENIAL_PR_INSTALLIMAGE: ${CI_REGISTRY_IMAGE}:ci-install-${CI_COMMIT_REF_SLUG}-UBUNTU_XENIAL - - CCACHE_DIR: $CI_PROJECT_DIR/ccache - - GIT_SUBMODULE_STRATEGY: none - - # bitbake variables - BITBAKE_IMAGE: ${METAUPDATER_REGISTRY_IMAGE}:ci-master-bitbake - BITBAKE_CHECKOUT_IMAGE: ${METAUPDATER_REGISTRY_IMAGE}:ci-master-checkout - -include: - - template: SAST.gitlab-ci.yml - - template: Secret-Detection.gitlab-ci.yml - - template: Dependency-Scanning.gitlab-ci.yml - - template: License-Scanning.gitlab-ci.yml - - project: 'olp/edge/ota/connect/client/meta-updater' - ref: 'master' - file: 'scripts/ci/gitlab/docker.yml' - - project: 'olp/edge/ota/connect/client/meta-updater' - ref: 'master' - file: 'scripts/ci/gitlab/checkout.yml' - - project: 'olp/edge/ota/connect/client/meta-updater' - ref: 'master' - file: 'scripts/ci/gitlab/tests.yml' - -Docker Setup: - image: docker:18 - stage: docker - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - services: - - docker:18-dind - before_script: - - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - script: - # main images - - docker pull "$UBUNTU_BIONIC_PR_IMAGE" || docker pull "$UBUNTU_BIONIC_MASTER_IMAGE" || true - - docker build --pull --cache-from "$UBUNTU_BIONIC_MASTER_IMAGE" --cache-from "$UBUNTU_BIONIC_PR_IMAGE" -f "$CI_PROJECT_DIR/docker/Dockerfile.ubuntu.bionic" -t "$UBUNTU_BIONIC_PR_IMAGE" . - - docker push "$UBUNTU_BIONIC_PR_IMAGE" - - - docker pull "$UBUNTU_XENIAL_PR_IMAGE" || docker pull "$UBUNTU_XENIAL_MASTER_IMAGE" || true - - docker build --pull --cache-from "$UBUNTU_XENIAL_MASTER_IMAGE" --cache-from "$UBUNTU_XENIAL_PR_IMAGE" -f "$CI_PROJECT_DIR/docker/Dockerfile.ubuntu.xenial" -t "$UBUNTU_XENIAL_PR_IMAGE" . - - docker push "$UBUNTU_XENIAL_PR_IMAGE" - - - docker pull "$UBUNTU_FOCAL_PR_IMAGE" || docker pull "$UBUNTU_FOCAL_MASTER_IMAGE" || true - - docker build --pull --cache-from "$UBUNTU_FOCAL_MASTER_IMAGE" --cache-from "$UBUNTU_FOCAL_PR_IMAGE" -f "$CI_PROJECT_DIR/docker/Dockerfile.ubuntu.focal" -t "$UBUNTU_FOCAL_PR_IMAGE" . - - docker push "$UBUNTU_FOCAL_PR_IMAGE" - - # used for install tests - - docker pull "$UBUNTU_BIONIC_PR_INSTALLIMAGE" || docker pull "$UBUNTU_BIONIC_MASTER_INSTALLIMAGE" || true - - docker build --pull --cache-from "$UBUNTU_BIONIC_MASTER_INSTALLIMAGE" --cache-from "$UBUNTU_BIONIC_PR_INSTALLIMAGE" -f "$CI_PROJECT_DIR/docker/Dockerfile-test-install.ubuntu.bionic" -t "$UBUNTU_BIONIC_PR_INSTALLIMAGE" . - - docker push "$UBUNTU_BIONIC_PR_INSTALLIMAGE" - - - docker pull "$UBUNTU_XENIAL_PR_INSTALLIMAGE" || docker pull "$UBUNTU_XENIAL_MASTER_INSTALLIMAGE" || true - - docker build --pull --cache-from "$UBUNTU_XENIAL_MASTER_INSTALLIMAGE" --cache-from "$UBUNTU_XENIAL_PR_INSTALLIMAGE" -f "$CI_PROJECT_DIR/docker/Dockerfile-test-install.ubuntu.xenial" -t "$UBUNTU_XENIAL_PR_INSTALLIMAGE" . - - docker push "$UBUNTU_XENIAL_PR_INSTALLIMAGE" - -# static scans: - -license_scanning: - stage: static scans - -bandit-sast: - stage: static scans - needs: [] - -flawfinder-sast: - stage: static scans - needs: [] - -secret_detection: - stage: static scans - needs: [] - -coverage: - variables: - GIT_CLONE_PATH: $CI_BUILDS_DIR/aktualizr-coverage-$CI_JOB_ID - GIT_SUBMODULE_STRATEGY: 'recursive' - - TEST_BUILD_DIR: 'build-coverage' - TEST_CMAKE_BUILD_TYPE: 'Valgrind' - TEST_WITH_COVERAGE: '1' - TEST_WITH_P11: '1' - TEST_WITH_FAULT_INJECTION: '1' - TEST_SOTA_PACKED_CREDENTIALS: "$CI_PROJECT_DIR/credentials.zip" - image: "$UBUNTU_BIONIC_PR_IMAGE" - stage: test - needs: ["Docker Setup"] - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - cache: - key: "$CI_JOB_NAME" - paths: - - ccache/ - artifacts: - paths: - - build-coverage/coverage/ - reports: - junit: build-coverage/report.xml - script: - - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip $CI_PROJECT_DIR/credentials.zip - - ./scripts/test.sh - - xsltproc -o build-coverage/report.xml ./third_party/junit/ctest2junit.xsl build-coverage/Testing/**/Test.xml > /dev/null - -nop11: - variables: - GIT_CLONE_PATH: $CI_BUILDS_DIR/aktualizr-nop11-$CI_JOB_ID - GIT_SUBMODULE_STRATEGY: 'recursive' - - TEST_BUILD_DIR: 'build-nop11' - TEST_CMAKE_BUILD_TYPE: 'Debug' - TEST_WITH_TESTSUITE: '0' - image: "$UBUNTU_BIONIC_PR_IMAGE" - stage: test - needs: ["Docker Setup"] - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - cache: - key: "$CI_JOB_NAME" - paths: - - ccache/ - script: - - ./scripts/test.sh - -focal-build-static: - variables: - GIT_CLONE_PATH: $CI_BUILDS_DIR/aktualizr-focal-build-static-$CI_JOB_ID - GIT_SUBMODULE_STRATEGY: 'recursive' - - TEST_BUILD_DIR: 'build-ubuntu-focal' - TEST_CC: 'clang' - # should run with valgrind but some leaks are still unfixed - # TEST_CMAKE_BUILD_TYPE = 'Valgrind' - TEST_CMAKE_BUILD_TYPE: 'Debug' - TEST_TESTSUITE_ONLY: 'crypto' - TEST_WITH_STATICTESTS: '1' - TEST_WITH_DOCS: '1' - image: "$UBUNTU_FOCAL_PR_IMAGE" - stage: test - needs: ["Docker Setup"] - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - cache: - key: "$CI_JOB_NAME" - paths: - - ccache/ - artifacts: - paths: - - build-ubuntu-focal/docs/doxygen/ - reports: - junit: build-ubuntu-focal/report.xml - script: - - ./scripts/test.sh - - xsltproc -o build-ubuntu-focal/report.xml ./third_party/junit/ctest2junit.xsl build-ubuntu-focal/Testing/**/Test.xml > /dev/null - - -.u-pkg: - needs: ["Docker Setup"] - stage: test - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - cache: - key: "$CI_JOB_NAME" - paths: - - ccache/ - script: - - mkdir -p $TEST_INSTALL_DESTDIR - - ./scripts/build_ubuntu.sh - -bionic-pkg: - extends: .u-pkg - variables: - GIT_CLONE_PATH: $CI_BUILDS_DIR/aktualizr-bionic-pkg-$CI_JOB_ID - GIT_SUBMODULE_STRATEGY: 'recursive' - - TEST_BUILD_DIR: 'build-bionic' - TEST_INSTALL_RELEASE_NAME: '-ubuntu_18.04' - TEST_INSTALL_DESTDIR: "$CI_PROJECT_DIR/build-bionic/pkg" - image: "$UBUNTU_BIONIC_PR_IMAGE" - artifacts: - paths: - - build-bionic/pkg - -xenial-pkg: - extends: .u-pkg - variables: - GIT_CLONE_PATH: $CI_BUILDS_DIR/aktualizr-xenial-pkg-$CI_JOB_ID - GIT_SUBMODULE_STRATEGY: 'recursive' - - TEST_BUILD_DIR: 'build-xenial' - TEST_INSTALL_RELEASE_NAME: '-ubuntu_16.04' - TEST_INSTALL_DESTDIR: "$CI_PROJECT_DIR/build-xenial/pkg" - image: "$UBUNTU_XENIAL_PR_IMAGE" - artifacts: - paths: - - build-xenial/pkg - - -.pkg-test: - stage: pkg-test - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - script: - - ./scripts/test_install_garage_deploy.sh - - ./scripts/test_install_aktualizr.sh - -bionic-pkg-test: - extends: .pkg-test - variables: - TEST_INSTALL_DESTDIR: "$CI_PROJECT_DIR/build-bionic/pkg" - needs: ["bionic-pkg"] - image: "$UBUNTU_BIONIC_PR_INSTALLIMAGE" - -xenial-pkg-test: - extends: .pkg-test - variables: - TEST_INSTALL_DESTDIR: "$CI_PROJECT_DIR/build-xenial/pkg" - needs: ["xenial-pkg"] - image: "$UBUNTU_XENIAL_PR_INSTALLIMAGE" - - -# -- yocto tests - -OE Docker setup: - extends: .bb_docker_remote - stage: docker - variables: - GIT_CHECKOUT: 'false' - only: - variables: - - $OE_PTEST - - $CI_COMMIT_REF_NAME == "master" - - $CI_COMMIT_REF_NAME =~ /^\d\d\d\d\.\d+(-\w+)?$/ - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - - -OE Checkout: - extends: .bb_checkout - stage: oe-checkout - needs: [] - variables: - GIT_CHECKOUT: 'false' - only: - variables: - - $OE_PTEST - - $CI_COMMIT_REF_NAME == "master" - - $CI_COMMIT_REF_NAME =~ /^\d\d\d\d\.\d+(-\w+)?$/ - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - variables: - MANIFEST: dunfell - -Ptest qemux86_64: - extends: .oe-selftest - stage: oe-test - variables: - GIT_CHECKOUT: false - dependencies: - - OE Checkout - allow_failure: true - only: - variables: - - $OE_PTEST - - $CI_COMMIT_REF_NAME == "master" - - $CI_COMMIT_REF_NAME =~ /^\d\d\d\d\.\d+(-\w+)?$/ - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - variables: - TEST_BUILD_DIR: 'build-ptest' - OE_SELFTESTS: 'updater_qemux86_64_ptest' - TEST_AKTUALIZR_REMOTE: '1' - # note: tag will take precedence if not empty - TEST_AKTUALIZR_TAG: $CI_COMMIT_TAG - TEST_AKTUALIZR_BRANCH: $CI_COMMIT_REF_NAME - TEST_AKTUALIZR_REV: $CI_COMMIT_SHA - -# -- create GitHub releases - -github-release: - image: "$UBUNTU_BIONIC_PR_IMAGE" - stage: deploy - # focal-build-static is needed for doxygen - needs: ["bionic-pkg", "xenial-pkg", "focal-build-static"] - script: - # github release - - ./scripts/make_src_archive.sh ./aktualizr_src-$CI_COMMIT_TAG.tar.gz - - >- - ./scripts/publish_github_rls.py $CI_COMMIT_TAG - build-xenial/pkg/garage_deploy-ubuntu_16.04.deb - build-xenial/pkg/aktualizr-ubuntu_16.04.deb - build-bionic/pkg/garage_deploy-ubuntu_18.04.deb - build-bionic/pkg/aktualizr-ubuntu_18.04.deb - aktualizr_src-$CI_COMMIT_TAG.tar.gz - # github doxygen - - ./scripts/publish_github_docs.sh - only: - variables: - - $CI_COMMIT_TAG =~ /^\d\d\d\d\.\d+(-\w+)?$/ - except: - - /^20\d\d\.\d\d?-docs$/ - variables: - DOX_DOCS: 'build-ubuntu-focal/docs/doxygen/html' - TEST_BUILD_DIR: 'build-gh-rls' - -# -- publish coverage results on gitlab pages - -pages: - stage: deploy - needs: ["coverage"] - script: - - mv build-coverage/coverage/ public/ - artifacts: - paths: - - public - expire_in: 7 days - # would be nice to have on a per-PR basis: # https://gitlab.com/gitlab-org/gitlab-ce/issues/35141 - only: - - master - - -# -- e2e - -app-docker-image: - image: docker:18 - services: - - docker:18-dind - stage: deploy - only: - - master - variables: - GIT_SUBMODULE_STRATEGY: recursive - needs: [] - allow_failure: true - before_script: - - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - script: - - cp ./docker/Dockerfile.aktualizr ./ci/gitlab/Dockerfile - - docker build --build-arg AKTUALIZR_BASE=$UBUNTU_BIONIC_MASTER_IMAGE -t $CI_REGISTRY_IMAGE/app:ci-$CI_COMMIT_REF_SLUG -f ./ci/gitlab/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/app:ci-$CI_COMMIT_REF_SLUG - -trigger-device-farm-pipeline: - stage: trigger - only: - - master - trigger: olp/edge/ota/testing/device-farm - -# -- otf - -trigger-otf-pipeline: - image: "$UBUNTU_BIONIC_PR_IMAGE" - stage: trigger - when: on_success - needs: ["github-release"] - script: - - curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[TEST_JOB_ONLY]=true" https://main.gitlab.in.here.com/api/v4/projects/163/trigger/pipeline - only: - variables: - - $CI_COMMIT_REF_NAME =~ /^\d\d\d\d\.\d+(-\w+)?$/ - except: - - /^20\d\d\.\d\d?-docs$/ - - /^docs\// - allow_failure: true - -trigger-docsite-build: - stage: trigger - only: - - /^20\d\d\.\d\d?-docs$/ - trigger: - project: olp/edge/ota/documentation/ota-connect-docs - branch: master - -trigger-osx-build: - stage: trigger - trigger: - project: olp/edge/ota/connect/client/homebrew-otaconnect - branch: master - rules: - - if: $OSX_BUILD - when: always - -build-osx-release: - stage: trigger - needs: ["github-release"] - variables: - VERSION: "$CI_COMMIT_TAG" - REVISION: "$CI_COMMIT_SHA" - RELEASE_BASE_URL: "https://github.com/advancedtelematic/aktualizr/releases/download" - GITHUB_REPOSITORY: "uptane/aktualizr" - GITHUB_TOKEN: "$GITHUB_API_TOKEN" - HOMEBREW_GITHUB_API_TOKEN: "$GITHUB_API_TOKEN" - FORMULA_DIR: "/usr/local/Homebrew/Library/Taps/advancedtelematic/homebrew-otaconnect" - FORMULA_FILE: "${FORMULA_DIR}/aktualizr.rb" - before_script: - - brew uninstall -f aktualizr - - brew untap advancedtelematic/otaconnect - - rm -rf $(brew --cache)/aktualizr--git - - brew install ghr - script: - # clone a repo that contains the aktualizr formula - - brew tap advancedtelematic/otaconnect - # update Version and Revision in the formula - - sed -i '' -E "s/ version = \"20[1-2][0-9].[0-9]+\"/ version = \"${VERSION}\"/" ${FORMULA_FILE} - - sed -i '' -E "s/ revision = \".*\"/ revision = \"${REVISION}\"/" ${FORMULA_FILE} - # build aktualizr - - brew install -v --build-bottle aktualizr - - aktualizr --version - # create aktualizr bottle - an archive/tar.gz file along with a json file containing its metadata - - brew bottle --json --no-rebuild --force-core-tap --root-url=${RELEASE_BASE_URL}/${VERSION} aktualizr - # undo changes in the formula (version and revision) - - git -C ${FORMULA_DIR} stash - # update the formula with sha256 hash of the new bottle (the archive file) - - brew bottle --merge --write --no-commit ./aktualizr--${VERSION}.mojave.bottle.json - # update Version and Revision in the formula and create a pull request with the updated formula - # that contains the new version, revision and sha256 of the bottle/archive file - - brew bump-formula-pr -v -d -f --tag=${VERSION} --revision=${REVISION} --no-browse aktualizr - # add the bottle file to the github release artifacts - - mv aktualizr--${VERSION}.mojave.bottle.tar.gz aktualizr-${VERSION}.mojave.bottle.tar.gz - - ghr -u "${GITHUB_REPOSITORY%/*}" -r "${GITHUB_REPOSITORY#*/}" ${VERSION} aktualizr-${VERSION}.mojave.bottle.tar.gz - rules: - - if: $CI_COMMIT_TAG =~ /^20\d\d\.\d\d?-docs$/ - when: never - - if: $OSX_RELEASE && $CI_COMMIT_TAG =~ /^\d\d\d\d\.\d+(-\w+)?$/ - when: on_success - tags: - - osx diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 00d4bc41f..000000000 --- a/codecov.yml +++ /dev/null @@ -1,10 +0,0 @@ -coverage: - status: - project: - default: - threshold: 0.3% -ignore: - - "third_party" - - "tests" - - "**/*_test.cc" - - "src/libaktualizr-c/test" diff --git a/docker/Dockerfile-test-install.ubuntu.bionic b/docker/Dockerfile-test-install.ubuntu.bionic deleted file mode 100644 index 379647180..000000000 --- a/docker/Dockerfile-test-install.ubuntu.bionic +++ /dev/null @@ -1,26 +0,0 @@ -FROM ubuntu:bionic -LABEL Description="Ubuntu Bionic package testing dockerfile" - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && apt-get -y install debian-archive-keyring -RUN apt-get update && apt-get install -y --no-install-suggests --no-install-recommends \ - libarchive13 \ - libboost-log1.65.1 \ - libboost-program-options1.65.1 \ - libboost-system1.65.1 \ - libboost-test1.65.1 \ - libboost-thread1.65.1 \ - libc6 \ - libcurl4 \ - libglib2.0-0 \ - libsodium23 \ - libsqlite3-0 \ - libssl1.1 \ - libstdc++6 \ - lshw \ - openjdk-8-jre \ - python3 \ - zip - -ADD ./scripts /scripts diff --git a/docker/Dockerfile-test-install.ubuntu.xenial b/docker/Dockerfile-test-install.ubuntu.xenial deleted file mode 100644 index 330f21b5a..000000000 --- a/docker/Dockerfile-test-install.ubuntu.xenial +++ /dev/null @@ -1,26 +0,0 @@ -FROM ubuntu:xenial -LABEL Description="Ubuntu Xenial package testing dockerfile" - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && apt-get -y install debian-archive-keyring -RUN apt-get update && apt-get install -y --no-install-suggests --no-install-recommends \ - libarchive13 \ - libboost-log1.58.0 \ - libboost-program-options1.58.0 \ - libboost-system1.58.0 \ - libboost-test1.58.0 \ - libboost-thread1.58.0 \ - libc6 \ - libcurl3 \ - libglib2.0-0 \ - libsodium18 \ - libsqlite3-0 \ - libssl1.0.0 \ - libstdc++6 \ - lshw \ - openjdk-8-jre \ - python3 \ - zip - -ADD ./scripts /scripts diff --git a/docker/Dockerfile.aktualizr b/docker/Dockerfile.aktualizr deleted file mode 100644 index d80b07758..000000000 --- a/docker/Dockerfile.aktualizr +++ /dev/null @@ -1,10 +0,0 @@ -ARG AKTUALIZR_BASE=advancedtelematic/aktualizr-base -FROM $AKTUALIZR_BASE -LABEL Description="Aktualizr application dockerfile" - -ADD . /aktualizr -WORKDIR /aktualizr/build - -RUN cmake -DFAULT_INJECTION=on -DBUILD_SOTA_TOOLS=on -DBUILD_DEB=on -DCMAKE_BUILD_TYPE=Debug .. -RUN make -j8 install -RUN ldconfig diff --git a/docker/Dockerfile.debian.bullseye b/docker/Dockerfile.debian.bullseye new file mode 100644 index 000000000..9ba75ca43 --- /dev/null +++ b/docker/Dockerfile.debian.bullseye @@ -0,0 +1,64 @@ +FROM debian:bullseye +LABEL Description="Aktualizr CI image using Debian Bullseye (oldstable)" + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get -y install --no-install-suggests --no-install-recommends debian-archive-keyring + + +# It is important to run these in the same RUN command, because otherwise +# Docker layer caching breaks us + +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1017558 +RUN apt-get update && apt-get -y install -\ + build-essential \ + asn1c \ + automake \ + bison \ + ccache \ + cmake \ + curl \ + g++ \ + git \ + jq \ + libarchive-dev \ + libboost-dev \ + libboost-log-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-test-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libengine-pkcs11-openssl \ + libglib2.0-dev \ + libgtest-dev \ + libostree-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + lshw \ + ninja-build \ + net-tools \ + opensc \ + ostree \ + pkg-config \ + psmisc \ + python3-dev \ + python3-gi \ + python3-openssl \ + python3-pip \ + python3-venv \ + sqlite3 \ + zip + + + +RUN useradd testuser + +WORKDIR /home/testuser +RUN chown testuser:testuser /home/testuser +COPY . /home/testuser/source +RUN chown -R testuser:testuser /home/testuser/source +USER testuser +RUN mkdir /home/testuser/.cache diff --git a/docker/docker-build.sh b/docker/docker-build.sh deleted file mode 100755 index 2fc7d9ed5..000000000 --- a/docker/docker-build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -# build aktualizr as a docker image - -REPO_ROOT="${1:-$(readlink -f "$(dirname "$0")/..")}" - -set -ex - -docker build -t advancedtelematic/aktualizr-base -f "$REPO_ROOT/docker/Dockerfile.ubuntu.bionic" "$REPO_ROOT" -docker build -t advancedtelematic/aktualizr-app -f "$REPO_ROOT/docker/Dockerfile.aktualizr" "$REPO_ROOT" diff --git a/scripts/build-and-test.sh b/scripts/build-and-test.sh new file mode 100755 index 000000000..3cc841030 --- /dev/null +++ b/scripts/build-and-test.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +set -euo pipefail + +# This expects to find the checked out source code in ./source +# It builds it into ./build and runs the tests. +# Test with +# docker build -t aktualizr-bullseye -f docker/Dockerfile.debian.bullseye +# docker run --mount=type=volume,source=ccache,destination=/home/testuser/.cache -it aktualizr-bullseye source/scripts/build-and-test.sh +cmake -G Ninja -S source -B build -DBUILD_SOTA_TOOLS=ON -DBUILD_OSTREE=ON +cd build +time ninja build_tests +ctest --output-on-failure -j 4 + diff --git a/scripts/build_ubuntu.sh b/scripts/build_ubuntu.sh deleted file mode 100755 index 03c8a56eb..000000000 --- a/scripts/build_ubuntu.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -exuo pipefail - -# configure test.sh -export GITREPO_ROOT=${1:-$(readlink -f "$(dirname "$0")/..")} -export TEST_INSTALL_DESTDIR=${TEST_INSTALL_DESTDIR:-/persistent} -export TEST_BUILD_DIR=${TEST_BUILD_DIR:-build-ubuntu} -export TEST_CMAKE_BUILD_TYPE=Release -export TEST_WITH_INSTALL_DEB_PACKAGES=1 -export TEST_WITH_OSTREE=0 -export TEST_WITH_TESTSUITE=0 - -# build and copy aktualizr.deb and garage_deploy.deb to $TEST_INSTALL_DESTDIR -mkdir -p "$TEST_INSTALL_DESTDIR" -# note: executables are stripped, following common conventions in .deb packages -LDFLAGS="-s" "$GITREPO_ROOT/scripts/test.sh" - -git -C "$GITREPO_ROOT" fetch --tags --unshallow || true -"$GITREPO_ROOT/scripts/get_version.sh" git "$GITREPO_ROOT" > "$TEST_INSTALL_DESTDIR/aktualizr-version" diff --git a/scripts/make_src_archive.sh b/scripts/make_src_archive.sh deleted file mode 100755 index 86d3e4329..000000000 --- a/scripts/make_src_archive.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -SCRIPTS_DIR=$(readlink -f "$(dirname "$0")") - -OUTPUT=$(realpath "$1") -REPO=$(realpath "${2:-.}") - -# Just in case this wasn't done before. -git -C "$REPO" submodule update --init --recursive - -python3 -m venv venv - -# shellcheck disable=SC1091 -. venv/bin/activate - -pip install 'git_archive_all==1.21.0' - -TMPDIR=$(mktemp -d) -trap 'rm -rf $TMPDIR' EXIT -cd "$TMPDIR" - -# store version in archive -"$SCRIPTS_DIR/get_version.sh" git "$REPO" > VERSION - -git-archive-all -C "$REPO" --extra VERSION "$OUTPUT" diff --git a/scripts/publish_github_docs.sh b/scripts/publish_github_docs.sh deleted file mode 100755 index 9ad351c98..000000000 --- a/scripts/publish_github_docs.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -GIT_REMOTE=https://github.com/advancedtelematic/aktualizr -DOX_DOCS=${DOX_DOCS:-$TEST_BUILD_DIR/docs/doxygen/html} -WORKTREE=${WORKTREE:-$TEST_BUILD_DIR/pages} -DRY_RUN=${DRY_RUN:-0} -DESCRIBE=$(git describe) -LAST_TAG=$(git describe --abbrev=0) - -set -x - -git remote add github_rls "$GIT_REMOTE" || true -git fetch github_rls -if ! [ -d "$WORKTREE" ]; then - mkdir -p "$WORKTREE" - git worktree add "$WORKTREE" github_rls/gh-pages -fi - -gitcommit() ( - export GIT_AUTHOR_NAME="HERE OTA Gitlab CI" - export GIT_AUTHOR_EMAIL="gitlab@example.org" - export GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME - export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL - git commit "$@" -) - -DOX_DOCS=$(realpath "$DOX_DOCS") -WORKTREE=$(realpath "$WORKTREE") - -( -cd "$WORKTREE" - -git reset --hard github_rls/gh-pages - -# create release directory -if [ -d "$WORKTREE/$LAST_TAG" ]; then - echo "Docs for $LAST_TAG already published, skipping..." -else - cp -r "$DOX_DOCS" "$WORKTREE/$LAST_TAG" - git add "$WORKTREE/$LAST_TAG" - gitcommit -m "$LAST_TAG release" -fi - -# create last snapshot - -# cleanup old snapshot -find . \( -regex './[^/]*' -and -type f -and -not -path ./.git \) -or \( -path './search/*' \) -exec git rm -r {} + -cp -ar "$DOX_DOCS/." . -git add . -if git diff --cached --quiet; then - echo "Docs already updated to the latest version, skipping..." -else - gitcommit -m "Update docs to latest ($DESCRIBE)" -fi - -if [ "$DRY_RUN" != 1 ]; then - git config credential.${GIT_REMOTE}.username "$GITHUB_API_USER" - # shellcheck disable=SC2016 - git config credential.${GIT_REMOTE}.helper '!f() { echo "password=$(echo $GITHUB_API_TOKEN)"; }; f' - git push github_rls HEAD:gh-pages -fi -) diff --git a/scripts/publish_github_rls.py b/scripts/publish_github_rls.py deleted file mode 100755 index 716c27f22..000000000 --- a/scripts/publish_github_rls.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 - -import json -import mimetypes -import os -import os.path -import re -import sys -import time -import urllib.request - - -def urlopen_retry(req): - delay = 1 - last_exc = Exception() - for k in range(5): - try: - return urllib.request.urlopen(req) - except urllib.error.HTTPError as e: - if e.code < 500: - raise - last_exc = e - time.sleep(delay) - delay *= 2 - raise last_exc - - -def main(): - if len(sys.argv) < 2: - print("usage: {} RLS_TAG [assets]".format(sys.argv[0])) - - rls_tag = sys.argv[1] - if "GITHUB_API_TOKEN" not in os.environ: - raise RuntimeError("Please define $GITHUB_API_TOKEN") - api_token = os.environ["GITHUB_API_TOKEN"] - files = sys.argv[2:] - - req = urllib.request.Request( - "https://api.github.com/repos/uptane/aktualizr/releases/tags/{}".format( - rls_tag - ), - headers={ - "Accept": "application/vnd.github.v3+json", - "Authorization": "token {}".format(api_token), - "Content-Type": "application/json", - }, - method="GET", - ) - try: - with urlopen_retry(req) as f: - json.loads(f.read()) - except urllib.error.HTTPError as e: - if e.code != 404: - raise - else: - print("release already exist, nothing to do...") - return 0 - - # create release - c = {"tag_name": rls_tag, "name": rls_tag, "body": "", "draft": False} - req = urllib.request.Request( - "https://api.github.com/repos/uptane/aktualizr/releases", - data=json.dumps(c).encode(), - headers={ - "Accept": "application/vnd.github.v3+json", - "Authorization": "token {}".format(api_token), - "Content-Type": "application/json", - }, - method="POST", - ) - with urlopen_retry(req) as f: - resp = json.loads(f.read()) - - upload_url = re.sub("{.*}", "", resp["upload_url"]) - - for fn in files: - bn = os.path.basename(fn) - url = upload_url + "?name={}".format(bn) - with open(fn, "rb") as f: - req = urllib.request.Request( - url, - data=f, - headers={ - "Accept": "application/vnd.github.v3+json", - "Authorization": "token {}".format(api_token), - "Content-Length": str(os.path.getsize(fn)), - "Content-Type": mimetypes.guess_type(bn)[0], - }, - method="POST", - ) - urlopen_retry(req) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/test.sh b/scripts/test.sh index b10327d6e..facd2e73a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -8,7 +8,6 @@ GITREPO_ROOT="${1:-$(readlink -f "$(dirname "$0")/..")}" TEST_BUILD_DIR=${TEST_BUILD_DIR:-build-test} TEST_WITH_STATICTESTS=${TEST_WITH_STATICTESTS:-0} TEST_WITH_BUILD=${TEST_WITH_BUILD:-1} -TEST_WITH_INSTALL_DEB_PACKAGES=${TEST_WITH_INSTALL_DEB_PACKAGES:-0} TEST_WITH_TESTSUITE=${TEST_WITH_TESTSUITE:-1} TEST_WITH_DOCS=${TEST_WITH_DOCS:-0} @@ -37,7 +36,6 @@ git config --global --add safe.directory /home/runner/work/aktualizr/aktualizr CMAKE_ARGS=() CMAKE_ARGS+=("-G$TEST_CMAKE_GENERATOR") CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${TEST_CMAKE_BUILD_TYPE}") -if [[ $TEST_WITH_COVERAGE = 1 ]]; then CMAKE_ARGS+=("-DBUILD_WITH_CODE_COVERAGE=ON"); fi if [[ $TEST_WITH_SOTA_TOOLS = 1 ]]; then CMAKE_ARGS+=("-DBUILD_SOTA_TOOLS=ON"); fi if [[ $TEST_WITH_P11 = 1 ]]; then CMAKE_ARGS+=("-DBUILD_P11=ON") @@ -151,45 +149,15 @@ if [[ $TEST_WITH_BUILD = 1 ]]; then fi fi -if [[ $TEST_WITH_INSTALL_DEB_PACKAGES = 1 ]]; then - echo ">> Building debian package" +if [[ $TEST_WITH_TESTSUITE = 1 ]]; then + echo ">> Running test suite" if [[ $TEST_DRYRUN != 1 ]]; then set -x - run_make package || add_failure "make package" - - # install garage-deploy - cp ./*garage_deploy.deb "${TEST_INSTALL_DESTDIR}/garage_deploy${TEST_INSTALL_RELEASE_NAME}.deb" - - # install aktualizr.deb - cp ./*aktualizr.deb "${TEST_INSTALL_DESTDIR}/aktualizr${TEST_INSTALL_RELEASE_NAME}.deb" + run_make check || add_failure "testsuite" set +x fi fi -if [[ $TEST_WITH_TESTSUITE = 1 ]]; then - if [[ $TEST_WITH_COVERAGE = 1 ]]; then - echo ">> Running test suite with coverage" - if [[ $TEST_DRYRUN != 1 ]]; then - set -x - run_make coverage || add_failure "testsuite with coverage" - - if [[ -n ${CODECOV_TOKEN:-} ]]; then - bash <(curl -s https://codecov.io/bash) -f '!*/#usr*' -f '!*/^#third_party*' -R "${GITREPO_ROOT}" -s . > /dev/null - else - echo "Skipping codecov.io upload" - fi - set +x - fi - else - echo ">> Running test suite" - if [[ $TEST_DRYRUN != 1 ]]; then - set -x - run_make check || add_failure "testsuite" - set +x - fi - fi -fi - if [[ $TEST_WITH_DOCS = 1 ]]; then echo ">> Running make docs" if [[ $TEST_DRYRUN != 1 ]]; then diff --git a/scripts/test_install_aktualizr.sh b/scripts/test_install_aktualizr.sh deleted file mode 100755 index 995607aed..000000000 --- a/scripts/test_install_aktualizr.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/bash - -set -exuo pipefail - -TEST_INSTALL_DESTDIR=${TEST_INSTALL_DESTDIR:-/persistent} - -dpkg-deb -I "$TEST_INSTALL_DESTDIR"/aktualizr*.deb && dpkg -i "$TEST_INSTALL_DESTDIR"/aktualizr*.deb -akt_version=$(aktualizr --version) -(grep "$(cat "$TEST_INSTALL_DESTDIR"/aktualizr-version)" <<< "$akt_version") || (echo "$akt_version"; false) - diff --git a/scripts/test_install_garage_deploy.sh b/scripts/test_install_garage_deploy.sh deleted file mode 100755 index 789c88e2a..000000000 --- a/scripts/test_install_garage_deploy.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -set -exuo pipefail - -TEST_INSTALL_DESTDIR=${TEST_INSTALL_DESTDIR:-/persistent} - -dpkg -i "$TEST_INSTALL_DESTDIR"/garage_deploy*.deb -garage-deploy --version -garage-sign --help diff --git a/src/aktualizr_info/aktualizr_info_test.cc b/src/aktualizr_info/aktualizr_info_test.cc index 6b15c0547..6f4c3753e 100644 --- a/src/aktualizr_info/aktualizr_info_test.cc +++ b/src/aktualizr_info/aktualizr_info_test.cc @@ -1,6 +1,7 @@ #include #include +#include #include "libaktualizr/config.h" #include "storage/sqlstorage.h" diff --git a/src/aktualizr_secondary/aktualizr_secondary_test.cc b/src/aktualizr_secondary/aktualizr_secondary_test.cc index 7094e019f..98705261d 100644 --- a/src/aktualizr_secondary/aktualizr_secondary_test.cc +++ b/src/aktualizr_secondary/aktualizr_secondary_test.cc @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include "aktualizr_secondary_file.h" #include "crypto/keymanager.h" diff --git a/src/cert_provider/cert_provider_test.cc b/src/cert_provider/cert_provider_test.cc index 913ccecab..b1c4336be 100644 --- a/src/cert_provider/cert_provider_test.cc +++ b/src/cert_provider/cert_provider_test.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include "cert_provider_test.h" diff --git a/src/libaktualizr/config/config_test.cc b/src/libaktualizr/config/config_test.cc index 0c1ae8e02..79a8e6e51 100644 --- a/src/libaktualizr/config/config_test.cc +++ b/src/libaktualizr/config/config_test.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include "bootstrap/bootstrap.h" #include "crypto/crypto.h" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ffcf09610..4e99310f0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -197,6 +197,9 @@ add_test(NAME test_ip_secondary COMMAND ${PROJECT_SOURCE_DIR}/tests/ipsecondary_test.py --build-dir ${PROJECT_BINARY_DIR} --src-dir ${PROJECT_SOURCE_DIR}) set_tests_properties(test_ip_secondary PROPERTIES LABELS "noptest") +# This test flakes when run in parallel. Force it to run serially +# TODO Fix this test! +set_tests_properties(test_ip_secondary PROPERTIES RUN_SERIAL ON) add_test(NAME test_ip_secondary_rotation COMMAND ${PROJECT_SOURCE_DIR}/tests/ipsecondary_rotation_test.py