Merge branch 'main' into dependabot/pip/zstandard-0.23.0

.github/workflows/test.yaml

@@ -16,8 +16,15 @@ on:
 
 jobs:
   pytest_with_coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test_base:
+          - ubuntu-18.04
+          - ubuntu-20.04
+          - ubuntu-22.04
     steps:
       - name: Checkout commit
         uses: actions/checkout@v4
@@ -26,20 +33,23 @@ jobs:
           fetch-depth: 0
       - name: Build ota-test_base docker image
         run: |
-          docker compose -f docker/docker-compose_tests.yml build
+          docker compose -f docker/docker-compose_tests.yml build tester-${{ matrix.test_base }}
       - name: Execute pytest with coverage trace under ota-test_base container
         run: |
           mkdir -p test_result
-          docker compose -f docker/docker-compose_tests.yml up --abort-on-container-exit
+          docker compose -f docker/docker-compose_tests.yml run --rm tester-${{ matrix.test_base }}
       - name: SonarCloud Scan
         uses: SonarSource/sonarcloud-github-action@master
         continue-on-error: true
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        if: ${{ matrix.test_base == 'ubuntu-22.04' }}
+
       # export the coverage report to the comment!
       - name: Add coverage report to PR comment
         continue-on-error: true
         uses: MishaKav/[email protected]
         with:
           pytest-xml-coverage-path: test_result/coverage.xml
           junitxml-path: test_result/pytest.xml
+        if: ${{ matrix.test_base == 'ubuntu-22.04' }}

.pre-commit-config.yaml

@@ -22,15 +22,15 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-bugbear==24.2.6
           - flake8-comprehensions
           - flake8-simplify
   - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: "2.1.3"
+    rev: "2.2.1"
     hooks:
       - id: pyproject-fmt
         # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version

docker/docker-compose_tests.yml

@@ -1,9 +1,43 @@
 services:
-  tester:
+  tester-ubuntu-18.04:
+    build:
+      context: ../
+      dockerfile: ./docker/test_base/Dockerfile_ubuntu-18.04
+      args:
+        - UBUNTU_BASE=ubuntu:18.04
+    image: ota-test_base:ubuntu1804
+    network_mode: bridge
+    container_name: ota-test
+    environment:
+      OUTPUT_DIR: /test_result
+      CERTS_DIR: /certs
+    volumes:
+      - ..:/otaclient_src:ro
+      - ../test_result:/test_result:rw
+
+  tester-ubuntu-20.04:
+    build:
+      context: ../
+      dockerfile: ./docker/test_base/Dockerfile
+      args:
+        - UBUNTU_BASE=ubuntu:20.04
+    image: ota-test_base:ubuntu2004
+    network_mode: bridge
+    container_name: ota-test
+    environment:
+      OUTPUT_DIR: /test_result
+      CERTS_DIR: /certs
+    volumes:
+      - ..:/otaclient_src:ro
+      - ../test_result:/test_result:rw
+
+  tester-ubuntu-22.04:
     build:
       context: ../
       dockerfile: ./docker/test_base/Dockerfile
-    image: ota-test_base
+      args:
+        - UBUNTU_BASE=ubuntu:22.04
+    image: ota-test_base:ubuntu2204
     network_mode: bridge
     container_name: ota-test
     environment:

docker/test_base/Dockerfile

@@ -1,17 +1,47 @@
-FROM ubuntu:20.04
+ARG UBUNTU_BASE
+
+#
+# ------ stage 1: prepare base image ------ #
+#
+
+FROM ${UBUNTU_BASE} AS builder
+
+SHELL ["/bin/bash", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement"
+
+# special treatment to the ota-image: create file that needs url escaping
+# NOTE: include special identifiers #?; into the pathname
+RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}"
+
+# install required packages
+RUN set -eux; \
+    apt-get update -qq; \
+    apt-get install -y linux-image-generic; \
+    apt-get clean; \
+    rm -rf \
+        /tmp/* \
+        /var/lib/apt/lists/* \
+        /var/tmp/* \
+
+#
+# ------ stage 2: prepare test environment ------ #
+#
+ARG UBUNTU_BASE
+
+FROM ${UBUNTU_BASE}
+
 SHELL ["/bin/bash", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
-ARG KERNEL_VERSION="5.8.0-53-generic"
-ARG BASE_IMG_URL="http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04.1-base-amd64.tar.gz"
-ARG OTA_METADATA_REPO="https://github.com/tier4/ota-metadata"
+
+ENV OTA_METADATA_REPO="https://github.com/tier4/ota-metadata"
 ENV OTA_IMAGE_SERVER_ROOT="/ota-image"
 ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data"
 ENV CERTS_DIR="/certs"
 ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement"
 
-
 # install required packages
-RUN set -eu; \
+RUN set -eux; \
     apt-get update -qq; \
     apt-get install -y -qq --no-install-recommends \
         python3-minimal \
@@ -23,36 +53,25 @@ RUN set -eu; \
 	    gcc \
         wget \
         git; \
-    apt-get install -y -qq linux-image-${KERNEL_VERSION}
-
-# prepare ubuntu base
-WORKDIR ${OTA_IMAGE_SERVER_ROOT}
-RUN set -eu; \
-    wget -q -O /tmp/base_image.tar.gz ${BASE_IMG_URL}; \
-    mkdir -p ${OTA_IMAGE_DIR}; \
-    tar zxf /tmp/base_image.tar.gz -C ${OTA_IMAGE_DIR}; \
-    cp -a \
-        /boot/vmlinuz-${KERNEL_VERSION} \
-        /boot/initrd.img-${KERNEL_VERSION} \
-        /boot/config-${KERNEL_VERSION} \
-        /boot/System.map-${KERNEL_VERSION} ${OTA_IMAGE_DIR}/boot
-
-# special treatment to the ota-image: create file that needs url escaping
-# NOTE: include special identifiers #?; into the pathname
-RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}"
+    apt-get install -y -qq linux-image-generic; \
+    apt-get clean
 
 # install hatch
-RUN set -eu; \
+RUN set -eux; \
     python3 -m pip install --no-cache-dir -q -U pip; \
     python3 -m pip install --no-cache-dir -U hatch
 
+WORKDIR ${OTA_IMAGE_SERVER_ROOT}
+
+COPY --from=builder / /${OTA_IMAGE_DIR}
+
 # generate test certs and sign key
 COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/
 RUN set -eu; \
     mkdir -p "${CERTS_DIR}"; \
     pushd /tmp/certs; \
     ./gen_certs.sh; \
-    cp * "${CERTS_DIR}"; \
+    cp ./* "${CERTS_DIR}"; \
     popd
 
 # build the test OTA image
@@ -62,8 +81,11 @@ RUN set -eu; \
     git clone ${OTA_METADATA_REPO}; \
     python3 -m venv ota-metadata/.venv; \
     source ota-metadata/.venv/bin/activate; \
+    python3 -m pip install --no-cache-dir -U pip; \
     python3 -m pip install --no-cache-dir -q \
         -r ota-metadata/metadata/ota_metadata/requirements.txt; \
+    # patch the ignore files
+    echo "" > ota-metadata/metadata/ignore.txt; \
     python3 ota-metadata/metadata/ota_metadata/metadata_gen.py \
         --target-dir data \
         --compressed-dir data.zst \
@@ -77,7 +99,7 @@ RUN set -eu; \
     cp ota-metadata/metadata/persistents.txt .
 
 # cleanup
-RUN set -eu; \
+RUN set -eux; \
     apt-get clean; \
     rm -rf \
         /tmp/* \

docker/test_base/Dockerfile_ubuntu-18.04

@@ -0,0 +1,115 @@
+ARG UBUNTU_BASE
+
+#
+# ------ stage 1: prepare base image ------ #
+#
+
+FROM ${UBUNTU_BASE} AS builder
+
+SHELL ["/bin/bash", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement"
+
+# special treatment to the ota-image: create file that needs url escaping
+# NOTE: include special identifiers #?; into the pathname
+RUN echo -n "${SPECIAL_FILE}" > "/${SPECIAL_FILE}"
+
+# install required packages
+RUN set -eux; \
+    apt-get update -qq; \
+    apt-get install -y linux-image-generic; \
+    apt-get clean; \
+    rm -rf \
+        /tmp/* \
+        /var/lib/apt/lists/* \
+        /var/tmp/* \
+
+#
+# ------ stage 2: prepare test environment ------ #
+#
+ARG UBUNTU_BASE
+
+FROM ${UBUNTU_BASE}
+
+SHELL ["/bin/bash", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV OTA_METADATA_REPO="https://github.com/tier4/ota-metadata"
+ENV OTA_IMAGE_SERVER_ROOT="/ota-image"
+ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data"
+ENV CERTS_DIR="/certs"
+ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement"
+
+# install required packages
+RUN set -eux; \
+    apt-get update -qq; \
+    apt-get install -y -qq --no-install-recommends \
+        python3-minimal \
+        python3-pip \
+        python3-venv \
+        python3-dev \
+	    libcurl4-openssl-dev \
+	    libssl-dev \
+	    gcc \
+        wget \
+        git \
+        python3.8-venv; \
+    apt-get install -y -qq linux-image-generic; \
+    apt-get clean
+
+# install hatch to the system
+RUN set -eux; \
+    python3.8 -m pip install --no-cache-dir -q -U pip; \
+    python3.8 -m pip install --no-cache-dir -U hatch
+
+WORKDIR ${OTA_IMAGE_SERVER_ROOT}
+
+COPY --from=builder / /${OTA_IMAGE_DIR}
+
+# generate test certs and sign key
+COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/
+RUN set -eux; \
+    mkdir -p "${CERTS_DIR}"; \
+    pushd /tmp/certs; \
+    ./gen_certs.sh; \
+    cp ./* "${CERTS_DIR}"; \
+    popd
+
+# build the test OTA image
+RUN set -eux; \
+    cp "${CERTS_DIR}"/sign.key sign.key; \
+    cp "${CERTS_DIR}"/sign.pem sign.pem; \
+    git clone ${OTA_METADATA_REPO}; \
+    python3.8 -m venv ota-metadata/.venv; \
+    source ota-metadata/.venv/bin/activate; \
+    python3.8 -m pip install --no-cache-dir -U pip; \
+    python3.8 -m pip install --no-cache-dir -q \
+        -r ota-metadata/metadata/ota_metadata/requirements.txt; \
+    # patch the ignore files
+    echo "" > ota-metadata/metadata/ignore.txt; \
+    python3.8 ota-metadata/metadata/ota_metadata/metadata_gen.py \
+        --target-dir data \
+        --compressed-dir data.zst \
+        --ignore-file ota-metadata/metadata/ignore.txt; \
+    python3.8 ota-metadata/metadata/ota_metadata/metadata_sign.py \
+        --sign-key sign.key \
+        --cert-file sign.pem \
+        --persistent-file ota-metadata/metadata/persistents.txt \
+        --rootfs-directory data \
+        --compressed-rootfs-directory data.zst; \
+    cp ota-metadata/metadata/persistents.txt .
+
+# cleanup
+RUN set -eux; \
+    apt-get clean; \
+    rm -rf \
+        /tmp/* \
+        /var/lib/apt/lists/* \
+        /var/tmp/* \
+        ota-metadata
+
+# copy and setup the entry_point.sh
+COPY ./docker/test_base/entry_point.sh /entry_point.sh
+RUN chmod +x /entry_point.sh
+
+ENTRYPOINT [ "/entry_point.sh" ]