From 45d2b582011ccdd026505477362c199f948287a6 Mon Sep 17 00:00:00 2001 From: Huaqi Fang <578567190@qq.com> Date: Mon, 20 Nov 2023 11:55:35 +0800 Subject: [PATCH] ci: update and optimize github workflow to use Nuclei Qemu 2023.10 1. More SoC and Core are supported 2. Build docker is support in each dev* naming branch Squashed commit of the following: commit 7fe22c9feb72eb03414dcc28a402bcd79e787101 Author: Huaqi Fang <578567190@qq.com> Date: Mon Nov 20 09:43:49 2023 +0800 ci: update missing qemu libraries Signed-off-by: Huaqi Fang <578567190@qq.com> commit 57331f26e43c33daa3837b084aa4db6342f8aca5 Author: Huaqi Fang <578567190@qq.com> Date: Fri Nov 17 14:35:15 2023 +0800 ci: try to fix linux sdk docker build Signed-off-by: Huaqi Fang <578567190@qq.com> commit 9b1aac027cf64c5753a0a49810b80cfe8c56bad2 Author: Huaqi Fang <578567190@qq.com> Date: Fri Nov 17 11:28:37 2023 +0800 ci: fix show qemu Signed-off-by: Huaqi Fang <578567190@qq.com> commit 9aec229a6784770e755c11865d9abdac4ded4792 Author: Huaqi Fang <578567190@qq.com> Date: Fri Nov 17 10:16:42 2023 +0800 ci: Fix missing ignore steps for demosoc - u900* Signed-off-by: Huaqi Fang <578567190@qq.com> commit c95689dd87be8d45382045281dbe92d78856c526 Author: Huaqi Fang <578567190@qq.com> Date: Fri Nov 17 10:12:48 2023 +0800 ci: Update docker.yml commit 3ed1b0a3c599eadd2e2e61de4c5f399d01b8231c Author: Huaqi Fang <578567190@qq.com> Date: Fri Nov 17 10:10:53 2023 +0800 ci: Update github ci workflow using Nuclei Qemu 2023.10 Signed-off-by: Huaqi Fang <578567190@qq.com> Signed-off-by: Huaqi Fang <578567190@qq.com> --- .github/apt.txt | 2 +- .github/linuxsdk.Dockerfile | 19 ++++++++++---- .github/run_qemu.sh | 2 +- .github/workflows/build.yml | 48 ++++++++++++++++++++++-------------- .github/workflows/docker.yml | 17 +++++++------ 5 files changed, 55 insertions(+), 33 deletions(-) diff --git a/.github/apt.txt b/.github/apt.txt index f7007ef..784b82e 100644 --- a/.github/apt.txt +++ b/.github/apt.txt @@ -1,3 +1,3 @@ build-essential git python3 python3-pip autotools-dev make cmake texinfo bison minicom flex liblz4-tool libgmp-dev libmpfr-dev libmpc-dev gawk libz-dev libssl-dev libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev -libncursesw5-dev libncursesw5 mtools wget cpio zip unzip rsync bc sudo device-tree-compiler +libncursesw5-dev libncursesw5 mtools wget cpio zip unzip rsync bc sudo device-tree-compiler libpng16-16 libasound2 diff --git a/.github/linuxsdk.Dockerfile b/.github/linuxsdk.Dockerfile index 91afe12..5e4fbe7 100644 --- a/.github/linuxsdk.Dockerfile +++ b/.github/linuxsdk.Dockerfile @@ -19,20 +19,26 @@ RUN xargs apt install -y < /home/apt.txt RUN rm -f /home/apt.txt -RUN apt autoclean - RUN ln -s /lib/x86_64-linux-gnu/libgmp.so /lib/x86_64-linux-gnu/libgmp.so.3 COPY pipreq.txt /home/ +RUN pip3 install -U pip + RUN pip install -r /home/pipreq.txt RUN rm -f /home/pipreq.txt +# Cleanup apt/pip cache +RUN apt autoclean +RUN apt clean +RUN pip3 cache purge + # create USER with PASS ARG USER=nuclei ARG PASS=riscv123 -ARG QEMUVER=2022.12 +ARG QEMUVER=2023.10 +ARG BRANCH=dev_nuclei_next RUN groupadd --system $USER @@ -52,9 +58,12 @@ RUN cd prebuilt && tar --no-same-owner -xzf nuclei-qemu.tar.gz ENV PATH "/home/$USER/prebuilt/qemu/bin:$PATH" -RUN git clone https://github.com/Nuclei-Software/nuclei-linux-sdk +RUN ldd `which qemu-system-riscv64` + +RUN git clone -b $BRANCH https://github.com/Nuclei-Software/nuclei-linux-sdk -RUN cd nuclei-linux-sdk && git remote add gitee https://gitee.com/Nuclei-Software/nuclei-linux-sdk +# gitee mirror no longer works +#RUN cd nuclei-linux-sdk && git remote add gitee https://gitee.com/Nuclei-Software/nuclei-linux-sdk RUN cd nuclei-linux-sdk && git submodule init && git submodule update --recursive --init diff --git a/.github/run_qemu.sh b/.github/run_qemu.sh index b78b7b9..a5b601e 100644 --- a/.github/run_qemu.sh +++ b/.github/run_qemu.sh @@ -1,5 +1,5 @@ cd nuclei-linux-sdk -timeout --foreground -s SIGTERM 4m make run_qemu > >(tee run_qemu.log) +timeout --foreground -s SIGTERM 5m make run_qemu > >(tee run_qemu.log) # check pass or not if cat run_qemu.log | grep "Run /init" ; then diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6eab61b..096374b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,14 +24,14 @@ jobs: strategy: matrix: soc: [demosoc, evalsoc] - core: [ux600, ux600fd] + core: [ux900, ux900fd, u900, u900fd] boot_mode: [sd, flash] # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Linux SDK Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: recursive @@ -43,8 +43,8 @@ jobs: mv source.tar.gz $base/ - name: Upload source - if: matrix.core == 'ux600' && matrix.boot_mode == 'sd' && matrix.soc == 'demosoc' - uses: actions/upload-artifact@v2.2.0 + if: matrix.core == 'ux900fd' && matrix.boot_mode == 'sd' && matrix.soc == 'evalsoc' + uses: actions/upload-artifact@v3 with: name: nuclei_linux_sdk_source path: source.tar.gz @@ -52,7 +52,7 @@ jobs: continue-on-error: true - name: Caching packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: buildroot/dl key: build @@ -65,29 +65,38 @@ jobs: python3 python3-pip mtools sudo pip3 install git-archive-all mkdir -p buildroot/dl - QEMUVER=2022.12 + QEMUVER=2023.10 if [ ! -f buildroot/dl/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz ] ; then - wget -P buildroot/dl https://nucleisys.com/upload/files/toochain/qemu/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz + wget --progress=dot -P buildroot/dl https://nucleisys.com/upload/files/toochain/qemu/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz fi mkdir -p prebuilt_tools tar -xzf buildroot/dl/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz -C prebuilt_tools # Build bootimages - name: Build bootimages for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }} + if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }} run: | export SOC=${{ matrix.soc }} CORE=${{ matrix.core }} BOOT_MODE=${{ matrix.boot_mode }} echo "Generate QEMU Disk" - echo ">>> Apply workaround: change TIMERCLK_FREQ from 32768 to 4000000" + #echo ">>> Apply workaround: change TIMERCLK_FREQ from 32768 to 4000000" #sed -i "s/32768/4000000/" conf/$SOC/*.dts - export TIMER_HZ=4000000 + #export TIMER_HZ=4000000 make genstamp make freeloader cp work/$SOC/freeloader/freeloader.elf work/$SOC/freeloader_qemu.elf make bootimages make DISK_SIZE=128 gendisk RUNQEMUSH=work/$SOC/run_qemu.sh - echo "qemu-system-riscv64 -M nuclei_u,download=flashxip -smp 8 -m 256M -bios freeloader_qemu.elf -nographic -drive file=disk.img,if=sd,format=raw" > $RUNQEMUSH + SHOWQEMUSH=work/$SOC/show_qemu.sh + qemucmd=qemu-system-riscv32 + if [[ $CORE == *ux* ]] ; then + qemucmd=qemu-system-riscv64 + fi + echo "which $qemucmd" > $SHOWQEMUSH + echo "$qemucmd --version" >> $SHOWQEMUSH + echo "$qemucmd -M nuclei_${SOC},download=flashxip -cpu nuclei-${CORE},ext=$ARCH_EXT -smp 8 -m 2g -bios freeloader_qemu.elf -nographic -drive file=disk.img,if=sd,format=raw" > $RUNQEMUSH chmod +x $RUNQEMUSH + chmod +x $SHOWQEMUSH unset TIMER_HZ echo ">>> Reset workaround and source code" git reset --hard @@ -96,22 +105,25 @@ jobs: # Run on qemu - name: Run and test on qemu for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }} - if: matrix.soc == 'demosoc' + if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }} run: | QEMU_PATH=$(pwd)/prebuilt_tools/linux_qemu/bin:$(pwd)/prebuilt_tools/qemu/bin export PATH=${QEMU_PATH}:$PATH export SOC=${{ matrix.soc }} CORE=${{ matrix.core }} BOOT_MODE=${{ matrix.boot_mode }} cd work/$SOC + # show qemu version + bash show_qemu.sh # $(cat run_qemu.sh) is workaround for directly run bash run_qemu.sh # If do kill it will just kill bash process, the qemu process is not killed # SIGTERM is better for kill qemu - timeout --foreground -s SIGTERM 8m $(cat run_qemu.sh) > >(tee run_qemu.log) || { + timeout --foreground -s SIGTERM 5m $(cat run_qemu.sh) > >(tee run_qemu.log) || { if cat run_qemu.log | grep "Run /init" ; then echo "Kernel boot successfully" ; else echo "Kernel boot failed" && exit 1; fi; if cat run_qemu.log | grep "Welcome to" ; then echo "Pass simulation" && exit 0; else echo "Failed init process" && exit 1; fi; } - name: Upload bootimages for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }} - uses: actions/upload-artifact@v2.2.0 + if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }} + uses: actions/upload-artifact@v3 with: name: bootimages_${{ matrix.soc }}_${{ matrix.core }}_${{ matrix.boot_mode }} path: | @@ -120,8 +132,8 @@ jobs: work/${{ matrix.soc }}/buildstamp.txt - name: Upload qemu images for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }} - if: matrix.soc == 'demosoc' - uses: actions/upload-artifact@v2.2.0 + if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }} + uses: actions/upload-artifact@v3 with: name: qemu_images_${{ matrix.soc }}_${{ matrix.core }}_${{ matrix.boot_mode }} path: | @@ -130,11 +142,11 @@ jobs: work/${{ matrix.soc }}/freeloader_qemu.elf work/${{ matrix.soc }}/buildstamp.txt work/${{ matrix.soc }}/disk.img - work/${{ matrix.soc }}/run_qemu.sh + work/${{ matrix.soc }}/*.sh - name: Upload cached packages - if: matrix.core == 'ux600' && matrix.boot_mode == 'sd' && matrix.soc == 'demosoc' - uses: actions/upload-artifact@v2.2.0 + if: matrix.core == 'ux900fd' && matrix.boot_mode == 'sd' && matrix.soc == 'evalsoc' + uses: actions/upload-artifact@v3 with: name: build_caching path: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c150e4a..587976f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,14 +2,14 @@ name: Build Linux SDK Docker Image on: push: - branches: [ dev_nuclei_next ] + branches: [ dev* ] paths-ignore: - README.md - LICENSE - .gitlab-ci.yml - .gitignore pull_request: - branches: [ dev_nuclei_next ] + branches: [ dev* ] jobs: @@ -22,19 +22,20 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Linux SDK Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: false - name: Build Docker Image run: | cd .github - docker build . -f linuxsdk.Dockerfile -t linuxsdk:latest + docker build . -f linuxsdk.Dockerfile -t linuxsdk:${{github.ref_name}} --build-arg BRANCH=${{github.ref_name}} docker images - name: Test Docker Image + if: ${{ startsWith(github.ref_name, 'dev_') && github.event_name != 'pull_request' }} run: | - docker run -i linuxsdk:latest bash run_qemu.sh + docker run -i linuxsdk:${{github.ref_name}} bash run_qemu.sh - name: Publish Docker Image if: ${{ github.event_name != 'pull_request' }} @@ -42,6 +43,6 @@ jobs: echo "Login to docker hub" docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PWD }} docker.io DOCKER_REPO=docker.io/nucleisoftware/linuxsdk - docker tag linuxsdk:latest ${DOCKER_REPO}:ci - docker push ${DOCKER_REPO}:ci - echo "Docker image is pushed to https://hub.docker.com/repository/docker/nucleisoftware/linuxsdk" + docker tag linuxsdk:${{github.ref_name}} ${DOCKER_REPO}:${{github.ref_name}} + docker push ${DOCKER_REPO}:${{github.ref_name}} + echo "Docker image is pushed to https://hub.docker.com/repository/docker/nucleisoftware/linuxsdk, tag ${{github.ref_name}}"