ci: update and optimize github workflow to use Nuclei Qemu 2023.10

1. More SoC and Core are supported 2. Build docker is support in each dev* naming branch Squashed commit of the following: commit 7fe22c9 Author: Huaqi Fang <[email protected]> Date: Mon Nov 20 09:43:49 2023 +0800 ci: update missing qemu libraries Signed-off-by: Huaqi Fang <[email protected]> commit 57331f2 Author: Huaqi Fang <[email protected]> Date: Fri Nov 17 14:35:15 2023 +0800 ci: try to fix linux sdk docker build Signed-off-by: Huaqi Fang <[email protected]> commit 9b1aac0 Author: Huaqi Fang <[email protected]> Date: Fri Nov 17 11:28:37 2023 +0800 ci: fix show qemu Signed-off-by: Huaqi Fang <[email protected]> commit 9aec229 Author: Huaqi Fang <[email protected]> Date: Fri Nov 17 10:16:42 2023 +0800 ci: Fix missing ignore steps for demosoc - u900* Signed-off-by: Huaqi Fang <[email protected]> commit c95689d Author: Huaqi Fang <[email protected]> Date: Fri Nov 17 10:12:48 2023 +0800 ci: Update docker.yml commit 3ed1b0a Author: Huaqi Fang <[email protected]> Date: Fri Nov 17 10:10:53 2023 +0800 ci: Update github ci workflow using Nuclei Qemu 2023.10 Signed-off-by: Huaqi Fang <[email protected]> Signed-off-by: Huaqi Fang <[email protected]>
Nuclei-Software · Nov 20, 2023 · 88a4dec · 88a4dec
1 parent 2651604
commit 88a4dec
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 33 deletions.
diff --git a/.github/apt.txt b/.github/apt.txt
@@ -1,3 +1,3 @@
 build-essential git python3 python3-pip autotools-dev make cmake texinfo bison minicom flex liblz4-tool
 libgmp-dev libmpfr-dev libmpc-dev gawk libz-dev libssl-dev libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev
-libncursesw5-dev libncursesw5 mtools wget cpio zip unzip rsync bc sudo device-tree-compiler
+libncursesw5-dev libncursesw5 mtools wget cpio zip unzip rsync bc sudo device-tree-compiler libpng16-16 libasound2
diff --git a/.github/linuxsdk.Dockerfile b/.github/linuxsdk.Dockerfile
@@ -19,8 +19,6 @@ RUN xargs apt install -y < /home/apt.txt
 
 RUN rm -f /home/apt.txt
 
-RUN apt autoclean
-
 RUN ln -s /lib/x86_64-linux-gnu/libgmp.so /lib/x86_64-linux-gnu/libgmp.so.3
 
 COPY pipreq.txt /home/
@@ -29,10 +27,16 @@ RUN pip install -r /home/pipreq.txt
 
 RUN rm -f /home/pipreq.txt
 
+# Cleanup apt/pip cache
+RUN apt autoclean
+RUN apt clean
+RUN pip3 cache purge
+
 # create USER with PASS
 ARG USER=nuclei
 ARG PASS=riscv123
-ARG QEMUVER=2022.12
+ARG QEMUVER=2023.10
+ARG BRANCH=dev_nuclei_next
 
 RUN groupadd --system $USER
 
@@ -52,9 +56,12 @@ RUN cd prebuilt && tar --no-same-owner -xzf nuclei-qemu.tar.gz
 
 ENV PATH "/home/$USER/prebuilt/qemu/bin:$PATH"
 
-RUN git clone https://github.com/Nuclei-Software/nuclei-linux-sdk
+RUN ldd `which qemu-system-riscv64`
+
+RUN git clone -b $BRANCH https://github.com/Nuclei-Software/nuclei-linux-sdk
 
-RUN cd nuclei-linux-sdk && git remote add gitee https://gitee.com/Nuclei-Software/nuclei-linux-sdk
+# gitee mirror no longer works
+#RUN cd nuclei-linux-sdk && git remote add gitee https://gitee.com/Nuclei-Software/nuclei-linux-sdk
 
 RUN cd nuclei-linux-sdk && git submodule init && git submodule update --recursive --init
 

diff --git a/.github/run_qemu.sh b/.github/run_qemu.sh
@@ -1,5 +1,5 @@
 cd nuclei-linux-sdk
-timeout --foreground -s SIGTERM 4m make run_qemu > >(tee run_qemu.log)
+timeout --foreground -s SIGTERM 5m make run_qemu > >(tee run_qemu.log)
 
 # check pass or not
 if cat run_qemu.log | grep "Run /init" ; then

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,14 +24,14 @@ jobs:
     strategy:
       matrix:
         soc: [demosoc, evalsoc]
-        core: [ux600, ux600fd]
+        core: [ux900, ux900fd, u900, u900fd]
         boot_mode: [sd, flash]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout Linux SDK Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
@@ -43,16 +43,16 @@ jobs:
           mv source.tar.gz $base/
 
       - name: Upload source
-        if: matrix.core == 'ux600' && matrix.boot_mode == 'sd' && matrix.soc == 'demosoc'
-        uses: actions/upload-artifact@v2.2.0
+        if: matrix.core == 'ux900fd' && matrix.boot_mode == 'sd' && matrix.soc == 'evalsoc'
+        uses: actions/upload-artifact@v3
         with:
           name: nuclei_linux_sdk_source
           path: source.tar.gz
         # continue build on error
         continue-on-error: true
 
       - name: Caching packages
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: buildroot/dl
           key: build
@@ -65,29 +65,38 @@ jobs:
             python3 python3-pip mtools
           sudo pip3 install git-archive-all
           mkdir -p buildroot/dl
-          QEMUVER=2022.12
+          QEMUVER=2023.10
           if [ ! -f buildroot/dl/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz ] ; then
-            wget -P buildroot/dl https://nucleisys.com/upload/files/toochain/qemu/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz
+            wget --progress=dot -P buildroot/dl https://nucleisys.com/upload/files/toochain/qemu/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz
           fi
           mkdir -p prebuilt_tools
           tar -xzf buildroot/dl/nuclei-qemu-${QEMUVER}-linux-x64.tar.gz -C prebuilt_tools
 
       # Build bootimages
       - name: Build bootimages for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }}
+        if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }}
         run: |
           export SOC=${{ matrix.soc }} CORE=${{ matrix.core }} BOOT_MODE=${{ matrix.boot_mode }}
           echo "Generate QEMU Disk"
-          echo ">>> Apply workaround: change TIMERCLK_FREQ from 32768 to 4000000"
+          #echo ">>> Apply workaround: change TIMERCLK_FREQ from 32768 to 4000000"
           #sed -i "s/32768/4000000/" conf/$SOC/*.dts
-          export TIMER_HZ=4000000
+          #export TIMER_HZ=4000000
           make genstamp
           make freeloader
           cp work/$SOC/freeloader/freeloader.elf work/$SOC/freeloader_qemu.elf
           make bootimages
           make DISK_SIZE=128 gendisk
           RUNQEMUSH=work/$SOC/run_qemu.sh
-          echo "qemu-system-riscv64 -M nuclei_u,download=flashxip -smp 8 -m 256M -bios freeloader_qemu.elf -nographic -drive file=disk.img,if=sd,format=raw" > $RUNQEMUSH
+          SHOWQEMUSH=work/$SOC/show_qemu.sh
+          qemucmd=qemu-system-riscv32
+          if [[ $CORE == *ux* ]] ; then
+              qemucmd=qemu-system-riscv64
+          fi
+          echo "which $qemucmd" > $SHOWQEMUSH
+          echo "$qemucmd --version" >> $SHOWQEMUSH
+          echo "$qemucmd -M nuclei_${SOC},download=flashxip -cpu nuclei-${CORE},ext=$ARCH_EXT -smp 8 -m 2g -bios freeloader_qemu.elf -nographic -drive file=disk.img,if=sd,format=raw" > $RUNQEMUSH
           chmod +x $RUNQEMUSH
+          chmod +x $SHOWQEMUSH
           unset TIMER_HZ
           echo ">>> Reset workaround and source code"
           git reset --hard
@@ -96,22 +105,25 @@ jobs:
 
       # Run on qemu
       - name: Run and test on qemu for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }}
-        if: matrix.soc == 'demosoc'
+        if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }}
         run: |
           QEMU_PATH=$(pwd)/prebuilt_tools/linux_qemu/bin:$(pwd)/prebuilt_tools/qemu/bin
           export PATH=${QEMU_PATH}:$PATH
           export SOC=${{ matrix.soc }} CORE=${{ matrix.core }} BOOT_MODE=${{ matrix.boot_mode }}
           cd work/$SOC
+          # show qemu version
+          bash show_qemu.sh
           # $(cat run_qemu.sh) is workaround for directly run bash run_qemu.sh
           # If do kill it will just kill bash process, the qemu process is not killed
           # SIGTERM is better for kill qemu
-          timeout --foreground -s SIGTERM 8m $(cat run_qemu.sh) > >(tee run_qemu.log) || {
+          timeout --foreground -s SIGTERM 5m $(cat run_qemu.sh) > >(tee run_qemu.log) || {
               if cat run_qemu.log | grep "Run /init" ; then echo "Kernel boot successfully" ; else echo "Kernel boot failed" && exit 1; fi;
               if cat run_qemu.log | grep "Welcome to" ; then echo "Pass simulation" && exit 0; else echo "Failed init process" && exit 1; fi;
           }
 
       - name: Upload bootimages for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }}
-        uses: actions/[email protected]
+        if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }}
+        uses: actions/upload-artifact@v3
         with:
           name: bootimages_${{ matrix.soc }}_${{ matrix.core }}_${{ matrix.boot_mode }}
           path: |
@@ -120,8 +132,8 @@ jobs:
             work/${{ matrix.soc }}/buildstamp.txt
 
       - name: Upload qemu images for ${{ matrix.soc }} - ${{ matrix.core }} - ${{ matrix.boot_mode }}
-        if: matrix.soc == 'demosoc'
-        uses: actions/upload-artifact@v2.2.0
+        if: ${{ !( matrix.soc == 'demosoc' && startsWith(matrix.core, 'u900')) }}
+        uses: actions/upload-artifact@v3
         with:
           name: qemu_images_${{ matrix.soc }}_${{ matrix.core }}_${{ matrix.boot_mode }}
           path: |
@@ -130,11 +142,11 @@ jobs:
             work/${{ matrix.soc }}/freeloader_qemu.elf
             work/${{ matrix.soc }}/buildstamp.txt
             work/${{ matrix.soc }}/disk.img
-            work/${{ matrix.soc }}/run_qemu.sh
+            work/${{ matrix.soc }}/*.sh
 
       - name: Upload cached packages
-        if: matrix.core == 'ux600' && matrix.boot_mode == 'sd' && matrix.soc == 'demosoc'
-        uses: actions/upload-artifact@v2.2.0
+        if: matrix.core == 'ux900fd' && matrix.boot_mode == 'sd' && matrix.soc == 'evalsoc'
+        uses: actions/upload-artifact@v3
         with:
           name: build_caching
           path: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -2,14 +2,14 @@ name: Build Linux SDK Docker Image
 
 on:
   push:
-    branches: [ dev_nuclei_next ]
+    branches: [ dev* ]
     paths-ignore:
     - README.md
     - LICENSE
     - .gitlab-ci.yml
     - .gitignore
   pull_request:
-    branches: [ dev_nuclei_next ]
+    branches: [ dev* ]
 
 jobs:
 
@@ -22,26 +22,27 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout Linux SDK Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: false
 
       - name: Build Docker Image
         run: |
           cd .github
-          docker build . -f linuxsdk.Dockerfile -t linuxsdk:latest
+          docker build . -f linuxsdk.Dockerfile -t linuxsdk:${{github.ref_name}} --build-arg BRANCH=${{github.ref_name}}
           docker images
 
       - name: Test Docker Image
+        if: ${{ startsWith(github.ref_name, 'dev_') && github.event_name != 'pull_request' }}
         run: |
-          docker run -i linuxsdk:latest bash run_qemu.sh
+          docker run -i linuxsdk:${{github.ref_name}} bash run_qemu.sh
 
       - name: Publish Docker Image
         if: ${{ github.event_name != 'pull_request' }}
         run: |
           echo "Login to docker hub"
           docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PWD }} docker.io
           DOCKER_REPO=docker.io/nucleisoftware/linuxsdk
-          docker tag linuxsdk:latest ${DOCKER_REPO}:ci
-          docker push ${DOCKER_REPO}:ci
-          echo "Docker image is pushed to https://hub.docker.com/repository/docker/nucleisoftware/linuxsdk"
+          docker tag linuxsdk:${{github.ref_name}} ${DOCKER_REPO}:${{github.ref_name}}
+          docker push ${DOCKER_REPO}:${{github.ref_name}}
+          echo "Docker image is pushed to https://hub.docker.com/repository/docker/nucleisoftware/linuxsdk, tag ${{github.ref_name}}"