Merge pull request #1608 from CARV-ICS-FORTH/ci-updates

Update build workflow
riscv-collab · Nov 21, 2024 · 20f6153 · 20f6153
2 parents a4b355b + 404de92
commit 20f6153
Show file tree

Hide file tree

Showing 6 changed files with 234 additions and 77 deletions.
diff --git a/.github/cleanup-rootfs.sh b/.github/cleanup-rootfs.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# There is no need to wait for this to finish, at least the rm part can be done
+# while the process moves forward (for apt we'll need to wait for it to finish
+# before we install dependencies later on, but it'll only give us a 1-3GBs so
+# we can skip it.
+WAIT=0
+RMONLY=1
+
+PACKAGES=(
+	"firefox"
+	"google-chrome-stable"
+	"microsoft-edge-stable"
+	"php-pear"
+	"ruby-full"
+	"^aspnetcore-.*"
+	"^dotnet-.*"
+	"powershell*"
+)
+
+PATHS=(
+	"/opt/hostedtoolcache"
+	"/usr/local/.ghcup/"
+	"/usr/share/swift"
+	"/usr/local/lib/android"
+	"/usr/local/share/edge_driver"
+	"/usr/local/share/gecko_driver"
+	"/usr/local/share/chromedriver-linux64"
+	"/usr/local/share/chromium"
+	"/home/linuxbrew"
+	"/usr/local/share/vcpkg"
+	"/usr/share/kotlinc"
+	"/usr/local/bin/minikube"
+)
+
+
+function cleanup_packages()
+{
+	if [[ ${RMONLY} == 0 ]]; then
+		apt-get purge -y "${PACKAGES[@]}"
+		apt-get autoremove --purge -y
+		apt-get clean
+	fi
+}
+
+function cleanup_paths()
+{
+	for i in "${PATHS[@]}"; do
+		rm -rf "${i}" &
+	done
+	if [[ ${WAIT} == 1 ]]; then
+		wait
+	fi
+}
+
+if [[ ${WAIT} == 1 ]]; then
+	echo "---=== Before ===---"
+	df -hT
+	cleanup_packages
+	cleanup_paths
+	echo "---=== After ===---"
+	df -hT
+else
+	cleanup_packages
+	cleanup_paths
+fi
diff --git a/.github/dedup-dir.sh b/.github/dedup-dir.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+function deduplicate_files() {
+	local DUPLICATES=()
+	local DIR=${1}
+	local IFS=$'\n'
+	local LINK_CHECK=""
+
+	readarray -t DUPLICATES < <(for i in `find ${DIR} -type f ! -empty`; do sha1sum ${i}; done | sort | uniq -w 40 --all-repeated=separate)
+
+	for ((i=1; i < ${#DUPLICATES[@]}; i++ )); do
+		if [[ ${DUPLICATES[$i]} == "" ]]; then
+			continue
+		elif [[ ${DUPLICATES[$i-1]} = "" ]]; then
+			continue
+		else
+			LINK_CHECK=$(ls -li "${DUPLICATES[$i]:42}" "${DUPLICATES[$i-1]:42}" |awk '{print $1}' | uniq | wc -l)
+			if [[ ${LINK_CHECK} != "1" ]]; then
+				ln -f "${DUPLICATES[$i-1]:42}" "${DUPLICATES[$i]:42}"
+			fi
+		fi
+	done
+
+	return 0
+}
+
+deduplicate_files ${1}
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,16 +1,67 @@
 name: Build
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
   pull_request:
     branches:
       - master
 
+env:
+  submodule_paths: |
+    binutils
+    dejagnu
+    gcc
+    gdb
+    glibc
+    llvm
+    musl
+    newlib
+    pk
+    qemu
+    spike
+    uclibc-ng
+    .git/modules
+
 jobs:
+  submodule_cache:
+    name: Initialize submodule cache
+    runs-on: ubuntu-latest
+    outputs:
+      key: ${{ steps.keygen.outputs.smcache_key }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Remove unneeded frameworks to recover disk space
+        run: sudo ./.github/cleanup-rootfs.sh
+
+      - name: Generate submodule cache key
+        id: keygen
+        run: echo "smcache_key=smcache-$(printf $(git submodule | sha1sum))" >> $GITHUB_OUTPUT
+
+      - name: Setup submodule cache
+        id: smcache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.submodule_paths }}
+          key: ${{ steps.keygen.outputs.smcache_key }}
+
+      - name: Checkout required submodules
+        if: steps.smcache.outputs.cache-hit != 'true'
+        run: git submodule update --init -j $(nproc) --depth 1 $(echo ${submodule_paths} | sed '$d' | tr '\n' ' ')
+
+      - name: Storage size optimization
+        if: steps.smcache.outputs.cache-hit != 'true'
+        run: |
+          git submodule foreach 'git maintenance run'
+
   build:
     runs-on: ${{ matrix.os }}
+    needs: [submodule_cache]
+    env:
+      smcache_key: ${{ needs.submodule_cache.outputs.key }}
     strategy:
       matrix:
         os:     [ubuntu-22.04, ubuntu-24.04]
@@ -23,47 +74,46 @@ jobs:
           - mode: uclibc
             compiler: llvm
     steps:
-      - name: Remove unneeded frameworks to recover disk space
-        run: |
-          echo "-- Before --"
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          echo "-- After --"
-          df -h
-
       - uses: actions/checkout@v4
 
+      - name: Remove unneeded frameworks to recover disk space
+        run: sudo ./.github/cleanup-rootfs.sh
+
       - name: install dependencies
         run: sudo ./.github/setup-apt.sh
 
+      - name: Load submodule cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.submodule_paths }}
+          key: ${{ env.smcache_key }}
+
       - name: build toolchain
         run: |
           TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
-          BUILD_TOOLCHAIN="./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}"
+          BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}"
           if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm
             $BUILD_TOOLCHAIN --enable-llvm
           else
             $BUILD_TOOLCHAIN
           fi
-          sudo make -j $(nproc) ${{ matrix.mode }}
+          sudo mkdir /mnt/riscv
+          sudo chown runner:runner /mnt/riscv
+          make -j $(nproc) ${{ matrix.mode }}
+
+      - name: tarball build
+        run: |
+          du -s -h /mnt/riscv
+          ./.github/dedup-dir.sh /mnt/riscv/
+          XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/
 
       - name: make report
         if: |
           matrix.os == 'ubuntu-24.04'
           && (matrix.mode == 'linux' || matrix.mode == 'newlib')
           && matrix.compiler == 'gcc'
         run: |
-          sudo make report-${{ matrix.mode }} -j $(nproc)
-
-      - name: recover space
-        run: |
-          sudo du -hs / 2> /dev/null || true
-          sudo rm -rf binutils dejagnu gcc gdb glibc llvm musl newlib pk qemu spike uclibc-ng || true
-          sudo du -hs / 2> /dev/null || true
-
-      - name: tarball build
-        run: tar czvf riscv.tar.gz -C /opt/ riscv/
+          make report-${{ matrix.mode }} -j $(nproc)
 
       - name: generate prebuilt toolchain name
         id:   toolchain-name-generator
@@ -84,35 +134,40 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }}
-          path: riscv.tar.gz
+          path: riscv.tar.xz
 
   test-sim:
     runs-on: ${{ matrix.os }}
+    needs: [submodule_cache]
+    env:
+      smcache_key: ${{ needs.submodule_cache.outputs.key }}
     strategy:
       matrix:
         os:     [ubuntu-24.04]
         mode:   [newlib]
         target: [rv64gc-lp64d]
         sim:    [spike]
     steps:
-      - name: Remove unneeded frameworks to recover disk space
-        run: |
-          echo "-- Before --"
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          echo "-- After --"
-          df -h
-
       - uses: actions/checkout@v4
 
+      - name: Remove unneeded frameworks to recover disk space
+        run: sudo ./.github/cleanup-rootfs.sh
+
       - name: install dependencies
         run: sudo ./.github/setup-apt.sh
 
+      - name: Load submodule cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.submodule_paths }}
+          key: ${{ env.smcache_key }}
+
       - name: build toolchain
         run: |
           TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
-          ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }}
+          ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }}
+          sudo mkdir /mnt/riscv
+          sudo chown runner:runner /mnt/riscv
           make -j $(nproc) ${{ matrix.mode }}
 
       - name: make report
@@ -121,38 +176,46 @@ jobs:
   build-multilib:
     if: ${{ false }} # Disable until multilib errors are triaged
     runs-on: ${{ matrix.os }}
+    needs: [submodule_cache]
+    env:
+      smcache_key: ${{ needs.submodule_cache.outputs.key }}
     strategy:
       matrix:
         os:     [ubuntu-24.04]
         mode:   [newlib, linux]
         target: [rv64gc-lp64d]
     steps:
-      - name: Remove unneeded frameworks to recover disk space
-        run: |
-          echo "-- Before --"
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          echo "-- After --"
-          df -h
-
       - uses: actions/checkout@v4
 
+      - name: Remove unneeded frameworks to recover disk space
+        run: sudo ./.github/cleanup-rootfs.sh
+
       - name: install dependencies
         run: sudo ./.github/setup-apt.sh
 
+      - name: Load submodule cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.submodule_paths }}
+          key: ${{ env.smcache_key }}
+
       - name: build toolchain
         run: |
           TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
-          ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib
-          sudo make -j $(nproc) ${{ matrix.mode }}
+          ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib
+          sudo mkdir /mnt/riscv
+          sudo chown runner:runner /mnt/riscv
+          make -j $(nproc) ${{ matrix.mode }}
 
-      - name: make report
+      - name: tarball build
         run: |
-          sudo make report-${{ matrix.mode }} -j $(nproc)
+          du -s -h /mnt/riscv
+          ./.github/dedup-dir.sh /mnt/riscv/
+          XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/
 
-      - name: tarball build
-        run: tar czvf riscv.tar.gz -C /opt/ riscv/
+      - name: make report
+        run: |
+          make report-${{ matrix.mode }} -j $(nproc)
 
       - name: generate prebuilt toolchain name
         id:   toolchain-name-generator
@@ -173,4 +236,4 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }}
-          path: riscv.tar.gz
+          path: riscv.tar.xz