From 5f3764fb43fb9869f09d19878f5113e5ee01cb10 Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Wed, 30 Aug 2023 20:06:42 -0700 Subject: [PATCH 1/6] ci(docker-build): update/clean up layer scripts --- .../docker/layer-00.00-base-dependencies.sh | 3 + .github/docker/layer-00.10-base-daggerio.sh | 3 + .github/docker/layer-02.20-exercism-nodejs.sh | 27 +++--- .../layer-10.00-exercism-gcc_clang_llvm.sh | 83 +++++++++--------- .github/docker/layer-15.00-exercism-rust.sh | 87 ++++++++++--------- 5 files changed, 109 insertions(+), 94 deletions(-) diff --git a/.github/docker/layer-00.00-base-dependencies.sh b/.github/docker/layer-00.00-base-dependencies.sh index 7e0789e6..5850014f 100755 --- a/.github/docker/layer-00.00-base-dependencies.sh +++ b/.github/docker/layer-00.00-base-dependencies.sh @@ -1,4 +1,7 @@ #!/bin/sh +# +# .github/docker/layer-00.00-base-dependencies.sh +# # this path from for the container # shellcheck disable=SC1091 diff --git a/.github/docker/layer-00.10-base-daggerio.sh b/.github/docker/layer-00.10-base-daggerio.sh index 2f617481..b1733952 100755 --- a/.github/docker/layer-00.10-base-daggerio.sh +++ b/.github/docker/layer-00.10-base-daggerio.sh @@ -1,4 +1,7 @@ #!/bin/bash +# +# .github/docker/layer-00.10-base-daggerio.sh +# set -o pipefail diff --git a/.github/docker/layer-02.20-exercism-nodejs.sh b/.github/docker/layer-02.20-exercism-nodejs.sh index 65272efe..f12db9fd 100755 --- a/.github/docker/layer-02.20-exercism-nodejs.sh +++ b/.github/docker/layer-02.20-exercism-nodejs.sh @@ -1,4 +1,7 @@ #!/bin/bash +# +# .github/docker/layer-02.20-exercism-nodejs.sh +# set -o pipefail @@ -6,21 +9,21 @@ set -o pipefail # shellcheck disable=SC1091 . /.github/docker/include -layer_begin "$@" +main() { + layer_begin "$@" -declare -a PACKAGES -PACKAGES=( - nodejs - npm -) + declare -a PACKAGES + PACKAGES=( + nodejs + npm + ) -declare -a MODULES -MODULES=( - json2yaml - yarn -) + declare -a MODULES + MODULES=( + json2yaml + yarn + ) -main() { apt update echo apt install -y "${PACKAGES[@]}" diff --git a/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh b/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh index be7f8fb2..24fb13b8 100755 --- a/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh +++ b/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh @@ -1,4 +1,7 @@ #!/bin/bash +# +# .github/docker/layer-10.00-exercism-gcc_clang_llvm.sh +# set -o pipefail @@ -8,48 +11,48 @@ set -o pipefail # shellcheck disable=SC1091 . /.github/docker/include -layer_begin "$@" +main() { + layer_begin "$@" -declare -a PACKAGES -PACKAGES=( - build-essential - ccls - clangd-16 - clang-16 - clang-16-doc - clang-format-16 - clang-tidy-16 - clang-tools-16 - cmake - gcc - gcovr - gettext - gnu-standards - g++ - lcov - libboost-all-dev - libclang1-16 - libclang-16-dev - libclang-common-16-dev - libclang-rt-16-dev - libc++abi-16-dev - libc++-16-dev - libfuzzer-16-dev - libllvm16 - libllvm-16-ocaml-dev - libssl-dev - libunwind-16-dev - lldb-16 - lld-16 - llvm-16 - llvm-16-dev - llvm-16-runtime - make - pkg-config - python3-clang-16 -) + declare -a PACKAGES + PACKAGES=( + build-essential + ccls + clangd-16 + clang-16 + clang-16-doc + clang-format-16 + clang-tidy-16 + clang-tools-16 + cmake + gcc + gcovr + gettext + gnu-standards + g++ + lcov + libboost-all-dev + libclang1-16 + libclang-16-dev + libclang-common-16-dev + libclang-rt-16-dev + libc++abi-16-dev + libc++-16-dev + libfuzzer-16-dev + libllvm16 + libllvm-16-ocaml-dev + libssl-dev + libunwind-16-dev + lldb-16 + lld-16 + llvm-16 + llvm-16-dev + llvm-16-runtime + make + pkg-config + python3-clang-16 + ) -main() { # wget -q https://apt.llvm.org/llvm.sh # chmod -v +x llvm.sh # ./llvm.sh 16 all diff --git a/.github/docker/layer-15.00-exercism-rust.sh b/.github/docker/layer-15.00-exercism-rust.sh index f494b89a..54e65917 100755 --- a/.github/docker/layer-15.00-exercism-rust.sh +++ b/.github/docker/layer-15.00-exercism-rust.sh @@ -1,4 +1,7 @@ #!/bin/bash +# +# .github/docker/layer-15.00-exercism-rust.sh +# set -o pipefail @@ -8,49 +11,49 @@ set -o pipefail # shellcheck disable=SC1091 . /.github/docker/include -layer_begin "$@" - -declare -a PACKAGES -PACKAGES=( - clang-16 - clang-tidy-16 - clang-tools-16 - cmake - g++ - gcovr - lcov - libllvm-16-ocaml-dev - libllvm16 - libssl-dev - lld-16 - llvm-16 - llvm-16-dev - llvm-16-doc - llvm-16-examples - llvm-16-runtime -) - -declare -a CRATES -CRATES=( - cargo-audit - cargo-edit - cargo-fix - cargo-fuzz - cargo-kcov - cargo-llvm-cov - cargo-tarpaulin - grcov - zellij -) - -declare -a COMPONENTS -COMPONENTS=( - clippy - llvm-tools-x86_64-unknown-linux-gnu - rustfmt -) - main() { + layer_begin "$@" + + declare -a PACKAGES + PACKAGES=( + clang-16 + clang-tidy-16 + clang-tools-16 + cmake + g++ + gcovr + lcov + libllvm-16-ocaml-dev + libllvm16 + libssl-dev + lld-16 + llvm-16 + llvm-16-dev + llvm-16-doc + llvm-16-examples + llvm-16-runtime + ) + + declare -a CRATES + CRATES=( + cargo-audit + cargo-edit + cargo-fix + cargo-fuzz + cargo-kcov + cargo-llvm-cov + cargo-tarpaulin + grcov + zellij + ) + + declare -a COMPONENTS + COMPONENTS=( + clippy + llvm-tools-x86_64-unknown-linux-gnu + rustfmt + ) + echo Running: apt install -y "${PACKAGES[@]}" time apt install -y "${PACKAGES[@]}" || exit printf "\n" From 33552ace2408b06477529789836137199aab525f Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Wed, 30 Aug 2023 20:09:16 -0700 Subject: [PATCH 2/6] ci(docker-build): add image info to job summary --- .../docker/layer-00.00-base-dependencies.sh | 5 ++ .github/docker/layer-99.00-summary.sh | 59 +++++++++++++++++++ Dockerfile | 2 + 3 files changed, 66 insertions(+) create mode 100755 .github/docker/layer-99.00-summary.sh diff --git a/.github/docker/layer-00.00-base-dependencies.sh b/.github/docker/layer-00.00-base-dependencies.sh index 5850014f..9395a78c 100755 --- a/.github/docker/layer-00.00-base-dependencies.sh +++ b/.github/docker/layer-00.00-base-dependencies.sh @@ -12,6 +12,11 @@ main() { PACKAGES="apt-utils bash coreutils curl jq moreutils sudo tmux util-linux vim-nox" + printf "Collecting apt installed packages:\n" + echo Running: apt list --installed \> /root/apt-pkgs-start.txt + apt list --installed >/root/apt-pkgs-start.txt + printf "\n" + echo apt install -y "${PACKAGES}" # shellcheck disable=SC2086 apt install -y ${PACKAGES} || exit diff --git a/.github/docker/layer-99.00-summary.sh b/.github/docker/layer-99.00-summary.sh new file mode 100755 index 00000000..b918d52f --- /dev/null +++ b/.github/docker/layer-99.00-summary.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# .github/docker/layer-99.00-summary.sh +# + +set -o pipefail + +# this path from for the container +# shellcheck disable=SC1091 +. /.github/docker/include + +main() { + layer_begin "$@" + + printf "OS Info:\n" + tail -n 1000 /etc/os-release /etc/debian_version | paste /dev/null - + printf "\n" + + printf "Collecting apt installed packages:\n" + echo Running: apt list --installed \> /root/apt-pkgs-end.txt + apt list --installed >/root/apt-pkgs-end.txt + printf "\n" + + printf "Show apt package diff:\n" + echo Running: diff -uNr /root/apt-pkgs-{start,end}.txt + diff -uNr /root/apt-pkgs-{start,end}.txt + printf "\n" + + # shellcheck disable=SC1090 + source ~/.cargo/env + + printf "Show cargo packages:\n" + echo Running cargo install --list + cargo install --list | paste /dev/null - + printf "\n" + + #printf "Show golang packages:\n" + #echo Running ls "$(go env GOPATH)/bin" + ## shellcheck disable=SC2012 + #ls "$(go env GOPATH)/bin/" | paste /dev/null - + #printf "\n" + + printf "Show npm packages:\n" + echo Running npm list --global + npm list --global | paste /dev/null - + printf "\n" + + layer_end "$@" +} + +main "${@}" |& tee /root/layer-15.00-exercism-rust.log + +if [[ -n ${GITHUB_STEP_SUMMARY} ]]; then + { + printf "\`\`\`text\n" + cat /root/layer-15.00-exercism-rust.log + printf "\`\`\`\n" + } >>"${GITHUB_STEP_SUMMARY}" +fi diff --git a/Dockerfile b/Dockerfile index 87943f22..48b6b239 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ RUN .github/docker/layer-10.00-exercism-gcc_clang_llvm.sh ci-generic-debian RUN .github/docker/layer-15.00-exercism-rust.sh ci-generic-debian +RUN .github/docker/layer-99.00-summary.sh ci-generic-debian + # Copies your code file from your action repository to the filesystem path `/` of the container COPY .github/docker/entrypoint.sh /entrypoint.sh From 149542822e7756b5802dfc094bc7bde5a2fef718 Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Wed, 30 Aug 2023 21:41:06 -0700 Subject: [PATCH 3/6] ci(docker-build): fix entry point to the image can be used locally --- .github/docker/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/docker/entrypoint.sh b/.github/docker/entrypoint.sh index 55187887..35a13149 100755 --- a/.github/docker/entrypoint.sh +++ b/.github/docker/entrypoint.sh @@ -6,3 +6,5 @@ printf "%d args:" "$#" printf " '%s'" "$@" printf "\n" + +"$@" From ac8fb16d8ff8307a11173ac39b5c8b769c308d06 Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Wed, 30 Aug 2023 23:01:06 -0700 Subject: [PATCH 4/6] ci(docker-build): fix layer_{begin,end} usage in layer scripts --- .github/docker/layer-00.00-base-dependencies.sh | 4 ++-- .github/docker/layer-00.10-base-daggerio.sh | 4 ++-- .github/docker/layer-02.20-exercism-nodejs.sh | 4 ++-- .github/docker/layer-10.00-exercism-gcc_clang_llvm.sh | 4 ++-- .github/docker/layer-15.00-exercism-rust.sh | 4 ++-- .github/docker/layer-99.00-summary.sh | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/docker/layer-00.00-base-dependencies.sh b/.github/docker/layer-00.00-base-dependencies.sh index 9395a78c..f11e3152 100755 --- a/.github/docker/layer-00.00-base-dependencies.sh +++ b/.github/docker/layer-00.00-base-dependencies.sh @@ -8,7 +8,7 @@ . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" PACKAGES="apt-utils bash coreutils curl jq moreutils sudo tmux util-linux vim-nox" @@ -22,7 +22,7 @@ main() { apt install -y ${PACKAGES} || exit printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" 2>&1 | tee /root/layer-00.00-base-dependencies.log diff --git a/.github/docker/layer-00.10-base-daggerio.sh b/.github/docker/layer-00.10-base-daggerio.sh index b1733952..df48fd82 100755 --- a/.github/docker/layer-00.10-base-daggerio.sh +++ b/.github/docker/layer-00.10-base-daggerio.sh @@ -10,13 +10,13 @@ set -o pipefail . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" echo curl -sSfL https://releases.dagger.io/dagger/install.sh \| sh curl -sSfL https://releases.dagger.io/dagger/install.sh | sh || exit printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" |& tee /root/layer-00.10-base-daggerio.log diff --git a/.github/docker/layer-02.20-exercism-nodejs.sh b/.github/docker/layer-02.20-exercism-nodejs.sh index f12db9fd..adfa8789 100755 --- a/.github/docker/layer-02.20-exercism-nodejs.sh +++ b/.github/docker/layer-02.20-exercism-nodejs.sh @@ -10,7 +10,7 @@ set -o pipefail . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" declare -a PACKAGES PACKAGES=( @@ -34,7 +34,7 @@ main() { npm install --global "${MODULES[@]}" || exit printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" |& tee /root/layer-02.20-exercism-nodejs.log diff --git a/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh b/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh index 24fb13b8..2d731675 100755 --- a/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh +++ b/.github/docker/layer-10.00-exercism-gcc_clang_llvm.sh @@ -12,7 +12,7 @@ set -o pipefail . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" declare -a PACKAGES PACKAGES=( @@ -79,7 +79,7 @@ EOF apt install -y "${PACKAGES[@]}" || exit printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" |& tee /root/layer-10.00-exercism-gcc_clang_llvm.log diff --git a/.github/docker/layer-15.00-exercism-rust.sh b/.github/docker/layer-15.00-exercism-rust.sh index 54e65917..1e3064fb 100755 --- a/.github/docker/layer-15.00-exercism-rust.sh +++ b/.github/docker/layer-15.00-exercism-rust.sh @@ -12,7 +12,7 @@ set -o pipefail . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" declare -a PACKAGES PACKAGES=( @@ -123,7 +123,7 @@ main() { cargo cache --info printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" |& tee /root/layer-15.00-exercism-rust.log diff --git a/.github/docker/layer-99.00-summary.sh b/.github/docker/layer-99.00-summary.sh index b918d52f..95a049b1 100755 --- a/.github/docker/layer-99.00-summary.sh +++ b/.github/docker/layer-99.00-summary.sh @@ -10,7 +10,7 @@ set -o pipefail . /.github/docker/include main() { - layer_begin "$@" + layer_begin "${0}" "$@" printf "OS Info:\n" tail -n 1000 /etc/os-release /etc/debian_version | paste /dev/null - @@ -45,7 +45,7 @@ main() { npm list --global | paste /dev/null - printf "\n" - layer_end "$@" + layer_end "${0}" "$@" } main "${@}" |& tee /root/layer-15.00-exercism-rust.log From 252e15099ce71e5ccc0dfdcdf560641fde778dc4 Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Wed, 30 Aug 2023 23:03:53 -0700 Subject: [PATCH 5/6] ci(docker-build): add more image clean up steps to layers --- .github/docker/include | 20 +++++++++++++------ .../docker/layer-00.00-base-dependencies.sh | 13 ++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/docker/include b/.github/docker/include index b13c31f3..aae0f40f 100644 --- a/.github/docker/include +++ b/.github/docker/include @@ -14,15 +14,15 @@ layer_begin() { cat /etc/ci-image-info printf "\n" - echo apt update + echo Running: apt update apt update || exit printf "\n" - echo apt list --upgradable + echo Running: apt list --upgradable apt list --upgradable || exit printf "\n" - echo apt upgrade -y + echo Running: apt upgrade -y apt upgrade -y || exit printf "\n" @@ -32,11 +32,19 @@ layer_begin() { layer_end() { printf "=== %s:%s start ===\n\n" "${0}" "layer_end()" - echo apt-get clean - apt-get clean || exit + echo Running: apt autoremove -y + apt autoremove -y || exit printf "\n" - echo rm -rf /var/lib/apt/lists/* + echo Running: apt autoclean -y + apt autoclean -y || exit + printf "\n" + + echo Running: apt-get clean -y + apt-get clean -y || exit + printf "\n" + + echo Running: rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/* || exit printf "\n" diff --git a/.github/docker/layer-00.00-base-dependencies.sh b/.github/docker/layer-00.00-base-dependencies.sh index f11e3152..9376bb54 100755 --- a/.github/docker/layer-00.00-base-dependencies.sh +++ b/.github/docker/layer-00.00-base-dependencies.sh @@ -17,6 +17,19 @@ main() { apt list --installed >/root/apt-pkgs-start.txt printf "\n" + echo Running: apt remove -y "$(apt list --installed | grep -e xorg -e xserver -e qt | cut -f1 -d/)" + # shellcheck disable=SC2046 + apt remove -y $(apt list --installed | grep -e xorg -e xserver -e qt | cut -f1 -d/) || exit + printf "\n" + + echo Running: apt-get purge libx11.* libqt.* + apt-get purge libx11.* libqt.* || exit + printf "\n" + + echo Running: apt-get remove -y x11-common + apt-get remove -y x11-common || exit + printf "\n" + echo apt install -y "${PACKAGES}" # shellcheck disable=SC2086 apt install -y ${PACKAGES} || exit From d3da6e2b4c434d94e01ecf2b48c0b1ef79277c00 Mon Sep 17 00:00:00 2001 From: Victor Payno Date: Thu, 31 Aug 2023 08:51:41 -0700 Subject: [PATCH 6/6] ci(docker-build): add cargo-cache to rust layer --- .github/docker/layer-15.00-exercism-rust.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker/layer-15.00-exercism-rust.sh b/.github/docker/layer-15.00-exercism-rust.sh index 1e3064fb..963f9438 100755 --- a/.github/docker/layer-15.00-exercism-rust.sh +++ b/.github/docker/layer-15.00-exercism-rust.sh @@ -37,6 +37,7 @@ main() { declare -a CRATES CRATES=( cargo-audit + cargo-cache cargo-edit cargo-fix cargo-fuzz