From 659c2bdf4f94220526669208570df1e627662116 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 21:24:13 -0500 Subject: [PATCH 01/12] Dockerfile: reformat multiline commands I know this feels cursed, but this is the best way I've seen multiline commands formatted in a Dockerfile so we're doing that here before anything else. Signed-off-by: Randolph Sapp --- Dockerfile | 197 +++++++++++++++++++++++++++-------------------------- 1 file changed, 99 insertions(+), 98 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22bfc30..eaac73b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,51 +11,51 @@ ARG INSTALL_GCC=0 #--- END START ARG DEBIAN_FRONTEND noninteractive -RUN apt-get update && \ - apt-get install -y \ - build-essential \ - wget \ - gcc \ - ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ - flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - llvm \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ - libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ - libparmap-ocaml-dev \ - libpython3.11 \ - libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ - python3 \ - python3-ruamel.yaml \ - aria2 +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + wget \ + gcc \ + ccache \ + ncurses-dev \ + xz-utils \ + libssl-dev \ + bc \ + flex \ + libelf-dev \ + bison \ + libyaml-dev \ + python3-pip \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + libz-dev \ + libssl-dev \ + flex \ + bison \ + pkg-config \ + sqlite3 \ + libsqlite3-dev \ + llvm \ + autoconf \ + pkg-config \ + ocaml-nox \ + ocaml-findlib \ + menhir \ + libmenhir-ocaml-dev \ + ocaml-native-compilers \ + libpcre-ocaml-dev \ + libparmap-ocaml-dev \ + libpython3.11 \ + libpython3.11-dev \ + libgmp-dev \ + libmpc-dev \ + diffstat \ + yamllint \ + swig \ + python3 \ + python3-ruamel.yaml \ + aria2 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 @@ -78,62 +78,63 @@ LABEL org.opencontainers.image.source https://github.com/nmenon/kernel_patch_ver ARG DEBIAN_FRONTEND noninteractive -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - gnupg2 \ - build-essential \ - wget \ - gcc \ - ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ - flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ - libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ - libparmap-ocaml-dev \ - libpython3.11 \ - libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ - python3 \ - python3-dev \ - python3-ruamel.yaml +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg2 \ + build-essential \ + wget \ + gcc \ + ccache \ + ncurses-dev \ + xz-utils \ + libssl-dev \ + bc \ + flex \ + libelf-dev \ + bison \ + libyaml-dev \ + python3-pip \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + libz-dev \ + libssl-dev \ + flex \ + bison \ + pkg-config \ + sqlite3 \ + libsqlite3-dev \ + autoconf \ + pkg-config \ + ocaml-nox \ + ocaml-findlib \ + menhir \ + libmenhir-ocaml-dev \ + ocaml-native-compilers \ + libpcre-ocaml-dev \ + libparmap-ocaml-dev \ + libpython3.11 \ + libpython3.11-dev \ + libgmp-dev \ + libmpc-dev \ + diffstat \ + yamllint \ + swig \ + python3 \ + python3-dev \ + python3-ruamel.yaml # Add our llvm repo configs COPY llvm-config / -RUN apt-get update && apt-get install -y --no-install-recommends\ - llvm \ - clang \ - lld - -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + llvm \ + clang \ + lld + +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 From a208fe25c758ecdd03d201a857aec35bd698b262 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 21:28:38 -0500 Subject: [PATCH 02/12] Dockerfile: apt cleanup should occur with install Every command run creates a new commit. Temporary files must be cleaned in the same command they are generated in to avoid committing things. Signed-off-by: Randolph Sapp --- Dockerfile | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index eaac73b..1f41eda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,15 @@ RUN apt-get update \ swig \ python3 \ python3-ruamel.yaml \ - aria2 + aria2 \ + && echo "**** cleanup ****" \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /var/log/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 @@ -123,7 +131,15 @@ RUN apt-get update \ swig \ python3 \ python3-dev \ - python3-ruamel.yaml + python3-ruamel.yaml \ + && echo "**** cleanup ****" \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /var/log/* # Add our llvm repo configs COPY llvm-config / @@ -131,10 +147,16 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ llvm \ clang \ - lld + lld \ + && echo "**** cleanup ****" \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /var/log/* -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 From fec09311dd47cdb393524387ea2f73755239ffce Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 21:33:44 -0500 Subject: [PATCH 03/12] Dockerfile: remove duplicate sections We don't need to build 2 different images in the middle of this. There is substantial overlab between the build tools required to boostrap our tools in build-env.sh and the build tools required to build the kernel. Since we clean up after ourselves in build-env.sh there's no reason to make a temporary build container for this and then rip the entire rootfs out for the actual kpv container. In addition this prevents accidentally breaking apt with intermediary state files and clobbering the build cache. Signed-off-by: Randolph Sapp --- Dockerfile | 139 ++++++++++++----------------------------------------- 1 file changed, 31 insertions(+), 108 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f41eda..79d554d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,50 +12,48 @@ ARG INSTALL_GCC=0 ARG DEBIAN_FRONTEND noninteractive RUN apt-get update \ - && apt-get install -y \ + && apt-get install -y --no-install-recommends \ + aria2 \ + autoconf \ + bc \ + bison \ build-essential \ - wget \ - gcc \ + ca-certificates \ ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ + diffstat \ flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ + gcc \ + gettext \ + gnupg2 \ libcurl4-gnutls-dev \ + libelf-dev \ libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - llvm \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ + libgmp-dev \ libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ + libmpc-dev \ libparmap-ocaml-dev \ + libpcre-ocaml-dev \ libpython3.11 \ libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ + libsqlite3-dev \ + libssl-dev \ + libyaml-dev \ + libz-dev \ + menhir \ + ncurses-dev \ + ocaml-findlib \ + ocaml-native-compilers \ + ocaml-nox \ + pkg-config \ python3 \ + python3-dev \ + python3-pip \ python3-ruamel.yaml \ - aria2 \ + sqlite3 \ + swig \ + wget \ + xz-utils \ + yamllint \ && echo "**** cleanup ****" \ && apt-get autoremove \ && apt-get clean \ @@ -67,80 +65,12 @@ RUN apt-get update \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -COPY other-configs/ / - COPY build-env.sh /tmp RUN INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh -FROM $BASE_DISTRO - -ARG INSTALL_GCC=0 - # Publish the source repository LABEL org.opencontainers.image.source https://github.com/nmenon/kernel_patch_verify -#--- PROXY SETUP START -# COPY proxy-configuration/ / -# RUN export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils -#--- END START - - -ARG DEBIAN_FRONTEND noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - gnupg2 \ - build-essential \ - wget \ - gcc \ - ccache \ - ncurses-dev \ - xz-utils \ - libssl-dev \ - bc \ - flex \ - libelf-dev \ - bison \ - libyaml-dev \ - python3-pip \ - libcurl4-gnutls-dev \ - libexpat1-dev \ - gettext \ - libz-dev \ - libssl-dev \ - flex \ - bison \ - pkg-config \ - sqlite3 \ - libsqlite3-dev \ - autoconf \ - pkg-config \ - ocaml-nox \ - ocaml-findlib \ - menhir \ - libmenhir-ocaml-dev \ - ocaml-native-compilers \ - libpcre-ocaml-dev \ - libparmap-ocaml-dev \ - libpython3.11 \ - libpython3.11-dev \ - libgmp-dev \ - libmpc-dev \ - diffstat \ - yamllint \ - swig \ - python3 \ - python3-dev \ - python3-ruamel.yaml \ - && echo "**** cleanup ****" \ - && apt-get autoremove \ - && apt-get clean \ - && rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* \ - /var/log/* - # Add our llvm repo configs COPY llvm-config / RUN apt-get update \ @@ -157,15 +87,8 @@ RUN apt-get update \ /var/tmp/* \ /var/log/* - -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - COPY other-configs/ / -COPY --from=0 /usr/local/ /usr/local/ - -RUN ldconfig /usr/local/lib - COPY kernel_patch_verify /usr/bin/kernel_patch_verify WORKDIR /workdir From c1de18fa9d1c3454eabccb71036954ec586d1891 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 21:40:21 -0500 Subject: [PATCH 04/12] Dockerfile: use python-is-python3 instead Use python-is-python3 instead of manually tweaking alternatives entries. Signed-off-by: Randolph Sapp --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79d554d..518e5f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get update \ ocaml-native-compilers \ ocaml-nox \ pkg-config \ + python-is-python3 \ python3 \ python3-dev \ python3-pip \ @@ -63,8 +64,6 @@ RUN apt-get update \ /var/tmp/* \ /var/log/* -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - COPY build-env.sh /tmp RUN INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh From 1c62cd6ef168eaf619e158dc4985722ca454f259 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 21:43:24 -0500 Subject: [PATCH 05/12] Dockerfile: bind mount build-env.sh in run Bind mount this script instead of copying it in. Cuts down on what's committed. Signed-off-by: Randolph Sapp --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 518e5f4..38c1f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,8 +64,8 @@ RUN apt-get update \ /var/tmp/* \ /var/log/* -COPY build-env.sh /tmp -RUN INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh +RUN --mount=type=bind,source=build-env.sh,target=/tmp/build-env.sh \ + INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh # Publish the source repository LABEL org.opencontainers.image.source https://github.com/nmenon/kernel_patch_verify From 78bf5bbca0156b0f9c8bff4348e1942ccc245a73 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 22:30:06 -0500 Subject: [PATCH 06/12] build-env.sh: verbose downloads Had some issues with aria2. Now I want status during builds. Signed-off-by: Randolph Sapp --- build-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-env.sh b/build-env.sh index 5644723..a995bbd 100755 --- a/build-env.sh +++ b/build-env.sh @@ -18,7 +18,7 @@ export COCCI_TAG=1.1.1 # https://github.com/devicetree-org/dt-schema/tags export DTSCHEMA_REV=v2024.02 -ARIA_OPTS=(--timeout=180 --retry-wait=10 -m 0 -x 10 -j 10) +ARIA_OPTS=( --summary-interval=5 --timeout=180 --retry-wait=10 -m 0 -x 10 -j 10 ) download_build_install_git() { @@ -52,7 +52,7 @@ download_build_install_python_deps() clone_and_cd() { cd /tmp && - git clone --depth=1 --branch "$1" "$2" "$3" && + git clone --progress --depth=1 --branch "$1" "$2" "$3" && cd /tmp/"$3" return $? } From d67f5eef32a5b52f1ff84da7922440d136fd0584 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 22:31:56 -0500 Subject: [PATCH 07/12] Dockerfile: add automake dependency Required for coccinelle build. Signed-off-by: Randolph Sapp --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 38c1f81..3d7c71c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ aria2 \ autoconf \ + automake \ bc \ bison \ build-essential \ From 54855e97402fb5fb8fd4d2bab4f9678503cd8a19 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Tue, 19 Mar 2024 22:53:32 -0500 Subject: [PATCH 08/12] build-env.sh: purge the pip cache Signed-off-by: Randolph Sapp --- build-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-env.sh b/build-env.sh index a995bbd..dd3f1f2 100755 --- a/build-env.sh +++ b/build-env.sh @@ -47,6 +47,7 @@ download_build_install_python_deps() # scripts/spdxcheck.py dependencies python -m pip install --upgrade --break-system-packages ply gitpython yamllint rfc3987 pylibfdt python -m pip install --break-system-packages git+https://github.com/devicetree-org/dt-schema.git@$DTSCHEMA_REV + rm -rf "/tmp/.cache/" } clone_and_cd() From 8cb4f58d0020e97a89a49cfa64374e9bba61d2b8 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Thu, 11 Apr 2024 18:19:00 -0500 Subject: [PATCH 09/12] build-env.sh: install toolchains to /opt Install the toolchains to /opt instead of /usr/local. I noticed in the shellcheck PR that this was happening but wanted to reserve functional changes for later. This is the preferred location for these things anyway. Signed-off-by: Randolph Sapp --- build-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-env.sh b/build-env.sh index dd3f1f2..171b629 100755 --- a/build-env.sh +++ b/build-env.sh @@ -119,7 +119,7 @@ download_and_install_armgcc_64() cd /tmp mkdir -p /opt/cross-gcc-linux-9/ aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL" - tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" + tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" rm -f /tmp/"$FILE" } @@ -132,7 +132,7 @@ download_and_install_armgcc_32() cd /tmp mkdir -p /opt/cross-gcc-linux-9/ aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL" - tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" + tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE" rm -f /tmp/"$FILE" } From 00bbc4759987186a731be3412c84b3b1c9d172df Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Wed, 20 Mar 2024 22:28:27 -0500 Subject: [PATCH 10/12] all: use entrypoint id remapping instead Switch everything to use entrypoint remapping. Use dumb-init to clean up any potential forks and gosu to switch user and execute command. Gosu is preferred over standard su because it ignores command line arguments and handles shell commands as well as binary paths. This uses the internal user and group "developer". Signed-off-by: Randolph Sapp --- Dockerfile | 14 ++++++++++++++ kernel_patch_verify | 5 +++++ kp_common | 22 +++++++++++----------- kps | 8 ++------ kpv | 8 ++------ other-configs/init | 43 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 23 deletions(-) create mode 100755 other-configs/init diff --git a/Dockerfile b/Dockerfile index 3d7c71c..2347376 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,10 +22,12 @@ RUN apt-get update \ ca-certificates \ ccache \ diffstat \ + dumb-init \ flex \ gcc \ gettext \ gnupg2 \ + gosu \ libcurl4-gnutls-dev \ libelf-dev \ libexpat1-dev \ @@ -89,6 +91,18 @@ RUN apt-get update \ COPY other-configs/ / +RUN echo "**** create developer user and make our folders ****" \ + && useradd -u 1000 -U -d /config -s /bin/false developer \ + && usermod -G users developer \ + && mkdir /workdir && chown developer:developer /workdir \ + && mkdir /config && chown developer:developer /config + +ENTRYPOINT ["/init"] + +CMD ["/usr/bin/bash"] + +VOLUME /workdir + COPY kernel_patch_verify /usr/bin/kernel_patch_verify WORKDIR /workdir diff --git a/kernel_patch_verify b/kernel_patch_verify index 906c223..d8bf923 100755 --- a/kernel_patch_verify +++ b/kernel_patch_verify @@ -26,6 +26,11 @@ source /etc/profile shopt -s expand_aliases +# extend the path with the supplied extra directories +if [ -n "$KP_PATH" ]; then + export PATH=${KP_PATH}:${PATH} +fi + ccache=$(which ccache) # We would rather that we hit cache more often, than rebuild.. diff --git a/kp_common b/kp_common index 00a10ff..93f7650 100644 --- a/kp_common +++ b/kp_common @@ -1,6 +1,6 @@ #!/bin/bash -export USER_ID GROUP_ID PATH DOCKER_MOUNT_DIRS IMAGE_ID +export DOCKER_MOUNT_DIRS IMG_NAME KP_PATH # Check if Docker image exists @@ -38,9 +38,6 @@ else fi fi -USER_ID=$(id -u) -GROUP_ID=$(id -g) - DOCKER_MOUNT_DIRS=() DOCKER_MOUNT_DIRS+=(-v /tmp:/tmp) DOCKER_MOUNT_DIRS+=(-v /opt:/opt) @@ -57,11 +54,14 @@ if [ "$GIT_WORKTREE_COMMONDIR" != ".git" ]; then DOCKER_MOUNT_DIRS+=(-v "$GIT_WORKTREE_COMMONDIR":"$GIT_WORKTREE_COMMONDIR") fi -# Run our image to add our swuser -docker run "$IMG_NAME" /bin/bash -c "groupadd -r swuser -g $GROUP_ID && useradd -u $USER_ID -r -g swuser -d /workdir -s /sbin/nologin -c \"Docker kernel patch user\" swuser" -# Get the container ID of the last run container (above) -CONTAINER_ID=$(docker ps -lq) -# Commit the container state (returns an image_id with sha256: prefix cut off) -IMAGE_ID=$(docker commit "$CONTAINER_ID" | cut -c8-) +# list of paths to append to the PATH variable in the container +extra_paths=( + /workdir/scripts/dtc + /opt/cross-gcc-linux-13/bin + /opt/cross-gcc-linux-12/bin + /opt/cross-gcc-linux-11/bin + /opt/cross-gcc-linux-10/bin + /opt/cross-gcc-linux-9/bin +) -PATH=/workdir/scripts/dtc:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/opt/cross-gcc-linux-13/bin:/opt/cross-gcc-linux-12/bin:/opt/cross-gcc-linux-11/bin:/opt/cross-gcc-linux-10/bin:/opt/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-10/bin +KP_PATH=$(IFS=:; printf '%s' "${extra_paths[*]}") diff --git a/kps b/kps index 6a7a07b..5bb6515 100755 --- a/kps +++ b/kps @@ -4,11 +4,7 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" # If we wanted to get to bash shell: docker run --rm -ti \ - --user "$USER_ID":"$GROUP_ID" \ - -e PATH \ + -e KP_PATH -e CROSS_COMPILE -e ARCH \ "${DOCKER_MOUNT_DIRS[@]}" \ - "$IMAGE_ID" \ + "$IMG_NAME" \ bash --init-file /etc/profile - -# TODO: we can reuse this image for future runs, for now just clean up after ourselves -docker rmi "$IMAGE_ID" diff --git a/kpv b/kpv index 2de6290..74b439a 100755 --- a/kpv +++ b/kpv @@ -3,11 +3,7 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" docker run --rm -ti \ - --user "$USER_ID":"$GROUP_ID" \ - -e PATH \ + -e KP_PATH -e CROSS_COMPILE -e ARCH \ "${DOCKER_MOUNT_DIRS[@]}" \ - "$IMAGE_ID" \ + "$IMG_NAME" \ kernel_patch_verify -S /usr/local/smatch/bin/k_sm_check_script "$@" - -# TODO: we can reuse this image for future runs, for now just clean up after ourselves -docker rmi "$IMAGE_ID" diff --git a/other-configs/init b/other-configs/init new file mode 100755 index 0000000..de2c99a --- /dev/null +++ b/other-configs/init @@ -0,0 +1,43 @@ +#!/bin/bash + +get_attribs() { + local file_stats file_to_test useful_attribs + if file_to_test=$(realpath "$1") && [[ $2 =~ ^[0-9]+$ ]] ; then + useful_attribs=$(stat "$file_to_test" -t) + read -r -a file_stats <<< "${useful_attribs#"$file_to_test"}" + echo "${file_stats["$2"]}" + else + return 1 + fi +} + +get_build_uid() { + get_attribs /workdir 3 +} + +get_build_gid() { + get_attribs /workdir 4 +} + +if NEW_GID=$(get_build_gid) && NEW_UID=$(get_build_uid); then + # bypass everything if podman is remapping the id to root + if [ "${NEW_UID}" == "0" ]; then + if [ "$(id -u)" == "0" ]; then + exec dumb-init -- "$@" + else + echo "Unable to resolve ns mapping!" + fi + fi + + # change the uid and gid of developer otherwise + [ "$NEW_GID" != "$(id -g developer)" ] && groupmod -g "${NEW_GID}" developer + [ "$NEW_UID" != "$(id -u developer)" ] && usermod -u "${NEW_UID}" developer +else + echo "Not able to detect UID/GID for remapping!" +fi + +if [ "$(id -u)" == "$(id -u developer)" ]; then + exec dumb-init -- "$@" +else + exec dumb-init -- gosu developer "$@" +fi From e043a2535349fb94f6a66c1df586d5212c889a18 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Sat, 23 Mar 2024 01:44:48 -0500 Subject: [PATCH 11/12] kp*: try to preserve relative git paths Attempt to handle relative git paths, used by submodules, worktrees, and other custom git configurations. Fixes #12. Signed-off-by: Randolph Sapp --- kp_common | 36 ++++++++++++++++++++++++++---------- kps | 2 +- kpv | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/kp_common b/kp_common index 93f7650..701d464 100644 --- a/kp_common +++ b/kp_common @@ -1,6 +1,6 @@ #!/bin/bash -export DOCKER_MOUNT_DIRS IMG_NAME KP_PATH +export DOCKER_ARGS IMG_NAME KP_PATH # Check if Docker image exists @@ -38,20 +38,36 @@ else fi fi -DOCKER_MOUNT_DIRS=() -DOCKER_MOUNT_DIRS+=(-v /tmp:/tmp) -DOCKER_MOUNT_DIRS+=(-v /opt:/opt) -DOCKER_MOUNT_DIRS+=(-v "$CCACHEDIR":/ccache) -DOCKER_MOUNT_DIRS+=(-v "$(pwd)":/workdir) +DOCKER_ARGS=() +DOCKER_ARGS+=(-v /tmp:/tmp) +DOCKER_ARGS+=(-v /opt:/opt) +DOCKER_ARGS+=(-v "$CCACHEDIR":/ccache) # Check if current directory is a git directory -if ! GIT_WORKTREE_COMMONDIR=$(git rev-parse --git-common-dir); then +if ! git rev-parse --is-inside-work-tree > /dev/null; then exit 1 fi -# Mount parent directory if its a worktree -if [ "$GIT_WORKTREE_COMMONDIR" != ".git" ]; then - DOCKER_MOUNT_DIRS+=(-v "$GIT_WORKTREE_COMMONDIR":"$GIT_WORKTREE_COMMONDIR") +ROOT_GIT=$(git rev-parse --show-toplevel) +# if a submodule or worktree then we need to fetch and preserve the gitdir path +if [ -f "${ROOT_GIT}/.git" ]; then + gitdir_str=$(grep -P -o '(?<=gitdir:\s).*' "${ROOT_GIT}/.git") + relative_str=$(echo "$gitdir_str" | grep -P -o '^(../)*') + if [ -n "$relative_str" ]; then + # relative paths need to be preserved + common_path=$(realpath "$ROOT_GIT/$relative_str") + relative_pwd=$(realpath --relative-to="$common_path" "$PWD") + DOCKER_ARGS+=(-v "$common_path":"/workdir") + DOCKER_ARGS+=(-w "/workdir/${relative_pwd}") + else + # absolute paths can be passed through + gitdir_path=$(git rev-parse --git-common-dir) + DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir") + DOCKER_ARGS+=(-v "$gitdir_path":"$gitdir_path") + fi +else + # normal git directory, just toss everything in + DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir") fi # list of paths to append to the PATH variable in the container diff --git a/kps b/kps index 5bb6515..b5adaab 100755 --- a/kps +++ b/kps @@ -5,6 +5,6 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" # If we wanted to get to bash shell: docker run --rm -ti \ -e KP_PATH -e CROSS_COMPILE -e ARCH \ - "${DOCKER_MOUNT_DIRS[@]}" \ + "${DOCKER_ARGS[@]}" \ "$IMG_NAME" \ bash --init-file /etc/profile diff --git a/kpv b/kpv index 74b439a..7d936cc 100755 --- a/kpv +++ b/kpv @@ -4,6 +4,6 @@ source "$(dirname "$(readlink -f "$0")")/kp_common" docker run --rm -ti \ -e KP_PATH -e CROSS_COMPILE -e ARCH \ - "${DOCKER_MOUNT_DIRS[@]}" \ + "${DOCKER_ARGS[@]}" \ "$IMG_NAME" \ kernel_patch_verify -S /usr/local/smatch/bin/k_sm_check_script "$@" From 7e529e0ebd832c61b716a9879f88538fffb4f4d8 Mon Sep 17 00:00:00 2001 From: Randolph Sapp Date: Thu, 11 Apr 2024 18:27:15 -0500 Subject: [PATCH 12/12] all: purge proxy configurations This should not be necessary and should be removed for multiple reasons. Use the docker proxy config [1] instead. [1] https://docs.docker.com/network/proxy/ Signed-off-by: Randolph Sapp --- Dockerbuild.md | 7 -- Dockerfile | 7 -- .../etc/apt/apt.conf.d/99-corporate-proxy | 9 --- .../etc/profile.d/99-corporate-env.sh | 6 -- proxy-configuration/usr/bin/git-tunnel.sh | 73 ------------------- 5 files changed, 102 deletions(-) delete mode 100644 proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy delete mode 100755 proxy-configuration/etc/profile.d/99-corporate-env.sh delete mode 100755 proxy-configuration/usr/bin/git-tunnel.sh diff --git a/Dockerbuild.md b/Dockerbuild.md index ba8234f..f72303c 100644 --- a/Dockerbuild.md +++ b/Dockerbuild.md @@ -5,13 +5,6 @@ Quite often we need to build quite a bunch of applications to get the very latest tools and environment. Instead of hand holding every developer to get the latest environment, let's use Docker to bootstrap a common build environment. -Proxy setup -=========== - -If you are working in an environment where an http proxy is necessary, update -the files in `proxy-configuration` as required. With out this, you cannot -install the various packages needed to build the Docker image. - Versions of packages ==================== diff --git a/Dockerfile b/Dockerfile index 2347376..50b532f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,6 @@ FROM $BASE_DISTRO ARG INSTALL_GCC=0 -# In case of Proxy based environment, leave the following enabled. -# in Direct internet cases, comment out the following two lines. -#--- PROXY SETUP START -# COPY proxy-configuration/ / -# RUN export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils -#--- END START - ARG DEBIAN_FRONTEND noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy b/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy deleted file mode 100644 index ed6a852..0000000 --- a/proxy-configuration/etc/apt/apt.conf.d/99-corporate-proxy +++ /dev/null @@ -1,9 +0,0 @@ -// Use when Providing Corporate proxy environment -Acquire -{ - http { - Proxy "http://webproxy.ext.ti.com:80"; - Proxy::artifactory.itg.ti.com "DIRECT"; - } -} - diff --git a/proxy-configuration/etc/profile.d/99-corporate-env.sh b/proxy-configuration/etc/profile.d/99-corporate-env.sh deleted file mode 100755 index 1203128..0000000 --- a/proxy-configuration/etc/profile.d/99-corporate-env.sh +++ /dev/null @@ -1,6 +0,0 @@ - -export http_proxy=http://webproxy.ext.ti.com:80 -export https_proxy=http://webproxy.ext.ti.com:80 -export ftp_proxy=http://webproxy.ext.ti.com:80 -export no_proxy=.ti.com -export GIT_PROXY_COMMAND=/usr/bin/git-tunnel.sh diff --git a/proxy-configuration/usr/bin/git-tunnel.sh b/proxy-configuration/usr/bin/git-tunnel.sh deleted file mode 100755 index 6ea0c3b..0000000 --- a/proxy-configuration/usr/bin/git-tunnel.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Export GIT_PROXY_COMMAND TO THIS -# usable with settings: -# no_proxy="$no_proxy,ti.com" -# http_proxy="http://wwwgate.ti.com:80/" -# OR -# http_proxy="http://wwwgate.ti.com:80" -# OR -# http_proxy=http://proxyconfig.itg.ti.com/proxy.pac - -# Apps required -WGET=/usr/bin/wget -CORKSCREW=/usr/bin/corkscrew -SOCKET=/usr/bin/socket -# Special note on pacparse -# http://code.google.com/p/pacparser/ -# I hand statically built this for ubuntu 9.10 -# but you can clone and install this original install -# will be pactester - just rename the following to use -PACPARSE=pacparse - -use_proxy=1 - -# Set this up if you are going to use this seamlessly -# over vpn -use_at_home=0 - -if [ $use_at_home -eq 1 ]; then - k=`ifconfig|grep tun0` - if [ -n "$k" ]; then - use_proxy=1 - else - use_proxy=0 - fi -fi - -while [ -n "$no_proxy" ]; do - no_proxy_entry=${no_proxy%%,*} - if [ "$1" != "${1%$no_proxy_entry}" ]; then - #echo "no_proxy match: $1 is in $no_proxy_entry" - use_proxy=0 - break; - fi - if [ "$no_proxy_entry" = "$no_proxy" ]; then - break - fi - no_proxy=${no_proxy#*,} -done - -if [ $use_proxy = 0 ]; then - (cat | $SOCKET $1 $2) -else - if [ -z "$GIT_PROXY_HOST" -o -z "$GIT_PROXY_PORT" ]; then - if [ -z "$http_proxy" ]; then - echo "FAILED ($1 $2)!! no proxy options? yet need proxy?" 1>&2 - fi - pac=`echo "$http_proxy"|grep "pac$"` - if [ -n "$pac" ]; then - #parse pac file - LOC="http://$1" - $WGET -O /tmp/proxy.pac "$http_proxy" - http_proxy=`$PACPARSE -p /tmp/proxy.pac -u $LOC|cut -d ' ' -f2` - else - http_proxy=`echo $http_proxy|tr -d '/'|sed -e "s/http://g"` - fi - #parse normal http_proxy config - export GIT_PROXY_HOST=`echo "$http_proxy"|cut -d ':' -f1` - export GIT_PROXY_PORT=`echo "$http_proxy"|cut -d ':' -f2` - fi - exec $CORKSCREW $GIT_PROXY_HOST $GIT_PROXY_PORT "$@" -fi - -