refactor: general updates to the hamlet image

- Move to debian bookworm as the base image - Remove the hamlet shim engine as calling commands directly has been deprecated and no longer supported - Remove Java8 and install java11 and java17 for jenv by default - Remove docker-compose and move to the docker cli plugin for compose - Move to Python3.11 as the base python install - Move to Node16 as the base node install - Move to Java17 for the jenkins agent
hamlet-io · Sep 20, 2023 · 2d32456 · 2d32456
1 parent 3882344
commit 2d32456
Show file tree

Hide file tree

Showing 10 changed files with 101 additions and 172 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -2,54 +2,53 @@
 # Base CI image
 # This image is a general purpose CI image that also includes hamlet
 # -------------------------------------------------------------------
-FROM buildpack-deps:bullseye-scm AS base
+FROM buildpack-deps:bookworm-scm AS base
 
 USER root
 
 # Basic Package installs
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        # setup apt for different sources
-        apt-utils apt-transport-https \
-        ca-certificates \
-        gnupg2 \
-        software-properties-common \
-        git git-lfs \
-        # Standard linux tools
-        tar zip unzip \
-        less vim sudo \
-        iputils-ping \
-        # hamlet req
-        jq graphviz \
-        # Python/PyEnv Reqs
-        make build-essential libssl-dev zlib1g-dev \
-        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
-        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
-        # Builder Req
-        libpq-dev libcurl4-openssl-dev \
-        libedit-dev \
-   && rm -rf /var/lib/apt/lists/*
+    # setup apt for different sources
+    apt-utils apt-transport-https \
+    ca-certificates gnupg2 \
+    software-properties-common \
+    git git-lfs \
+    # Standard linux tools
+    tar zip unzip less vim sudo iputils-ping \
+    # hamlet req
+    jq \
+    # Python/PyEnv Reqs
+    make build-essential libssl-dev zlib1g-dev \
+    libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+    # Builder Req
+    libpq-dev libcurl4-openssl-dev \
+    libedit-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # Add docker to apt-get
-RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
-        && add-apt-repository \
-        "deb [arch=amd64] https://download.docker.com/linux/debian \
-        $(lsb_release -cs) \
-        stable"
+RUN install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg && \
+    echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        docker-ce-cli docker-compose-plugin \
-        && rm -rf /var/lib/apt/lists/
+    docker-ce-cli docker-compose-plugin docker-buildx-plugin  \
+    && rm -rf /var/lib/apt/lists/
 
 # Add various java versions via apt-get
 RUN curl -fsSL https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - \
-        && add-apt-repository \
-        "deb [arch=amd64] https://packages.adoptium.net/artifactory/deb \
-        $(lsb_release -cs) \
-        main"
+    && add-apt-repository \
+    "deb [arch=amd64] https://packages.adoptium.net/artifactory/deb \
+    $(lsb_release -cs) \
+    main"
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        temurin-8-jdk temurin-11-jdk \
-        && rm -rf /var/lib/apt/lists/
+    temurin-11-jdk temurin-17-jdk\
+    && rm -rf /var/lib/apt/lists/
 
 RUN echo "alias docker-compose='docker compose'" >> /etc/bash.bashrc
 
@@ -58,16 +57,16 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
 
 # AWSCliv2 Install
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \
-        && unzip "/tmp/awscliv2.zip" -d "/tmp/" && /tmp/aws/install && rm -rf /tmp/aws/
+    && unzip "/tmp/awscliv2.zip" -d "/tmp/" && /tmp/aws/install && rm -rf /tmp/aws/
 
 ### Scripts for user env and entrypoint
 COPY scripts/ /opt/tools/scripts/
 COPY scripts/entrypoint.sh /entrypoint.sh
 
 # Sudo support for apt-get installs
 RUN /usr/sbin/groupadd appenv \
-        && echo '#Allow everyone in appenv group to install packages' \
-        && echo '%appenv ALL = NOPASSWD : /usr/bin/apt-get' >> /etc/sudoers
+    && echo '#Allow everyone in appenv group to install packages' \
+    && echo '%appenv ALL = NOPASSWD : /usr/bin/apt-get' >> /etc/sudoers
 
 ENTRYPOINT [ "/entrypoint.sh" ]
 CMD [ "/bin/bash" ]
@@ -84,8 +83,8 @@ ARG HAMLETUID=1003
 ARG HOME=/home/hamlet
 
 RUN useradd -u 1003 --shell /bin/bash --create-home hamlet \
-        && chown hamlet:hamlet /home/hamlet && chmod u+rwx $HOME \
-        && usermod -aG appenv hamlet
+    && chown hamlet:hamlet /home/hamlet && chmod u+rwx $HOME \
+    && usermod -aG appenv hamlet
 
 USER hamlet
 WORKDIR $HOME
@@ -95,15 +94,6 @@ ENV PATH=$HOME/.pyenv/bin:$HOME/.pyenv/versions:$HOME/.pyenv/shims:$PATH
 ENV PATH=$HOME/.jenv/bin:$HOME/.jenv/versions:$HOME/.jenv/shims:$PATH
 ENV PYENV_ROOT=$HOME/.pyenv NODENV_ROOT=$HOME/.nodenv JENV_ROOT=$HOME/.jenv
 
-ENV GENERATION_PLUGIN_DIRS="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-aws;$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-azure" \
-        GENERATION_WRAPPER_LOCAL_JAVA="false" \
-        GENERATION_WRAPPER_SCRIPT_FILE="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-wrapper/freemarker-wrapper-Linux/bin/freemarker-wrapper" \
-        GENERATION_WRAPPER_JAR_FILE="" \
-        GENERATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash" \
-        GENERATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/cli" \
-        AUTOMATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation/jenkins/aws" \
-        AUTOMATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation"
-
 RUN echo 'export PS1='\''\033[0;32m\]\[\033[0m\033[0;32m\]\u\[\033[0;36m\] @ \w\[\033[0;32m\]\n$(git branch 2>/dev/null | grep "^*" | cut -d " " -f2)\[\033[0;32m\]└─\[\033[0m\033[0;32m\] \$\[\033[0m\033[0;32m\]\[\033[0m\] '\''' >> /home/hamlet/.bashrc
 RUN mkdir -p ${HOME}/cmdb
 
@@ -114,7 +104,7 @@ RUN /opt/tools/scripts/setup_user_env.sh
 # Jenkins Inbound Agent
 # ----------------------
 
-FROM jenkins/inbound-agent:latest-jdk11 as jenkins-agent
+FROM jenkins/inbound-agent:latest-jdk17 as jenkins-agent
 
 USER root
 
@@ -123,48 +113,55 @@ ARG HOME=/home/jenkins
 
 # Basic Package installs
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        # setup apt for different sources
-        apt-utils apt-transport-https \
-        ca-certificates \
-        gnupg2 \
-        software-properties-common \
-        git git-lfs \
-        # Standard linux tools
-        tar zip unzip \
-        less vim sudo \
-        iputils-ping \
-        # hamlet req
-        jq graphviz \
-        # Python/PyEnv Reqs
-        make build-essential libssl-dev zlib1g-dev \
-        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
-        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
-        # Builder Req
-        libpq-dev libcurl4-openssl-dev \
-        libedit-dev \
-   && rm -rf /var/lib/apt/lists/*
+    # setup apt for different sources
+    apt-utils apt-transport-https \
+    ca-certificates \
+    gnupg2 \
+    software-properties-common \
+    git git-lfs \
+    # Standard linux tools
+    tar zip unzip \
+    less vim sudo \
+    iputils-ping \
+    # hamlet req
+    jq graphviz \
+    # Python/PyEnv Reqs
+    make build-essential libssl-dev zlib1g-dev \
+    libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+    # Builder Req
+    libpq-dev libcurl4-openssl-dev \
+    libedit-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN chsh jenkins --shell /bin/bash
 
 # Add docker to apt-get
-RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
-        && add-apt-repository \
-        "deb [arch=amd64] https://download.docker.com/linux/debian \
-        $(lsb_release -cs) \
-        stable"
+RUN install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg && \
+    echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        docker-ce-cli docker-compose-plugin \
-        && rm -rf /var/lib/apt/lists/
+    docker-ce-cli docker-compose-plugin \
+    && rm -rf /var/lib/apt/lists/
 
 # Add various java versions via apt-get
 RUN curl -fsSL https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - \
-        && add-apt-repository \
-        "deb [arch=amd64] https://packages.adoptium.net/artifactory/deb \
-        $(lsb_release -cs) \
-        main"
+    && add-apt-repository \
+    "deb [arch=amd64] https://packages.adoptium.net/artifactory/deb \
+    $(lsb_release -cs) \
+    main"
+
+# Fix for slim docker images not including the man directories
+RUN mkdir -p /usr/share/man/man1
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        temurin-8-jdk temurin-11-jdk \
-        && rm -rf /var/lib/apt/lists/
+    temurin-11-jdk temurin-17-jdk \
+    && rm -rf /var/lib/apt/lists/
 
 RUN echo "alias docker-compose='docker compose'" >> /etc/bash.bashrc
 
@@ -173,35 +170,28 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
 
 # AWSCliv2 Install
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \
-        && unzip "/tmp/awscliv2.zip" -d "/tmp/" && /tmp/aws/install && rm -rf /tmp/aws/
+    && unzip "/tmp/awscliv2.zip" -d "/tmp/" && /tmp/aws/install && rm -rf /tmp/aws/
 
 ### Scripts for user env and entrypoint
 COPY scripts/ /opt/tools/scripts/
 COPY scripts/jenkins-agent/entrypoint.sh /entrypoint.sh
 
 # Sudo support for apt-get installs
 RUN /usr/sbin/groupadd appenv \
-        && echo '#Allow everyone in appenv group to install packages' \
-        && echo '%appenv ALL = NOPASSWD : /usr/bin/apt-get' >> /etc/sudoers \
-        && usermod -aG appenv jenkins
+    && echo '#Allow everyone in appenv group to install packages' \
+    && echo '%appenv ALL = NOPASSWD : /usr/bin/apt-get' >> /etc/sudoers \
+    && usermod -aG appenv jenkins
 
 USER jenkins
 WORKDIR $HOME
 
+ENV JENKINS_AGENT_WORKDIR=${HOME} JENKINS_JAVA_BIN=/opt/java/openjdk/bin/java JENKINS_WEB_SOCKET=true
+
 ENV PATH=$HOME/.nodenv/bin:$HOME/.nodenv/versions:$HOME/.nodenv/shims:$PATH
 ENV PATH=$HOME/.pyenv/bin:$HOME/.pyenv/versions:$HOME/.pyenv/shims:$PATH
 ENV PATH=$HOME/.jenv/bin:$HOME/.jenv/versions:$HOME/.jenv/shims:$PATH
 ENV PYENV_ROOT=$HOME/.pyenv NODENV_ROOT=$HOME/.nodenv JENV_ROOT=$HOME/.jenv
 
-ENV GENERATION_PLUGIN_DIRS="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-aws;$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-azure" \
-        GENERATION_WRAPPER_LOCAL_JAVA="false" \
-        GENERATION_WRAPPER_SCRIPT_FILE="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-wrapper/freemarker-wrapper-Linux/bin/freemarker-wrapper" \
-        GENERATION_WRAPPER_JAR_FILE="" \
-        GENERATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash" \
-        GENERATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/cli" \
-        AUTOMATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation/jenkins/aws" \
-        AUTOMATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation"
-
 ## Setup the user specific tooling
 RUN /opt/tools/scripts/setup_user_env.sh
 
@@ -222,21 +212,21 @@ ARG PIPELINESUID=1000
 ARG HOME=/home/azp
 
 RUN useradd -u ${PIPELINESUID} --shell /bin/bash --create-home azp \
-        && chown azp:azp $HOME \
-        && chmod u+rwx $HOME \
-        && usermod -aG appenv azp \
-        && usermod -aG sudo azp
+    && chown azp:azp $HOME \
+    && chmod u+rwx $HOME \
+    && usermod -aG appenv azp \
+    && usermod -aG sudo azp
 
 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux
 RUN apt-get update \
-        && apt-get install -y --no-install-recommends \
-                libcurl4 \
-                libunwind8 \
-        && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y --no-install-recommends \
+    libcurl4 \
+    libunwind8 \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY scripts/azpipelines-agent/start /usr/local/bin/start
 RUN chmod +x /usr/local/bin/start \
-        && chown azp:azp /usr/local/bin/start
+    && chown azp:azp /usr/local/bin/start
 
 USER azp
 
@@ -246,15 +236,6 @@ ENV PATH=$HOME/.pyenv/bin:$HOME/.pyenv/versions:$HOME/.pyenv/shims:$PATH
 ENV PATH=$HOME/.jenv/bin:$HOME/.jenv/versions:$HOME/.jenv/shims:$PATH
 ENV PYENV_ROOT=$HOME/.pyenv NODENV_ROOT=$HOME/.nodenv JENV_ROOT=$HOME/.jenv
 
-ENV GENERATION_PLUGIN_DIRS="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-aws;$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-plugin-azure" \
-        GENERATION_WRAPPER_LOCAL_JAVA="false" \
-        GENERATION_WRAPPER_SCRIPT_FILE="$HOME/.hamlet/engine/engines/bundled_shim/shim/engine-wrapper/freemarker-wrapper-Linux/bin/freemarker-wrapper" \
-        GENERATION_WRAPPER_JAR_FILE="" \
-        GENERATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash" \
-        GENERATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/cli" \
-        AUTOMATION_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation/jenkins/aws" \
-        AUTOMATION_BASE_DIR="$HOME/.hamlet/engine/engines/bundled_shim/shim/executor-bash/automation"
-
 ## Setup the user specific tooling
 RUN /opt/tools/scripts/setup_user_env.sh
 

diff --git a/scripts/azpipelines-agent/start b/scripts/azpipelines-agent/start
@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-# Set the shim environment variables
-. /opt/tools/scripts/shim_setup.sh
-
 if [ -z "$AZP_URL" ]; then
   echo 1>&2 "error: missing AZP_URL environment variable"
   exit 1

diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh
@@ -1,7 +1,4 @@
 #!/usr/bin/env bash
 set -e
 
-# Set the shim environment variables
-. /opt/tools/scripts/shim_setup.sh
-
 exec "$@"
diff --git a/scripts/jenkins-agent/entrypoint.sh b/scripts/jenkins-agent/entrypoint.sh
@@ -36,9 +36,6 @@
 #                              the agent skips connecting to an HTTP(S) port for connection info.
 # * JENKINS_PROTOCOLS:         Specify the remoting protocols to attempt when instanceIdentity is provided.
 
-# Set the shim environment variables
-. /opt/tools/scripts/shim_setup.sh
-
 # Allows for running extra startup commands
 if [ -n "${STARTUP_COMMANDS:-}" ]; then
 	echo "Running extra startup commands...."

diff --git a/scripts/shim_setup.sh b/scripts/shim_setup.sh
diff --git a/scripts/user_env/0_jenv.sh b/scripts/user_env/0_jenv.sh
@@ -7,7 +7,8 @@ git clone --depth 1 https://github.com/jenv/jenv.git "${JENV_ROOT}"
 
 # pyenv install
 eval "$(jenv init -)"
-jenv add /usr/lib/jvm/temurin-8-jdk-amd64/
 jenv add /usr/lib/jvm/temurin-11-jdk-amd64/
+jenv add /usr/lib/jvm/temurin-17-jdk-amd64/
 
 jenv global system
+jenv enable-plugin maven
diff --git a/scripts/user_env/0_nodenv.sh b/scripts/user_env/0_nodenv.sh
@@ -17,7 +17,7 @@ git clone --depth 1 https://github.com/nodenv/nodenv-default-packages.git "${NOD
 echo "yarn" >> "${NODENV_ROOT}/default-packages"
 
 # nodenv install
-NODE_VERSION=14.19.0
+NODE_VERSION="16.20.2"
 eval "$(nodenv init -)"
 nodenv install "${NODE_VERSION}"
 nodenv global "${NODE_VERSION}"