From 7a116834ae83235b62d0a67007057ce3cdc4d39c Mon Sep 17 00:00:00 2001 From: illuminatus Date: Sat, 21 Oct 2023 17:39:53 -0700 Subject: [PATCH 1/4] Docker optional backup and restore with documentation (#1692) * Make backups optional with env ENABLE vars. Use a dedicated backup directory instead of priv. * Docker optional backup and restore documentation --- docs/docker/tips.md | 23 +++++++++++++++++++++++ files/docker/node/addons/entrypoint.sh | 22 ++++++++++++++-------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/docs/docker/tips.md b/docs/docker/tips.md index f27621049..d3fc30a5d 100644 --- a/docs/docker/tips.md +++ b/docs/docker/tips.md @@ -57,3 +57,26 @@ cardanocommunity/cardano-node:latest # Mandatory: image to run #### Un-Official Docker managment cli tool - [Lazydocker](https://github.com/jesseduffield/lazydocker) + +### Docker backups and restores + +The docker container has an optional backup and restore functionality that can be used to backup the `/opt/cardano/cnode/db` directory. To have the +backup persist longer than the countainer, the backup directory should be mounted as a volume. + +[!NOTE] +The backup and restore functionality is disabled by default. + +[!WARNING] +Make sure adequate space exists on the host as the backup will double the space consumed by the database. + +#### Creating a Backup + +When the container is started with the **ENABLE_BACKUP** environment variable set to **Y** the container will automatically create a +backup in the `/opt/cardano/cnode/backup/$NETWORK-db` directory. The backup will be created when the container is started and if the +backup directory is smaller than the db directory. + +#### Restoring from a Backup + +When the container is started with the **ENABLE_RESTORE** environment variable set to **Y** the container will automatically restore +the latest backup from the `/opt/cardano/cnode/backup/$NETWORK-db` directory. The database will be restored when the container is started +and if the backup directory is larger than the db directory. \ No newline at end of file diff --git a/files/docker/node/addons/entrypoint.sh b/files/docker/node/addons/entrypoint.sh index f91b737d6..8247eac40 100755 --- a/files/docker/node/addons/entrypoint.sh +++ b/files/docker/node/addons/entrypoint.sh @@ -16,15 +16,21 @@ echo "NETWORK: $NETWORK $POOL_NAME $TOPOLOGY"; echo "NODE: $HOSTNAME - Port:$CNODE_PORT - $POOL_NAME"; cardano-node --version; -dbsize=$(du -s ${CNODE_HOME}/db | awk '{print $1}') -bksizedb=$(du -s $CNODE_HOME/priv/$NETWORK-db 2>/dev/null | awk '{print $1}') +if [[ "${ENABLE_BACKUP}" == "Y" ]] || [[ "${ENABLE_RESTORE}" == "Y" ]]; then + [[ ! -d "${CNODE_HOME}"/backup/$NETWORK-db ]] && mkdir -p $CNODE_HOME/backup/$NETWORK-db + dbsize=$(du -s $CNODE_HOME/db | awk '{print $1}') + bksizedb=$(du -s $CNODE_HOME/backup/$NETWORK-db 2>/dev/null | awk '{print $1}') + if [[ "${ENABLE_RESTORE}" == "Y" ]] && [[ "$dbsize" -lt "$bksizedb" ]]; then + echo "Backup Started" + cp -rf "${CNODE_HOME}"/backup/"${NETWORK}"-db/* "${CNODE_HOME}"/db 2>/dev/null + echo "Backup Finished" + fi -if [[ "$dbsize" -lt "$bksizedb" ]]; then -cp -rf $CNODE_HOME/priv/$NETWORK-db/* ${CNODE_HOME}/db 2>/dev/null -fi - -if [[ "$dbsize" -gt "$bksizedb" ]]; then -cp -rf $CNODE_HOME/db/* $CNODE_HOME/priv/$NETWORK-db/ 2>/dev/null + if [[ "${ENABLE_BACKUP}" == "Y" ]] && [[ "$dbsize" -gt "$bksizedb" ]]; then + echo "Restore Started" + cp -rf "${CNODE_HOME}"/db/* "${CNODE_HOME}"/backup/"${NETWORK}"-db/ 2>/dev/null + echo "Restore Finished" + fi fi # Customisation From c27b8eca5c30c287d3805a105a8db75cd7d851d1 Mon Sep 17 00:00:00 2001 From: illuminatus Date: Sun, 22 Oct 2023 13:56:12 -0700 Subject: [PATCH 2/4] Update premerge.yml (#1694) * Change to Maximize build space action. * Remove Haskell --- .github/workflows/premerge.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/premerge.yml b/.github/workflows/premerge.yml index 06328eee8..a7329beb5 100644 --- a/.github/workflows/premerge.yml +++ b/.github/workflows/premerge.yml @@ -23,21 +23,14 @@ jobs: BRANCH: ${{ github.event.inputs.branch || '' }} if: github.event.pull_request.draft == false steps: - - name: Provide additional free space - run: | - # Workaround to provide additional free space for builds. - # https://github.com/actions/virtual-environments/issues/2840 - sudo apt-get update -y - sudo apt-get remove -y '^dotnet-.*' - sudo apt-get remove -y 'php.*' - sudo apt-get remove -y azure-cli google-cloud-sdk google-chrome-stable firefox powershell mono-devel - sudo apt-get autoremove -y - sudo apt-get clean - sudo rm -rf "/usr/share/dotnet" - sudo rm -rf "/usr/local/lib/android" - sudo rm -rf "/opt/ghc" - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: ' true' - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: From 10b08d98f7ebbb41f48e22793a77aa38ce6d4141 Mon Sep 17 00:00:00 2001 From: illuminatus Date: Sun, 22 Oct 2023 16:04:38 -0700 Subject: [PATCH 3/4] Slim container builds (#1676) * Reorder steps to reduce container size * Ensure ownership and read/execute after final ADD * move all apt installs to earlier step/layer * split the os level dependencies from the static binaries * Fix Typo * restore scripts to end of dockerfile --------- Co-authored-by: RdLrT <3169068+rdlrt@users.noreply.github.com> --- files/docker/node/dockerfile_bin | 64 +++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/files/docker/node/dockerfile_bin b/files/docker/node/dockerfile_bin index ef28aa4a4..b6aa4104e 100644 --- a/files/docker/node/dockerfile_bin +++ b/files/docker/node/dockerfile_bin @@ -20,30 +20,60 @@ ENV \ PATH=/opt/cardano/cnode/scripts:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/home/guild/.local/bin \ GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt +RUN apt-get update && apt-get install --no-install-recommends -y locales apt-utils sudo \ + && apt install -y curl wget gnupg git udev \ + && apt-get -y purge \ + && apt-get -y clean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \ + && locale-gen \ + && echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc \ + && echo "export LANG=en_US.UTF-8" >> ~/.bashrc \ + && echo "export LANGUAGE=en_US.UTF-8" >> ~/.bashrc \ + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +RUN adduser --disabled-password --gecos '' guild \ + && adduser guild sudo \ + && mkdir -pv /home/guild/.local/ /home/guild/.scripts/ + + RUN set -x && apt update \ + && apt-get update \ && mkdir -p /root/.local/bin \ - && apt install -y curl wget gnupg apt-utils git udev \ && wget https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/master/scripts/cnode-helper-scripts/guild-deploy.sh \ && export SUDO='N' \ && export UPDATE_CHECK='N' \ && export SKIP_DBSYNC_DOWNLOAD='Y' \ - && chmod +x ./guild-deploy.sh && ./guild-deploy.sh -b master -s pdcowx \ + && chmod +x ./guild-deploy.sh && ./guild-deploy.sh -b master -s p \ && ls /opt/ \ && mkdir -p $CNODE_HOME/priv/files \ - && apt-get update && apt-get install --no-install-recommends -y locales apt-utils \ - && sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \ - && locale-gen \ - && echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc \ - && echo "export LANG=en_US.UTF-8" >> ~/.bashrc \ - && echo "export LANGUAGE=en_US.UTF-8" >> ~/.bashr \ - && apt-get install -y procps libcap2 libselinux1 libc6 libsodium-dev ncurses-bin iproute2 curl wget apt-utils xz-utils netbase sudo coreutils dnsutils net-tools procps tcptraceroute bc usbip sqlite3 python3 tmux jq ncurses-base libtool autoconf git gnupg tcptraceroute util-linux less openssl bsdmainutils dialog vim \ - && apt-get -y remove libpq-dev build-essential pkg-config libffi-dev libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev make g++ && apt-get -y purge && apt-get -y clean && apt-get -y autoremove && rm -rf /var/lib/apt/lists/* \ + && apt-get -y remove libpq-dev build-essential pkg-config libffi-dev libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev make g++ \ + && apt-get -y purge \ + && apt-get -y clean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* + + +RUN set -x && export SUDO='N' \ + && export UPDATE_CHECK='N' \ + && export SKIP_DBSYNC_DOWNLOAD='Y' \ + && ./guild-deploy.sh -b master -s dcowx \ && cd /usr/bin \ && wget http://www.vdberg.org/~richard/tcpping \ && chmod 755 tcpping \ - && adduser --disabled-password --gecos '' guild \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ - && adduser guild sudo + && chown -R guild:guild $CNODE_HOME/* \ + && mv /root/.local/bin /home/guild/.local/ \ + && chown -R guild:guild /home/guild/.* \ + && chmod a+x /home/guild/.scripts/*.sh /opt/cardano/cnode/scripts/*.sh + +# Add final tools in a separate layer to shrink the largest layer +RUN apt-get update \ + && apt-get install -y procps libcap2 libselinux1 libc6 libsodium-dev ncurses-bin iproute2 xz-utils netbase coreutils dnsutils net-tools procps tcptraceroute bc usbip sqlite3 python3 tmux jq ncurses-base libtool autoconf tcptraceroute util-linux less openssl bsdmainutils dialog vim \ + && apt-get -y purge \ + && apt-get -y clean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* USER guild WORKDIR /home/guild @@ -66,13 +96,11 @@ ADD https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/master/files/ ADD https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/master/scripts/cnode-helper-scripts/guild-deploy.sh /opt/cardano/cnode/scripts/ ADD https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/master/files/docker/node/addons/entrypoint.sh ./ -RUN sudo chown -R guild:guild $CNODE_HOME/* \ - && mkdir /home/guild/.local/ \ - && sudo mv /root/.local/bin /home/guild/.local/ \ - && sudo chown -R guild:guild /home/guild/.* \ - && sudo chmod a+x /home/guild/.scripts/*.sh /opt/cardano/cnode/scripts/*.sh /home/guild/entrypoint.sh +RUN sudo chmod a+rx /home/guild/.scripts/*.sh /opt/cardano/cnode/scripts/*.sh /home/guild/entrypoint.sh \ + && sudo chown -R guild:guild /home/guild/.* $CNODE_HOME/* HEALTHCHECK --start-period=5m --interval=5m --timeout=100s CMD /home/guild/.scripts/healthcheck.sh ENTRYPOINT ["./entrypoint.sh"] + From a1c37b615f5042c6596122a93f2a2a57e831e754 Mon Sep 17 00:00:00 2001 From: illuminatus Date: Mon, 23 Oct 2023 06:44:09 -0700 Subject: [PATCH 4/4] Premerge space adjustment, and missing quotes. (#1695) split space between system and build path --- .github/workflows/premerge.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/premerge.yml b/.github/workflows/premerge.yml index a7329beb5..6aff5f2f0 100644 --- a/.github/workflows/premerge.yml +++ b/.github/workflows/premerge.yml @@ -26,11 +26,18 @@ jobs: - name: Maximize build space uses: easimon/maximize-build-space@master with: - root-reserve-mb: 512 + root-reserve-mb: 30720 swap-size-mb: 1024 remove-dotnet: 'true' remove-android: 'true' remove-haskell: ' true' + remove-codeql: 'true' + - name: Provide additional free space + run: | + # Workaround to provide additional free space for builds + # https://github.com/actions/virtual-environments/issues/2840 + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" + df -h - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -41,7 +48,7 @@ jobs: - name: Define BRANCH, COMMIT and G_ACCOUNT in environment run: | echo "G_ACCOUNT=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV - if [[ -z ${{ env.BRANCH }} ]]; then + if [[ -z "${{ env.BRANCH }}" ]]; then echo "BRANCH=${GITHUB_HEAD_REF}" >> $GITHUB_ENV echo "COMMIT=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV else