doc: How to Build Relocatable Postgres 17 for Alpine / musl

[coolaj86]

Table of Contents

• How to Install
• Goal
• How to Build
• References

How to Install

Install Dependencies

# required
sudo apk add openssl tzdata

# optional
sudo apk add llvm15

Webi (macOS, Linux, etc)

curl -sS https://webi.sh/postgres | sh

Manually

curl -L -O https://github.com/bnnanet/postgresql-releases/releases/download/REL_17_0/postgres-17.0.0-x86_64-linux.tar.gz
tar xvf ./postgres-17.0.0-x86_64-linux.tar.gz
mv ./postgres-17.0.0-x86_64-linux ~/.local/opt/postgres-17.0.0
ln -s 'postgres-17.0.0' ~/.local/opt/postgres

export PATH="$HOME/.local/opt/postgres/bin:$PATH"

Goal

Build postgres on one musl Linux box that will also run on others -- without having to install a specific version of libicu, etc.

Build PostgreSQL from Source for Alpine / musl Linux

Build Dependencies

sudo apk add alpine-sdk
sudo apk add bison flex patchelf
sudo apk add clang15 llvm15-dev openssl-dev perl readline-dev

# for bundling with postgres distributable
sudo apk add icu-data-full icu-libs icu-dev

Build

Update: This script is now maintained as https://github.com/bnnanet/pg-essentials/blob/main/pg-build-linux

#!/bin/sh
#shellcheck disable=SC2016,SC2155
set -e
set -u

g_vendor="${1:-}"
g_ver="${2:-}"
g_patch="${POSTGRES_PATCH_VERSION:-0}"
g_llvmver="${POSTGRES_LLVM_VERSION:-15}"

g_semver="${g_ver}.${g_patch}"

#g_exts="$(ls ./cortrib/ | grep -v 'README|*.mk|perl|python|ossp|start-scripts|xml2')"
g_exts="amcheck auth_delay auto_explain basebackup_to_shell basic_archive bloom btree_gin btree_gist citext cube dblink dict_int dict_xsyn earthdistance file_fdw fuzzystrmatch hstore isn lo ltree pageinspect passwordcheck pg_buffercache pg_freespacemap pg_prewarm pg_stat_statements pg_surgery pg_trgm pg_visibility pg_walinspect pgcrypto pgrowlocks pgstattuple postgres_fdw seg sslinfo tablefunc tcn test_decoding tsm_system_rows tsm_system_time unaccent"
g_exts_only="intarray intagg oid2name spi vacuumlo"

g_prof=""
g_libc="gnu"
if ldd /bin/ls | grep -q 'musl'; then
    g_libc="musl"
fi

# linux
g_platform="$(uname -s | tr '[:upper:]' '[:lower:]')" # darwin
# x86_64 or arm64
g_arch="$(uname -m)"
if test "arm64" = "${g_arch}"; then
    g_arch="aarch64"
fi
# aarch64-linux
g_target="${g_prof}${g_arch}-${g_platform}-${g_libc}"

g_libdir="/usr/lib/${g_arch}-linux-${g_libc}"
if ! test -e "${g_libdir}"; then
    g_libdir="/usr/lib"
fi

main() { (
    if test -z "${g_vendor}"; then
        echo ""
        echo "USAGE"
        echo "    pg-build-linux <vendor-name> <pg-version>"
        echo ""
        echo "EXAMPLE"
        echo "    pg-build-linux 'custom' 17.0"
        echo ""
        echo "ENVs"
        echo "    Use ENVs to set the (cosmetic) patch version and optional LLVM version"
        echo "    POSTGRES_PATCH_VERSION=0"
        echo "    POSTGRES_LLVM_VERSION=15"
        echo ""
        return 1
    fi

    echo ""
    echo "Installing build dependencies ..."
    if ! test -e ./"postgresql-${g_ver}-${g_target}"; then
        sleep 1
        if command -v apt-get > /dev/null; then
            fn_install_build_deps_apt
        elif command -v apk > /dev/null; then
            fn_install_build_deps_apk
        else
            echo "warn: unknown package manager: you moust install dependencies manually"
        fi
    fi

    echo ""
    echo ""
    echo "Downloading PostgreSQL ${g_ver} source ..."
    sleep 1
    fn_download_pg

    echo ""
    echo ""
    echo "Building 'postgres+psql' in ./postgresql-${g_ver}-${g_target}/"
    sleep 1
    fn_build_pg

    echo ""
    echo "Bundling 'postgres+psql' libs in ~/relocatable/postgres-${g_semver}-${g_target}/lib/"
    echo "     and 'psql' libs in ~/relocatable/psql-${g_ver}-${g_target}/lib/"
    sleep 1
    fn_bundle_icu
    fn_bundle_libs ~/relocatable/"postgres-${g_semver}-${g_target}"/lib/
    fn_bundle_libs ~/relocatable/"psql-${g_semver}-${g_target}"/lib/

    echo ""
    echo "Updating 'postgres+psql' linker paths and resigning ..."
    echo "     and 'psql' linker paths and resigning ..."
    sleep 1
    #fn_patch_rpath_extensions ~/relocatable/"postgres-${g_semver}-${g_target}"
    fn_patch_rpath_libs_server ~/relocatable/"postgres-${g_semver}-${g_target}"
    fn_patch_rpath_libs ~/relocatable/"postgres-${g_semver}-${g_target}"
    fn_patch_rpath_libs ~/relocatable/"psql-${g_semver}-${g_target}"

    echo ""
    echo "Creating distributable packages for 'postgres+psql' and 'psql'"
    sleep 1
    fn_package

    rm -rf ~/relocatable
    echo "Done"

    b_pad="$(echo "${g_target}" | tr '[:graph:]' ' ')"
    echo ""
    echo "To start from scratch, remove the following:"
    echo "    ./postgresql-${g_ver}.tar.gz ${b_pad}# official source"
    echo "    ./postgresql-${g_ver}-${g_target}/      # intermediate build files"
    echo "    ./postgres-${g_ver}-${g_target}.tar.gz  # distributable server+client"
    echo "    ./psql-${g_ver}-${g_target}.tar.gz      # distributable client"
    echo ""
); }

fn_install_build_deps_apt() { (
    cmd_sudo=''
    if command -v sudo > /dev/null; then
        cmd_sudo='sudo'
    fi

    $cmd_sudo apt-get update
    $cmd_sudo apt-get install -y \
        tzdata \
        wget \
        build-essential bison flex \
        binutils patchelf \
        "clang-${g_llvmver}" "llvm-${g_llvmver}-dev" \
        libicu-dev libreadline-dev zlib1g-dev \
        libssl-dev \
        liblz4-dev libzstd-dev
); }

fn_install_build_deps_apk() { (
    cmd_sudo=''
    if command -v sudo > /dev/null; then
        cmd_sudo='sudo'
    fi

    $cmd_sudo apk update
    $cmd_sudo apk add \
        openssl tzdata \
        wget \
        alpine-sdk bison flex perl \
        binutils patchelf \
        "clang${g_llvmver}" "llvm${g_llvmver}-dev" \
        icu-dev readline-dev zlib-dev \
        openssl-dev lz4-dev zstd-dev \
        icu-data-full icu-libs lz4 zlib zstd
); }

fn_download_pg() { (
    if ! test -f ./"postgresql-${g_ver}".tar.gz; then
        (
            cd /tmp/
            curl -L -O "https://ftp.postgresql.org/pub/source/v${g_ver}/postgresql-${g_ver}.tar.gz"
        )
        mv /tmp/"postgresql-${g_ver}".tar.gz .
    fi

    #rm -rf ./"postgresql-${g_ver}-${g_target}"/

    if ! test -d ./"postgresql-${g_ver}-${g_target}"/; then
        echo "Unpacking ./postgresql-${g_ver}.tar.gz ..."
        tar xzf ./"postgresql-${g_ver}".tar.gz
        mv ./"postgresql-${g_ver}"/ ./"postgresql-${g_ver}-${g_target}"/
    fi
); }

fn_build_pg() { (
    export CLANG="/usr/bin/clang-${g_llvmver}"
    export LLVM_CONFIG="/usr/bin/llvm-config-${g_llvmver}"
    if command -v apk > /dev/null; then
        export LLVM_CONFIG="/usr/bin/llvm${g_llvmver}-config"
    fi

    export ICU_CFLAGS="-I/usr/include"
    export ICU_LIBS="-L${g_libdir} -licui18n -licuuc -licudata"

    export LZ4_CFLAGS="-I/usr/include"
    export LZ4_LIBS="-L${g_libdir} -llz4"

    export ZSTD_CFLAGS="-I/usr/include"
    export ZSTD_LIBS="-L${g_libdir} -lzstd"

    b_arch="$(echo "${g_arch}" | tr _ -)"
    if test "${b_arch}" = "x86_64"; then
        b_arch="x86_64_v2"
    elif test "${b_arch}" = "aarch64"; then
        b_arch="armv8-a"
    fi
    export CFLAGS="-march=${b_arch} -mtune=generic -O2 -pipe -fstack-protector-strong -flto=auto -I/usr/include"
    export CXXFLAGS="${CFLAGS}"
    export CPPFLAGS="${CFLAGS}"
    export LDFLAGS="-L${g_libdir}"

    export LD_RUN_PATH='$ORIGIN/../lib'

    cd ./"postgresql-${g_ver}-${g_target}"/ || return 1

    # Clean (uncomment when needed)
    #make clean

    # Configure

    # turned on: llvm,lz4,ssl,zstd
    # disabled: -
    # not turned off: icu,readline,zlib,spinlocks,atomics
    # built-in: -
    # custom-location: tzdata
    # not turned on: gssapi,ldap,nls,ossp,pam,perl,python,selinux,systemd,tcl,xml,xslt
    if ! test -f ./config.status; then
        ./configure \
            --prefix="${HOME}/relocatable/pgsql-${g_semver}-${g_target}" \
            --exec-prefix="${HOME}/relocatable/pgsql-${g_semver}-${g_target}" \
            --disable-rpath \
            --with-llvm \
            --with-lz4 \
            --with-ssl=openssl \
            --with-system-tzdata=/usr/share/zoneinfo \
            --with-zstd \
            --with-extra-version=" ${g_vendor} +icu,llvm-${g_llvmver},lz4,openssl-3,readline,zlib,zstd -tzdata"
    fi

    # Build
    make -j"$(nproc --ignore=1)"
    for b_ext in $g_exts; do
        (
            echo ""
            echo ""
            echo "#### Building Extension ${b_ext} ####"
            sleep 0.3
            cd ./contrib/"${b_ext}"
            make
        )
    done
    for b_ext in $g_exts_only; do
        (
            echo ""
            echo ""
            echo "#### Building Extension ${b_ext} ####"
            sleep 0.3
            cd ./contrib/"${b_ext}"
            make
        )
    done

    # Install
    rm -rf ~/relocatable
    mkdir -p ~/relocatable

    # Server + Client
    make install
    for b_ext in $g_exts; do
        (
            echo ""
            echo ""
            echo "#### Installing Extension ${b_ext} ####"
            sleep 0.3
            cd ./contrib/"${b_ext}"
            make install
        )
    done
    for b_ext in $g_exts_only; do
        (
            echo ""
            echo ""
            echo "#### Installing Extension ${b_ext} ####"
            sleep 0.3
            cd ./contrib/"${b_ext}"
            make install
        )
    done
    mv ~/relocatable/"pgsql-${g_semver}-${g_target}" ~/relocatable/"postgres-${g_semver}-${g_target}"
    rm -rf ~/relocatable/"postgres-${g_semver}-${g_target}"/include

    # (Mostly) Client Tools
    b_client_targets="./src/bin/ ./src/include/ ./src/interfaces/"
    # this excludes './doc/' due to failing linter errors
    for b_target in ${b_client_targets}; do
        echo ""
        echo ""
        echo "#### Installing client ${b_target} ####"
        sleep 0.3
        make -C "${b_target}" install
    done
    mv ~/relocatable/"pgsql-${g_semver}-${g_target}" ~/relocatable/"psql-${g_semver}-${g_target}"
    rm -rf ~/relocatable/"psql-${g_semver}-${g_target}"/include
); }

fn_bundle_icu() { (
    cp -RPp \
        "${g_libdir}"/libicuuc.so* \
        "${g_libdir}"/libicui18n.so* \
        "${g_libdir}"/libicudata.so* \
        ~/relocatable/"postgres-${g_semver}-${g_target}"/lib/ 2> /dev/null || true
); }

fn_bundle_libs() { (
    # Everything EXCEPT:
    #   - llvm (due to size and number of items)
    #   - openssl (for security)
    #   - tzdata (due to ubiquiti)

    a_pgqsl_dir="${1}"

    cp -RPp "${g_libdir}"/liblz4.so* "${a_pgqsl_dir}" 2> /dev/null || true
    cp -RPp "${g_libdir}"/libreadline.so* "${a_pgqsl_dir}" 2> /dev/null || true

    if test -e /lib/libz.so; then
        # alpine installs libz.so to /lib/
        cp -RPp /lib/libz.so* "${a_pgqsl_dir}" 2> /dev/null || true
    else
        cp -RPp "${g_libdir}"/libz.so* "${a_pgqsl_dir}" 2> /dev/null || true
    fi

    cp -RPp "${g_libdir}"/libzstd*.so* "${a_pgqsl_dir}" 2> /dev/null || true
); }

fn_patch_rpath() { (
    patchelf --set-rpath '$ORIGIN/../lib' "${1}"
); }

fn_patch_rpath_libs_server() { (
    b_pgdir="${HOME}/relocatable/postgres-${g_semver}-${g_target}"

    fn_patch_rpath "${b_pgdir}"/lib/libicuuc.so.*.*
    fn_patch_rpath "${b_pgdir}"/lib/libicui18n.so.*.*
    fn_patch_rpath "${b_pgdir}"/lib/libicudata.so.*.*

    # for b_ext in $g_exts; do
    #     fn_patch_rpath "${b_pgdir}"/lib/"${b_ext}".so
    # done
); }

fn_patch_rpath_libs() { (
    # Each of these can be debugged to ensure they don't link to other things
    #   readelf -d ~/relocatable/"postgres-${g_semver}-${g_target}"/lib/libfoo.so.0.0
    #
    # They each still rely on slow-changing libraries, such as:
    #   libstdc++.so.6, libm.so.6, libgcc_s.so.1,
    #   libc.so.6, ld-linux-aarch64.so.1, libtinfo.so.6
    # but these are likely to exist on a standard system, and to match versions

    a_pgdir="${1}"

    fn_patch_rpath "${a_pgdir}"/lib/liblz4.so.*.*
    fn_patch_rpath "${a_pgdir}"/lib/libreadline.so.*.*
    fn_patch_rpath "${a_pgdir}"/lib/libz.so.*.*
    fn_patch_rpath "${a_pgdir}"/lib/libzstd.so.*.*
); }

fn_package() { (
    echo ""
    tar czf ./"postgres-${g_semver}-${g_target}".tar.gz \
        -C ~/relocatable/ ./"postgres-${g_semver}-${g_target}"/
    echo "    ./postgres-${g_semver}-${g_target}.tar.gz"

    tar czf ./"psql-${g_semver}-${g_target}".tar.gz \
        -C ~/relocatable/ ./"psql-${g_semver}-${g_target}"/
    echo "    ./psql-${g_semver}-${g_target}.tar.gz"
    echo ""
); }

main

[coolaj86]

closing as sticky documentation