From 0556b7468b812d86549e5ea8d4942dd549127715 Mon Sep 17 00:00:00 2001 From: endline Date: Fri, 6 Dec 2024 11:27:30 -0700 Subject: [PATCH 01/13] Port audiusd changes from various fog branches https://github.com/AudiusProject/audius-protocol/pull/10441/files#diff-cca22bcc46b804b83cf7374b01f6d6a3634c6eee755bae2d623bb2018348ed08 --- cmd/audiusd/Dockerfile | 12 +++++++++--- cmd/audiusd/entrypoint.sh | 9 ++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/audiusd/Dockerfile b/cmd/audiusd/Dockerfile index 13be1c76bdf..993352f03ae 100644 --- a/cmd/audiusd/Dockerfile +++ b/cmd/audiusd/Dockerfile @@ -1,4 +1,9 @@ -FROM debian:bookworm AS cpp-builder +FROM debian:bullseye AS cpp-builder + +RUN apt-get update && \ + apt-get install -y curl gnupg2 lsb-release && \ + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/postgresql.list RUN apt-get update && apt-get install -y \ build-essential \ @@ -19,6 +24,7 @@ RUN apt-get update && apt-get install -y \ libavformat-dev \ libavutil-dev \ libswresample-dev \ + libavresample-dev \ libsamplerate0-dev \ libtag1-dev \ libchromaprint-dev \ @@ -73,11 +79,11 @@ RUN g++ -o /bin/analyze-key /app/cpp/keyfinder.cpp \ RUN g++ -o /bin/analyze-bpm /app/cpp/bpm-analyzer.cpp \ -I/usr/include/eigen3 -I/usr/local/include/essentia -I/usr/local/include \ -L/usr/local/lib \ - -lessentia -ltag -lyaml -lfftw3 -lfftw3f -lavcodec -lavformat -lavutil -lavfilter -lsamplerate -lswresample -lpthread -lz -lchromaprint && \ + -lessentia -ltag -lyaml -lfftw3 -lfftw3f -lavcodec -lavformat -lavutil -lavfilter -lsamplerate -lavresample -lpthread -lz -lchromaprint && \ chmod +x /bin/analyze-bpm -FROM golang:1.22-bookworm AS go-builder +FROM golang:1.22-bullseye AS go-builder WORKDIR /app diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index 430f328c7b9..4085f774b5c 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -14,7 +14,10 @@ source_env_file() { [[ -z "$key" ]] && continue # only set variables that are not already defined (prioritize docker-passed env) if [ -z "${!key}" ]; then - export "$key"="$value" + # strip quotations + val="${value%\"}" + val="${val#\"}" + export "$key"="$val" fi done < "$file" else @@ -29,12 +32,12 @@ source_env_file "$OVERRIDE_ENV_FILE" POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} POSTGRES_DB=${POSTGRES_DB:-audiusd} POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/var/lib/postgresql/data} -export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audius_creator_node?sslmode=disable} +export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audiusd?sslmode=disable} export uptimeDataDir=${uptimeDataDir:-/data/bolt} export audius_core_root_dir=${audius_core_root_dir:-/data/audiusd} export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost} -if [ ! -d "$POSTGRES_DATA_DIR" ]; then +if [ ! -d "$POSTGRES_DATA_DIR" ] || [ -z "$(ls -A "$POSTGRES_DATA_DIR" 2>/dev/null)" ]; then echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." su - postgres -c "/usr/lib/postgresql/*/bin/initdb -D $POSTGRES_DATA_DIR" From 40bab829c157559ad35dd3a85fde8ad552ce15c6 Mon Sep 17 00:00:00 2001 From: endline Date: Fri, 6 Dec 2024 11:42:32 -0700 Subject: [PATCH 02/13] Make local postgres work out of the box --- cmd/audiusd/Dockerfile | 6 ++++ cmd/audiusd/entrypoint.sh | 71 ++++++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/cmd/audiusd/Dockerfile b/cmd/audiusd/Dockerfile index 993352f03ae..5557b9d11a6 100644 --- a/cmd/audiusd/Dockerfile +++ b/cmd/audiusd/Dockerfile @@ -115,6 +115,12 @@ EXPOSE 80 EXPOSE 443 EXPOSE 26656 +# Make sure the data directory exists and has correct permissions +RUN mkdir -p /var/lib/postgresql/data && \ + chown -R postgres:postgres /var/lib/postgresql/data && \ + chmod -R 700 /var/lib/postgresql/data + +VOLUME ["/var/lib/postgresql/data"] VOLUME ["/data"] ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index 4085f774b5c..a4f15361fbb 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -1,5 +1,7 @@ #!/bin/bash +# Default to prod if NETWORK is not set +NETWORK=${NETWORK:-prod} ENV_FILE="/env/${NETWORK}.env" OVERRIDE_ENV_FILE="/env/override.env" @@ -37,35 +39,56 @@ export uptimeDataDir=${uptimeDataDir:-/data/bolt} export audius_core_root_dir=${audius_core_root_dir:-/data/audiusd} export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost} -if [ ! -d "$POSTGRES_DATA_DIR" ] || [ -z "$(ls -A "$POSTGRES_DATA_DIR" 2>/dev/null)" ]; then - echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." - su - postgres -c "/usr/lib/postgresql/*/bin/initdb -D $POSTGRES_DATA_DIR" - - echo "Updating PostgreSQL configuration for password authentication..." - sed -i "s/peer/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf" - sed -i "s/md5/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf" -fi - -chown -R postgres:postgres "$POSTGRES_DATA_DIR" -chmod -R u+rwx,g-rwx,o-rwx "$POSTGRES_DATA_DIR" +setup_postgres() { + # Find PostgreSQL binaries directory + PG_BIN="/usr/lib/postgresql/15/bin" + + # Check if directory exists but is not initialized + if [ -d "$POSTGRES_DATA_DIR" ] && ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then + echo "Directory exists but is not initialized. Cleaning up..." + rm -rf "$POSTGRES_DATA_DIR"/* + fi + + # Initialize PostgreSQL if needed + if [ ! -d "$POSTGRES_DATA_DIR" ] || ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then + echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." + mkdir -p "$POSTGRES_DATA_DIR" + chown postgres:postgres "$POSTGRES_DATA_DIR" + + # Initialize the database + su - postgres -c "$PG_BIN/initdb -D $POSTGRES_DATA_DIR" + + # Configure authentication + sed -i "s/peer/trust/g; s/md5/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf" + + # Configure logging + sed -i "s|#log_destination = 'stderr'|log_destination = 'stderr'|; \ + s|#logging_collector = on|logging_collector = off|" \ + "$POSTGRES_DATA_DIR/postgresql.conf" + fi -echo "Configuring PostgreSQL to log to stderr for docker capture..." -sed -i "s|#log_destination = 'stderr'|log_destination = 'stderr'|" "$POSTGRES_DATA_DIR/postgresql.conf" -sed -i "s|#logging_collector = on|logging_collector = off|" "$POSTGRES_DATA_DIR/postgresql.conf" + # Set permissions + chown -R postgres:postgres "$POSTGRES_DATA_DIR" + chmod -R 700 "$POSTGRES_DATA_DIR" -echo "Starting PostgreSQL service..." -su - postgres -c "/usr/lib/postgresql/*/bin/pg_ctl -D $POSTGRES_DATA_DIR -o '-c config_file=$POSTGRES_DATA_DIR/postgresql.conf' start" + # Start PostgreSQL + echo "Starting PostgreSQL service..." + su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR start" -until su - postgres -c "pg_isready -q"; do - echo "Waiting for PostgreSQL to start..." - sleep 2 -done + # Wait for PostgreSQL to be ready + until su - postgres -c "$PG_BIN/pg_isready -q"; do + echo "Waiting for PostgreSQL to start..." + sleep 2 + done -echo "Setting up PostgreSQL user and database..." -su - postgres -c "psql -c \"ALTER USER postgres WITH PASSWORD '$POSTGRES_PASSWORD';\"" -su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB'\" | grep -q 1 || psql -c 'CREATE DATABASE $POSTGRES_DB;'" + # Setup database + echo "Setting up PostgreSQL user and database..." + su - postgres -c "psql -c \"ALTER USER postgres WITH PASSWORD '$POSTGRES_PASSWORD';\"" + su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB'\" | grep -q 1 || \ + psql -c \"CREATE DATABASE $POSTGRES_DB;\"" +} -su - postgres -c "/usr/lib/postgresql/*/bin/pg_ctl -D $POSTGRES_DATA_DIR -o '-c config_file=$POSTGRES_DATA_DIR/postgresql.conf' restart" +setup_postgres echo "Starting audiusd..." exec /bin/audiusd "$@" From 7be27939b1061dddb79a6a14a35b5e98fd7496ab Mon Sep 17 00:00:00 2001 From: endline Date: Fri, 6 Dec 2024 11:59:26 -0700 Subject: [PATCH 03/13] Update readme --- cmd/audiusd/Dockerfile | 10 ++++---- cmd/audiusd/README.md | 26 ++++++++++++++++++--- cmd/audiusd/entrypoint.sh | 48 +++++++++++++++++++++++---------------- cmd/audiusd/env/dev.env | 2 +- cmd/audiusd/env/prod.env | 2 +- cmd/audiusd/env/stage.env | 2 +- 6 files changed, 59 insertions(+), 31 deletions(-) diff --git a/cmd/audiusd/Dockerfile b/cmd/audiusd/Dockerfile index 5557b9d11a6..b6f2c23ea40 100644 --- a/cmd/audiusd/Dockerfile +++ b/cmd/audiusd/Dockerfile @@ -115,12 +115,12 @@ EXPOSE 80 EXPOSE 443 EXPOSE 26656 -# Make sure the data directory exists and has correct permissions -RUN mkdir -p /var/lib/postgresql/data && \ - chown -R postgres:postgres /var/lib/postgresql/data && \ - chmod -R 700 /var/lib/postgresql/data +# Set up data directory structure +RUN mkdir -p /data/postgres && \ + chown -R postgres:postgres /data/postgres && \ + chmod -R 700 /data/postgres -VOLUME ["/var/lib/postgresql/data"] +# Single volume mount point VOLUME ["/data"] ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/cmd/audiusd/README.md b/cmd/audiusd/README.md index abf63f2a639..a012de8a59b 100644 --- a/cmd/audiusd/README.md +++ b/cmd/audiusd/README.md @@ -7,7 +7,7 @@ A golang implementation of the audius protocol. Minimal example to run a node and sync it to the audius mainnet. ```bash -docker run --rm -ti -p 80:80 audius/audiusd:latest +docker run --rm -ti -p 80:80 audius/audiusd:current open http://localhost/console/overview ``` @@ -17,8 +17,8 @@ open http://localhost/console/overview To operate a [registered](https://docs.audius.org/node-operator/setup/registration/) node requires the minimal config below. ```bash -# directory for data persistence -mkdir ~/.audiusd +# directory for data and configuration persistence +mkdir -p ~/.audiusd cat < ~/.audiusd/override.env creatorNodeEndpoint=https:// @@ -31,6 +31,12 @@ EOF docker run -d -ti --env-file ~/.audiusd/override.env -v ~/.audiusd/data:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:latest ``` +If you are migrating from an **existing registered production node**, you will want to pay attention to the persistent volume mount point. Which will likely look something more like this: + +```bash +docker run -d -ti --env-file ~/.audiusd/override.env -v /var/k8s:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:latest +``` + ### P2P Ports Port `26656` must be open and accessible for your node to fully participate in the Audius network, enabling it to propose blocks, vote in consensus, and relay transactions to other nodes. @@ -44,3 +50,17 @@ To enable TLS, set `ENABLE_TLS=true` in your environment. This will instruct `au For this to function correctly, the following conditions must be met: - Your service must be publicly accessible via the URL specified in the `creatorNodeEndpoint` environment variable. - Your service must be reachable on both port `:80` and port `:443` + +## Development + +``` +make build-audiusd-local + +# sync a local node to stage +docker run --rm -ti -p 80:80 -e NETWORK=stage audius/audiusd:$(git rev-parse HEAD) +open http://localhost/console/overview + +# network defaults to prod out of box, for an unregistered, RPC node +# tag would be "current" after this PR merges +docker run --rm -ti -p 80:80 audius/audiusd:$(git rev-parse HEAD) +``` diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index a4f15361fbb..aa498362d03 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -1,30 +1,38 @@ #!/bin/bash -# Default to prod if NETWORK is not set -NETWORK=${NETWORK:-prod} +# Set default network to prod if not specified +NETWORK="${NETWORK:-prod}" ENV_FILE="/env/${NETWORK}.env" OVERRIDE_ENV_FILE="/env/override.env" +# Validate environment files exist +if [ ! -f "$ENV_FILE" ]; then + echo "Error: Network environment file not found at $ENV_FILE" + exit 1 +fi + # source environment variables without overwriting existing ones source_env_file() { local file=$1 - if [ -f "$file" ]; then - echo "Sourcing environment variables from $file" - while IFS='=' read -r key value || [ -n "$key" ]; do - # skip lines that are comments or empty - [[ "$key" =~ ^#.*$ ]] && continue - [[ -z "$key" ]] && continue - # only set variables that are not already defined (prioritize docker-passed env) - if [ -z "${!key}" ]; then - # strip quotations - val="${value%\"}" - val="${val#\"}" - export "$key"="$val" - fi - done < "$file" - else - echo "Environment file $file not found!" + if [ ! -f "$file" ]; then + echo "Environment file $file not found" + return fi + + echo "Loading environment from $file" + while IFS='=' read -r key value || [ -n "$key" ]; do + # skip comments and empty lines + [[ "$key" =~ ^#.*$ ]] && continue + [[ -z "$key" ]] && continue + + # only set if not already defined (prioritize docker-passed env) + if [ -z "${!key}" ]; then + # strip quotations + val="${value%\"}" + val="${val#\"}" + export "$key"="$val" + fi + done < "$file" } source_env_file "$ENV_FILE" @@ -33,10 +41,10 @@ source_env_file "$OVERRIDE_ENV_FILE" # minimum values for a core node to just run POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} POSTGRES_DB=${POSTGRES_DB:-audiusd} -POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/var/lib/postgresql/data} +POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/data/postgres} export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audiusd?sslmode=disable} export uptimeDataDir=${uptimeDataDir:-/data/bolt} -export audius_core_root_dir=${audius_core_root_dir:-/data/audiusd} +export audius_core_root_dir=${audius_core_root_dir:-/data/core} export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost} setup_postgres() { diff --git a/cmd/audiusd/env/dev.env b/cmd/audiusd/env/dev.env index 42ad74c5413..6afec6f1bee 100644 --- a/cmd/audiusd/env/dev.env +++ b/cmd/audiusd/env/dev.env @@ -12,5 +12,5 @@ ethProviderUrl=http://eth-ganache.devnet.audius-d ethRegistryAddress=0xABbfF712977dB51f9f212B85e8A4904c818C2b63 MEDIORUM_ENV=sandbox identityService=https://identity.devnet.audius-d -audius_core_root_dir=/audius-core +audius_core_root_dir=/data/core uptimeDataDir=/bolt diff --git a/cmd/audiusd/env/prod.env b/cmd/audiusd/env/prod.env index 8ec955b32d4..96c62ee6692 100644 --- a/cmd/audiusd/env/prod.env +++ b/cmd/audiusd/env/prod.env @@ -45,5 +45,5 @@ rewardsManagerProgramPda=71hWFVYokLaN1PNYzTAWi13EfJ7Xt9VbSWUKsXUT8mxE rewardsManagerTokenPda=3V9opXNpHmPPymKeq7CYD8wWMH8wzFXmqEkNdzfsZhYq # Core -audius_core_root_dir=/audius-core +audius_core_root_dir=/data/core uptimeDataDir=/bolt diff --git a/cmd/audiusd/env/stage.env b/cmd/audiusd/env/stage.env index dc2a166afd8..5720f789494 100644 --- a/cmd/audiusd/env/stage.env +++ b/cmd/audiusd/env/stage.env @@ -40,5 +40,5 @@ rewardsManagerProgramPda=GaiG9LDYHfZGqeNaoGRzFEnLiwUT7WiC6sA6FDJX9ZPq rewardsManagerTokenPda=HJQj8P47BdA7ugjQEn45LaESYrxhiZDygmukt8iumFZJ # Core -audius_core_root_dir=/audius-core +audius_core_root_dir=/data/core uptimeDataDir=/bolt From a1a358945bba869b61cb371b2a77c898b303773c Mon Sep 17 00:00:00 2001 From: endline Date: Fri, 6 Dec 2024 13:44:47 -0700 Subject: [PATCH 04/13] Update tag --- Makefile | 2 +- cmd/audiusd/README.md | 1 + cmd/audiusd/main.go | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bd6813efd89..b2b59bf59dd 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ build-push-wrapper: .PHONY: build-audiusd-local build-push-audiusd build-audiusd-local: - docker build -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./ + docker build -t audius/audiusd:$(AD_TAG) -t audius/audiusd:current -f ./cmd/audiusd/Dockerfile ./ build-push-audiusd: DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build --push -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./ diff --git a/cmd/audiusd/README.md b/cmd/audiusd/README.md index a012de8a59b..f2ada2319d9 100644 --- a/cmd/audiusd/README.md +++ b/cmd/audiusd/README.md @@ -20,6 +20,7 @@ To operate a [registered](https://docs.audius.org/node-operator/setup/registrati # directory for data and configuration persistence mkdir -p ~/.audiusd +# note that as on now, only creator nodes are supported cat < ~/.audiusd/override.env creatorNodeEndpoint=https:// delegateOwnerWallet= diff --git a/cmd/audiusd/main.go b/cmd/audiusd/main.go index eb7fd845ebe..55908b4f155 100644 --- a/cmd/audiusd/main.go +++ b/cmd/audiusd/main.go @@ -130,14 +130,24 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { e.AutoTLSManager.Cache = autocert.DirCache("/data/var/www/.cache") e.Pre(middleware.HTTPSRedirect()) + httpsPort := os.Getenv("AUDIUSD_HTTPS_PORT") + if httpsPort == "" { + httpsPort = "443" + } + + httpPort := os.Getenv("AUDIUSD_HTTP_PORT") + if httpPort == "" { + httpPort = "80" + } + go func() { - if err := e.StartAutoTLS(":443"); err != nil && err != http.ErrServerClosed { + if err := e.StartAutoTLS(":" + httpsPort); err != nil && err != http.ErrServerClosed { e.Logger.Fatal("shutting down the server") } }() go func() { - if err := e.Start(":80"); err != nil && err != http.ErrServerClosed { + if err := e.Start(":" + httpPort); err != nil && err != http.ErrServerClosed { e.Logger.Fatal("HTTP server failed") } }() From b35848a4388027b51b9ac786001436a2b72147c9 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 08:04:23 -0700 Subject: [PATCH 05/13] Use single mount point for audiusd (audio + pg data) --- cmd/audiusd/Dockerfile | 12 ++++++------ cmd/audiusd/README.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/audiusd/Dockerfile b/cmd/audiusd/Dockerfile index b6f2c23ea40..a7a8610b142 100644 --- a/cmd/audiusd/Dockerfile +++ b/cmd/audiusd/Dockerfile @@ -105,7 +105,12 @@ COPY ./cmd/audiusd/env/dev.env ./cmd/audiusd/env/stage.env ./cmd/audiusd/env/pro COPY ./cmd/audiusd/entrypoint.sh /bin/entrypoint.sh RUN chmod +x /bin/entrypoint.sh -RUN mkdir -p /data && chown -R postgres:postgres /data +# Set up data directory structure with proper permissions +RUN mkdir -p /data && \ + mkdir -p /data/postgres && \ + chown -R postgres:postgres /data/postgres && \ + chmod -R 700 /data/postgres + RUN localedef -i en_US -f UTF-8 en_US.UTF-8 ARG git_sha @@ -115,11 +120,6 @@ EXPOSE 80 EXPOSE 443 EXPOSE 26656 -# Set up data directory structure -RUN mkdir -p /data/postgres && \ - chown -R postgres:postgres /data/postgres && \ - chmod -R 700 /data/postgres - # Single volume mount point VOLUME ["/data"] diff --git a/cmd/audiusd/README.md b/cmd/audiusd/README.md index f2ada2319d9..ef2495202d8 100644 --- a/cmd/audiusd/README.md +++ b/cmd/audiusd/README.md @@ -29,13 +29,13 @@ spOwnerWallet= ENABLE_STORAGE=true EOF -docker run -d -ti --env-file ~/.audiusd/override.env -v ~/.audiusd/data:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:latest +docker run -d -ti --env-file ~/.audiusd/override.env -v ~/.audiusd/data:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:current ``` If you are migrating from an **existing registered production node**, you will want to pay attention to the persistent volume mount point. Which will likely look something more like this: ```bash -docker run -d -ti --env-file ~/.audiusd/override.env -v /var/k8s:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:latest +docker run -d -ti --env-file ~/.audiusd/override.env -v /var/k8s:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:current ``` ### P2P Ports From 2417f4dde91d70144e6c3b27804d75a4c13b0fa8 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 08:54:21 -0700 Subject: [PATCH 06/13] Fix entrypoint pg --- cmd/audiusd/entrypoint.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index aa498362d03..a4d17f883cc 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -51,17 +51,17 @@ setup_postgres() { # Find PostgreSQL binaries directory PG_BIN="/usr/lib/postgresql/15/bin" - # Check if directory exists but is not initialized - if [ -d "$POSTGRES_DATA_DIR" ] && ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then - echo "Directory exists but is not initialized. Cleaning up..." - rm -rf "$POSTGRES_DATA_DIR"/* - fi + # Ensure parent data directory exists with correct permissions + mkdir -p /data + chown postgres:postgres /data - # Initialize PostgreSQL if needed - if [ ! -d "$POSTGRES_DATA_DIR" ] || ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then + # Ensure postgres data directory exists with correct permissions + mkdir -p "$POSTGRES_DATA_DIR" + chown postgres:postgres "$POSTGRES_DATA_DIR" + + # Check if directory is empty or not initialized + if [ -z "$(ls -A $POSTGRES_DATA_DIR)" ] || ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." - mkdir -p "$POSTGRES_DATA_DIR" - chown postgres:postgres "$POSTGRES_DATA_DIR" # Initialize the database su - postgres -c "$PG_BIN/initdb -D $POSTGRES_DATA_DIR" @@ -75,7 +75,7 @@ setup_postgres() { "$POSTGRES_DATA_DIR/postgresql.conf" fi - # Set permissions + # Set permissions (after initialization or for existing data) chown -R postgres:postgres "$POSTGRES_DATA_DIR" chmod -R 700 "$POSTGRES_DATA_DIR" From dbd06b0622e5da107056fe6dbeb3921e26563c94 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 09:39:24 -0700 Subject: [PATCH 07/13] Allow https://localhost and https:// to audiusd echoserver --- cmd/audiusd/main.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cmd/audiusd/main.go b/cmd/audiusd/main.go index 55908b4f155..3bc1d99685e 100644 --- a/cmd/audiusd/main.go +++ b/cmd/audiusd/main.go @@ -12,6 +12,7 @@ import ( "syscall" "log" + "net" "github.com/AudiusProject/audius-protocol/pkg/core" "github.com/AudiusProject/audius-protocol/pkg/core/common" @@ -126,7 +127,26 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { e.Any("/*", echo.WrapHandler(mediorumProxy)) if enableTLS { - e.AutoTLSManager.HostPolicy = autocert.HostWhitelist(hostUrl.Hostname()) + // Get server's IP addresses + addrs, err := net.InterfaceAddrs() + if err != nil { + e.Logger.Warn("Failed to get interface addresses:", err) + } + + // Build whitelist starting with hostname and localhost + whitelist := []string{hostUrl.Hostname(), "localhost"} + + // Add all non-loopback IPv4 addresses + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ip4 := ipnet.IP.To4(); ip4 != nil { + whitelist = append(whitelist, ip4.String()) + } + } + } + + e.Logger.Info("TLS host whitelist:", whitelist) + e.AutoTLSManager.HostPolicy = autocert.HostWhitelist(whitelist...) e.AutoTLSManager.Cache = autocert.DirCache("/data/var/www/.cache") e.Pre(middleware.HTTPSRedirect()) From 4054be55cef169064aa639492c8a2bd089324a8e Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 10:49:04 -0700 Subject: [PATCH 08/13] Set log_level madness --- cmd/audiusd/main.go | 23 ++++++++++++++++++++++- pkg/core/core.go | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/audiusd/main.go b/cmd/audiusd/main.go index 3bc1d99685e..c5add03955d 100644 --- a/cmd/audiusd/main.go +++ b/cmd/audiusd/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/hex" + "log/slog" "net/http" "net/http/httputil" "net/url" @@ -30,7 +31,27 @@ func main() { tlsEnabled := getEnvBool("ENABLE_TLS", false) storageEnabled := getEnvBool("ENABLE_STORAGE", false) - logger := common.NewLogger(nil) + var slogLevel slog.Level + if logLevel := os.Getenv("audiusd_log_level"); logLevel != "" { + switch logLevel { + case "debug": + slogLevel = slog.LevelDebug + case "info": + slogLevel = slog.LevelInfo + case "warn": + slogLevel = slog.LevelWarn + case "error": + slogLevel = slog.LevelError + default: + slogLevel = slog.LevelWarn + } + } else { + slogLevel = slog.LevelInfo + } + + logger := common.NewLogger(&slog.HandlerOptions{ + Level: slogLevel, + }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pkg/core/core.go b/pkg/core/core.go index 67a37348d6c..ca6f17595da 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -320,6 +320,7 @@ func setupNode(logger *common.Logger) (*config.Config, *cconfig.Config, error) { // https://docs.cometbft.com/main/references/config/config.toml#log_level cometConfig.LogLevel = envConfig.LogLevel + logger.Infof("Setting cometConfig.LogLevel = envConfig.LogLevel: %s, %s", cometConfig.LogLevel, envConfig.LogLevel) // postgres indexer config cometConfig.TxIndex.Indexer = "psql" From d857e2a12a1386d844419d85c68ae7b9a88c6267 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 11:40:09 -0700 Subject: [PATCH 09/13] Make sure we persist the echo cert --- cmd/audiusd/main.go | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/audiusd/main.go b/cmd/audiusd/main.go index c5add03955d..3c5bee4d4f4 100644 --- a/cmd/audiusd/main.go +++ b/cmd/audiusd/main.go @@ -168,7 +168,7 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { e.Logger.Info("TLS host whitelist:", whitelist) e.AutoTLSManager.HostPolicy = autocert.HostWhitelist(whitelist...) - e.AutoTLSManager.Cache = autocert.DirCache("/data/var/www/.cache") + e.AutoTLSManager.Cache = autocert.DirCache(getEnvString("audius_core_root_dir", "/audius-core") + "/echo/cache") e.Pre(middleware.HTTPSRedirect()) httpsPort := os.Getenv("AUDIUSD_HTTPS_PORT") @@ -199,23 +199,12 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { return e.Start(":80") } -func keyGen() (pKey string, addr string) { - privateKey, err := crypto.GenerateKey() - if err != nil { - log.Fatalf("Failed to generate private key: %v", err) - } - privateKeyBytes := crypto.FromECDSA(privateKey) - privateKeyStr := hex.EncodeToString(privateKeyBytes) - address := crypto.PubkeyToAddress(privateKey.PublicKey) - return privateKeyStr, address.Hex() -} - -func getEnvBool(key string, defaultVal bool) bool { +func getEnv[T any](key string, defaultVal T, parse func(string) (T, error)) T { val, ok := os.LookupEnv(key) if !ok { return defaultVal } - parsed, err := strconv.ParseBool(val) + parsed, err := parse(val) if err != nil { log.Printf("Invalid value for %s: %v, defaulting to %v", key, val, defaultVal) return defaultVal @@ -223,6 +212,14 @@ func getEnvBool(key string, defaultVal bool) bool { return parsed } +func getEnvString(key, defaultVal string) string { + return getEnv(key, defaultVal, func(s string) (string, error) { return s, nil }) +} + +func getEnvBool(key string, defaultVal bool) bool { + return getEnv(key, defaultVal, strconv.ParseBool) +} + func getHostUrl() (*url.URL, error) { ep := os.Getenv("creatorNodeEndpoint") if ep == "" { @@ -233,3 +230,14 @@ func getHostUrl() (*url.URL, error) { } return url.Parse(ep) } + +func keyGen() (pKey string, addr string) { + privateKey, err := crypto.GenerateKey() + if err != nil { + log.Fatalf("Failed to generate private key: %v", err) + } + privateKeyBytes := crypto.FromECDSA(privateKey) + privateKeyStr := hex.EncodeToString(privateKeyBytes) + address := crypto.PubkeyToAddress(privateKey.PublicKey) + return privateKeyStr, address.Hex() +} From 3651a735ef99a3822cc4b930ef77b3a4a4c842f3 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 12:02:59 -0700 Subject: [PATCH 10/13] Add docs on CF proxy --- cmd/audiusd/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/audiusd/README.md b/cmd/audiusd/README.md index ef2495202d8..39e8eb1e749 100644 --- a/cmd/audiusd/README.md +++ b/cmd/audiusd/README.md @@ -52,6 +52,15 @@ For this to function correctly, the following conditions must be met: - Your service must be publicly accessible via the URL specified in the `creatorNodeEndpoint` environment variable. - Your service must be reachable on both port `:80` and port `:443` +**CLOUDFLARE PROXY** + +If you are using Cloudflare Proxy, and want to use auto TLS, you will need to start with DNS-only mode: + - Configure Cloudflare in DNS-only mode initially (not proxied) + - Let the node obtain its LetsEncrypt certificate (requires HTTP access) + - Once certificate is obtained, you can enable Cloudflare proxy + +See Cloudflare [ssl-mode docs](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/) for more details. + ## Development ``` From e237959b3d905df9ca8dfd3321301563fa967afc Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 12:35:50 -0700 Subject: [PATCH 11/13] Cleanup pg entrypoint --- cmd/audiusd/entrypoint.sh | 66 +++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index a4d17f883cc..c669f353a76 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -21,13 +21,9 @@ source_env_file() { echo "Loading environment from $file" while IFS='=' read -r key value || [ -n "$key" ]; do - # skip comments and empty lines [[ "$key" =~ ^#.*$ ]] && continue [[ -z "$key" ]] && continue - - # only set if not already defined (prioritize docker-passed env) if [ -z "${!key}" ]; then - # strip quotations val="${value%\"}" val="${val#\"}" export "$key"="$val" @@ -38,48 +34,62 @@ source_env_file() { source_env_file "$ENV_FILE" source_env_file "$OVERRIDE_ENV_FILE" -# minimum values for a core node to just run -POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} -POSTGRES_DB=${POSTGRES_DB:-audiusd} +# Set database name based on creatorNodeEndpoint +if [ -n "$creatorNodeEndpoint" ]; then + POSTGRES_DB="audius_creator_node" +elif [ -n "$audius_discprov_url" ]; then + POSTGRES_DB="audius_discovery" +else + POSTGRES_DB="audiusd" +fi + +# Set other defaults +POSTGRES_USER="postgres" +POSTGRES_PASSWORD="postgres" POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/data/postgres} -export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audiusd?sslmode=disable} +export dbUrl="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?sslmode=disable" export uptimeDataDir=${uptimeDataDir:-/data/bolt} export audius_core_root_dir=${audius_core_root_dir:-/data/core} export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost} setup_postgres() { - # Find PostgreSQL binaries directory PG_BIN="/usr/lib/postgresql/15/bin" - # Ensure parent data directory exists with correct permissions + # Ensure directories exist with correct permissions mkdir -p /data - chown postgres:postgres /data - - # Ensure postgres data directory exists with correct permissions mkdir -p "$POSTGRES_DATA_DIR" - chown postgres:postgres "$POSTGRES_DATA_DIR" + chown -R postgres:postgres /data + chown -R postgres:postgres "$POSTGRES_DATA_DIR" + chmod -R 700 "$POSTGRES_DATA_DIR" - # Check if directory is empty or not initialized + # Initialize if needed if [ -z "$(ls -A $POSTGRES_DATA_DIR)" ] || ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." - - # Initialize the database su - postgres -c "$PG_BIN/initdb -D $POSTGRES_DATA_DIR" - # Configure authentication + # Configure authentication and logging sed -i "s/peer/trust/g; s/md5/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf" - - # Configure logging sed -i "s|#log_destination = 'stderr'|log_destination = 'stderr'|; \ s|#logging_collector = on|logging_collector = off|" \ "$POSTGRES_DATA_DIR/postgresql.conf" - fi - # Set permissions (after initialization or for existing data) - chown -R postgres:postgres "$POSTGRES_DATA_DIR" - chmod -R 700 "$POSTGRES_DATA_DIR" + # Only set up database and user on fresh initialization + echo "Setting up PostgreSQL user and database..." + # Start PostgreSQL temporarily to create user and database + su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR start" + until su - postgres -c "$PG_BIN/pg_isready -q"; do + sleep 1 + done + + su - postgres -c "psql -c \"ALTER USER ${POSTGRES_USER} WITH PASSWORD '${POSTGRES_PASSWORD}';\"" + su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '${POSTGRES_DB}'\" | grep -q 1 || \ + psql -c \"CREATE DATABASE ${POSTGRES_DB};\"" + + # Stop PostgreSQL to restart it properly + su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR stop" + fi - # Start PostgreSQL + # Always start PostgreSQL echo "Starting PostgreSQL service..." su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR start" @@ -88,12 +98,6 @@ setup_postgres() { echo "Waiting for PostgreSQL to start..." sleep 2 done - - # Setup database - echo "Setting up PostgreSQL user and database..." - su - postgres -c "psql -c \"ALTER USER postgres WITH PASSWORD '$POSTGRES_PASSWORD';\"" - su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB'\" | grep -q 1 || \ - psql -c \"CREATE DATABASE $POSTGRES_DB;\"" } setup_postgres From ad7fc803bff9bc67472c1cef2dd939854e97a9d9 Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 15:53:04 -0700 Subject: [PATCH 12/13] Do not use embedded pg for discovery --- cmd/audiusd/entrypoint.sh | 2 -- cmd/audiusd/main.go | 23 ++++++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index c669f353a76..3c661693470 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -37,8 +37,6 @@ source_env_file "$OVERRIDE_ENV_FILE" # Set database name based on creatorNodeEndpoint if [ -n "$creatorNodeEndpoint" ]; then POSTGRES_DB="audius_creator_node" -elif [ -n "$audius_discprov_url" ]; then - POSTGRES_DB="audius_discovery" else POSTGRES_DB="audiusd" fi diff --git a/cmd/audiusd/main.go b/cmd/audiusd/main.go index 3c5bee4d4f4..9ecb2eaf785 100644 --- a/cmd/audiusd/main.go +++ b/cmd/audiusd/main.go @@ -113,6 +113,17 @@ func main() { } func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { + + httpPort := os.Getenv("AUDIUSD_HTTP_PORT") + if httpPort == "" { + httpPort = "80" + } + + httpsPort := os.Getenv("AUDIUSD_HTTPS_PORT") + if httpsPort == "" { + httpsPort = "443" + } + e := echo.New() e.Use(middleware.Logger()) @@ -171,16 +182,6 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { e.AutoTLSManager.Cache = autocert.DirCache(getEnvString("audius_core_root_dir", "/audius-core") + "/echo/cache") e.Pre(middleware.HTTPSRedirect()) - httpsPort := os.Getenv("AUDIUSD_HTTPS_PORT") - if httpsPort == "" { - httpsPort = "443" - } - - httpPort := os.Getenv("AUDIUSD_HTTP_PORT") - if httpPort == "" { - httpPort = "80" - } - go func() { if err := e.StartAutoTLS(":" + httpsPort); err != nil && err != http.ErrServerClosed { e.Logger.Fatal("shutting down the server") @@ -196,7 +197,7 @@ func startEchoProxyWithOptionalTLS(hostUrl *url.URL, enableTLS bool) error { return nil } - return e.Start(":80") + return e.Start(":" + httpPort) } func getEnv[T any](key string, defaultVal T, parse func(string) (T, error)) T { From 8aee1a19435e780fec2ce1c21d3dc503f1685f1a Mon Sep 17 00:00:00 2001 From: endline Date: Thu, 12 Dec 2024 18:10:34 -0700 Subject: [PATCH 13/13] Use /audius-core for default dir for maximum compat --- cmd/audiusd/env/dev.env | 2 +- cmd/audiusd/env/prod.env | 2 +- cmd/audiusd/env/stage.env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/audiusd/env/dev.env b/cmd/audiusd/env/dev.env index 6afec6f1bee..42ad74c5413 100644 --- a/cmd/audiusd/env/dev.env +++ b/cmd/audiusd/env/dev.env @@ -12,5 +12,5 @@ ethProviderUrl=http://eth-ganache.devnet.audius-d ethRegistryAddress=0xABbfF712977dB51f9f212B85e8A4904c818C2b63 MEDIORUM_ENV=sandbox identityService=https://identity.devnet.audius-d -audius_core_root_dir=/data/core +audius_core_root_dir=/audius-core uptimeDataDir=/bolt diff --git a/cmd/audiusd/env/prod.env b/cmd/audiusd/env/prod.env index 96c62ee6692..8ec955b32d4 100644 --- a/cmd/audiusd/env/prod.env +++ b/cmd/audiusd/env/prod.env @@ -45,5 +45,5 @@ rewardsManagerProgramPda=71hWFVYokLaN1PNYzTAWi13EfJ7Xt9VbSWUKsXUT8mxE rewardsManagerTokenPda=3V9opXNpHmPPymKeq7CYD8wWMH8wzFXmqEkNdzfsZhYq # Core -audius_core_root_dir=/data/core +audius_core_root_dir=/audius-core uptimeDataDir=/bolt diff --git a/cmd/audiusd/env/stage.env b/cmd/audiusd/env/stage.env index 5720f789494..dc2a166afd8 100644 --- a/cmd/audiusd/env/stage.env +++ b/cmd/audiusd/env/stage.env @@ -40,5 +40,5 @@ rewardsManagerProgramPda=GaiG9LDYHfZGqeNaoGRzFEnLiwUT7WiC6sA6FDJX9ZPq rewardsManagerTokenPda=HJQj8P47BdA7ugjQEn45LaESYrxhiZDygmukt8iumFZJ # Core -audius_core_root_dir=/data/core +audius_core_root_dir=/audius-core uptimeDataDir=/bolt