Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Infra Fog: Protocol #10641

Merged
merged 13 commits into from
Dec 13, 2024
Merged

Infra Fog: Protocol #10641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
0556b74
Port audiusd changes from various fog branches
Dec 6, 2024
40bab82
Make local postgres work out of the box
Dec 6, 2024
7be2793
Update readme
Dec 6, 2024
a1a3589
Update tag
Dec 6, 2024
b35848a
Use single mount point for audiusd (audio + pg data)
Dec 12, 2024
2417f4d
Fix entrypoint pg
Dec 12, 2024
dbd06b0
Allow https://localhost and https://<node.ip.ad.dr> to audiusd echose…
Dec 12, 2024
4054be5
Set log_level madness
Dec 12, 2024
d857e2a
Make sure we persist the echo cert
Dec 12, 2024
3651a73
Add docs on CF proxy
Dec 12, 2024
e237959
Cleanup pg entrypoint
Dec 12, 2024
ad7fc80
Do not use embedded pg for discovery
Dec 12, 2024
8aee1a1
Use /audius-core for default dir for maximum compat
Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ build-push-wrapper:

.PHONY: build-audiusd-local build-push-audiusd
build-audiusd-local:
docker build -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./
docker build -t audius/audiusd:$(AD_TAG) -t audius/audiusd:current -f ./cmd/audiusd/Dockerfile ./

build-push-audiusd:
DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build --push -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./
Expand Down
20 changes: 16 additions & 4 deletions cmd/audiusd/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
FROM debian:bookworm AS cpp-builder
endline marked this conversation as resolved.
Show resolved Hide resolved
FROM debian:bullseye AS cpp-builder

RUN apt-get update && \
apt-get install -y curl gnupg2 lsb-release && \
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg && \
echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/postgresql.list

RUN apt-get update && apt-get install -y \
build-essential \
Expand All @@ -19,6 +24,7 @@ RUN apt-get update && apt-get install -y \
libavformat-dev \
libavutil-dev \
libswresample-dev \
libavresample-dev \
libsamplerate0-dev \
libtag1-dev \
libchromaprint-dev \
Expand Down Expand Up @@ -73,11 +79,11 @@ RUN g++ -o /bin/analyze-key /app/cpp/keyfinder.cpp \
RUN g++ -o /bin/analyze-bpm /app/cpp/bpm-analyzer.cpp \
-I/usr/include/eigen3 -I/usr/local/include/essentia -I/usr/local/include \
-L/usr/local/lib \
-lessentia -ltag -lyaml -lfftw3 -lfftw3f -lavcodec -lavformat -lavutil -lavfilter -lsamplerate -lswresample -lpthread -lz -lchromaprint && \
-lessentia -ltag -lyaml -lfftw3 -lfftw3f -lavcodec -lavformat -lavutil -lavfilter -lsamplerate -lavresample -lpthread -lz -lchromaprint && \
chmod +x /bin/analyze-bpm


FROM golang:1.22-bookworm AS go-builder
FROM golang:1.22-bullseye AS go-builder

WORKDIR /app

Expand All @@ -99,7 +105,12 @@ COPY ./cmd/audiusd/env/dev.env ./cmd/audiusd/env/stage.env ./cmd/audiusd/env/pro
COPY ./cmd/audiusd/entrypoint.sh /bin/entrypoint.sh
RUN chmod +x /bin/entrypoint.sh

RUN mkdir -p /data && chown -R postgres:postgres /data
# Set up data directory structure with proper permissions
RUN mkdir -p /data && \
mkdir -p /data/postgres && \
chown -R postgres:postgres /data/postgres && \
chmod -R 700 /data/postgres

RUN localedef -i en_US -f UTF-8 en_US.UTF-8

ARG git_sha
Expand All @@ -109,6 +120,7 @@ EXPOSE 80
EXPOSE 443
EXPOSE 26656

# Single volume mount point
VOLUME ["/data"]

ENTRYPOINT ["/bin/entrypoint.sh"]
38 changes: 34 additions & 4 deletions cmd/audiusd/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ A golang implementation of the audius protocol.
Minimal example to run a node and sync it to the audius mainnet.

```bash
docker run --rm -ti -p 80:80 audius/audiusd:latest
docker run --rm -ti -p 80:80 audius/audiusd:current

open http://localhost/console/overview
```
Expand All @@ -17,9 +17,10 @@ open http://localhost/console/overview
To operate a [registered](https://docs.audius.org/node-operator/setup/registration/) node requires the minimal config below.

```bash
# directory for data persistence
mkdir ~/.audiusd
# directory for data and configuration persistence
mkdir -p ~/.audiusd

# note that as on now, only creator nodes are supported
cat <<EOF > ~/.audiusd/override.env
creatorNodeEndpoint=https://
delegateOwnerWallet=
Expand All @@ -28,7 +29,13 @@ spOwnerWallet=
ENABLE_STORAGE=true
EOF

docker run -d -ti --env-file ~/.audiusd/override.env -v ~/.audiusd/data:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:latest
docker run -d -ti --env-file ~/.audiusd/override.env -v ~/.audiusd/data:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:current
```

If you are migrating from an **existing registered production node**, you will want to pay attention to the persistent volume mount point. Which will likely look something more like this:

```bash
docker run -d -ti --env-file ~/.audiusd/override.env -v /var/k8s:/data -p 80:80 -p 443:443 -p 26656:26656 audius/audiusd:current
```

### P2P Ports
Expand All @@ -44,3 +51,26 @@ To enable TLS, set `ENABLE_TLS=true` in your environment. This will instruct `au
For this to function correctly, the following conditions must be met:
- Your service must be publicly accessible via the URL specified in the `creatorNodeEndpoint` environment variable.
- Your service must be reachable on both port `:80` and port `:443`

**CLOUDFLARE PROXY**

If you are using Cloudflare Proxy, and want to use auto TLS, you will need to start with DNS-only mode:
- Configure Cloudflare in DNS-only mode initially (not proxied)
- Let the node obtain its LetsEncrypt certificate (requires HTTP access)
- Once certificate is obtained, you can enable Cloudflare proxy

See Cloudflare [ssl-mode docs](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/) for more details.
Comment on lines +55 to +62
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

excellent


## Development

```
make build-audiusd-local
# sync a local node to stage
docker run --rm -ti -p 80:80 -e NETWORK=stage audius/audiusd:$(git rev-parse HEAD)
open http://localhost/console/overview
# network defaults to prod out of box, for an unregistered, RPC node
# tag would be "current" after this PR merges
docker run --rm -ti -p 80:80 audius/audiusd:$(git rev-parse HEAD)
```
122 changes: 79 additions & 43 deletions cmd/audiusd/entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,68 +1,104 @@
#!/bin/bash

# Set default network to prod if not specified
NETWORK="${NETWORK:-prod}"
ENV_FILE="/env/${NETWORK}.env"
OVERRIDE_ENV_FILE="/env/override.env"

# Validate environment files exist
if [ ! -f "$ENV_FILE" ]; then
echo "Error: Network environment file not found at $ENV_FILE"
exit 1
fi

# source environment variables without overwriting existing ones
source_env_file() {
local file=$1
if [ -f "$file" ]; then
echo "Sourcing environment variables from $file"
while IFS='=' read -r key value || [ -n "$key" ]; do
# skip lines that are comments or empty
[[ "$key" =~ ^#.*$ ]] && continue
[[ -z "$key" ]] && continue
# only set variables that are not already defined (prioritize docker-passed env)
if [ -z "${!key}" ]; then
export "$key"="$value"
fi
done < "$file"
else
echo "Environment file $file not found!"
if [ ! -f "$file" ]; then
echo "Environment file $file not found"
return
fi

echo "Loading environment from $file"
while IFS='=' read -r key value || [ -n "$key" ]; do
[[ "$key" =~ ^#.*$ ]] && continue
[[ -z "$key" ]] && continue
if [ -z "${!key}" ]; then
val="${value%\"}"
val="${val#\"}"
export "$key"="$val"
fi
done < "$file"
}

source_env_file "$ENV_FILE"
source_env_file "$OVERRIDE_ENV_FILE"

# minimum values for a core node to just run
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
POSTGRES_DB=${POSTGRES_DB:-audiusd}
POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/var/lib/postgresql/data}
export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audius_creator_node?sslmode=disable}
# Set database name based on creatorNodeEndpoint
if [ -n "$creatorNodeEndpoint" ]; then
POSTGRES_DB="audius_creator_node"
else
POSTGRES_DB="audiusd"
fi

# Set other defaults
POSTGRES_USER="postgres"
POSTGRES_PASSWORD="postgres"
POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/data/postgres}
export dbUrl="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?sslmode=disable"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this not override dbUrl needed for remote postgres, e.g. in dn4?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good eye!.
yeah i need to come back and work out how to handle discovery.
we will use a much simpler entrypoint control flow there (i.e. skip the entire postgres part of this script) as discovery will still use a pg container.
furthermore, discovery only needs the core app and not all the encoding deps, so we may produce a different docker artifact altogether. that would be say 10mb and not 1.5gb!

export uptimeDataDir=${uptimeDataDir:-/data/bolt}
export audius_core_root_dir=${audius_core_root_dir:-/data/audiusd}
export audius_core_root_dir=${audius_core_root_dir:-/data/core}
export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost}

if [ ! -d "$POSTGRES_DATA_DIR" ]; then
echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..."
su - postgres -c "/usr/lib/postgresql/*/bin/initdb -D $POSTGRES_DATA_DIR"
setup_postgres() {
PG_BIN="/usr/lib/postgresql/15/bin"

# Ensure directories exist with correct permissions
mkdir -p /data
mkdir -p "$POSTGRES_DATA_DIR"
chown -R postgres:postgres /data
chown -R postgres:postgres "$POSTGRES_DATA_DIR"
chmod -R 700 "$POSTGRES_DATA_DIR"

echo "Updating PostgreSQL configuration for password authentication..."
sed -i "s/peer/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf"
sed -i "s/md5/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf"
fi

chown -R postgres:postgres "$POSTGRES_DATA_DIR"
chmod -R u+rwx,g-rwx,o-rwx "$POSTGRES_DATA_DIR"

echo "Configuring PostgreSQL to log to stderr for docker capture..."
sed -i "s|#log_destination = 'stderr'|log_destination = 'stderr'|" "$POSTGRES_DATA_DIR/postgresql.conf"
sed -i "s|#logging_collector = on|logging_collector = off|" "$POSTGRES_DATA_DIR/postgresql.conf"
# Initialize if needed
if [ -z "$(ls -A $POSTGRES_DATA_DIR)" ] || ! [ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]; then
echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..."
su - postgres -c "$PG_BIN/initdb -D $POSTGRES_DATA_DIR"

# Configure authentication and logging
sed -i "s/peer/trust/g; s/md5/trust/g" "$POSTGRES_DATA_DIR/pg_hba.conf"
sed -i "s|#log_destination = 'stderr'|log_destination = 'stderr'|; \
s|#logging_collector = on|logging_collector = off|" \
"$POSTGRES_DATA_DIR/postgresql.conf"

echo "Starting PostgreSQL service..."
su - postgres -c "/usr/lib/postgresql/*/bin/pg_ctl -D $POSTGRES_DATA_DIR -o '-c config_file=$POSTGRES_DATA_DIR/postgresql.conf' start"
# Only set up database and user on fresh initialization
echo "Setting up PostgreSQL user and database..."
# Start PostgreSQL temporarily to create user and database
su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR start"
until su - postgres -c "$PG_BIN/pg_isready -q"; do
sleep 1
done

su - postgres -c "psql -c \"ALTER USER ${POSTGRES_USER} WITH PASSWORD '${POSTGRES_PASSWORD}';\""
su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '${POSTGRES_DB}'\" | grep -q 1 || \
psql -c \"CREATE DATABASE ${POSTGRES_DB};\""

# Stop PostgreSQL to restart it properly
su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR stop"
fi

until su - postgres -c "pg_isready -q"; do
echo "Waiting for PostgreSQL to start..."
sleep 2
done
# Always start PostgreSQL
echo "Starting PostgreSQL service..."
su - postgres -c "$PG_BIN/pg_ctl -D $POSTGRES_DATA_DIR start"

echo "Setting up PostgreSQL user and database..."
su - postgres -c "psql -c \"ALTER USER postgres WITH PASSWORD '$POSTGRES_PASSWORD';\""
su - postgres -c "psql -tc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB'\" | grep -q 1 || psql -c 'CREATE DATABASE $POSTGRES_DB;'"
# Wait for PostgreSQL to be ready
until su - postgres -c "$PG_BIN/pg_isready -q"; do
echo "Waiting for PostgreSQL to start..."
sleep 2
done
}

su - postgres -c "/usr/lib/postgresql/*/bin/pg_ctl -D $POSTGRES_DATA_DIR -o '-c config_file=$POSTGRES_DATA_DIR/postgresql.conf' restart"
setup_postgres

echo "Starting audiusd..."
exec /bin/audiusd "$@"
Loading