diff --git a/Makefile b/Makefile index ab41b558207..1babffe2037 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ build-push-wrapper: .PHONY: build-audiusd-local build-push-audiusd build-audiusd-local: - docker build -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./ + DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./ build-push-audiusd: DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build --push -t audius/audiusd:$(AD_TAG) -f ./cmd/audiusd/Dockerfile ./ diff --git a/cmd/audiusd/entrypoint.sh b/cmd/audiusd/entrypoint.sh index 430f328c7b9..2ff8927667c 100644 --- a/cmd/audiusd/entrypoint.sh +++ b/cmd/audiusd/entrypoint.sh @@ -14,7 +14,10 @@ source_env_file() { [[ -z "$key" ]] && continue # only set variables that are not already defined (prioritize docker-passed env) if [ -z "${!key}" ]; then - export "$key"="$value" + # strip quotations + val="${value%\"}" + val="${val#\"}" + export "$key"="$val" fi done < "$file" else @@ -29,12 +32,12 @@ source_env_file "$OVERRIDE_ENV_FILE" POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} POSTGRES_DB=${POSTGRES_DB:-audiusd} POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR:-/var/lib/postgresql/data} -export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audius_creator_node?sslmode=disable} +export dbUrl=${dbUrl:-postgresql://postgres:postgres@localhost:5432/audiusd?sslmode=disable} export uptimeDataDir=${uptimeDataDir:-/data/bolt} export audius_core_root_dir=${audius_core_root_dir:-/data/audiusd} export creatorNodeEndpoint=${creatorNodeEndpoint:-http://localhost} -if [ ! -d "$POSTGRES_DATA_DIR" ]; then +if [ ! -d "$POSTGRES_DATA_DIR" ] || [ -z "$(ls -A "$POSTGRES_DATA_DIR")" ]; then echo "Initializing PostgreSQL data directory at $POSTGRES_DATA_DIR..." su - postgres -c "/usr/lib/postgresql/*/bin/initdb -D $POSTGRES_DATA_DIR" diff --git a/dev-tools/templates/devnet.yaml b/dev-tools/templates/devnet.yaml index cb7c000ca92..461989c9626 100644 --- a/dev-tools/templates/devnet.yaml +++ b/dev-tools/templates/devnet.yaml @@ -13,6 +13,7 @@ nodes: httpPort: 4000 httpsPort: 4001 version: prerelease + isLocalhost: true privateKey: 21118f9a6de181061a2abd549511105adb4877cf9026f271092e6813b7cf58ab wallet: 0x0D38e653eC28bdea5A2296fD5940aaB2D0B8875c rewardsWallet: 0xb3c66e682Bf9a85F6800c769AC5A05c18C3F331d diff --git a/go.mod b/go.mod index fdb270732cf..15d33085a59 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( ) require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/go-openapi/errors v0.22.0 github.com/go-openapi/runtime v0.28.0 github.com/go-openapi/strfmt v0.23.0 @@ -137,6 +136,7 @@ require ( github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect diff --git a/pkg/orchestration/docker.go b/pkg/orchestration/docker.go index 5f5a38ef3f8..84e65e9407c 100644 --- a/pkg/orchestration/docker.go +++ b/pkg/orchestration/docker.go @@ -45,6 +45,15 @@ var ( }, } + devnetHosts = []string{ + "creator-1.devnet.audius-d:172.100.0.1", + "discovery-1.devnet.audius-d:172.100.0.1", + "identity.devnet.audius-d:172.100.0.1", + "eth-ganache.devnet.audius-d:172.100.0.1", + "acdc-ganache.devnet.audius-d:172.100.0.1", + "solana-test-validator.devnet.audius-d:172.100.0.1", + } + semverRegex = regexp.MustCompile(`\d+\.\d+\.\d+`) ) @@ -169,14 +178,7 @@ func runNode( if nconf.DeployOn == conf.Devnet { hostConfig.NetworkMode = "deployments_devnet" - hostConfig.ExtraHosts = []string{ - "creator-1.devnet.audius-d:172.100.0.1", - "discovery-1.devnet.audius-d:172.100.0.1", - "identity.devnet.audius-d:172.100.0.1", - "eth-ganache.devnet.audius-d:172.100.0.1", - "acdc-ganache.devnet.audius-d:172.100.0.1", - "solana-test-validator.devnet.audius-d:172.100.0.1", - } + hostConfig.ExtraHosts = devnetHosts containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"} } @@ -365,6 +367,214 @@ func runNode( return nil } +func runNodeWrapperless( + host string, + config conf.NodeConfig, + nconf conf.NetworkConfig, +) error { + if config.Type != conf.Content { + return logger.Errorf("Unsupported node type %s", config.Type) + } + + execHost := host + if config.IsLocalhost { + execHost = "localhost" + } + containerName := host + if host == "localhost" { + containerName = "audiusd" + } + + dockerClient, err := getDockerClient(execHost) + if err != nil { + return logger.Error(err) + } + defer dockerClient.Close() + + if isContainerRunning(dockerClient, containerName) { + logger.Infof("audiusd container already running for %s", host) + logger.Infof("Use 'audius-ctl restart %s' for a clean restart", host) + return nil + } else if isContainerNameInUse(dockerClient, containerName) { + logger.Infof("stopped audiusd container '%s' exists on %s, removing and starting with current config", containerName, host) + if err := removeContainerByName(dockerClient, containerName); err != nil { + return logger.Error(err) + } + } + + logger.Infof("\nStarting %s\n", host) + + tag := config.Version + if tag == "" { + tag = "current" + } + + containerConfig := &container.Config{ + Image: fmt.Sprintf("audius/audiusd:%s", tag), + } + hostConfig := &container.HostConfig{ + Mounts: []mount.Mount{ + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/mediorum", + Target: "/tmp/mediorum", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/creator-node-backend", + Target: "/file_storage", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/creator-node-db-15", + Target: "/var/lib/postgresql/data", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/bolt", + Target: "/bolt", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/bolt", + Target: "/audius-core", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/env", + Target: "/env", + }, + }, + } + + var port uint = 80 + var tlsPort uint = 443 + if config.HttpPort != 0 { + port = config.HttpPort + } + if config.HttpsPort != 0 { + tlsPort = config.HttpsPort + } + httpPorts := fmt.Sprintf("%d:%d", port, port) + httpsPorts := fmt.Sprintf("%d:%d", tlsPort, tlsPort) + allPorts := []string{httpPorts, httpsPorts} + if config.HostPorts != "" { + allPorts = append(allPorts, strings.Split(config.HostPorts, ",")...) + } + + if config.CorePortP2P == 0 { + config.CorePortP2P = 26656 + } + + if config.CorePortRPC == 0 { + config.CorePortRPC = 26657 + } + + allPorts = append(allPorts, fmt.Sprintf("%d:26656", config.CorePortP2P), fmt.Sprintf("%d:26657", config.CorePortRPC)) + + portSet, portBindings, err := nat.ParsePortSpecs(allPorts) + if err != nil { + return logger.Error(err) + } + containerConfig.ExposedPorts = portSet + hostConfig.PortBindings = portBindings + + if nconf.DeployOn == conf.Devnet { + hostConfig.NetworkMode = "deployments_devnet" + hostConfig.ExtraHosts = devnetHosts + containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"} + } + + logger.Info("Generating configuration...") + + privateKey, err := NormalizedPrivateKey(execHost, config.PrivateKey) + if err != nil { + return logger.Error(err) + } + config.PrivateKey = privateKey + + override := config.ToOverrideEnv(host, nconf) + + // generate the override.env file locally + localOverrideFile, err := os.CreateTemp(".", fmt.Sprintf("%s*.env", host)) + if err != nil { + return logger.Error("Could not create local temp file:", err) + } + localOverridePath := localOverrideFile.Name() + localOverrideFile.Close() + + if err := appendRemoteConfig(execHost, override, config.RemoteConfigFile); err != nil { + return logger.Error(err) + } + if err := godotenv.Write(override, localOverridePath); err != nil { + return logger.Error(err) + } + if err := copyFileToHost(execHost, localOverridePath, "/var/k8s/env/override.env"); err != nil { + return logger.Errorf("Could not copy override config to host: %v", err) + } + if err := os.Remove(localOverridePath); err != nil { + return logger.Error(err) + } + + logger.Info("Pulling audiusd image...") + logger.Debugf("Using image %s", containerConfig.Image) + pullResp, err := dockerClient.ImagePull(context.Background(), containerConfig.Image, types.ImagePullOptions{}) + if err != nil { + logger.Warnf("Failed to pull image: %v", err) + logger.Warnf("Continuing with local image repository") + } else { + defer pullResp.Close() + if err := readAndLogCommandOutput(bufio.NewReader(pullResp)); err != nil { + return logger.Error("Error reading ImagePull output:", err) + } + } + + // create audiusd container + createResponse, err := dockerClient.ContainerCreate( + context.Background(), + containerConfig, + hostConfig, + nil, + nil, + containerName, + ) + if err != nil { + return logger.Error("Failed to create container:", err) + } + if err := dockerClient.ContainerStart( + context.Background(), + createResponse.ID, + container.StartOptions{}, + ); err != nil { + return logger.Error("Failed to start container:", err) + } + + // Wait for audius-d wrapper to be ready + ready := false + timeout := time.After(30 * time.Second) + for !ready { + select { + case <-timeout: + return logger.Error("Timeout waiting for audiusd container to start") + default: + inspect, err := dockerClient.ContainerInspect(context.Background(), createResponse.ID) + if err != nil { + return logger.Error("Could not get status of audiusd container:", err) + } + time.Sleep(3 * time.Second) + ready = inspect.State.Running + logger.Debugf("audiusd container status: %s", inspect.State.Status) + } + } + + logger.Info("Creating auto-update cron job") + if err := setAutoUpdateCron(host, config.Version, nconf.DeployOn); err != nil { + return logger.Error("Failed to add auto-update cron job:", err) + } + + return nil +} + func isContainerRunning(dockerClient *client.Client, containerName string) bool { containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{}) if err != nil { diff --git a/pkg/orchestration/run.go b/pkg/orchestration/run.go index 9dfb32565b8..31178463db3 100644 --- a/pkg/orchestration/run.go +++ b/pkg/orchestration/run.go @@ -4,12 +4,15 @@ import ( "bytes" "fmt" "io" + "math/rand" "net" "os" "os/exec" "strings" + "time" "github.com/AudiusProject/audius-protocol/pkg/conf" + "github.com/AudiusProject/audius-protocol/pkg/core/config" "github.com/AudiusProject/audius-protocol/pkg/logger" "github.com/AudiusProject/audius-protocol/pkg/register" "github.com/joho/godotenv" @@ -51,12 +54,22 @@ func RunAudiusNodes(nodes map[string]conf.NodeConfig, network conf.NetworkConfig } for host, nodeConfig := range nodes { - if err := runNode( - host, - nodeConfig, - network, - audiusdTagOverride, - ); err != nil { + var err error + if nodeConfig.Type == conf.Content { + err = runNodeWrapperless( + host, + nodeConfig, + network, + ) + } else { + err = runNode( + host, + nodeConfig, + network, + audiusdTagOverride, + ) + } + if err != nil { logger.Warnf("Error encountered starting node %s: %s", host, err.Error()) logger.Warnf("View full debug log at %s", logger.GetLogFilepath()) } else { @@ -171,3 +184,52 @@ func resolvesToLocalhost(host string) (bool, error) { } return false, nil } + +// WARNING: only effective when run by audius-ctl cli +func setAutoUpdateCron(host, version string, network conf.NetworkType) error { + var updateInterval string + rand.Seed(time.Now().UnixNano()) + if version == "prerelease" && network == conf.Testnet { + // Stage nodes should update continuously, slightly staggered + updateInterval = fmt.Sprintf("%d-59/5", rand.Intn(5)) + } else if config.Version == "edge" { + // Frequent, staggerd release of foundation and other canary nodes + updateInterval = fmt.Sprintf("%d-59/25", rand.Intn(25)) + } else { + // Hourly release for everything else + // starting 55 minutes from now (for randomness + prevent updates during CI) + fiveMinutesAgo := time.Now().Add(-5 * time.Minute).Minute() + updateInterval = fmt.Sprint(fiveMinutesAgo) + } + schedule := fmt.Sprintf("%s * * * *", updateInterval) + executable, err := os.Executable() + if err != nil { + return logger.Error("Failed to retrieve executable path:", err) + } + restartCommand := fmt.Sprintf("bash -c '%s restart -f %s'", executable, host) + comment := fmt.Sprintf("# audius auto-upgrade for %s", host) + cronJob := fmt.Sprintf("%s %s %s", schedule, restartCommand, comment) + script := fmt.Sprintf( + `(crontab -l | grep -v '%s'; echo "%s" ) | crontab - `, + host, + cronJob, + ) + if err := execLocal("bash", "-c", script); err != nil { + return logger.Error("Failed to execute crontab script:", err) + } + + logger.Debugf("auto-upgrade cron job added successfully for %s", host) + return nil +} + +func copyFileToHost(host, src, dest string) error { + var cmd *exec.Cmd + if host == "localhost" { + cmd = exec.Command("cp", src, dest) + } else { + cmd = exec.Command("scp", src, fmt.Sprintf("%s:%s", host, dest)) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +}