exp/services/ledgerexporter: Implement Docker based deployment (#5254)

.github/workflows/horizon.yml

@@ -156,47 +156,25 @@ jobs:
   ledger-exporter:
     name: Test and push the Ledger Exporter images
     runs-on: ubuntu-latest
-    if: false # Disable the job
     steps:
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
-      - name: Build and test Ledger Exporter images
-        # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
-        run: |
-          chmod 755 ./exp/lighthorizon/build/build.sh
-          mkdir $PWD/ledgerexport
-          # mkdir $PWD/index
-
-          ./exp/lighthorizon/build/build.sh ledgerexporter stellar latest false
-          docker run -e ARCHIVE_TARGET=file:///ledgerexport\
-                     -e START=5\
-                     -e END=150\
-                     -e NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"\
-                     -e CAPTIVE_CORE_CONFIG="/captive-core-pubnet.cfg"\
-                     -e HISTORY_ARCHIVE_URLS="https://history.stellar.org/prd/core-live/core_live_001"\
-                     -v $PWD/ledgerexport:/ledgerexport\
-                     stellar/lighthorizon-ledgerexporter
-
-          # # run map job
-          # docker run -e NETWORK_PASSPHRASE='pubnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e BATCH_SIZE=64 -e FIRST_CHECKPOINT=64 \
-          #            -e WORKER_COUNT=1 -e RUN_MODE=map -v $PWD/ledgerexport:/ledgermeta -e TXMETA_SOURCE=file:///ledgermeta -v $PWD/index:/index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch
-
-          # # run reduce job
-          # docker run -e NETWORK_PASSPHRASE='pubnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e MAP_JOB_COUNT=1 -e REDUCE_JOB_COUNT=1 \
-          #            -e WORKER_COUNT=1 -e RUN_MODE=reduce -v $PWD/index:/index -e INDEX_SOURCE_ROOT=file:///index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch
+      - name: Build Ledger Exporter docker
+        run: make -C exp/services/ledgerexporter docker-build
+
+      - name: Run Ledger Exporter test
+        run: make -C exp/services/ledgerexporter docker-test
 
       # Push images
-      - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lighthorizon'
+      - if: github.ref == 'refs/heads/master'
         name: Login to DockerHub
         uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lighthorizon'
+      - if: github.ref == 'refs/heads/master'
         name: Push to DockerHub
-        run: |
-          chmod 755 ./exp/lighthorizon/build/build.sh
-          ./exp/lighthorizon/build/build.sh ledgerexporter stellar latest true
+        run: make -C exp/services/ledgerexporter docker-push

Makefile

@@ -35,6 +35,9 @@ friendbot:
 horizon:
 	$(MAKE) -C services/horizon/ binary-build
 
+ledger-exporter:
+	$(MAKE) -C exp/services/ledgerexporter/ docker-build
+
 webauth:
 	$(MAKE) -C exp/services/webauth/ docker-build
 

exp/services/ledgerexporter/Makefile

@@ -0,0 +1,44 @@
+SUDO := $(shell docker version >/dev/null 2>&1 || echo "sudo")
+
+# https://github.com/opencontainers/image-spec/blob/master/annotations.md
+BUILD_DATE := $(shell date -u +%FT%TZ)
+VERSION ?= $(shell git rev-parse --short HEAD)
+DOCKER_IMAGE := stellar/ledger-exporter
+
+docker-build:
+	cd ../../../ && \
+	$(SUDO) docker build --platform linux/amd64 --pull --label org.opencontainers.image.created="$(BUILD_DATE)" \
+	--build-arg VERSION=$(VERSION) \
+$(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION)) \
+	-f exp/services/ledgerexporter/docker/Dockerfile \
+	-t $(DOCKER_IMAGE):$(VERSION) \
+	-t $(DOCKER_IMAGE):latest .
+
+docker-test:
+	# Create temp storage dir
+	$(SUDO) mkdir -p ${PWD}/storage/exporter-test
+
+	# Create test network for docker
+	$(SUDO) docker network create test-network
+
+	# Run the fake GCS server
+	$(SUDO) docker run -d --name fake-gcs-server -p 4443:4443 \
+		 -v ${PWD}/storage:/data --network test-network fsouza/fake-gcs-server -scheme http
+
+	# Run the ledger-exporter
+	$(SUDO) docker run --platform linux/amd64 -t --network test-network\
+		 -e NETWORK=pubnet \
+		 -e ARCHIVE_TARGET=gcs://exporter-test \
+		 -e START=1000 \
+		 -e END=2000 \
+		 -e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \
+		 $(DOCKER_IMAGE):$(VERSION)
+
+	$(SUDO) docker stop fake-gcs-server
+	$(SUDO) docker rm fake-gcs-server
+	$(SUDO) rm -rf ${PWD}/storage
+	$(SUDO) docker network rm test-network
+
+docker-push:
+	$(SUDO) docker push $(DOCKER_IMAGE):$(VERSION)
+	$(SUDO) docker push $(DOCKER_IMAGE):latest

exp/services/ledgerexporter/docker/Dockerfile

@@ -0,0 +1,36 @@
+FROM golang:1.22-bullseye AS builder
+
+WORKDIR /go/src/github.com/stellar/go
+
+COPY go.mod ./
+COPY go.sum ./
+
+RUN go mod download
+
+COPY . ./
+
+RUN go install github.com/stellar/go/exp/services/ledgerexporter
+
+FROM ubuntu:22.04
+ARG STELLAR_CORE_VERSION
+ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*}
+ENV STELLAR_CORE_BINARY_PATH /usr/bin/stellar-core
+
+ENV DEBIAN_FRONTEND=noninteractive
+# ca-certificates are required to make tls connections
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils
+RUN wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true apt-key add -
+RUN echo "deb https://apt.stellar.org focal stable" >/etc/apt/sources.list.d/SDF.list
+RUN echo "deb https://apt.stellar.org focal unstable" >/etc/apt/sources.list.d/SDF-unstable.list
+RUN apt-get update && apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
+RUN apt-get clean
+
+COPY exp/services/ledgerexporter/docker/start /
+
+RUN ["chmod", "+x", "/start"]
+
+COPY --from=builder /go/bin/ledgerexporter /usr/bin/ledgerexporter
+
+ENTRYPOINT ["/start"]
+
+

exp/services/ledgerexporter/docker/start

@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+set -e
+
+# Validation
+if [ -z "$ARCHIVE_TARGET" ]; then
+  echo "error: undefined ARCHIVE_TARGET env variable"
+  exit 1
+fi
+
+if [ -z "$NETWORK" ]; then
+  echo "error: undefined NETWORK env variable"
+  exit 1
+fi
+
+ledgers_per_file="${LEDGERS_PER_FILE:-1}"
+files_per_partition="${FILES_PER_PARTITION:-64000}"
+
+# Generate TOML configuration
+cat <<EOF > config.toml
+network = "${NETWORK}"
+destination_url = "${ARCHIVE_TARGET}"
+
+[exporter_config]
+  ledgers_per_file = $ledgers_per_file
+  files_per_partition = $files_per_partition
+EOF
+
+# Check if START or END variables are set
+if [[ -n "$START" || -n "$END" ]]; then
+    echo "START: $START END: $END"
+    /usr/bin/ledgerexporter --config-file config.toml --start $START --end $END
+# Check if FROM_LAST variable is set
+elif [[ -n "$FROM_LAST" ]]; then
+    echo "FROM_LAST: $FROM_LAST"
+    /usr/bin/ledgerexporter --config-file config.toml --from-last  $FROM_LAST
+else
+    echo "Error: No ledger range provided."
+    exit 1
+fi