exp/services/ledgerexporter: create go integration tests for sub commands

.github/workflows/horizon.yml

@@ -148,4 +148,3 @@ jobs:
       - if: github.ref == 'refs/heads/master'
         name: Push to DockerHub
         run: docker push stellar/horizon-verify-range:latest
-

.github/workflows/ledgerexporter-release.yml

@@ -10,17 +10,37 @@ jobs:
     name: Test and push the Ledger Exporter images
     runs-on: ubuntu-latest
     env:
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing
       STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v}
+      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v}  
     steps:
       - uses: actions/checkout@v3
         with:
           ref: github.sha
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
+      - name: Pull Quickstart image
+        shell: bash
+        run: |
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"    
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
 
       - name: Run Ledger Exporter test
-        run: make -C exp/services/ledgerexporter docker-test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+
+      - name: Build Ledger Exporter docker
+        run: make -C exp/services/ledgerexporter docker-build
 
       # Push images
       - name: Login to DockerHub

.github/workflows/ledgerexporter.yml

@@ -7,18 +7,36 @@ on:
 
 jobs:
   ledger-exporter:
-    name: Build and test Ledger Exporter image
+    name: Test Ledger Exporter
     runs-on: ubuntu-latest
     env:
-      STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing
     steps:
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$CAPTIVE_CORE_DEBIAN_PKG_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
+
+      - name: Pull Quickstart image
+        shell: bash
+        run: |
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
-
+
       - name: Run Ledger Exporter test
-        run: make -C exp/services/ledgerexporter docker-test
-
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 

exp/services/ledgerexporter/DEVELOPER_GUIDE.md

@@ -32,14 +32,27 @@ To achieve its goals, the ledger exporter uses the following architecture, which
 - An example implementation of `DataStore` for GCS, Google Cloud Storage. This plugin is located in the [support](https://github.com/stellar/go/tree/master/support/datastore) package. 
 - The ledger exporter currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
 
-## Build, Run and Test using Docker
+## Build and Run using Docker
 The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run the ledger exporter. 
 
 - Build: To build the Docker container, use the provided [Makefile](./Makefile). Simply run make `make docker-build` to build a new container after making any changes.
 
 - Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md).
 
-- Test: To test the Docker container, refer to the [docker-test](./Makefile) command for an example of how to use the [GCS emulator](https://github.com/fsouza/fake-gcs-server) for local testing.
+## Running Integration Tests:
+from top directory of stellar/go repo, run go test to launch ledger exporter integration
+tests.
+
+`LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true` is required environment variable to allow
+tests to run.
+
+Optional, tests will try to run `stellar-core` from o/s PATH for captive core, if not resolvable, then set `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/path/to/stellar-core` 
+
+Optional, can override the version of quickstart used to run standalone stellar network, p `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE=docker.io/stellar/quickstart:<tag>`. Ideally don't need to change this, but is available.
+
+```
+$ LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
+```
 
 ## Adding support for a new storage type
 Support for different data storage types are encapsulated as 'plugins', which are implementation of `DataStore` interface in a go package. To add a data storage plugin based on a new storage type (e.g. AWS S3), follow these steps:

exp/services/ledgerexporter/Makefile

@@ -14,32 +14,6 @@ $(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VE
 	-t $(DOCKER_IMAGE):$(VERSION) \
 	-t $(DOCKER_IMAGE):latest .
 
-docker-clean:
-	$(SUDO) docker stop fake-gcs-server || true
-	$(SUDO) docker rm fake-gcs-server || true
-	$(SUDO) rm -rf ${PWD}/storage || true
-	$(SUDO) docker network rm test-network || true
-
-docker-test: docker-clean
-	# Create temp storage dir
-	$(SUDO) mkdir -p ${PWD}/storage/exporter-test 
-
-	# Create test network for docker
-	$(SUDO) docker network create test-network
-
-	# Run the fake GCS server
-	$(SUDO) docker run -d --name fake-gcs-server -p 4443:4443 \
-		 -v ${PWD}/storage:/data --network test-network fsouza/fake-gcs-server -scheme http
-
-	# Run the ledger-exporter
-	$(SUDO) docker run --platform linux/amd64 -t --network test-network\
-		 -v ${PWD}/exp/services/ledgerexporter/docker/config.test.toml:/config.toml \
-		 -e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \
-		 $(DOCKER_IMAGE):$(VERSION) \
-		 scan-and-fill --start 1000 --end 2000
-
-	$(MAKE) docker-clean
-
 docker-push:
 	$(SUDO) docker push $(DOCKER_IMAGE):$(VERSION)
 	$(SUDO) docker push $(DOCKER_IMAGE):latest

exp/services/ledgerexporter/README.md

@@ -126,5 +126,4 @@ docker run --platform linux/amd64 -d \
 Arguments:
 - `--start <start_ledger>` (required): The starting ledger sequence number in the range to export.
 - `--end <end_ledger>` (required): The ending ledger sequence number in the range.
-- `--config-file <config_file_path>` (optional): The path to your configuration file, containing details like GCS bucket information. If not provided, the exporter will look for config.toml in the directory where you run the command.
-
+- `--config-file <config_file_path>` (optional): The path to your configuration file, containing details like GCS bucket information. If not provided, the exporter will look for config.toml in the directory where you run the command.

exp/services/ledgerexporter/internal/config.go

@@ -98,8 +98,8 @@ func NewConfig(settings RuntimeSettings, getCoreVersionFn ledgerbackend.CoreBuil
 	}
 	logger.Infof("Network Config Archive URLs: %v", config.StellarCoreConfig.HistoryArchiveUrls)
 	logger.Infof("Network Config Archive Passphrase: %v", config.StellarCoreConfig.NetworkPassphrase)
-	logger.Infof("Network Config Archive Stellar Core Binary Path: %v", config.StellarCoreConfig.StellarCoreBinaryPath)
-	logger.Infof("Network Config Archive Stellar Core Toml Config: %v", string(config.SerializedCaptiveCoreToml))
+	logger.Infof("Network Config Stellar Core Binary Path: %v", config.StellarCoreConfig.StellarCoreBinaryPath)
+	logger.Infof("Network Config Stellar Core Toml Config: %v", string(config.SerializedCaptiveCoreToml))
 
 	return config, nil
 }