services/horizon/internal/ingest: Add missing tables to TruncateInges… #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Horizon | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| jobs: | |
| integration: | |
| name: Integration tests | |
| strategy: | |
| matrix: | |
| os: [ubuntu-20.04, ubuntu-22.04] | |
| go: ["1.21", "1.22"] | |
| pg: [12] | |
| ingestion-backend: [captive-core, captive-core-remote-storage] | |
| protocol-version: [19, 20] | |
| runs-on: ${{ matrix.os }} | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.pg }} | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| env: | |
| HORIZON_INTEGRATION_TESTS_ENABLED: true | |
| HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} | |
| PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 20.2.0-1716.rc3.34d82fc00.focal | |
| PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:20.2.0-1716.rc3.34d82fc00.focal | |
| PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:20.3.0-rc1-52 | |
| PROTOCOL_19_CORE_DEBIAN_PKG_VERSION: 19.14.0-1500.5664eff4e.focal | |
| PROTOCOL_19_CORE_DOCKER_IMG: stellar/stellar-core:19.14.0-1500.5664eff4e.focal | |
| PGHOST: localhost | |
| PGPORT: 5432 | |
| PGUSER: postgres | |
| PGPASSWORD: postgres | |
| PGDATABASE: postgres | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # For pull requests, build and test the PR head not a merge of the PR with the destination. | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| # We need to full history for git-restore-mtime to know what modification dates to use. | |
| # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing). | |
| fetch-depth: "0" | |
| # In order to debug the integration tests, run 'touch continue' once you connect to the ssh session | |
| # | |
| # - name: Setup upterm session | |
| # uses: lhotari/action-upterm@d23c2722bdab893785c9fbeae314cbf080645bd7 | |
| # with: | |
| # ## limits ssh access and adds the ssh public key for the user which triggered the workflow | |
| # limit-access-to-actor: true | |
| # ## limits ssh access and adds the ssh public keys of the listed GitHub users | |
| # limit-access-to-users: <yourGithubUser> | |
| - uses: ./.github/actions/setup-go | |
| with: | |
| go-version: ${{ matrix.go }} | |
| - name: Pull and set Stellar Core image | |
| shell: bash | |
| run: | | |
| docker pull "$PROTOCOL_${{ matrix.protocol-version }}_CORE_DOCKER_IMG" | |
| echo HORIZON_INTEGRATION_TESTS_DOCKER_IMG="$PROTOCOL_${{ matrix.protocol-version }}_CORE_DOCKER_IMG" >> $GITHUB_ENV | |
| - if: ${{ matrix.protocol-version == '20' }} | |
| name: Pull and set Soroban RPC image | |
| shell: bash | |
| run: | | |
| docker pull "$PROTOCOL_${{ matrix.protocol-version }}_SOROBAN_RPC_DOCKER_IMG" | |
| echo HORIZON_INTEGRATION_TESTS_SOROBAN_RPC_DOCKER_IMG="$PROTOCOL_${{ matrix.protocol-version }}_SOROBAN_RPC_DOCKER_IMG" >> $GITHUB_ENV | |
| echo HORIZON_INTEGRATION_TESTS_ENABLE_SOROBAN_RPC=true >> $GITHUB_ENV | |
| - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }} | |
| name: Install and enable Captive Core | |
| run: | | |
| # Workaround for https://github.com/actions/virtual-environments/issues/5245, | |
| # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul | |
| # message about why the installation fails) | |
| sudo apt list --installed | grep libc++ | |
| sudo apt-get remove -y libc++1-* libc++abi1-* || true | |
| sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - | |
| sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list' | |
| sudo apt-get update && sudo apt-get install -y stellar-core="$PROTOCOL_${{ matrix.protocol-version }}_CORE_DEBIAN_PKG_VERSION" | |
| echo "Using stellar core version $(stellar-core version)" | |
| echo 'HORIZON_INTEGRATION_TESTS_ENABLE_CAPTIVE_CORE=true' >> $GITHUB_ENV | |
| echo 'HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/usr/bin/stellar-core' >> $GITHUB_ENV | |
| - if: ${{ matrix.ingestion-backend == 'captive-core-remote-storage' }} | |
| name: Setup Captive Core Remote Storage | |
| run: echo 'HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB=true' >> $GITHUB_ENV | |
| - name: Build Horizon reproducible build | |
| run: | | |
| go build -v -trimpath -buildvcs=false ./services/horizon | |
| touch empty | |
| - name: Calculate the source hash | |
| id: calculate_source_hash | |
| run: | | |
| combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1) | |
| echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV" | |
| - name: Restore Horizon binary and integration tests source hash to cache | |
| id: horizon_binary_tests_hash | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ./empty | |
| lookup-only: true | |
| key: ${{ env.COMBINED_SOURCE_HASH }} | |
| - if: ${{ steps.horizon_binary_tests_hash.outputs.cache-hit != 'true' }} | |
| run: go test -race -timeout 65m -v ./services/horizon/internal/integration/... | |
| - name: Save Horizon binary and integration tests source hash to cache | |
| if: ${{ success() && steps.horizon_binary_tests_hash.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/save@v3 | |
| with: | |
| path: ./empty | |
| key: ${{ env.COMBINED_SOURCE_HASH }} | |
| verify-range: | |
| name: Test (and push) verify-range image | |
| runs-on: ubuntu-22.04 | |
| env: | |
| STELLAR_CORE_VERSION: 19.14.0-1500.5664eff4e.focal | |
| CAPTIVE_CORE_STORAGE_PATH: /tmp | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # For pull requests, build and test the PR head not a merge of the PR with the destination. | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| - name: Build and test the Verify Range Docker image | |
| run: | | |
| docker build --build-arg="GO_VERSION=$(sed -En 's/^toolchain[[:space:]]+go([[:digit:].]+)$/\1/p' go.mod)" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/ | |
| # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run | |
| docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range | |
| # Push image | |
| - if: github.ref == 'refs/heads/master' | |
| name: Login to DockerHub | |
| uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - if: github.ref == 'refs/heads/master' | |
| name: Push to DockerHub | |
| run: docker push stellar/horizon-verify-range:latest | |
| ledger-exporter: | |
| name: Test and push the Ledger Exporter images | |
| runs-on: ubuntu-latest | |
| if: false # Disable the job | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # For pull requests, build and test the PR head not a merge of the PR with the destination. | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| - name: Build and test Ledger Exporter images | |
| # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run | |
| run: | | |
| chmod 755 ./exp/lighthorizon/build/build.sh | |
| mkdir $PWD/ledgerexport | |
| # mkdir $PWD/index | |
| ./exp/lighthorizon/build/build.sh ledgerexporter stellar latest false | |
| docker run -e ARCHIVE_TARGET=file:///ledgerexport\ | |
| -e START=5\ | |
| -e END=150\ | |
| -e NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"\ | |
| -e CAPTIVE_CORE_CONFIG="/captive-core-pubnet.cfg"\ | |
| -e HISTORY_ARCHIVE_URLS="https://history.stellar.org/prd/core-live/core_live_001"\ | |
| -v $PWD/ledgerexport:/ledgerexport\ | |
| stellar/lighthorizon-ledgerexporter | |
| # # run map job | |
| # docker run -e NETWORK_PASSPHRASE='pubnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e BATCH_SIZE=64 -e FIRST_CHECKPOINT=64 \ | |
| # -e WORKER_COUNT=1 -e RUN_MODE=map -v $PWD/ledgerexport:/ledgermeta -e TXMETA_SOURCE=file:///ledgermeta -v $PWD/index:/index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch | |
| # # run reduce job | |
| # docker run -e NETWORK_PASSPHRASE='pubnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e MAP_JOB_COUNT=1 -e REDUCE_JOB_COUNT=1 \ | |
| # -e WORKER_COUNT=1 -e RUN_MODE=reduce -v $PWD/index:/index -e INDEX_SOURCE_ROOT=file:///index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch | |
| # Push images | |
| - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lighthorizon' | |
| name: Login to DockerHub | |
| uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lighthorizon' | |
| name: Push to DockerHub | |
| run: | | |
| chmod 755 ./exp/lighthorizon/build/build.sh | |
| ./exp/lighthorizon/build/build.sh ledgerexporter stellar latest true |