Merge branch 'master' into dependabot/go_modules/go_modules-c9dd2c30c2

.github/pull_request_template.md

@@ -23,8 +23,7 @@ change is, and why it is being made, with enough context for anyone to understan
 
 ### Release planning
 
-* [ ] I've updated the relevant CHANGELOG ([here](services/horizon/CHANGELOG.md) for Horizon) if
-  needed with deprecations, added features, breaking changes, and DB schema changes.
+* [ ] I've reviewed the changes in this PR and if I consider them worthwhile for being mentioned on release notes then I have updated the relevant `CHANGELOG.md` within the  component folder structure. For example, if I changed horizon, then I updated ([services/horizon/CHANGELOG.md](services/horizon/CHANGELOG.md). I add a new line item describing the change and reference to this PR. If I don't update a CHANGELOG, I acknowledge this PR's change may not be mentioned in future release notes.  
 * [ ] I've decided if this PR requires a new major/minor version according to
   [semver](https://semver.org/), or if it's mainly a patch change. The PR is targeted at the next
   release branch if it's not a patch change.

.github/workflows/ledgerexporter-release.yml → .github/workflows/galexie-release.yml

@@ -1,31 +1,34 @@
-name: Ledger Exporter release
+name: Galexie Release 
 
 on:
   push:
-    tags: ['ledgerexporter-v*']
+    tags: ['galexie-v*']
 
 jobs:
 
   publish-docker:
-    name: Test and push the Ledger Exporter images
+    name: Test and push docker image
     runs-on: ubuntu-latest
     env:
-      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
-      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
+      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as STELLAR_CORE_VERSION
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
-      STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v} 
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      STELLAR_CORE_VERSION: 22.0.0-2138.721fd0a65.focal
     steps:
+      - name: Set VERSION
+        run: |
+          echo "VERSION=${GITHUB_REF_NAME#galexie-v}" >> $GITHUB_ENV
+
       - uses: actions/checkout@v3
         with:
-          ref: github.sha
+          ref: ${{ github.sha }}
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"    
+          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
       - name: Install captive core
         run: |
           # Workaround for https://github.com/actions/virtual-environments/issues/5245,
@@ -39,11 +42,11 @@ jobs:
           sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
           echo "Using stellar core version $(stellar-core version)"
 
-      - name: Run Ledger Exporter test
-        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+      - name: Run tests 
+        run: go test -v -race -run TestGalexieTestSuite ./services/galexie/...
 
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
+      - name: Build docker
+        run: make -C services/galexie docker-build
 
       # Push images
       - name: Login to DockerHub
@@ -53,4 +56,4 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Push to DockerHub
-        run: make -C exp/services/ledgerexporter docker-push
+        run: make -C services/galexie docker-push

.github/workflows/ledgerexporter.yml → .github/workflows/galexie.yml

@@ -1,22 +1,22 @@
-name: LedgerExporter
+name: Galexie
 
 on:
   push:
     branches: [master]
   pull_request:
 
 jobs:
-  ledger-exporter:
-    name: Test Ledger Exporter
+  galexie:
+    name: Test
     runs-on: ubuntu-latest
     env:
-      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
-      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      # this pins to a version of quickstart:testing that has the same version as LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.0.0-2138.721fd0a65.focal
+      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
+      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
     steps:
       - name: Install captive core
         run: |
@@ -34,12 +34,12 @@ jobs:
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
 
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: Run Ledger Exporter test
-        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+      - name: Run test
+        run: go test -v -race -run TestGalexieTestSuite ./services/galexie/... 

.github/workflows/horizon.yml

@@ -13,7 +13,7 @@ jobs:
         os: [ubuntu-20.04, ubuntu-22.04]
         go: ["1.21", "1.22"]
         pg: [12, 16]
-        protocol-version: [21]
+        protocol-version: [21, 22]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -33,9 +33,12 @@ jobs:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
       HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
-      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
-      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
-      PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
+      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal
+      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.3.1-2007.4ede19620.focal
+      PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.5.1
+      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.0.0-2138.721fd0a65.focal
+      PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.0.0-2138.721fd0a65.focal
+      PROTOCOL_22_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:22.0.0-rc3-101
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres
@@ -98,7 +101,7 @@ jobs:
       - name: Calculate the source hash
         id: calculate_source_hash
         run: |
-          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
+          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_21_RPC_DOCKER_IMG }}-${{ env.PROTOCOL_21_CORE_DEBIAN_PKG_VERSION }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
           echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
 
       - name: Restore Horizon binary and integration tests source hash to cache
@@ -123,7 +126,7 @@ jobs:
     name: Test (and push) verify-range image
     runs-on: ubuntu-22.04
     env:
-      STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal
+      STELLAR_CORE_VERSION: 21.3.1-2007.4ede19620.focal
       CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -5,7 +5,7 @@
 /services/horizon/captive-core
 /services/horizon/horizon
 /services/horizon/stellar-horizon
-/bucket-cache
+**bucket-cache/
 .vscode
 .idea
 debug

Makefile

@@ -18,7 +18,7 @@ XDRS = $(DOWNLOADABLE_XDRS) xdr/Stellar-exporter.x
 
 
 XDRGEN_COMMIT=e2cac557162d99b12ae73b846cf3d5bfe16636de
-XDR_COMMIT=70180d5e8d9caee9e8645ed8a38c36a8cf403cd9
+XDR_COMMIT=529d5176f24c73eeccfa5eba481d4e89c19b1181
 
 .PHONY: xdr xdr-clean xdr-update
 
@@ -34,8 +34,8 @@ friendbot:
 horizon:
 	$(MAKE) -C services/horizon/ binary-build
 
-ledger-exporter:
-	$(MAKE) -C exp/services/ledgerexporter/ docker-build
+galexie:
+	$(MAKE) -C services/galexie/ docker-build
 
 webauth:
 	$(MAKE) -C exp/services/webauth/ docker-build

clients/horizonclient/internal.go

@@ -1,13 +1,16 @@
 package horizonclient
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/stellar/go/protocols/horizon"
 	"github.com/stellar/go/support/clock"
 	"github.com/stellar/go/support/errors"
 )
@@ -27,10 +30,11 @@ func decodeResponse(resp *http.Response, object interface{}, horizonUrl string,
 	}
 	setCurrentServerTime(u.Hostname(), resp.Header["Date"], clock)
 
-	// While this part of code assumes that any error < 200 or error >= 300 is a Horizon problem, it is not
-	// true for the response from /transactions_async endpoint which does give these codes for certain responses
-	// from core.
-	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) && (resp.Request == nil || resp.Request.URL == nil || resp.Request.URL.Path != "/transactions_async") {
+	if isStatusCodeAnError(resp.StatusCode) {
+		if isAsyncTxSubRequest(resp) {
+			return decodeAsyncTxSubResponse(resp, object)
+		}
+
 		horizonError := &Error{
 			Response: resp,
 		}
@@ -47,6 +51,42 @@ func decodeResponse(resp *http.Response, object interface{}, horizonUrl string,
 	return
 }
 
+func isStatusCodeAnError(statusCode int) bool {
+	return !(statusCode >= 200 && statusCode < 300)
+}
+
+func isAsyncTxSubRequest(resp *http.Response) bool {
+	return resp.Request != nil && resp.Request.URL != nil && resp.Request.URL.Path == "/transactions_async"
+}
+
+func decodeAsyncTxSubResponse(resp *http.Response, object interface{}) error {
+	// We need to read the entire body in order to create 2 decoders later.
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "error reading response body")
+	}
+
+	// The first decoder converts the response to AsyncTransactionSubmissionResponse and checks
+	// the hash of the transaction. If the response was not a valid AsyncTransactionSubmissionResponse object,
+	// the hash of the converted object will be empty.
+	asyncRespDecoder := json.NewDecoder(bytes.NewReader(bodyBytes))
+	err = asyncRespDecoder.Decode(&object)
+	if asyncResp, ok := object.(*horizon.AsyncTransactionSubmissionResponse); err == nil && ok && asyncResp.Hash != "" {
+		return nil
+	}
+
+	// Create a new reader for the second decoding. The second decoder decodes to Horizon.Problem object.
+	problemDecoder := json.NewDecoder(bytes.NewReader(bodyBytes))
+	horizonError := Error{
+		Response: resp,
+	}
+	err = problemDecoder.Decode(&horizonError.Problem)
+	if err != nil {
+		return errors.Wrap(err, "error decoding horizon error")
+	}
+	return horizonError
+}
+
 // countParams counts the number of parameters provided
 func countParams(params ...interface{}) int {
 	counter := 0

clients/horizonclient/main_test.go

@@ -480,8 +480,6 @@ func TestAssetsRequest(t *testing.T) {
 		assert.Equal(t, record.Asset.Code, "ABC")
 		assert.Equal(t, record.Asset.Issuer, "GCLWGQPMKXQSPF776IU33AH4PZNOOWNAWGGKVTBQMIC5IMKUNP3E6NVU")
 		assert.Equal(t, record.PT, "1")
-		assert.Equal(t, record.NumAccounts, int32(3))
-		assert.Equal(t, record.Amount, "105.0000000")
 		assert.Equal(t, record.Flags.AuthRevocable, false)
 		assert.Equal(t, record.Flags.AuthRequired, true)
 		assert.Equal(t, record.Flags.AuthImmutable, false)