Karthik/5438/deprecate horizon fields for protocol 22

.gitignore

@@ -5,7 +5,7 @@
 /services/horizon/captive-core
 /services/horizon/horizon
 /services/horizon/stellar-horizon
-/bucket-cache
+**bucket-cache/
 .vscode
 .idea
 debug

clients/horizonclient/main_test.go

@@ -480,8 +480,6 @@ func TestAssetsRequest(t *testing.T) {
 		assert.Equal(t, record.Asset.Code, "ABC")
 		assert.Equal(t, record.Asset.Issuer, "GCLWGQPMKXQSPF776IU33AH4PZNOOWNAWGGKVTBQMIC5IMKUNP3E6NVU")
 		assert.Equal(t, record.PT, "1")
-		assert.Equal(t, record.NumAccounts, int32(3))
-		assert.Equal(t, record.Amount, "105.0000000")
 		assert.Equal(t, record.Flags.AuthRevocable, false)
 		assert.Equal(t, record.Flags.AuthRequired, true)
 		assert.Equal(t, record.Flags.AuthImmutable, false)

protocols/horizon/main.go

@@ -175,16 +175,12 @@ type AssetStat struct {
 	} `json:"_links"`
 
 	base.Asset
-	PT         string `json:"paging_token"`
-	ContractID string `json:"contract_id,omitempty"`
-	// Action needed in release: horizon-v3.0.0: deprecated field
-	NumAccounts          int32 `json:"num_accounts"`
-	NumClaimableBalances int32 `json:"num_claimable_balances"`
-	NumLiquidityPools    int32 `json:"num_liquidity_pools"`
-	NumContracts         int32 `json:"num_contracts"`
-	NumArchivedContracts int32 `json:"num_archived_contracts"`
-	// Action needed in release: horizon-v3.0.0: deprecated field
-	Amount                  string            `json:"amount"`
+	PT                      string            `json:"paging_token"`
+	ContractID              string            `json:"contract_id,omitempty"`
+	NumClaimableBalances    int32             `json:"num_claimable_balances"`
+	NumLiquidityPools       int32             `json:"num_liquidity_pools"`
+	NumContracts            int32             `json:"num_contracts"`
+	NumArchivedContracts    int32             `json:"num_archived_contracts"`
 	Accounts                AssetStatAccounts `json:"accounts"`
 	ClaimableBalancesAmount string            `json:"claimable_balances_amount"`
 	LiquidityPoolsAmount    string            `json:"liquidity_pools_amount"`
@@ -509,33 +505,30 @@ type Transaction struct {
 		// When TransactionSuccess is removed from the SDKs we can remove this HAL link
 		Transaction hal.Link `json:"transaction"`
 	} `json:"_links"`
-	ID                string    `json:"id"`
-	PT                string    `json:"paging_token"`
-	Successful        bool      `json:"successful"`
-	Hash              string    `json:"hash"`
-	Ledger            int32     `json:"ledger"`
-	LedgerCloseTime   time.Time `json:"created_at"`
-	Account           string    `json:"source_account"`
-	AccountMuxed      string    `json:"account_muxed,omitempty"`
-	AccountMuxedID    uint64    `json:"account_muxed_id,omitempty,string"`
-	AccountSequence   int64     `json:"source_account_sequence,string"`
-	FeeAccount        string    `json:"fee_account"`
-	FeeAccountMuxed   string    `json:"fee_account_muxed,omitempty"`
-	FeeAccountMuxedID uint64    `json:"fee_account_muxed_id,omitempty,string"`
-	FeeCharged        int64     `json:"fee_charged,string"`
-	MaxFee            int64     `json:"max_fee,string"`
-	OperationCount    int32     `json:"operation_count"`
-	EnvelopeXdr       string    `json:"envelope_xdr"`
-	ResultXdr         string    `json:"result_xdr"`
-	ResultMetaXdr     string    `json:"result_meta_xdr,omitempty"`
-	FeeMetaXdr        string    `json:"fee_meta_xdr"`
-	MemoType          string    `json:"memo_type"`
-	MemoBytes         string    `json:"memo_bytes,omitempty"`
-	Memo              string    `json:"memo,omitempty"`
-	Signatures        []string  `json:"signatures"`
-	// Action needed in release: horizon-v3.0.0: remove valid_(after|before)
-	ValidAfter         string                    `json:"valid_after,omitempty"`
-	ValidBefore        string                    `json:"valid_before,omitempty"`
+	ID                 string                    `json:"id"`
+	PT                 string                    `json:"paging_token"`
+	Successful         bool                      `json:"successful"`
+	Hash               string                    `json:"hash"`
+	Ledger             int32                     `json:"ledger"`
+	LedgerCloseTime    time.Time                 `json:"created_at"`
+	Account            string                    `json:"source_account"`
+	AccountMuxed       string                    `json:"account_muxed,omitempty"`
+	AccountMuxedID     uint64                    `json:"account_muxed_id,omitempty,string"`
+	AccountSequence    int64                     `json:"source_account_sequence,string"`
+	FeeAccount         string                    `json:"fee_account"`
+	FeeAccountMuxed    string                    `json:"fee_account_muxed,omitempty"`
+	FeeAccountMuxedID  uint64                    `json:"fee_account_muxed_id,omitempty,string"`
+	FeeCharged         int64                     `json:"fee_charged,string"`
+	MaxFee             int64                     `json:"max_fee,string"`
+	OperationCount     int32                     `json:"operation_count"`
+	EnvelopeXdr        string                    `json:"envelope_xdr"`
+	ResultXdr          string                    `json:"result_xdr"`
+	ResultMetaXdr      string                    `json:"result_meta_xdr,omitempty"`
+	FeeMetaXdr         string                    `json:"fee_meta_xdr"`
+	MemoType           string                    `json:"memo_type"`
+	MemoBytes          string                    `json:"memo_bytes,omitempty"`
+	Memo               string                    `json:"memo,omitempty"`
+	Signatures         []string                  `json:"signatures"`
 	Preconditions      *TransactionPreconditions `json:"preconditions,omitempty"`
 	FeeBumpTransaction *FeeBumpTransaction       `json:"fee_bump_transaction,omitempty"`
 	InnerTransaction   *InnerTransaction         `json:"inner_transaction,omitempty"`

services/horizon/internal/actions/asset_test.go

@@ -157,8 +157,6 @@ func TestAssetStats(t *testing.T) {
 		},
 		ClaimableBalancesAmount: "0.0000010",
 		LiquidityPoolsAmount:    "0.0000020",
-		Amount:                  "0.0000001",
-		NumAccounts:             usdAssetStat.NumAccounts,
 		ContractsAmount:         "0.0000000",
 		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
@@ -206,8 +204,6 @@ func TestAssetStats(t *testing.T) {
 		LiquidityPoolsAmount:    "0.0000000",
 		ContractsAmount:         "0.0000000",
 		ArchivedContractsAmount: "0.0000000",
-		Amount:                  "0.0000023",
-		NumAccounts:             etherAssetStat.NumAccounts,
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   etherAssetStat.AssetCode,
@@ -251,10 +247,8 @@ func TestAssetStats(t *testing.T) {
 		},
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
-		Amount:                  "0.0000001",
 		ContractsAmount:         "0.0000000",
 		ArchivedContractsAmount: "0.0000000",
-		NumAccounts:             otherUSDAssetStat.NumAccounts,
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   otherUSDAssetStat.AssetCode,
@@ -300,10 +294,8 @@ func TestAssetStats(t *testing.T) {
 		},
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
-		Amount:                  "0.0000111",
 		ContractsAmount:         "0.0000000",
 		ArchivedContractsAmount: "0.0000000",
-		NumAccounts:             eurAssetStat.NumAccounts,
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   eurAssetStat.AssetCode,
@@ -478,10 +470,8 @@ func TestAssetStatsIssuerDoesNotExist(t *testing.T) {
 		},
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
-		Amount:                  "0.0000001",
 		ContractsAmount:         "0.0000000",
 		ArchivedContractsAmount: "0.0000000",
-		NumAccounts:             usdAssetStat.NumAccounts,
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   usdAssetStat.AssetCode,

services/horizon/internal/integration/liquidity_pool_test.go

@@ -156,8 +156,6 @@ func TestLiquidityPoolHappyPath(t *testing.T) {
 	tt.Equal("credit_alphanum4", stat.Asset.Type)
 	tt.Equal("USD", stat.Asset.Code)
 	tt.Equal(master.Address(), stat.Asset.Issuer)
-	tt.Equal(int32(2), stat.NumAccounts)
-	tt.Equal("225.0000000", stat.Amount)
 	tt.Equal(int32(1), stat.NumLiquidityPools)
 	tt.Equal("775.0000000", stat.LiquidityPoolsAmount)
 

services/horizon/internal/integration/sac_test.go

@@ -1104,9 +1104,7 @@ func assertAssetStats(itest *integration.Test, expected assetStats) {
 	asset := assets.Embedded.Records[0]
 	assert.Equal(itest.CurrentTest(), expected.code, asset.Code)
 	assert.Equal(itest.CurrentTest(), expected.issuer, asset.Issuer)
-	assert.Equal(itest.CurrentTest(), expected.numAccounts, asset.NumAccounts)
 	assert.Equal(itest.CurrentTest(), expected.numAccounts, asset.Accounts.Authorized)
-	assert.Equal(itest.CurrentTest(), expected.balanceAccounts, amount.MustParse(asset.Amount))
 	assert.Equal(itest.CurrentTest(), expected.numContracts, asset.NumContracts)
 	assert.Equal(itest.CurrentTest(), expected.numArchivedContracts, asset.NumArchivedContracts)
 	assert.Equal(itest.CurrentTest(), expected.balanceContracts.String(), parseBalance(itest, asset.ContractsAmount).String())

services/horizon/internal/integration/transaction_preconditions_test.go

@@ -49,20 +49,11 @@ func TestTransactionPreconditionsTimeBounds(t *testing.T) {
 	txHistory, err := itest.Client().TransactionDetail(tx.Hash)
 	assert.NoError(t, err)
 
-	// Check that the deprecated string datetime fields match the new UNIX
-	// timestamp fields.
-	deprecatedHistoryMinTime, err := time.Parse(time.RFC3339, txHistory.ValidAfter)
-	assert.NoError(t, err)
-	deprecatedHistoryMaxTime, err := time.Parse(time.RFC3339, txHistory.ValidBefore)
-	assert.NoError(t, err)
-
 	historyMinTime, err := strconv.ParseInt(txHistory.Preconditions.TimeBounds.MinTime, 10, 64)
 	assert.NoError(t, err)
 	historyMaxTime, err := strconv.ParseInt(txHistory.Preconditions.TimeBounds.MaxTime, 10, 64)
 	assert.NoError(t, err)
 
-	assert.Equal(t, historyMinTime, deprecatedHistoryMinTime.UTC().Unix())
-	assert.Equal(t, historyMaxTime, deprecatedHistoryMaxTime.UTC().Unix())
 	assert.Equal(t, historyMinTime, txParams.Preconditions.TimeBounds.MinTime)
 	assert.Equal(t, historyMaxTime, txParams.Preconditions.TimeBounds.MaxTime)
 }

services/horizon/internal/resourceadapter/asset_stat.go

@@ -39,7 +39,6 @@ func PopulateAssetStat(
 	res.NumLiquidityPools = row.Accounts.LiquidityPools
 	res.NumContracts = row.Contracts.ActiveHolders
 	res.NumArchivedContracts = row.Contracts.ArchivedHolders
-	res.NumAccounts = row.NumAccounts
 	err = populateAssetStatBalances(res, row)
 	if err != nil {
 		return err
@@ -63,7 +62,6 @@ func PopulateAssetStat(
 }
 
 func populateAssetStatBalances(res *protocol.AssetStat, row history.AssetAndContractStat) (err error) {
-	res.Amount, err = amount.IntStringToAmount(row.Balances.Authorized)
 	if err != nil {
 		return errors.Wrap(err, "Invalid amount in PopulateAssetStat")
 	}

services/horizon/internal/resourceadapter/asset_stat_test.go

@@ -67,8 +67,6 @@ func TestPopulateExpAssetStat(t *testing.T) {
 	assert.Equal(t, "770000000000.0000000", res.LiquidityPoolsAmount)
 	assert.Equal(t, "90000000000.0000000", res.ContractsAmount)
 	assert.Equal(t, "70000000000.0000000", res.ArchivedContractsAmount)
-	assert.Equal(t, "10000000000000.0000000", res.Amount)
-	assert.Equal(t, int32(429), res.NumAccounts)
 	assert.Equal(t, horizon.AccountFlags{}, res.Flags)
 	assert.Equal(t, "https://xim.com/.well-known/stellar.toml", res.Links.Toml.Href)
 	assert.Equal(t, "", res.ContractID)
@@ -90,8 +88,6 @@ func TestPopulateExpAssetStat(t *testing.T) {
 	assert.Equal(t, "credit_alphanum4", res.Type)
 	assert.Equal(t, "XIM", res.Code)
 	assert.Equal(t, "GBZ35ZJRIKJGYH5PBKLKOZ5L6EXCNTO7BKIL7DAVVDFQ2ODJEEHHJXIM", res.Issuer)
-	assert.Equal(t, "10000000000000.0000000", res.Amount)
-	assert.Equal(t, int32(429), res.NumAccounts)
 	assert.Equal(
 		t,
 		horizon.AccountFlags{

services/horizon/internal/resourceadapter/transaction.go

@@ -66,10 +66,6 @@
 	dest.Preconditions = &protocol.TransactionPreconditions{}
 
 	if !row.TimeBounds.Null {
-		// Action needed in release: horizon-v3.0.0: remove ValidBefore and ValidAfter
-		dest.ValidBefore = timeString(row.TimeBounds.Upper)
-		dest.ValidAfter = timeString(row.TimeBounds.Lower)
-
 		dest.Preconditions.TimeBounds = &protocol.TransactionPreconditionsTimebounds{
 			MaxTime: timestampString(row.TimeBounds.Upper),
 			MinTime: timestampString(row.TimeBounds.Lower),
@@ -157,7 +153,7 @@
 	return base64.StdEncoding.EncodeToString([]byte(memo)), nil
 }
 
 func timeString(in null.Int) string {
 	if !in.Valid {
 		return ""
 	}

services/horizon/internal/resourceadapter/transaction_test.go

@@ -223,8 +223,6 @@ func TestPopulateTransaction_Preconditions(t *testing.T) {
 	assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false))
 	assert.NotEmpty(t, dest.ResultMetaXdr)
 	p := dest.Preconditions
-	assert.Equal(t, validAfter.Format(time.RFC3339), dest.ValidAfter)
-	assert.Equal(t, validBefore.Format(time.RFC3339), dest.ValidBefore)
 	assert.Equal(t, fmt.Sprint(validAfter.Unix()), p.TimeBounds.MinTime)
 	assert.Equal(t, fmt.Sprint(validBefore.Unix()), p.TimeBounds.MaxTime)
 	assert.Equal(t, minLedger, p.LedgerBounds.MinLedger)

services/regulated-assets-approval-server/internal/configureissuer/configureissuer_test.go

@@ -45,6 +45,9 @@ func TestSetup_accountAlreadyConfigured(t *testing.T) {
 			HomeDomain: "domain.test.com",
 			Sequence:   10,
 		}, nil)
+
+	dummyAsset := horizon.AssetStat{}
+	dummyAsset.Asset.Code = "FOO"
 	horizonMock.
 		On("Assets", horizonclient.AssetRequest{
 			ForAssetCode:   opts.AssetCode,
@@ -53,9 +56,7 @@ func TestSetup_accountAlreadyConfigured(t *testing.T) {
 		}).
 		Return(horizon.AssetsPage{
 			Embedded: struct{ Records []horizon.AssetStat }{
-				Records: []horizon.AssetStat{
-					{Amount: "0.0000001"},
-				},
+				Records: []horizon.AssetStat{dummyAsset},
 			},
 		}, nil)
 

services/ticker/internal/scraper/asset_scraper.go

@@ -23,23 +23,25 @@ import (
 
 // shouldDiscardAsset maps the criteria for discarding an asset from the asset index
 func shouldDiscardAsset(asset hProtocol.AssetStat, shouldValidateTOML bool) bool {
-	if asset.Amount == "" {
+	amt, _ := AddStringsToSum(asset.Balances.AuthorizedToMaintainLiabilities + asset.Balances.Unauthorized + asset.Balances.Authorized)
+	amount := strconv.FormatFloat(amt, 'f', -1, 64)
+
+	if amount == "" {
 		return true
 	}
-	f, _ := strconv.ParseFloat(asset.Amount, 64)
-	if f == 0.0 {
+	if amt == 0.0 {
 		return true
 	}
 	// [StellarX Ticker]: assets need at least some adoption to show up
-	if asset.NumAccounts < 10 {
+	if asset.Accounts.Unauthorized+asset.Accounts.Authorized+asset.Accounts.AuthorizedToMaintainLiabilities < 10 {
 		return true
 	}
 	if asset.Code == "REMOVE" {
 		return true
 	}
 	// [StellarX Ticker]: assets with at least 100 accounts get a pass,
 	// even with toml issues
-	if asset.NumAccounts >= 100 {
+	if asset.Accounts.Unauthorized+asset.Accounts.Authorized+asset.Accounts.AuthorizedToMaintainLiabilities >= 100 {
 		return false
 	}
 
@@ -143,14 +145,28 @@ func isDomainVerified(orgURL string, tomlURL string, hasCurrency bool) bool {
 	}
 	return true
 }
+func AddStringsToSum(values ...string) (float64, error) {
+	var sum float64
+	for _, s := range values {
+		if s == "" {
+			continue // Treat empty string as zero
+		}
+		value, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, fmt.Errorf("failed to convert '%s': %w", s, err)
+		}
+		sum += value
+	}
+	return sum, nil
+}
 
 // makeTomlAsset aggregates Horizon Data with TOML Data
 func makeFinalAsset(
 	asset hProtocol.AssetStat,
 	issuer TOMLIssuer,
 	errors []error,
 ) (t FinalAsset, err error) {
-	amount, err := strconv.ParseFloat(asset.Amount, 64)
+	amount, err := AddStringsToSum(asset.Balances.Authorized + asset.Balances.Unauthorized + asset.Balances.Unauthorized)
 	if err != nil {
 		return
 	}
@@ -159,7 +175,7 @@ func makeFinalAsset(
 		Type:          asset.Type,
 		Code:          asset.Code,
 		Issuer:        asset.Issuer,
-		NumAccounts:   asset.NumAccounts,
+		NumAccounts:   asset.Accounts.Authorized + asset.Accounts.Unauthorized + asset.Accounts.AuthorizedToMaintainLiabilities,
 		AuthRequired:  asset.Flags.AuthRequired,
 		AuthRevocable: asset.Flags.AuthRevocable,
 		Amount:        amount,