Include operation information in the ingest.Change struct

ingest/change.go

@@ -13,15 +13,76 @@ import (
 // It also provides some helper functions to quickly check if a given
 // change has occurred in an entry.
 //
-// If an entry is created: Pre is nil and Post is not nil.
-// If an entry is updated: Pre is not nil and Post is not nil.
-// If an entry is removed: Pre is not nil and Post is nil.
+// Change represents a modification to a ledger entry, capturing both the before and after states
+// of the entry along with the context that explains what caused the change. It is primarily used to
+// track changes during transactions and/or operations within a transaction
+// and can be helpful in identifying the specific cause of changes to the LedgerEntry state. (https://github.com/stellar/go/issues/5535
+//
+// Behavior:
+//
+//   - **Created entries**: Pre is nil, and Post is not nil.
+//
+//   - **Updated entries**: Both Pre and Post are non-nil.
+//
+//   - **Removed entries**: Pre is not nil, and Post is nil.
+//
+//     A `Change` can be caused primarily by either a transaction or by an operation within a transaction:
+//
+//   - **Operations**:
+//     Each successful operation can cause multiple ledger entry changes.
+//     For example, a path payment operation may affect the source and destination account entries,
+//     as well as potentially modify offers and/or liquidity pools.
+//
+//   - **Transactions**:
+//     Some ledger changes, such as those involving fees or account balances, may be caused by
+//     the transaction itself and may not be tied to a specific operation within a transaction.
+//     For instance, fees for all operations in a transaction are debited from the source account,
+//     triggering ledger changes without operation-specific details.
+//
+// Fields:
+//
+//   - Type: The type of the ledger entry being changed
+//
+//   - Pre: The state of the ledger entry before the change. This will be nil if the entry was created.
+//
+//   - Post: The state of the ledger entry after the change. This will be nil if the entry was removed.
+//
+//   - Reason: The reason for the ledger entry change, represented by LedgerEntryChangeReason.
+//
+//   - OperationIdx: The index of the operation in the transaction that caused the change.
+//     This field is relevant when the change is due to an operation within a transaction
+//     and won't be used when the change is compacted.
+//
+//   - Tx: A reference to the LedgerTransaction that caused the change.
+//     Contains information about Transaction, Hash, TxResultPair etc
+//
+//   - Lcm: The LedgerCloseMeta that precipitated the change.
+//     This is useful only when the Change is caused by an upgrade or by an eviction, i.e. outside a Transaction
+//     For changes caused by transaction or operations, look at the Tx field
+//
+//   - LedgerUpgrade: Information about the upgrade, if the change occurred as part of an upgrade.
 type Change struct {
-	Type xdr.LedgerEntryType
-	Pre  *xdr.LedgerEntry
-	Post *xdr.LedgerEntry
+	Type          xdr.LedgerEntryType
+	Pre           *xdr.LedgerEntry
+	Post          *xdr.LedgerEntry
+	Reason        LedgerEntryChangeReason
+	OperationIdx  uint32
+	Tx            *LedgerTransaction
+	Lcm           *xdr.LedgerCloseMeta
+	LedgerUpgrade *xdr.LedgerUpgrade
 }
 
+type LedgerEntryChangeReason uint16
+
+const (
+	Unknown LedgerEntryChangeReason = iota
+	Operation
+	Transaction
+	FeeChange
+	Upgrade
+	Eviction
+)
+
 // String returns a best effort string representation of the change.
 // If the Pre or Post xdr is invalid, the field will be omitted from the string.
 func (c Change) String() string {

ingest/ledger_change_reader.go

@@ -175,6 +175,7 @@ func (r *LedgerChangeReader) Read() (Change, error) {
 			r.pending = append(r.pending, metaChanges...)
 		}
 		return r.Read()
+
 	case evictionChangesState:
 		entries, err := r.lcm.EvictedPersistentLedgerEntries()
 		if err != nil {
@@ -185,21 +186,30 @@ func (r *LedgerChangeReader) Read() (Change, error) {
 			entry := entries[i]
 			// when a ledger entry is evicted it is removed from the ledger
 			changes[i] = Change{
-				Type: entry.Data.Type,
-				Pre:  &entry,
-				Post: nil,
+				Type:   entry.Data.Type,
+				Pre:    &entry,
+				Post:   nil,
+				Reason: Eviction,
+				Lcm:    &r.lcm,
 			}
 		}
 		sortChanges(changes)
 		r.pending = append(r.pending, changes...)
 		r.state++
 		return r.Read()
+
 	case upgradeChangesState:
 		// Get upgrade changes
 		if r.upgradeIndex < len(r.LedgerTransactionReader.lcm.UpgradesProcessing()) {
 			changes := GetChangesFromLedgerEntryChanges(
 				r.LedgerTransactionReader.lcm.UpgradesProcessing()[r.upgradeIndex].Changes,
 			)
+			ledgerUpgrades := r.LedgerTransactionReader.lcm.UpgradesProcessing()
+			for _, change := range changes {
+				change.Reason = Upgrade
+				change.Lcm = &r.lcm
+				change.LedgerUpgrade = &ledgerUpgrades[r.upgradeIndex].Upgrade
+			}
 			r.pending = append(r.pending, changes...)
 			r.upgradeIndex++
 			return r.Read()

ingest/ledger_transaction.go

@@ -17,6 +17,8 @@ type LedgerTransaction struct {
 	FeeChanges    xdr.LedgerEntryChanges
 	UnsafeMeta    xdr.TransactionMeta
 	LedgerVersion uint32
+	Lcm           *xdr.LedgerCloseMeta // This is read-only and not to be modified by downstream functions
+	Hash          *xdr.Hash
 }
 
 func (t *LedgerTransaction) txInternalError() bool {
@@ -26,7 +28,21 @@ func (t *LedgerTransaction) txInternalError() bool {
 // GetFeeChanges returns a developer friendly representation of LedgerEntryChanges
 // connected to fees.
 func (t *LedgerTransaction) GetFeeChanges() []Change {
-	return GetChangesFromLedgerEntryChanges(t.FeeChanges)
+	changes := GetChangesFromLedgerEntryChanges(t.FeeChanges)
+	for _, change := range changes {
+		change.Reason = FeeChange
+		change.Tx = t
+	}
+	return changes
+}
+
+func (t *LedgerTransaction) getTransactionChanges(ledgerEntryChanges xdr.LedgerEntryChanges) []Change {
+	changes := GetChangesFromLedgerEntryChanges(ledgerEntryChanges)
+	for _, change := range changes {
+		change.Reason = Transaction
+		change.Tx = t
+	}
+	return changes
 }
 
 // GetChanges returns a developer friendly representation of LedgerEntryChanges.
@@ -42,42 +58,48 @@ func (t *LedgerTransaction) GetChanges() ([]Change, error) {
 		return changes, errors.New("TransactionMeta.V=0 not supported")
 	case 1:
 		v1Meta := t.UnsafeMeta.MustV1()
-		txChanges := GetChangesFromLedgerEntryChanges(v1Meta.TxChanges)
+		// The var `txChanges` reflect the ledgerEntryChanges that are changed because of the transaction as a whole
+		txChanges := t.getTransactionChanges(v1Meta.TxChanges)
 		changes = append(changes, txChanges...)
 
 		// Ignore operations meta if txInternalError https://github.com/stellar/go/issues/2111
 		if t.txInternalError() && t.LedgerVersion <= 12 {
 			return changes, nil
 		}
 
-		for _, operationMeta := range v1Meta.Operations {
-			opChanges := GetChangesFromLedgerEntryChanges(
-				operationMeta.Changes,
-			)
+		// These changes reflect the ledgerEntry changes that were caused by the operations in the transaction
+		// Populate the operationInfo for these changes in the `Change` struct
+
+		operationMeta := v1Meta.Operations
+		//	operationMeta is a list of lists.
+		//	Each element in operationMeta is a list of ledgerEntryChanges
+		//	caused by the operation at that index of the element
+		for opIdx := range operationMeta {
+			opChanges := t.operationChanges(v1Meta.Operations, uint32(opIdx))
 			changes = append(changes, opChanges...)
 		}
 	case 2, 3:
 		var (
-			beforeChanges, afterChanges xdr.LedgerEntryChanges
-			operationMeta               []xdr.OperationMeta
+			txBeforeChanges, txAfterChanges xdr.LedgerEntryChanges
+			operationMeta                   []xdr.OperationMeta
 		)
 
 		switch t.UnsafeMeta.V {
 		case 2:
 			v2Meta := t.UnsafeMeta.MustV2()
-			beforeChanges = v2Meta.TxChangesBefore
-			afterChanges = v2Meta.TxChangesAfter
+			txBeforeChanges = v2Meta.TxChangesBefore
+			txAfterChanges = v2Meta.TxChangesAfter
 			operationMeta = v2Meta.Operations
 		case 3:
 			v3Meta := t.UnsafeMeta.MustV3()
-			beforeChanges = v3Meta.TxChangesBefore
-			afterChanges = v3Meta.TxChangesAfter
+			txBeforeChanges = v3Meta.TxChangesBefore
+			txAfterChanges = v3Meta.TxChangesAfter
 			operationMeta = v3Meta.Operations
 		default:
 			panic("Invalid meta version, expected 2 or 3")
 		}
 
-		txChangesBefore := GetChangesFromLedgerEntryChanges(beforeChanges)
+		txChangesBefore := t.getTransactionChanges(txBeforeChanges)
 		changes = append(changes, txChangesBefore...)
 
 		// Ignore operations meta and txChangesAfter if txInternalError
@@ -86,14 +108,15 @@ func (t *LedgerTransaction) GetChanges() ([]Change, error) {
 			return changes, nil
 		}
 
-		for _, operationMeta := range operationMeta {
-			opChanges := GetChangesFromLedgerEntryChanges(
-				operationMeta.Changes,
-			)
+		//	operationMeta is a list of lists.
+		//	Each element in operationMeta is a list of ledgerEntryChanges
+		//	caused by the operation at that index of the element
+		for opIdx := range operationMeta {
+			opChanges := t.operationChanges(operationMeta, uint32(opIdx))
 			changes = append(changes, opChanges...)
 		}
 
-		txChangesAfter := GetChangesFromLedgerEntryChanges(afterChanges)
+		txChangesAfter := t.getTransactionChanges(txAfterChanges)
 		changes = append(changes, txChangesAfter...)
 	default:
 		return changes, errors.New("Unsupported TransactionMeta version")
@@ -114,15 +137,13 @@ func (t *LedgerTransaction) GetOperation(index uint32) (xdr.Operation, bool) {
 // GetOperationChanges returns a developer friendly representation of LedgerEntryChanges.
 // It contains only operation changes.
 func (t *LedgerTransaction) GetOperationChanges(operationIndex uint32) ([]Change, error) {
-	changes := []Change{}
-
 	if t.UnsafeMeta.V == 0 {
-		return changes, errors.New("TransactionMeta.V=0 not supported")
+		return []Change{}, errors.New("TransactionMeta.V=0 not supported")
 	}
 
 	// Ignore operations meta if txInternalError https://github.com/stellar/go/issues/2111
 	if t.txInternalError() && t.LedgerVersion <= 12 {
-		return changes, nil
+		return []Change{}, nil
 	}
 
 	var operationMeta []xdr.OperationMeta
@@ -134,21 +155,26 @@ func (t *LedgerTransaction) GetOperationChanges(operationIndex uint32) ([]Change
 	case 3:
 		operationMeta = t.UnsafeMeta.MustV3().Operations
 	default:
-		return changes, errors.New("Unsupported TransactionMeta version")
+		return []Change{}, errors.New("Unsupported TransactionMeta version")
 	}
 
-	return operationChanges(operationMeta, operationIndex), nil
+	return t.operationChanges(operationMeta, operationIndex), nil
 }
 
-func operationChanges(ops []xdr.OperationMeta, index uint32) []Change {
+func (t *LedgerTransaction) operationChanges(ops []xdr.OperationMeta, index uint32) []Change {
 	if int(index) >= len(ops) {
-		return []Change{}
+		return []Change{} // TODO - operations_processor somehow seems to be failing without this
 	}
 
 	operationMeta := ops[index]
-	return GetChangesFromLedgerEntryChanges(
-		operationMeta.Changes,
-	)
+	changes := GetChangesFromLedgerEntryChanges(operationMeta.Changes)
+
+	for _, change := range changes {
+		change.Reason = Operation
+		change.Tx = t
+		change.OperationIdx = index
+	}
+	return changes
 }
 
 // GetDiagnosticEvents returns all contract events emitted by a given operation.

ingest/ledger_transaction_reader.go

@@ -94,6 +94,8 @@ func (reader *LedgerTransactionReader) Read() (LedgerTransaction, error) {
 		UnsafeMeta:    reader.lcm.TxApplyProcessing(i),
 		FeeChanges:    reader.lcm.FeeProcessing(i),
 		LedgerVersion: uint32(reader.lcm.LedgerHeaderHistoryEntry().Header.LedgerVersion),
+		Lcm:           &reader.lcm,
+		Hash:          &hash,
 	}, nil
 }
 

ingest/ledger_transaction_test.go

@@ -336,7 +336,7 @@ func TestFeeMetaAndOperationsChangesSeparate(t *testing.T) {
 	assert.Equal(t, operationChanges[0].Pre.Data.MustAccount().Balance, xdr.Int64(300))
 	assert.Equal(t, operationChanges[0].Post.Data.MustAccount().Balance, xdr.Int64(400))
 
-	// Ignore operation meta if tx result is txInternalError
+	// Ignore operation meta if Tx result is txInternalError
 	// https://github.com/stellar/go/issues/2111
 	tx.Result.Result.Result.Code = xdr.TransactionResultCodeTxInternalError
 	metaChanges, err = tx.GetChanges()

services/horizon/internal/ingest/processors/operations_processor.go

@@ -324,7 +324,7 @@ func addAccountAndMuxedAccountDetails(result map[string]interface{}, a xdr.Muxed
 		// _muxed_id fields should had ideally been stored in the DB as a string instead of uint64
 		// due to Javascript not being able to handle them, see https://github.com/stellar/go/issues/3714
 		// However, we released this code in the wild before correcting it. Thus, what we do is
-		// work around it (by preprocessing it into a string) in Operation.UnmarshalDetails()
+		// work around it (by preprocessing it into a string) in OperationChange.UnmarshalDetails()
 		result[prefix+"_muxed_id"] = uint64(a.Med25519.Id)
 	}
 }

xdr/transaction_envelope.go

@@ -215,6 +215,10 @@ func (e TransactionEnvelope) Preconditions() Preconditions {
 // Note for fee bump transactions, Operations() returns the operations
 // of the inner transaction
 func (e TransactionEnvelope) Operations() []Operation {
+	// This is not expected to happen.
+	if (e == TransactionEnvelope{}) {
+		return []Operation{}
+	}
 	switch e.Type {
 	case EnvelopeTypeEnvelopeTypeTxFeeBump:
 		return e.FeeBump.Tx.InnerTx.V1.Tx.Operations