Include operation information in the ingest.Change struct (#5536)

This squash commit is the sum total of all the commits for this github issue - #5535

TL;dr 
Extend the ingest.Change struct to include information about the reason for the state change in the ledgerEntry.

* Include operation information in the Change struct

* Dont error out when operation index out of range

* Fix failing unit test

* Reafactor ledger_transaction.GetChanges() to use internal functions

* reformat comments

* Add all types of change reasons to the Change struct. Simplify LedgerTransaction.GetChanges()

* Add LCM and transaction info altogether in change entry

* Wrte help doc for Change Entry

* Add TxHash to LedgerTransaction struct

* reorg comments

* Comments cleanup

* Address PR review changes

* rename fields

* uncommit half baked changes

* Add helpers in intergration tests for creating captive core config

* fix updates to change struct

* Updates to integration.go

* Integration tests for change - part 1

* Undo all changes to parameters_test.go

* Make updates to comments and rename variables

* - Several changes to integration.go to pull out common functions from parameters_test.go to configure captive core
- Add flags to capture state for upgrade version in test.
- Add support to skip container deletion for debugging

- Add test for upgrade related changes

* cosmetic doc style changes

* check err before file close

* Add tx test and in change_test.go

* code cleanup

* Rework test fixtures

* reformat file

* Fix breaking test

* Address code review comments

ingest/change.go

@@ -13,15 +13,90 @@ import (
 // It also provides some helper functions to quickly check if a given
 // change has occurred in an entry.
 //
-// If an entry is created: Pre is nil and Post is not nil.
-// If an entry is updated: Pre is not nil and Post is not nil.
-// If an entry is removed: Pre is not nil and Post is nil.
+// Change represents a modification to a ledger entry, capturing both the before and after states
+// of the entry along with the context that explains what caused the change. It is primarily used to
+// track changes during transactions and/or operations within a transaction
+// and can be helpful in identifying the specific cause of changes to the LedgerEntry state. (https://github.com/stellar/go/issues/5535
+//
+// Behavior:
+//
+//   - **Created entries**: Pre is nil, and Post is not nil.
+//
+//   - **Updated entries**: Both Pre and Post are non-nil.
+//
+//   - **Removed entries**: Pre is not nil, and Post is nil.
+//
+//     A `Change` can be caused primarily by either a transaction or by an operation within a transaction:
+//
+//   - **Operations**:
+//     Each successful operation can cause multiple ledger entry changes.
+//     For example, a path payment operation may affect the source and destination account entries,
+//     as well as potentially modify offers and/or liquidity pools.
+//
+//   - **Transactions**:
+//     Some ledger changes, such as those involving fees or account balances, may be caused by
+//     the transaction itself and may not be tied to a specific operation within a transaction.
+//     For instance, fees for all operations in a transaction are debited from the source account,
+//     triggering ledger changes without operation-specific details.
 type Change struct {
+	// The type of the ledger entry being changed.
 	Type xdr.LedgerEntryType
-	Pre  *xdr.LedgerEntry
+
+	// The state of the LedgerEntry before the change. This will be nil if the entry was created.
+	Pre *xdr.LedgerEntry
+
+	// The state of the LedgerEntry after the change. This will be nil if the entry was removed.
 	Post *xdr.LedgerEntry
+
+	// Specifies why the change occurred, represented as a LedgerEntryChangeReason
+	Reason LedgerEntryChangeReason
+
+	// The index of the operation within the transaction that caused the change.
+	// This field is relevant only when the Reason is LedgerEntryChangeReasonOperation
+	// This field cannot be relied upon when the compactingChangeReader is used.
+	OperationIndex uint32
+
+	// The LedgerTransaction responsible for the change.
+	// It contains details such as transaction hash, envelope, result pair, and fees.
+	// This field is populated only when the Reason is one of:
+	// LedgerEntryChangeReasonTransaction, LedgerEntryChangeReasonOperation or LedgerEntryChangeReasonFee
+	Transaction *LedgerTransaction
+
+	// The LedgerCloseMeta that precipitated the change.
+	// This is useful only when the Change is caused by an upgrade or by an eviction, i.e. outside a transaction
+	// This field is populated only when the Reason is one of:
+	// LedgerEntryChangeReasonUpgrade or LedgerEntryChangeReasonEviction
+	// For changes caused by transaction or operations, look at the Transaction field
+	Ledger *xdr.LedgerCloseMeta
+
+	// Information about the upgrade, if the change occurred as part of an upgrade
+	// This field is relevant only when the Reason is LedgerEntryChangeReasonUpgrade
+	LedgerUpgrade *xdr.LedgerUpgrade
 }
 
+// LedgerEntryChangeReason represents the reason for a ledger entry change.
+type LedgerEntryChangeReason uint16
+
+const (
+	// LedgerEntryChangeReasonUnknown indicates an unknown or unsupported change reason
+	LedgerEntryChangeReasonUnknown LedgerEntryChangeReason = iota
+
+	// LedgerEntryChangeReasonOperation indicates a change caused by an operation in a transaction
+	LedgerEntryChangeReasonOperation
+
+	// LedgerEntryChangeReasonTransaction indicates a change caused by the transaction itself
+	LedgerEntryChangeReasonTransaction
+
+	// LedgerEntryChangeReasonFee indicates a change related to transaction fees.
+	LedgerEntryChangeReasonFee
+
+	// LedgerEntryChangeReasonUpgrade indicates a change caused by a ledger upgrade.
+	LedgerEntryChangeReasonUpgrade
+
+	// LedgerEntryChangeReasonEviction indicates a change caused by entry eviction.
+	LedgerEntryChangeReasonEviction
+)
+
 // String returns a best effort string representation of the change.
 // If the Pre or Post xdr is invalid, the field will be omitted from the string.
 func (c Change) String() string {

ingest/ledger_change_reader.go

@@ -175,6 +175,7 @@ func (r *LedgerChangeReader) Read() (Change, error) {
 			r.pending = append(r.pending, metaChanges...)
 		}
 		return r.Read()
+
 	case evictionChangesState:
 		entries, err := r.lcm.EvictedPersistentLedgerEntries()
 		if err != nil {
@@ -185,21 +186,30 @@ func (r *LedgerChangeReader) Read() (Change, error) {
 			entry := entries[i]
 			// when a ledger entry is evicted it is removed from the ledger
 			changes[i] = Change{
-				Type: entry.Data.Type,
-				Pre:  &entry,
-				Post: nil,
+				Type:   entry.Data.Type,
+				Pre:    &entry,
+				Post:   nil,
+				Reason: LedgerEntryChangeReasonEviction,
+				Ledger: &r.lcm,
 			}
 		}
 		sortChanges(changes)
 		r.pending = append(r.pending, changes...)
 		r.state++
 		return r.Read()
+
 	case upgradeChangesState:
 		// Get upgrade changes
 		if r.upgradeIndex < len(r.LedgerTransactionReader.lcm.UpgradesProcessing()) {
 			changes := GetChangesFromLedgerEntryChanges(
 				r.LedgerTransactionReader.lcm.UpgradesProcessing()[r.upgradeIndex].Changes,
 			)
+			ledgerUpgrades := r.LedgerTransactionReader.lcm.UpgradesProcessing()
+			for i := range changes {
+				changes[i].Reason = LedgerEntryChangeReasonUpgrade
+				changes[i].Ledger = &r.lcm
+				changes[i].LedgerUpgrade = &ledgerUpgrades[r.upgradeIndex].Upgrade
+			}
 			r.pending = append(r.pending, changes...)
 			r.upgradeIndex++
 			return r.Read()

ingest/ledger_transaction.go

@@ -17,6 +17,8 @@ type LedgerTransaction struct {
 	FeeChanges    xdr.LedgerEntryChanges
 	UnsafeMeta    xdr.TransactionMeta
 	LedgerVersion uint32
+	Ledger        xdr.LedgerCloseMeta // This is read-only and not to be modified by downstream functions
+	Hash          xdr.Hash
 }
 
 func (t *LedgerTransaction) txInternalError() bool {
@@ -26,7 +28,21 @@ func (t *LedgerTransaction) txInternalError() bool {
 // GetFeeChanges returns a developer friendly representation of LedgerEntryChanges
 // connected to fees.
 func (t *LedgerTransaction) GetFeeChanges() []Change {
-	return GetChangesFromLedgerEntryChanges(t.FeeChanges)
+	changes := GetChangesFromLedgerEntryChanges(t.FeeChanges)
+	for i := range changes {
+		changes[i].Reason = LedgerEntryChangeReasonFee
+		changes[i].Transaction = t
+	}
+	return changes
+}
+
+func (t *LedgerTransaction) getTransactionChanges(ledgerEntryChanges xdr.LedgerEntryChanges) []Change {
+	changes := GetChangesFromLedgerEntryChanges(ledgerEntryChanges)
+	for i := range changes {
+		changes[i].Reason = LedgerEntryChangeReasonTransaction
+		changes[i].Transaction = t
+	}
+	return changes
 }
 
 // GetChanges returns a developer friendly representation of LedgerEntryChanges.
@@ -42,42 +58,48 @@ func (t *LedgerTransaction) GetChanges() ([]Change, error) {
 		return changes, errors.New("TransactionMeta.V=0 not supported")
 	case 1:
 		v1Meta := t.UnsafeMeta.MustV1()
-		txChanges := GetChangesFromLedgerEntryChanges(v1Meta.TxChanges)
+		// The var `txChanges` reflect the ledgerEntryChanges that are changed because of the transaction as a whole
+		txChanges := t.getTransactionChanges(v1Meta.TxChanges)
 		changes = append(changes, txChanges...)
 
 		// Ignore operations meta if txInternalError https://github.com/stellar/go/issues/2111
 		if t.txInternalError() && t.LedgerVersion <= 12 {
 			return changes, nil
 		}
 
-		for _, operationMeta := range v1Meta.Operations {
-			opChanges := GetChangesFromLedgerEntryChanges(
-				operationMeta.Changes,
-			)
+		// These changes reflect the ledgerEntry changes that were caused by the operations in the transaction
+		// Populate the operationInfo for these changes in the `Change` struct
+
+		operationMeta := v1Meta.Operations
+		//	operationMeta is a list of lists.
+		//	Each element in operationMeta is a list of ledgerEntryChanges
+		//	caused by the operation at that index of the element
+		for opIdx := range operationMeta {
+			opChanges := t.operationChanges(v1Meta.Operations, uint32(opIdx))
 			changes = append(changes, opChanges...)
 		}
 	case 2, 3:
 		var (
-			beforeChanges, afterChanges xdr.LedgerEntryChanges
-			operationMeta               []xdr.OperationMeta
+			txBeforeChanges, txAfterChanges xdr.LedgerEntryChanges
+			operationMeta                   []xdr.OperationMeta
 		)
 
 		switch t.UnsafeMeta.V {
 		case 2:
 			v2Meta := t.UnsafeMeta.MustV2()
-			beforeChanges = v2Meta.TxChangesBefore
-			afterChanges = v2Meta.TxChangesAfter
+			txBeforeChanges = v2Meta.TxChangesBefore
+			txAfterChanges = v2Meta.TxChangesAfter
 			operationMeta = v2Meta.Operations
 		case 3:
 			v3Meta := t.UnsafeMeta.MustV3()
-			beforeChanges = v3Meta.TxChangesBefore
-			afterChanges = v3Meta.TxChangesAfter
+			txBeforeChanges = v3Meta.TxChangesBefore
+			txAfterChanges = v3Meta.TxChangesAfter
 			operationMeta = v3Meta.Operations
 		default:
 			panic("Invalid meta version, expected 2 or 3")
 		}
 
-		txChangesBefore := GetChangesFromLedgerEntryChanges(beforeChanges)
+		txChangesBefore := t.getTransactionChanges(txBeforeChanges)
 		changes = append(changes, txChangesBefore...)
 
 		// Ignore operations meta and txChangesAfter if txInternalError
@@ -86,14 +108,15 @@ func (t *LedgerTransaction) GetChanges() ([]Change, error) {
 			return changes, nil
 		}
 
-		for _, operationMeta := range operationMeta {
-			opChanges := GetChangesFromLedgerEntryChanges(
-				operationMeta.Changes,
-			)
+		//	operationMeta is a list of lists.
+		//	Each element in operationMeta is a list of ledgerEntryChanges
+		//	caused by the operation at that index of the element
+		for opIdx := range operationMeta {
+			opChanges := t.operationChanges(operationMeta, uint32(opIdx))
 			changes = append(changes, opChanges...)
 		}
 
-		txChangesAfter := GetChangesFromLedgerEntryChanges(afterChanges)
+		txChangesAfter := t.getTransactionChanges(txAfterChanges)
 		changes = append(changes, txChangesAfter...)
 	default:
 		return changes, errors.New("Unsupported TransactionMeta version")
@@ -114,15 +137,13 @@ func (t *LedgerTransaction) GetOperation(index uint32) (xdr.Operation, bool) {
 // GetOperationChanges returns a developer friendly representation of LedgerEntryChanges.
 // It contains only operation changes.
 func (t *LedgerTransaction) GetOperationChanges(operationIndex uint32) ([]Change, error) {
-	changes := []Change{}
-
 	if t.UnsafeMeta.V == 0 {
-		return changes, errors.New("TransactionMeta.V=0 not supported")
+		return []Change{}, errors.New("TransactionMeta.V=0 not supported")
 	}
 
 	// Ignore operations meta if txInternalError https://github.com/stellar/go/issues/2111
 	if t.txInternalError() && t.LedgerVersion <= 12 {
-		return changes, nil
+		return []Change{}, nil
 	}
 
 	var operationMeta []xdr.OperationMeta
@@ -134,21 +155,26 @@ func (t *LedgerTransaction) GetOperationChanges(operationIndex uint32) ([]Change
 	case 3:
 		operationMeta = t.UnsafeMeta.MustV3().Operations
 	default:
-		return changes, errors.New("Unsupported TransactionMeta version")
+		return []Change{}, errors.New("Unsupported TransactionMeta version")
 	}
 
-	return operationChanges(operationMeta, operationIndex), nil
+	return t.operationChanges(operationMeta, operationIndex), nil
 }
 
-func operationChanges(ops []xdr.OperationMeta, index uint32) []Change {
+func (t *LedgerTransaction) operationChanges(ops []xdr.OperationMeta, index uint32) []Change {
 	if int(index) >= len(ops) {
 		return []Change{}
 	}
 
 	operationMeta := ops[index]
-	return GetChangesFromLedgerEntryChanges(
-		operationMeta.Changes,
-	)
+	changes := GetChangesFromLedgerEntryChanges(operationMeta.Changes)
+
+	for i := range changes {
+		changes[i].Reason = LedgerEntryChangeReasonOperation
+		changes[i].Transaction = t
+		changes[i].OperationIndex = index
+	}
+	return changes
 }
 
 // GetDiagnosticEvents returns all contract events emitted by a given operation.

ingest/ledger_transaction_reader.go

@@ -94,6 +94,8 @@ func (reader *LedgerTransactionReader) Read() (LedgerTransaction, error) {
 		UnsafeMeta:    reader.lcm.TxApplyProcessing(i),
 		FeeChanges:    reader.lcm.FeeProcessing(i),
 		LedgerVersion: uint32(reader.lcm.LedgerHeaderHistoryEntry().Header.LedgerVersion),
+		Ledger:        reader.lcm,
+		Hash:          hash,
 	}, nil
 }