SOM: onchain report observation tracking

cmd/monitoring/main.go

@@ -57,27 +57,51 @@ func main() {
 		return
 	}
 
+	// per-feed sources
 	feedBalancesSourceFactory := monitoring.NewFeedBalancesSourceFactory(
 		chainReader,
 		logger.With(log, "component", "source-feed-balances"),
 	)
+	txDetailsSourceFactory := monitoring.NewTxDetailsSourceFactory(
+		chainReader,
+		logger.With(log, "component", "source-tx-details"),
+	)
+	monitor.SourceFactories = append(monitor.SourceFactories,
+		feedBalancesSourceFactory,
+		txDetailsSourceFactory,
+	)
+
+	// network sources
 	nodeBalancesSourceFactory := monitoring.NewNodeBalancesSourceFactory(
 		chainReader,
 		logger.With(log, "component", "source-node-balances"),
 	)
-	monitor.SourceFactories = append(monitor.SourceFactories, feedBalancesSourceFactory)
-	monitor.NetworkSourceFactories = append(monitor.NetworkSourceFactories, nodeBalancesSourceFactory)
+	monitor.NetworkSourceFactories = append(monitor.NetworkSourceFactories,
+		nodeBalancesSourceFactory,
+	)
 
+	// per-feed exporters
 	feedBalancesExporterFactory := exporter.NewFeedBalancesFactory(
 		logger.With(log, "component", "solana-prom-exporter"),
 		metrics.NewFeedBalances(logger.With(log, "component", "solana-metrics")),
 	)
+	reportObservationsFactory := exporter.NewReportObservationsFactory(
+		logger.With(log, "component", "solana-prome-exporter"),
+		metrics.NewReportObservations(logger.With(log, "component", "solana-metrics")),
+	)
+	monitor.ExporterFactories = append(monitor.ExporterFactories,
+		feedBalancesExporterFactory,
+		reportObservationsFactory,
+	)
+
+	// network exporters
 	nodeBalancesExporterFactory := exporter.NewNodeBalancesFactory(
 		logger.With(log, "component", "solana-prom-exporter"),
 		metrics.NewNodeBalances,
 	)
-	monitor.ExporterFactories = append(monitor.ExporterFactories, feedBalancesExporterFactory)
-	monitor.NetworkExporterFactories = append(monitor.NetworkExporterFactories, nodeBalancesExporterFactory)
+	monitor.NetworkExporterFactories = append(monitor.NetworkExporterFactories,
+		nodeBalancesExporterFactory,
+	)
 
 	monitor.Run()
 	log.Infow("monitor stopped")

go.mod

@@ -4,9 +4,9 @@ go 1.21
 
 require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
-	github.com/gagliardetto/binary v0.7.1
+	github.com/gagliardetto/binary v0.7.7
 	github.com/gagliardetto/gofuzz v1.2.2
-	github.com/gagliardetto/solana-go v1.4.1-0.20220428092759-5250b4abbb27
+	github.com/gagliardetto/solana-go v1.8.4
 	github.com/gagliardetto/treeout v0.1.4
 	github.com/gagliardetto/utilz v0.1.1
 	github.com/google/uuid v1.3.1
@@ -83,12 +83,14 @@ require (
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	go.mongodb.org/mongo-driver v1.15.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect
@@ -104,6 +106,7 @@ require (
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/grpc v1.58.3 // indirect

integration-tests/go.mod

@@ -7,7 +7,7 @@ replace github.com/smartcontractkit/chainlink-solana => ../
 require (
 	github.com/ethereum/go-ethereum v1.13.8
 	github.com/gagliardetto/binary v0.7.7
-	github.com/gagliardetto/solana-go v1.8.3
+	github.com/gagliardetto/solana-go v1.8.4
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
 	github.com/onsi/gomega v1.30.0
@@ -415,7 +415,7 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.9 // indirect
-	go.mongodb.org/mongo-driver v1.12.0 // indirect
+	go.mongodb.org/mongo-driver v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 // indirect
 	go.opentelemetry.io/collector/semconv v0.87.0 // indirect

integration-tests/go.sum

@@ -480,8 +480,8 @@ github.com/gagliardetto/binary v0.7.7 h1:QZpT38+sgoPg+TIQjH94sLbl/vX+nlIRA37pEyO
 github.com/gagliardetto/binary v0.7.7/go.mod h1:mUuay5LL8wFVnIlecHakSZMvcdqfs+CsotR5n77kyjM=
 github.com/gagliardetto/gofuzz v1.2.2 h1:XL/8qDMzcgvR4+CyRQW9UGdwPRPMHVJfqQ/uMvSUuQw=
 github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY=
-github.com/gagliardetto/solana-go v1.8.3 h1:YHcxw2nLNdjyy5iR+ZUcpljSrjZrWRr8OusuCTr70p4=
-github.com/gagliardetto/solana-go v1.8.3/go.mod h1:i+7aAyNDTHG0jK8GZIBSI4OVvDqkt2Qx+LklYclRNG8=
+github.com/gagliardetto/solana-go v1.8.4 h1:vmD/JmTlonyXGy39bAo0inMhmbdAwV7rXZtLDMZeodE=
+github.com/gagliardetto/solana-go v1.8.4/go.mod h1:i+7aAyNDTHG0jK8GZIBSI4OVvDqkt2Qx+LklYclRNG8=
 github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdFpgwaw=
 github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok=
 github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays=
@@ -1580,10 +1580,8 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
-github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
@@ -1632,8 +1630,8 @@ go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R7
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
 go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
-go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
-go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=

pkg/monitoring/config/feed_config.go

@@ -72,7 +72,7 @@ func (s SolanaFeedConfig) GetMultiply() *big.Int {
 	return s.Multiply
 }
 
-// GetID returns the state account's address as that uniquely
+// GetContractAddress returns the state account's address as that uniquely
 // identifies a feed on Solana. In Solana, a program is stateless and we
 // use the same program for all feeds so we can't use the program
 // account's address.

pkg/monitoring/exporter/feedbalances.go

@@ -29,7 +29,7 @@ type feedBalancesFactory struct {
 func (p *feedBalancesFactory) NewExporter(
 	params commonMonitoring.ExporterParams,
 ) (commonMonitoring.Exporter, error) {
-	return &feeBalances{
+	return &feedBalances{
 		params.ChainConfig,
 		params.FeedConfig,
 		p.log,
@@ -39,7 +39,7 @@ func (p *feedBalancesFactory) NewExporter(
 	}, nil
 }
 
-type feeBalances struct {
+type feedBalances struct {
 	chainConfig commonMonitoring.ChainConfig
 	feedConfig  commonMonitoring.FeedConfig
 
@@ -50,7 +50,7 @@ type feeBalances struct {
 	addresses   map[string]solana.PublicKey
 }
 
-func (p *feeBalances) Export(ctx context.Context, data interface{}) {
+func (p *feedBalances) Export(ctx context.Context, data interface{}) {
 	balances, isBalances := data.(types.Balances)
 	if !isBalances {
 		return
@@ -70,17 +70,17 @@ func (p *feeBalances) Export(ctx context.Context, data interface{}) {
 		}
 		p.metrics.SetBalance(
 			balance,
-			metrics.FeedBalanceInput{
-				BalanceAccountName: balanceAccountName,
-				AccountAddress:     address.String(),
-				FeedID:             p.feedConfig.GetContractAddress(),
-				ChainID:            p.chainConfig.GetChainID(),
-				ContractStatus:     p.feedConfig.GetContractStatus(),
-				ContractType:       p.feedConfig.GetContractType(),
-				FeedName:           p.feedConfig.GetName(),
-				FeedPath:           p.feedConfig.GetPath(),
-				NetworkID:          p.chainConfig.GetNetworkID(),
-				NetworkName:        p.chainConfig.GetNetworkName(),
+			balanceAccountName,
+			metrics.FeedInput{
+				AccountAddress: address.String(),
+				FeedID:         p.feedConfig.GetContractAddress(),
+				ChainID:        p.chainConfig.GetChainID(),
+				ContractStatus: p.feedConfig.GetContractStatus(),
+				ContractType:   p.feedConfig.GetContractType(),
+				FeedName:       p.feedConfig.GetName(),
+				FeedPath:       p.feedConfig.GetPath(),
+				NetworkID:      p.chainConfig.GetNetworkID(),
+				NetworkName:    p.chainConfig.GetNetworkName(),
 			},
 		)
 	}
@@ -90,21 +90,20 @@ func (p *feeBalances) Export(ctx context.Context, data interface{}) {
 	p.addresses = balances.Addresses
 }
 
-func (p *feeBalances) Cleanup(_ context.Context) {
+func (p *feedBalances) Cleanup(_ context.Context) {
 	p.addressesMu.Lock()
 	defer p.addressesMu.Unlock()
 	for balanceAccountName, address := range p.addresses {
-		p.metrics.Cleanup(metrics.FeedBalanceInput{
-			BalanceAccountName: balanceAccountName,
-			AccountAddress:     address.String(),
-			FeedID:             p.feedConfig.GetContractAddress(),
-			ChainID:            p.chainConfig.GetChainID(),
-			ContractStatus:     p.feedConfig.GetContractStatus(),
-			ContractType:       p.feedConfig.GetContractType(),
-			FeedName:           p.feedConfig.GetName(),
-			FeedPath:           p.feedConfig.GetPath(),
-			NetworkID:          p.chainConfig.GetNetworkID(),
-			NetworkName:        p.chainConfig.GetNetworkName(),
+		p.metrics.Cleanup(balanceAccountName, metrics.FeedInput{
+			AccountAddress: address.String(),
+			FeedID:         p.feedConfig.GetContractAddress(),
+			ChainID:        p.chainConfig.GetChainID(),
+			ContractStatus: p.feedConfig.GetContractStatus(),
+			ContractType:   p.feedConfig.GetContractType(),
+			FeedName:       p.feedConfig.GetName(),
+			FeedPath:       p.feedConfig.GetPath(),
+			NetworkID:      p.chainConfig.GetNetworkID(),
+			NetworkName:    p.chainConfig.GetNetworkName(),
 		})
 	}
 }

pkg/monitoring/exporter/feedbalances_test.go

@@ -37,34 +37,33 @@ func TestFeedBalances(t *testing.T) {
 		for _, accountName := range types.FeedBalanceAccountNames {
 			mockMetrics.On("SetBalance",
 				balances.Values[accountName],
-				metrics.FeedBalanceInput{
-					BalanceAccountName: accountName,
-					AccountAddress:     balances.Addresses[accountName].String(),
-					FeedID:             feedConfig.GetID(),
-					ChainID:            chainConfig.GetChainID(),
-					ContractStatus:     feedConfig.GetContractStatus(),
-					ContractType:       feedConfig.GetContractType(),
-					FeedName:           feedConfig.GetName(),
-					FeedPath:           feedConfig.GetPath(),
-					NetworkID:          chainConfig.GetNetworkID(),
-					NetworkName:        chainConfig.GetNetworkName(),
+				accountName,
+				metrics.FeedInput{
+					AccountAddress: balances.Addresses[accountName].String(),
+					FeedID:         feedConfig.GetID(),
+					ChainID:        chainConfig.GetChainID(),
+					ContractStatus: feedConfig.GetContractStatus(),
+					ContractType:   feedConfig.GetContractType(),
+					FeedName:       feedConfig.GetName(),
+					FeedPath:       feedConfig.GetPath(),
+					NetworkID:      chainConfig.GetNetworkID(),
+					NetworkName:    chainConfig.GetNetworkName(),
 				},
 			)
 		}
 		exporter.Export(ctx, balances)
 
 		for _, accountName := range types.FeedBalanceAccountNames {
-			mockMetrics.On("Cleanup", metrics.FeedBalanceInput{
-				BalanceAccountName: accountName,
-				AccountAddress:     balances.Addresses[accountName].String(),
-				FeedID:             feedConfig.GetID(),
-				ChainID:            chainConfig.GetChainID(),
-				ContractStatus:     feedConfig.GetContractStatus(),
-				ContractType:       feedConfig.GetContractType(),
-				FeedName:           feedConfig.GetName(),
-				FeedPath:           feedConfig.GetPath(),
-				NetworkID:          chainConfig.GetNetworkID(),
-				NetworkName:        chainConfig.GetNetworkName(),
+			mockMetrics.On("Cleanup", accountName, metrics.FeedInput{
+				AccountAddress: balances.Addresses[accountName].String(),
+				FeedID:         feedConfig.GetID(),
+				ChainID:        chainConfig.GetChainID(),
+				ContractStatus: feedConfig.GetContractStatus(),
+				ContractType:   feedConfig.GetContractType(),
+				FeedName:       feedConfig.GetName(),
+				FeedPath:       feedConfig.GetPath(),
+				NetworkID:      chainConfig.GetNetworkID(),
+				NetworkName:    chainConfig.GetNetworkName(),
 			})
 		}
 		exporter.Cleanup(ctx)

pkg/monitoring/exporter/reportobservations.go

@@ -0,0 +1,82 @@
+package exporter
+
+import (
+	"context"
+
+	commonMonitoring "github.com/smartcontractkit/chainlink-common/pkg/monitoring"
+	"github.com/smartcontractkit/chainlink-solana/pkg/monitoring/metrics"
+	"github.com/smartcontractkit/chainlink-solana/pkg/monitoring/types"
+)
+
+func NewReportObservationsFactory(
+	log commonMonitoring.Logger,
+	metrics metrics.ReportObservations,
+) commonMonitoring.ExporterFactory {
+	return &reportObservationsFactory{
+		log,
+		metrics,
+	}
+}
+
+type reportObservationsFactory struct {
+	log     commonMonitoring.Logger
+	metrics metrics.ReportObservations
+}
+
+func (p *reportObservationsFactory) NewExporter(
+	params commonMonitoring.ExporterParams,
+) (commonMonitoring.Exporter, error) {
+	return &reportObservations{
+		metrics.FeedInput{
+			AccountAddress: params.FeedConfig.GetContractAddress(),
+			FeedID:         params.FeedConfig.GetContractAddress(),
+			ChainID:        params.ChainConfig.GetChainID(),
+			ContractStatus: params.FeedConfig.GetContractStatus(),
+			ContractType:   params.FeedConfig.GetContractType(),
+			FeedName:       params.FeedConfig.GetName(),
+			FeedPath:       params.FeedConfig.GetPath(),
+			NetworkID:      params.ChainConfig.GetNetworkID(),
+			NetworkName:    params.ChainConfig.GetNetworkName(),
+		},
+		p.log,
+		p.metrics,
+	}, nil
+}
+
+type reportObservations struct {
+	label   metrics.FeedInput // static for each feed
+	log     commonMonitoring.Logger
+	metrics metrics.ReportObservations
+}
+
+func (p *reportObservations) Export(ctx context.Context, data interface{}) {
+	details, err := types.MakeTxDetails(data)
+	if err != nil {
+		return // skip if input could not be parsed
+	}
+
+	// skip on no updates
+	if len(details) == 0 {
+		return
+	}
+
+	// sanity check: find non-empty detail
+	// assumption: details ordered from latest -> earliest
+	var latest types.TxDetails
+	for _, d := range details {
+		if !d.Empty() {
+			latest = d
+			break
+		}
+	}
+	if latest.Empty() {
+		p.log.Errorw("exporter could not find non-empty TxDetails", "feed", p.label.ToPromLabels())
+		return
+	}
+
+	p.metrics.SetCount(latest.ObservationCount, p.label)
+}
+
+func (p *reportObservations) Cleanup(_ context.Context) {
+	p.metrics.Cleanup(p.label)
+}