Skip to content

Commit 75b81b0

Browse files
anserminoansermino
authored
Add basic prometheus metrics (ChainSafe#519)
- Adds counters for blocks processed and votes submitted - Starts prometheus collection on `/metrics`
1 parent edcf603 commit 75b81b0

21 files changed

+134
-80
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ rebuild-contracts:
6060
license:
6161
@echo " > \033[32mAdding license headers...\033[0m "
6262
GO111MODULE=off go get -u github.com/google/addlicense
63-
addlicense -c "ChainSafe Systems" -f ./copyright.txt -y 2020 .
63+
addlicense -c "ChainSafe Systems" -f ./scripts/header.txt -y 2020 .
6464

6565
## license-check: Checks for missing license headers
6666
license-check:

‎README.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,12 @@ For testing purposes, chainbridge provides 5 test keys. The can be used with `--
9797

9898
## Metrics
9999

100-
A basic health status check can be enabled with the `--metrics` flag (default port `8001`, use `--metricsPort` to specify).
100+
Basic metrics and a health status check can be enabled with the `--metrics` flag (default port `8001`, use `--metricsPort` to specify).
101101

102102
The endpoint `/health` will return the current block height and a timestamp of when it was processed. If the timestamp is at least 120 seconds old an error will be returned.
103103

104+
Prometheus metrics are served on `/metrics`.
105+
104106
# Chain Implementations
105107

106108
- Ethereum (Solidity): [chainbridge-solidity](https://github.com/ChainSafe/chainbridge-solidity)

‎chains/ethereum/chain.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func setupBlockstore(cfg *Config, kp *secp256k1.Keypair) (*blockstore.Blockstore
9090
return bs, nil
9191
}
9292

93-
func InitializeChain(chainCfg *core.ChainConfig, logger log15.Logger, sysErr chan<- error) (*Chain, error) {
93+
func InitializeChain(chainCfg *core.ChainConfig, logger log15.Logger, sysErr chan<- error, m *metrics.ChainMetrics) (*Chain, error) {
9494
cfg, err := parseChainConfig(chainCfg)
9595
if err != nil {
9696
return nil, err
@@ -163,10 +163,10 @@ func InitializeChain(chainCfg *core.ChainConfig, logger log15.Logger, sysErr cha
163163
cfg.startBlock = curr
164164
}
165165

166-
listener := NewListener(conn, cfg, logger, bs, stop, sysErr)
166+
listener := NewListener(conn, cfg, logger, bs, stop, sysErr, m)
167167
listener.setContracts(bridgeContract, erc20HandlerContract, erc721HandlerContract, genericHandlerContract)
168168

169-
writer := NewWriter(conn, cfg, logger, stop, sysErr)
169+
writer := NewWriter(conn, cfg, logger, stop, sysErr, m)
170170
writer.setContract(bridgeContract)
171171

172172
return &Chain{

‎chains/ethereum/chain_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func TestChain_ListenerShutdownOnFailure(t *testing.T) {
3939
},
4040
}
4141
sysErr := make(chan error)
42-
chain, err := InitializeChain(cfg, TestLogger, sysErr)
42+
chain, err := InitializeChain(cfg, TestLogger, sysErr, nil)
4343
if err != nil {
4444
t.Fatal(err)
4545
}
@@ -98,7 +98,7 @@ func TestChain_WriterShutdownOnFailure(t *testing.T) {
9898
},
9999
}
100100
sysErr := make(chan error)
101-
chain, err := InitializeChain(cfg, TestLogger, sysErr)
101+
chain, err := InitializeChain(cfg, TestLogger, sysErr, nil)
102102
if err != nil {
103103
t.Fatal(err)
104104
}

‎chains/ethereum/listener.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ type listener struct {
4343
stop <-chan int
4444
sysErr chan<- error // Reports fatal error to core
4545
latestBlock metrics.LatestBlock
46+
metrics *metrics.ChainMetrics
4647
}
4748

4849
// NewListener creates and returns a listener
49-
func NewListener(conn Connection, cfg *Config, log log15.Logger, bs blockstore.Blockstorer, stop <-chan int, sysErr chan<- error) *listener {
50+
func NewListener(conn Connection, cfg *Config, log log15.Logger, bs blockstore.Blockstorer, stop <-chan int, sysErr chan<- error, m *metrics.ChainMetrics) *listener {
5051
return &listener{
5152
cfg: *cfg,
5253
conn: conn,
@@ -55,6 +56,7 @@ func NewListener(conn Connection, cfg *Config, log log15.Logger, bs blockstore.B
5556
stop: stop,
5657
sysErr: sysErr,
5758
latestBlock: metrics.LatestBlock{LastUpdated: time.Now()},
59+
metrics: m,
5860
}
5961
}
6062

@@ -138,6 +140,9 @@ func (l *listener) pollBlocks() error {
138140
l.latestBlock.Height = big.NewInt(0).Set(latestBlock)
139141
l.latestBlock.LastUpdated = time.Now()
140142
retry = BlockRetryLimit
143+
if l.metrics != nil {
144+
l.metrics.BlocksProcessed.Inc()
145+
}
141146
}
142147
}
143148
}

‎chains/ethereum/listener_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func createTestListener(t *testing.T, config *Config, contracts *utils.DeployedC
6565
}
6666

6767
router := &MockRouter{msgs: make(chan msg.Message)}
68-
listener := NewListener(conn, &newConfig, TestLogger, &blockstore.EmptyStore{}, stop, sysErr)
68+
listener := NewListener(conn, &newConfig, TestLogger, &blockstore.EmptyStore{}, stop, sysErr, nil)
6969
listener.setContracts(bridgeContract, erc20HandlerContract, erc721HandlerContract, genericHandlerContract)
7070
listener.setRouter(router)
7171
// Start the listener

‎chains/ethereum/writer.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package ethereum
66
import (
77
"github.com/ChainSafe/ChainBridge/bindings/Bridge"
88
"github.com/ChainSafe/ChainBridge/chains"
9+
metrics "github.com/ChainSafe/ChainBridge/metrics/types"
910
"github.com/ChainSafe/chainbridge-utils/msg"
1011
"github.com/ChainSafe/log15"
1112
)
@@ -24,16 +25,18 @@ type writer struct {
2425
log log15.Logger
2526
stop <-chan int
2627
sysErr chan<- error // Reports fatal error to core
28+
metrics *metrics.ChainMetrics
2729
}
2830

2931
// NewWriter creates and returns writer
30-
func NewWriter(conn Connection, cfg *Config, log log15.Logger, stop <-chan int, sysErr chan<- error) *writer {
32+
func NewWriter(conn Connection, cfg *Config, log log15.Logger, stop <-chan int, sysErr chan<- error, m *metrics.ChainMetrics) *writer {
3133
return &writer{
32-
cfg: *cfg,
33-
conn: conn,
34-
log: log,
35-
stop: stop,
36-
sysErr: sysErr,
34+
cfg: *cfg,
35+
conn: conn,
36+
log: log,
37+
stop: stop,
38+
sysErr: sysErr,
39+
metrics: m,
3740
}
3841
}
3942

‎chains/ethereum/writer_methods.go

+3
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,9 @@ func (w *writer) voteProposal(m msg.Message, dataHash [32]byte) {
239239

240240
if err == nil {
241241
w.log.Info("Submitted proposal vote", "tx", tx.Hash(), "src", m.Source, "depositNonce", m.DepositNonce)
242+
if w.metrics != nil {
243+
w.metrics.VotesSubmitted.Inc()
244+
}
242245
return
243246
} else if err.Error() == ErrNonceTooLow.Error() || err.Error() == ErrTxUnderpriced.Error() {
244247
w.log.Debug("Nonce too low, will retry")

‎chains/ethereum/writer_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func createTestWriter(t *testing.T, cfg *Config, errs chan<- error) (*writer, fu
3333

3434
conn := newLocalConnection(t, cfg)
3535
stop := make(chan int)
36-
writer := NewWriter(conn, cfg, newTestLogger(cfg.name), stop, errs)
36+
writer := NewWriter(conn, cfg, newTestLogger(cfg.name), stop, errs, nil)
3737

3838
bridge, err := Bridge.NewBridge(cfg.bridgeContract, conn.Client())
3939
if err != nil {
@@ -112,7 +112,7 @@ func TestWriter_start_stop(t *testing.T) {
112112
defer conn.Close()
113113

114114
stop := make(chan int)
115-
writer := NewWriter(conn, aliceTestConfig, TestLogger, stop, nil)
115+
writer := NewWriter(conn, aliceTestConfig, TestLogger, stop, nil, nil)
116116

117117
err := writer.start()
118118
if err != nil {

‎chains/substrate/chain.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func checkBlockstore(bs *blockstore.Blockstore, startBlock uint64) (uint64, erro
5959
}
6060
}
6161

62-
func InitializeChain(cfg *core.ChainConfig, logger log15.Logger, sysErr chan<- error) (*Chain, error) {
62+
func InitializeChain(cfg *core.ChainConfig, logger log15.Logger, sysErr chan<- error, m *metrics.ChainMetrics) (*Chain, error) {
6363
kp, err := keystore.KeypairFromAddress(cfg.From, keystore.SubChain, cfg.KeystorePath, cfg.Insecure)
6464
if err != nil {
6565
return nil, err
@@ -102,8 +102,8 @@ func InitializeChain(cfg *core.ChainConfig, logger log15.Logger, sysErr chan<- e
102102
}
103103

104104
// Setup listener & writer
105-
l := NewListener(conn, cfg.Name, cfg.Id, startBlock, logger, bs, stop, sysErr)
106-
w := NewWriter(conn, logger, sysErr)
105+
l := NewListener(conn, cfg.Name, cfg.Id, startBlock, logger, bs, stop, sysErr, m)
106+
w := NewWriter(conn, logger, sysErr, m)
107107
return &Chain{
108108
cfg: cfg,
109109
conn: conn,

‎chains/substrate/listener.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,14 @@ type listener struct {
3030
stop <-chan int
3131
sysErr chan<- error
3232
latestBlock metrics.LatestBlock
33+
metrics *metrics.ChainMetrics
3334
}
3435

3536
// Frequency of polling for a new block
3637
var BlockRetryInterval = time.Second * 5
3738
var BlockRetryLimit = 5
3839

39-
func NewListener(conn *Connection, name string, id msg.ChainId, startBlock uint64, log log15.Logger, bs blockstore.Blockstorer, stop <-chan int, sysErr chan<- error) *listener {
40+
func NewListener(conn *Connection, name string, id msg.ChainId, startBlock uint64, log log15.Logger, bs blockstore.Blockstorer, stop <-chan int, sysErr chan<- error, m *metrics.ChainMetrics) *listener {
4041
return &listener{
4142
name: name,
4243
chainId: id,
@@ -48,6 +49,7 @@ func NewListener(conn *Connection, name string, id msg.ChainId, startBlock uint6
4849
stop: stop,
4950
sysErr: sysErr,
5051
latestBlock: metrics.LatestBlock{LastUpdated: time.Now()},
52+
metrics: m,
5153
}
5254
}
5355

@@ -165,6 +167,9 @@ func (l *listener) pollBlocks() error {
165167
l.latestBlock.Height = big.NewInt(0).SetUint64(currentBlock)
166168
l.latestBlock.LastUpdated = time.Now()
167169
retry = BlockRetryLimit
170+
if l.metrics != nil {
171+
l.metrics.BlocksProcessed.Inc()
172+
}
168173
}
169174
}
170175
}

‎chains/substrate/listener_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func newTestListener(client *utils.Client, conn *Connection) (*listener, chan er
3737
}
3838

3939
errs := make(chan error)
40-
l := NewListener(conn, "Alice", 1, startBlock, AliceTestLogger, &blockstore.EmptyStore{}, make(chan int), errs)
40+
l := NewListener(conn, "Alice", 1, startBlock, AliceTestLogger, &blockstore.EmptyStore{}, make(chan int), errs, nil)
4141
l.setRouter(r)
4242
err = l.start()
4343
if err != nil {

‎chains/substrate/test_helper_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,8 @@ func TestMain(m *testing.M) {
9393
if err != nil {
9494
panic(err)
9595
}
96-
alice := NewWriter(aliceConn, AliceTestLogger, wSysErr)
97-
bob := NewWriter(bobConn, BobTestLogger, wSysErr)
96+
alice := NewWriter(aliceConn, AliceTestLogger, wSysErr, nil)
97+
bob := NewWriter(bobConn, BobTestLogger, wSysErr, nil)
9898
context = testContext{
9999
client: client,
100100
listener: l,

‎chains/substrate/writer.go

+13-7
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
"github.com/ChainSafe/ChainBridge/chains"
13+
metrics "github.com/ChainSafe/ChainBridge/metrics/types"
1314
utils "github.com/ChainSafe/ChainBridge/shared/substrate"
1415
"github.com/ChainSafe/chainbridge-utils/msg"
1516
"github.com/ChainSafe/log15"
@@ -22,16 +23,18 @@ var AcknowledgeProposal utils.Method = utils.BridgePalletName + ".acknowledge_pr
2223
var TerminatedError = errors.New("terminated")
2324

2425
type writer struct {
25-
conn *Connection
26-
log log15.Logger
27-
sysErr chan<- error
26+
conn *Connection
27+
log log15.Logger
28+
sysErr chan<- error
29+
metrics *metrics.ChainMetrics
2830
}
2931

30-
func NewWriter(conn *Connection, log log15.Logger, sysErr chan<- error) *writer {
32+
func NewWriter(conn *Connection, log log15.Logger, sysErr chan<- error, m *metrics.ChainMetrics) *writer {
3133
return &writer{
32-
conn: conn,
33-
log: log,
34-
sysErr: sysErr,
34+
conn: conn,
35+
log: log,
36+
sysErr: sysErr,
37+
metrics: m,
3538
}
3639
}
3740

@@ -81,6 +84,9 @@ func (w *writer) ResolveMessage(m msg.Message) bool {
8184
time.Sleep(BlockRetryInterval)
8285
continue
8386
}
87+
if w.metrics != nil {
88+
w.metrics.VotesSubmitted.Inc()
89+
}
8490
return true
8591
} else {
8692
w.log.Debug("Ignoring proposal", "reason", reason, "nonce", prop.depositNonce, "source", prop.sourceId, "resource", prop.resourceId)

‎cmd/chainbridge/main.go

+27-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ package main
99

1010
import (
1111
"errors"
12+
"fmt"
13+
"net/http"
1214
"os"
1315
"strconv"
1416

@@ -17,8 +19,10 @@ import (
1719
"github.com/ChainSafe/ChainBridge/config"
1820
"github.com/ChainSafe/ChainBridge/core"
1921
"github.com/ChainSafe/ChainBridge/metrics/health"
22+
metrics "github.com/ChainSafe/ChainBridge/metrics/types"
2023
"github.com/ChainSafe/chainbridge-utils/msg"
2124
log "github.com/ChainSafe/log15"
25+
"github.com/prometheus/client_golang/prometheus/promhttp"
2226
"github.com/urfave/cli/v2"
2327
)
2428

@@ -176,11 +180,18 @@ func run(ctx *cli.Context) error {
176180
Opts: chain.Opts,
177181
}
178182
var newChain core.Chain
183+
var m *metrics.ChainMetrics
184+
179185
logger := log.Root().New("chain", chainConfig.Name)
186+
187+
if ctx.Bool(config.MetricsFlag.Name) {
188+
m = metrics.NewChainMetrics(chain.Name)
189+
}
190+
180191
if chain.Type == "ethereum" {
181-
newChain, err = ethereum.InitializeChain(chainConfig, logger, sysErr)
192+
newChain, err = ethereum.InitializeChain(chainConfig, logger, sysErr, m)
182193
} else if chain.Type == "substrate" {
183-
newChain, err = substrate.InitializeChain(chainConfig, logger, sysErr)
194+
newChain, err = substrate.InitializeChain(chainConfig, logger, sysErr, m)
184195
} else {
185196
return errors.New("unrecognized Chain Type")
186197
}
@@ -189,12 +200,24 @@ func run(ctx *cli.Context) error {
189200
return err
190201
}
191202
c.AddChain(newChain)
203+
192204
}
193205

194-
// Start metrics server
206+
// Start prometheus and health server
195207
if ctx.Bool(config.MetricsFlag.Name) {
196208
port := ctx.Int(config.MetricsPort.Name)
197-
go health.Start(port, c.Registry)
209+
h := health.NewHealthServer(port, c.Registry)
210+
211+
go func() {
212+
http.Handle("/metrics", promhttp.Handler())
213+
http.HandleFunc("/health", h.HealthStatus)
214+
err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
215+
if err == http.ErrServerClosed {
216+
log.Info("Health status server is shutting down", err)
217+
} else {
218+
log.Error("Error serving metrics", "err", err)
219+
}
220+
}()
198221
}
199222

200223
c.Start()

‎e2e/e2e_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -71,19 +71,19 @@ func createAndStartBridge(t *testing.T, name string, contractsA, contractsB *eth
7171
logger := log.Root().New()
7272
sysErr := make(chan error)
7373
ethACfg := eth.CreateConfig(name, EthAChainId, contractsA, eth.EthAEndpoint)
74-
ethA, err := ethChain.InitializeChain(ethACfg, logger.New("relayer", name, "chain", "ethA"), sysErr)
74+
ethA, err := ethChain.InitializeChain(ethACfg, logger.New("relayer", name, "chain", "ethA"), sysErr, nil)
7575
if err != nil {
7676
t.Fatal(err)
7777
}
7878

7979
subCfg := sub.CreateConfig(name, SubChainId)
80-
subA, err := subChain.InitializeChain(subCfg, logger.New("relayer", name, "chain", "sub"), sysErr)
80+
subA, err := subChain.InitializeChain(subCfg, logger.New("relayer", name, "chain", "sub"), sysErr, nil)
8181
if err != nil {
8282
t.Fatal(err)
8383
}
8484

8585
ethBCfg := eth.CreateConfig(name, EthBChainId, contractsB, eth.EthBEndpoint)
86-
ethB, err := ethChain.InitializeChain(ethBCfg, logger.New("relayer", name, "chain", "ethB"), sysErr)
86+
ethB, err := ethChain.InitializeChain(ethBCfg, logger.New("relayer", name, "chain", "ethB"), sysErr, nil)
8787
if err != nil {
8888
t.Fatal(err)
8989
}

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ require (
1212
github.com/deckarep/golang-set v1.7.1 // indirect
1313
github.com/ethereum/go-ethereum v1.9.17
1414
github.com/gorilla/websocket v1.4.2 // indirect
15-
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
15+
github.com/prometheus/client_golang v1.4.1
1616
github.com/rs/cors v1.7.0 // indirect
1717
github.com/stretchr/testify v1.4.0
1818
github.com/urfave/cli/v2 v2.2.0

0 commit comments

Comments
 (0)