feat(submitter): stateful submitter (#43)

In case of a restart, we want to avoid wasting fees if we have already
submitted transactions for a certain epoch.
Basic idea:
A crash occurs after sending both checkpoint transactions for epoch `n`
to the BTC and recording them in the database. Upon restarting, we find
that epoch `n` is still marked as sealed.

Before resending the transactions we first check our database and
confirm that the transactions for this epoch have already been sent.
Since the transactions were previously sent, the next step is to verify
their status on our Bitcoin node.

If the Bitcoin node is aware of the transactions and they are present in
the mempool, no further action is needed.
However, if the node does not recognize the transactions, this indicates
they were lost, and we must resend them to the network.



[References](#28 (comment))

Makefile

@@ -75,3 +75,9 @@ update-changelog:
 	./scripts/update_changelog.sh $(sinceTag) $(upcomingTag)
 
 .PHONY: build test test-e2e build-docker rm-docker mocks update-changelog
+
+proto-gen:
+	@$(call print, "Compiling protos.")
+	cd ./proto; ./gen_protos_docker.sh
+
+.PHONY: proto-gen

btcclient/client_wallet.go

@@ -121,3 +121,7 @@ func (c *Client) FundRawTransaction(tx *wire.MsgTx, opts btcjson.FundRawTransact
 func (c *Client) SignRawTransactionWithWallet(tx *wire.MsgTx) (*wire.MsgTx, bool, error) {
 	return c.Client.SignRawTransactionWithWallet(tx)
 }
+
+func (c *Client) GetRawTransaction(txHash *chainhash.Hash) (*btcutil.Tx, error) {
+	return c.Client.GetRawTransaction(txHash)
+}

btcclient/interface.go

@@ -38,4 +38,5 @@ type BTCWallet interface {
 	GetHighUTXOAndSum() (*btcjson.ListUnspentResult, float64, error)
 	FundRawTransaction(tx *wire.MsgTx, opts btcjson.FundRawTransactionOpts, isWitness *bool) (*btcjson.FundRawTransactionResult, error)
 	SignRawTransactionWithWallet(tx *wire.MsgTx) (*wire.MsgTx, bool, error)
+	GetRawTransaction(txHash *chainhash.Hash) (*btcutil.Tx, error)
 }

cmd/vigilante/cmd/submitter.go

@@ -62,6 +62,11 @@ func GetSubmitterCmd() *cobra.Command {
 			// register submitter metrics
 			submitterMetrics := metrics.NewSubmitterMetrics()
 
+			dbBackend, err := cfg.Submitter.DatabaseConfig.GetDbBackend()
+			if err != nil {
+				panic(err)
+			}
+
 			// create submitter
 			vigilantSubmitter, err := submitter.New(
 				&cfg.Submitter,
@@ -73,6 +78,7 @@ func GetSubmitterCmd() *cobra.Command {
 				cfg.Common.MaxRetrySleepTime,
 				cfg.Common.MaxRetryTimes,
 				submitterMetrics,
+				dbBackend,
 			)
 			if err != nil {
 				panic(fmt.Errorf("failed to create vigilante submitter: %w", err))

config/submitter.go

@@ -25,7 +25,8 @@ type SubmitterConfig struct {
 	PollingIntervalSeconds uint `mapstructure:"polling-interval-seconds"`
 	// ResendIntervalSeconds defines the time (in seconds) which the submitter awaits
 	// before resubmitting checkpoints to BTC
-	ResendIntervalSeconds uint `mapstructure:"resend-interval-seconds"`
+	ResendIntervalSeconds uint      `mapstructure:"resend-interval-seconds"`
+	DatabaseConfig        *DBConfig `mapstructure:"database-config"`
 }
 
 func (cfg *SubmitterConfig) Validate() error {

e2etest/monitor_e2e_test.go

@@ -61,6 +61,7 @@ func TestMonitorBootstrap(t *testing.T) {
 		tm.Config.Common.MaxRetrySleepTime,
 		tm.Config.Common.MaxRetryTimes,
 		metrics.NewSubmitterMetrics(),
+		testutil.MakeTestBackend(t),
 	)
 
 	vigilantSubmitter.Start()

e2etest/submitter_e2e_test.go

@@ -4,6 +4,7 @@
 package e2etest
 
 import (
+	"github.com/babylonlabs-io/vigilante/testutil"
 	"math/rand"
 	"testing"
 	"time"
@@ -64,6 +65,7 @@ func TestSubmitterSubmission(t *testing.T) {
 		tm.Config.Common.MaxRetrySleepTime,
 		tm.Config.Common.MaxRetryTimes,
 		metrics.NewSubmitterMetrics(),
+		testutil.MakeTestBackend(t),
 	)
 
 	vigilantSubmitter.Start()
@@ -138,6 +140,7 @@ func TestSubmitterSubmissionReplace(t *testing.T) {
 		tm.Config.Common.MaxRetrySleepTime,
 		tm.Config.Common.MaxRetryTimes,
 		metrics.NewSubmitterMetrics(),
+		testutil.MakeTestBackend(t),
 	)
 
 	vigilantSubmitter.Start()

proto/.clang-format

@@ -0,0 +1,7 @@
+---
+Language: Proto
+BasedOnStyle: Google
+IndentWidth: 4
+AllowShortFunctionsOnASingleLine: None
+SpaceBeforeParens: Always
+CompactNamespaces: false

proto/Dockerfile

@@ -0,0 +1,36 @@
+# If you change this value, please change it in the following files as well:
+# /.github/workflows/main.yaml
+# /Dockerfile
+# /dev.Dockerfile
+# /make/builder.Dockerfile
+# /tools/Dockerfile
+FROM golang:1.20.5-buster
+
+RUN apt-get update && apt-get install -y \
+  git \
+  protobuf-compiler='3.6.1*' \
+  clang-format='1:7.0*'
+
+# We don't want any default values for these variables to make sure they're
+# explicitly provided by parsing the go.mod file. Otherwise we might forget to
+# update them here if we bump the versions.
+ARG PROTOBUF_VERSION
+ARG GRPC_GATEWAY_VERSION
+
+ENV PROTOC_GEN_GO_GRPC_VERSION="v1.1.0"
+ENV GOCACHE=/tmp/build/.cache
+ENV GOMODCACHE=/tmp/build/.modcache
+
+RUN cd /tmp \
+  && mkdir -p /tmp/build/.cache \
+  && mkdir -p /tmp/build/.modcache \
+  && go install google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOBUF_VERSION} \
+  && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRPC_VERSION} \
+  && go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@${GRPC_GATEWAY_VERSION} \
+  && go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@${GRPC_GATEWAY_VERSION} \
+  && go install golang.org/x/tools/cmd/[email protected] \
+  && chmod -R 777 /tmp/build/
+
+WORKDIR /build
+
+CMD ["/bin/bash", "/build/proto/protocgen.sh"]

proto/checkpoint.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+
+package proto;
+
+option go_package = "github.com/babylonlabs-io/vigilante/proto";
+
+// StoredCheckpoint holds two transactions and an epoch number
+message StoredCheckpoint {
+  bytes tx1 = 1; // wire.MsgTx serialized as bytes
+  bytes tx2 = 2;
+  uint64 epoch = 3;
+}

proto/gen_protos_docker.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Directory of the script file, independent of where it's called from.
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+PROTOBUF_VERSION=$(go list -f '{{.Version}}' -m google.golang.org/protobuf)
+GRPC_GATEWAY_VERSION=$(go list -f '{{.Version}}' -m github.com/grpc-ecosystem/grpc-gateway/v2)
+
+echo "Building protobuf compiler docker image..."
+docker build -t vigilante-protobuf-builder \
+  --build-arg PROTOBUF_VERSION="$PROTOBUF_VERSION" \
+  --build-arg GRPC_GATEWAY_VERSION="$GRPC_GATEWAY_VERSION" \
+  .
+
+echo "Compiling and formatting *.proto files..."
+docker run \
+  --rm \
+  --user "$UID:$(id -g)" \
+  -e UID=$UID \
+  -e SUBSERVER_PREFIX \
+  -v "$DIR/../:/build" \
+ vigilante-protobuf-builder

proto/protocgen.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+# generate compiles the *.pb.go stubs from the *.proto files.
+function generate() {
+  echo "Generating vigilatne protos"
+
+  PROTOS="checkpoint.proto"
+
+  # For each of the sub-servers, we then generate their protos, but a restricted
+  # set as they don't yet require REST proxies, or swagger docs.
+  for file in $PROTOS; do
+    DIRECTORY=$(dirname "${file}")
+    echo "Generating protos from ${file}, into ${DIRECTORY}"
+
+    # Generate the protos.
+    protoc --go_out . --go_opt paths=source_relative \
+      --go-grpc_out . --go-grpc_opt paths=source_relative \
+      "${file}" --proto_path=$GOPATH/src/ --proto_path=.
+  done
+}
+
+# format formats the *.proto files with the clang-format utility.
+function format() {
+  echo "Formatting protos"
+  #| xargs -0 clang-format --style=file -i
+  find . -name "*.proto" -print0 | xargs -0
+}
+
+# Compile and format the proto package.
+pushd proto
+format
+generate
+popd
+
+if [[ "$COMPILE_MOBILE" == "1" ]]; then
+  pushd mobile
+  ./gen_bindings.sh $FALAFEL_VERSION
+  popd
+fi

submitter/relayer/change_address_test.go

@@ -1,6 +1,7 @@
 package relayer_test
 
 import (
+	"github.com/babylonlabs-io/vigilante/testutil"
 	"testing"
 
 	"github.com/btcsuite/btcd/btcjson"
@@ -49,7 +50,7 @@ func TestGetChangeAddress(t *testing.T) {
 	logger, err := config.NewRootLogger("auto", "debug")
 	require.NoError(t, err)
 	testRelayer := relayer.New(wallet, []byte("bbnt"), btctxformatter.CurrentVersion, submitterAddr,
-		submitterMetrics.RelayerMetrics, nil, &cfg, logger)
+		submitterMetrics.RelayerMetrics, nil, &cfg, logger, testutil.MakeTestBackend(t))
 
 	// 1. only SegWit Bech32 addresses
 	segWitBech32Addrs := append(SegWitBech32p2wshAddrsStr, SegWitBech32p2wpkhAddrsStr...)