Add support for universal nonce, fixes/improves relayer and depends o…

…n latest QGB contract (#497)

* adds AttestationRequestI part 1

* adds AttestationRequestI part 2: state machine + orchestrator + relayer changes

* use latest  QGB contract part 1

* partially working abci + orchestrator

* partially working orchestrator + relayer + unit tests refactor

* ugly but working e2e orchestrator tests

* use latest qgb contract + update tests

* update qgb e2e tests

* update qgb e2e tests

* fix relayer matching commitement signatures with valset validators part 1

* fix relayer matching signatures

* refactor + cleanup + making sure things work

* fixes tests + removes unnecessary code

* fixes docs in proto

* uses new method instead of direct struct init for valset/dc confirms

* increase qgb e2e test time

* final nits

* increase QGB E2E github test timeout

* remove dead code maxValsetRequestsReturned and maxDataCommitmentRequestsReturned

* ignores full long behaviour e2e tests

* Typos x/qgb/orchestrator/errors.go

Co-authored-by: Evan Forbes <[email protected]>

* nits

* use require instead of assert

* remove unnecessary qgb dc requests + nits

Co-authored-by: Evan Forbes <[email protected]>

.github/workflows/test.yml

@@ -268,6 +268,6 @@ jobs:
             go.mod
             go.sum
       - name: Test
-        run: go test -test.timeout 30m -failfast -v github.com/celestiaorg/celestia-app/e2e/qgb
+        run: go test -test.timeout 60m -failfast -v github.com/celestiaorg/celestia-app/e2e/qgb
         env:
           QGB_INTEGRATION_TEST: true

e2e/qgb/Makefile

@@ -5,4 +5,4 @@ PACKAGES=$(shell go list ./...)
 all: test
 
 test:
-	@QGB_INTEGRATION_TEST=true go test -mod=readonly -failfast -test.timeout 30m -v $(PACKAGES)
+	@QGB_INTEGRATION_TEST=true go test -mod=readonly -failfast -test.timeout 60m -v $(PACKAGES)

e2e/qgb/deployer_test.go

@@ -39,11 +39,7 @@ func TestDeployer(t *testing.T) {
 	// FIXME should we use the evm client here or go for raw queries?
 	evmClient := orchestrator.NewEvmClient(nil, *bridge, nil, network.EVMRPC)
 
-	vsNonce, err := evmClient.StateLastValsetNonce(&bind.CallOpts{Context: network.Context})
+	eventNonce, err := evmClient.StateLastEventNonce(&bind.CallOpts{Context: network.Context})
 	assert.NoError(t, err)
-	assert.Equal(t, uint64(0), vsNonce)
-
-	dcNonce, err := evmClient.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: network.Context})
-	assert.NoError(t, err)
-	assert.Equal(t, uint64(0), dcNonce)
+	assert.Equal(t, uint64(0), eventNonce)
 }

e2e/qgb/docker-compose.yml

@@ -27,17 +27,21 @@ services:
       context: ../..
     depends_on:
       - core0
+    environment:
+      - MONIKER=core0
+      # eth address: 0x966e6f22781EF6a6A82BBB4DB3df8E225DfD9488
+      - PRIVATE_KEY=da6ed55cb2894ac2c9c10209c09de8e8b9d109b910338d5bf3d747a7e1fc9eb9
+      - TENDERMINT_RPC=tcp://core0:26657
+      - CELESTIA_GRPC=core0:9090
+    entrypoint: [
+      "/bin/bash"
+    ]
     command: [
-        "orchestrator", "-p", "/opt",
-        "-x", "qgb-e2e",
-        "-d", "da6ed55cb2894ac2c9c10209c09de8e8b9d109b910338d5bf3d747a7e1fc9eb9",
-        "--keyring-account", "core0",
-        "-t", "tcp://core0:26657",
-        "-c", "core0:9090"
+      "/opt/start_orchestrator_after_validator_created.sh"
     ]
-    # Eth Address: 0x966e6f22781EF6a6A82BBB4DB3df8E225DfD9488:da6ed55cb2894ac2c9c10209c09de8e8b9d109b910338d5bf3d747a7e1fc9eb9
     volumes:
       - ${PWD}/celestia-app/core0/keyring-test:/opt/keyring-test:ro
+      - ${PWD}/scripts/start_orchestrator_after_validator_created.sh:/opt/start_orchestrator_after_validator_created.sh:ro
 
   core1:
     container_name: core1
@@ -187,6 +191,7 @@ services:
     image: trufflesuite/ganache-cli
     command: [
       "--port=8545",
+      "--verbose",
       "--host=0.0.0.0",
       "--networkId=1234",
       "--chainId=1234",

e2e/qgb/full_test.go

@@ -3,9 +3,9 @@ package e2e
 import (
 	"context"
 	"github.com/celestiaorg/celestia-app/x/qgb/orchestrator"
-	"github.com/celestiaorg/celestia-app/x/qgb/types"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"os"
 	"testing"
 	"time"
@@ -15,7 +15,8 @@ import (
 // the valsets and data commitments are relayed correctly.
 // currently, it takes around 10min to reach 120 block.
 func TestFullLongBehaviour(t *testing.T) {
-	if os.Getenv("QGB_INTEGRATION_TEST") != TRUE {
+	// Separating it from other e2e as it takes around 20min to finish
+	if os.Getenv("QGB_INTEGRATION_FULL_TEST") != TRUE {
 		t.Skip("Skipping QGB integration tests")
 	}
 
@@ -33,27 +34,27 @@ func TestFullLongBehaviour(t *testing.T) {
 	HandleNetworkError(t, network, err, false)
 
 	// check whether the four validators are up and running
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
 	// check whether all the validators are up and running
-	lastValsets, err := querier.QueryLastValsets(network.Context)
+	latestValset, err := querier.QueryLatestValset(network.Context)
 	assert.NoError(t, err)
-	assert.Equal(t, 4, len(lastValsets[0].Members))
+	require.NotNil(t, latestValset)
+	assert.Equal(t, 4, len(latestValset.Members))
 
 	// check whether the QGB contract was deployed
 	bridge, err := network.GetLatestDeployedQGBContract(network.Context)
 	HandleNetworkError(t, network, err, false)
 
 	evmClient := orchestrator.NewEvmClient(nil, *bridge, nil, network.EVMRPC)
 
-	// check whether the relayer relayed all data commitments
-	dcNonce, err := evmClient.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: network.Context})
+	// check whether the relayer relayed all attestations
+	eventNonce, err := evmClient.StateLastEventNonce(&bind.CallOpts{Context: network.Context})
 	assert.NoError(t, err)
-	assert.GreaterOrEqual(t, dcNonce, 100/types.DataCommitmentWindow)
 
-	// check whether the relayer relayed all valsets
-	vsNonce, err := evmClient.StateLastValsetNonce(&bind.CallOpts{Context: network.Context})
+	// attestations are either data commitments or valsets
+	latestNonce, err := querier.QueryLatestAttestationNonce(network.Context)
 	assert.NoError(t, err)
-	assert.GreaterOrEqual(t, vsNonce, lastValsets[0].Nonce)
+	assert.GreaterOrEqual(t, eventNonce, latestNonce-1)
 }

e2e/qgb/orchestrator_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/celestiaorg/celestia-app/x/qgb/orchestrator"
 	"github.com/celestiaorg/celestia-app/x/qgb/types"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"os"
 	"testing"
 )
@@ -36,13 +37,13 @@ func TestOrchestratorWithOneValidator(t *testing.T) {
 	HandleNetworkError(t, network, err, false)
 
 	// FIXME should we use the querier here or go for raw queries?
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
 	vsConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE0ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
-	assert.NotNil(t, vsConfirm)
+	require.NotNil(t, vsConfirm)
 	// assert that it carries the right eth address
 	assert.Equal(t, CORE0EVMADDRESS, vsConfirm.EthAddress)
 
@@ -92,11 +93,11 @@ func TestOrchestratorWithTwoValidators(t *testing.T) {
 	err = network.WaitForOrchestratorToStart(network.Context, CORE1ACCOUNTADDRESS)
 	HandleNetworkError(t, network, err, false)
 
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
 	// check core0 submited the valset confirm
-	core0ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 2, CORE0ACCOUNTADDRESS)
+	core0ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE0ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core0ValsetConfirm)
@@ -117,7 +118,7 @@ func TestOrchestratorWithTwoValidators(t *testing.T) {
 	assert.Equal(t, CORE0EVMADDRESS, core0DataCommitmentConfirm.EthAddress)
 
 	// check core1 submited the valset confirm
-	core1ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 2, CORE1ACCOUNTADDRESS)
+	core1ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE1ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core1ValsetConfirm)
@@ -169,14 +170,11 @@ func TestOrchestratorWithMultipleValidators(t *testing.T) {
 	err = network.WaitForOrchestratorToStart(network.Context, CORE3ACCOUNTADDRESS)
 	HandleNetworkError(t, network, err, false)
 
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
-	lastValsets, err := querier.QueryLastValsets(ctx)
-	assert.NoError(t, err)
-
 	// check core0 submited the valset confirm
-	core0ValsetConfirm, err := querier.QueryValsetConfirm(ctx, lastValsets[0].Nonce, CORE0ACCOUNTADDRESS)
+	core0ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE0ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core0ValsetConfirm)
@@ -197,7 +195,7 @@ func TestOrchestratorWithMultipleValidators(t *testing.T) {
 	assert.Equal(t, CORE0EVMADDRESS, core0DataCommitmentConfirm.EthAddress)
 
 	// check core1 submited the valset confirm
-	core1ValsetConfirm, err := querier.QueryValsetConfirm(ctx, lastValsets[0].Nonce, CORE1ACCOUNTADDRESS)
+	core1ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE1ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core1ValsetConfirm)
@@ -218,7 +216,7 @@ func TestOrchestratorWithMultipleValidators(t *testing.T) {
 	assert.Equal(t, CORE1EVMADDRESS, core1DataCommitmentConfirm.EthAddress)
 
 	// check core2 submited the valset confirm
-	core2ValsetConfirm, err := querier.QueryValsetConfirm(ctx, lastValsets[0].Nonce, CORE2ACCOUNTADDRESS)
+	core2ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE2ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core2ValsetConfirm)
@@ -240,7 +238,7 @@ func TestOrchestratorWithMultipleValidators(t *testing.T) {
 	assert.Equal(t, CORE2EVMADDRESS, core2DataCommitmentConfirm.EthAddress)
 
 	// check core3 submited the valset confirm
-	core3ValsetConfirm, err := querier.QueryValsetConfirm(ctx, lastValsets[0].Nonce, CORE3ACCOUNTADDRESS)
+	core3ValsetConfirm, err := querier.QueryValsetConfirm(ctx, 1, CORE3ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, core3ValsetConfirm)
@@ -303,7 +301,7 @@ func TestOrchestratorReplayOld(t *testing.T) {
 	HandleNetworkError(t, network, err, false)
 
 	// FIXME should we use the querier here or go for raw queries?
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
 	// check core0 submitted valset 1 confirm
@@ -315,7 +313,7 @@ func TestOrchestratorReplayOld(t *testing.T) {
 	assert.Equal(t, CORE0EVMADDRESS, vs1Core0Confirm.EthAddress)
 
 	// check core0 submitted valset 2 confirm
-	vs2Core0Confirm, err := querier.QueryValsetConfirm(ctx, 2, CORE0ACCOUNTADDRESS)
+	vs2Core0Confirm, err := querier.QueryValsetConfirm(ctx, 1, CORE0ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, vs2Core0Confirm)
@@ -331,7 +329,7 @@ func TestOrchestratorReplayOld(t *testing.T) {
 	assert.Equal(t, CORE1EVMADDRESS, vs1Core1Confirm.EthAddress)
 
 	// check core1 submitted valset 2 confirm
-	vs2Core1Confirm, err := querier.QueryValsetConfirm(ctx, 2, CORE1ACCOUNTADDRESS)
+	vs2Core1Confirm, err := querier.QueryValsetConfirm(ctx, 1, CORE1ACCOUNTADDRESS)
 	// assert the confirm exist
 	assert.NoError(t, err)
 	assert.NotNil(t, vs2Core1Confirm)

e2e/qgb/qgb_network.go

@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"github.com/celestiaorg/celestia-app/x/qgb/orchestrator"
 	"github.com/celestiaorg/celestia-app/x/qgb/types"
-	wrapper "github.com/celestiaorg/quantum-gravity-bridge/ethereum/solidity/wrappers/QuantumGravityBridge.sol"
+	wrapper "github.com/celestiaorg/quantum-gravity-bridge/wrappers/QuantumGravityBridge.sol"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/google/uuid"
+	"github.com/tendermint/spm/cosmoscmd"
 	"github.com/tendermint/tendermint/rpc/client/http"
 	"github.com/testcontainers/testcontainers-go"
 	"math/big"
@@ -27,6 +28,7 @@ type QGBNetwork struct {
 	EVMRPC        string
 	TendermintRPC string
 	CelestiaGRPC  string
+	EncCfg        cosmoscmd.EncodingConfig
 }
 
 func NewQGBNetwork(_ctx context.Context) (*QGBNetwork, error) {
@@ -42,6 +44,7 @@ func NewQGBNetwork(_ctx context.Context) (*QGBNetwork, error) {
 		EVMRPC:        "http://localhost:8545",
 		TendermintRPC: "tcp://localhost:26657",
 		CelestiaGRPC:  "localhost:9090",
+		EncCfg:        orchestrator.MakeEncodingConfig(),
 	}
 
 	// trap Ctrl+C or ctx.Done()
@@ -363,7 +366,7 @@ func (network QGBNetwork) WaitForBlockWithCustomTimeout(
 // and for any nonce, but would require adding a new method to the querier. Don't think it is worth it now as
 // the number of valsets that will be signed is trivial and reaching 0 would be in no time).
 func (network QGBNetwork) WaitForOrchestratorToStart(_ctx context.Context, accountAddress string) error {
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	if err != nil {
 		return err
 	}
@@ -450,7 +453,7 @@ func (network QGBNetwork) WaitForRelayerToStart(_ctx context.Context, bridge *wr
 			}
 			return ctx.Err()
 		default:
-			nonce, err := bridge.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: ctx})
+			nonce, err := bridge.StateEventNonce(&bind.CallOpts{Context: ctx})
 			if err == nil && nonce != nil && nonce.Int64() >= 1 {
 				return nil
 			}

e2e/qgb/relayer_test.go

@@ -40,13 +40,9 @@ func TestRelayerWithOneValidator(t *testing.T) {
 	// FIXME should we use the evm client here or go for raw queries?
 	evmClient := orchestrator.NewEvmClient(nil, *bridge, nil, network.EVMRPC)
 
-	vsNonce, err := evmClient.StateLastValsetNonce(&bind.CallOpts{Context: ctx})
+	vsNonce, err := evmClient.StateLastEventNonce(&bind.CallOpts{Context: ctx})
 	assert.NoError(t, err)
-	assert.Equal(t, uint64(1), vsNonce)
-
-	dcNonce, err := evmClient.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: ctx})
-	assert.NoError(t, err)
-	assert.Equal(t, uint64(1), dcNonce)
+	assert.GreaterOrEqual(t, vsNonce, uint64(2))
 }
 
 func TestRelayerWithTwoValidators(t *testing.T) {
@@ -91,13 +87,9 @@ func TestRelayerWithTwoValidators(t *testing.T) {
 	// FIXME should we use the evm client here or go for raw queries?
 	evmClient := orchestrator.NewEvmClient(nil, *bridge, nil, network.EVMRPC)
 
-	dcNonce, err := evmClient.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: ctx})
-	assert.NoError(t, err)
-	assert.Equal(t, uint64(1), dcNonce)
-
-	vsNonce, err := evmClient.StateLastValsetNonce(&bind.CallOpts{Context: ctx})
+	dcNonce, err := evmClient.StateLastEventNonce(&bind.CallOpts{Context: ctx})
 	assert.NoError(t, err)
-	assert.Equal(t, uint64(2), vsNonce)
+	assert.GreaterOrEqual(t, dcNonce, uint64(3))
 }
 
 func TestRelayerWithMultipleValidators(t *testing.T) {
@@ -132,12 +124,12 @@ func TestRelayerWithMultipleValidators(t *testing.T) {
 	HandleNetworkError(t, network, err, false)
 
 	// check whether the four validators are up and running
-	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil)
+	querier, err := orchestrator.NewQuerier(network.CelestiaGRPC, network.TendermintRPC, nil, network.EncCfg)
 	HandleNetworkError(t, network, err, false)
 
-	lastValsets, err := querier.QueryLastValsets(ctx)
+	latestValset, err := querier.QueryLatestValset(network.Context)
 	assert.NoError(t, err)
-	assert.Equal(t, 4, len(lastValsets[0].Members))
+	assert.Equal(t, 4, len(latestValset.Members))
 
 	bridge, err := network.GetLatestDeployedQGBContract(network.Context)
 	HandleNetworkError(t, network, err, false)
@@ -148,11 +140,7 @@ func TestRelayerWithMultipleValidators(t *testing.T) {
 	// FIXME should we use the evm client here or go for raw queries?
 	evmClient := orchestrator.NewEvmClient(nil, *bridge, nil, network.EVMRPC)
 
-	dcNonce, err := evmClient.StateLastDataRootTupleRootNonce(&bind.CallOpts{Context: ctx})
-	assert.NoError(t, err)
-	assert.Equal(t, uint64(1), dcNonce)
-
-	vsNonce, err := evmClient.StateLastValsetNonce(&bind.CallOpts{Context: ctx})
+	dcNonce, err := evmClient.StateLastEventNonce(&bind.CallOpts{Context: ctx})
 	assert.NoError(t, err)
-	assert.GreaterOrEqual(t, uint64(2), vsNonce)
+	assert.GreaterOrEqual(t, dcNonce, uint64(3))
 }

go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/celestiaorg/nmt v0.8.0
-	github.com/celestiaorg/quantum-gravity-bridge v1.0.4-0.20220324221947-cc0560976538
+	github.com/celestiaorg/quantum-gravity-bridge v1.2.0
 	github.com/cosmos/cosmos-sdk v0.44.3
 	github.com/cosmos/ibc-go v1.2.2
 	github.com/ethereum/go-ethereum v1.10.12
@@ -13,6 +13,9 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/pkg/errors v0.9.1
+	github.com/regen-network/cosmos-proto v0.3.1
+	github.com/rs/zerolog v1.26.0
 	github.com/spf13/cast v1.4.1
 	github.com/spf13/cobra v1.3.0
 	github.com/stretchr/testify v1.7.0
@@ -119,7 +122,6 @@ require (
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
@@ -128,12 +130,9 @@ require (
 	github.com/prometheus/tsdb v0.7.1 // indirect
 	github.com/rakyll/statik v0.1.7 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
-	github.com/regen-network/cosmos-proto v0.3.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.1 // indirect
-	github.com/rogpeppe/go-internal v1.7.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
-	github.com/rs/zerolog v1.26.0 // indirect
 	github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa // indirect
 	github.com/shirou/gopsutil v3.21.6+incompatible // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect