Skip to content

Commit

Permalink
CCIP-3461: Wiring mainnet smoke test with traffic check
Browse files Browse the repository at this point in the history
  • Loading branch information
@b-gopalswami
b-gopalswami committed Sep 20, 2024
1 parent 2a08e17 commit 293fca4
Show file tree
Hide file tree
Showing 9 changed files with 237 additions and 146 deletions.
44 changes: 26 additions & 18 deletions .github/workflows/ccip-live-network-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ on:
required: false
type: choice
options:
- 'load'
- 'smoke'
- 'load'
test_secrets_override_key:
description: 'Key to run tests with custom test secrets'
required: false
Expand All @@ -41,19 +41,21 @@ env:
E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
# Default private key test secret loaded from Github Secret as only security team has access to it.
# this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets.
E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_BLAST_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_CELO_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}


jobs:
build-chainlink:
Expand Down Expand Up @@ -133,8 +135,8 @@ jobs:
matrix:
config: [mainnet.toml]
needs: [ build-chainlink, build-test-image ]
# if the event is a scheduled event or the test type is load and no previous job failed
if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'load') && !contains(needs.*.result, 'failure') }}
# if the event is a workflow_dispatch event and the test type is load and no previous job failed
if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'load' && !contains(needs.*.result, 'failure') }}
permissions:
issues: read
checks: write
Expand Down Expand Up @@ -207,8 +209,8 @@ jobs:
RR_CPU: 4
DETACH_RUNNER: true
TEST_TRIGGERED_BY: ccip-load-test-ci
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
with:
test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
Expand All @@ -233,8 +235,8 @@ jobs:
environment: integration
runs-on: ubuntu-latest
needs: [ build-chainlink, build-test-image ]
# if the event is a scheduled event or the test type is load and no previous job failed
if: ${{ github.event_name == 'workflow_dispatch' && inputs.test_type == 'smoke' && !contains(needs.*.result, 'failure') }}
# if the event is a scheduled event or the test type is smoke and no previous job failed
if: ${{ (github.event_name == 'schedule' || inputs.test_type == 'smoke') && !contains(needs.*.result, 'failure') }}
permissions:
issues: read
checks: write
Expand Down Expand Up @@ -276,6 +278,12 @@ jobs:
echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
fi
if [[ "${{ github.event_name }}" == "schedule" ]]; then
BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ./integration-tests/ccip-tests/testconfig/override/${{ matrix.config }})
echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
echo "SLACK_USER=${{ secrets.QA_SLACK_USER }}" >> $GITHUB_ENV
fi
- name: step summary
shell: bash
run: |
Expand All @@ -301,11 +309,11 @@ jobs:
RR_MEM: 8Gi
RR_CPU: 4
TEST_TRIGGERED_BY: ccip-smoke-test-ci
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }},${{ steps.set_override_config.outputs.base_64_override }}
E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
with:
test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 3h -count=1 -p 30 -json -run ^TestSmokeCCIPForBidirectionalLane$ ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
# Other default test secrets loaded from dotenv Github Secret.
test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }}
Expand Down
46 changes: 25 additions & 21 deletions integration-tests/ccip-tests/actions/ccip_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1648,44 +1648,48 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe

// IsPastRequestTriggeredWithinTimeframe determines the average block time and calculates the block numbers
// within the specified timeframe. It then uses FilterCCIPSendRequested to identify the past events.
func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(ctx context.Context, timeframe *commonconfig.Duration) (*time.Time, error) {
func (sourceCCIP *SourceCCIPModule) IsPastRequestTriggeredWithinTimeframe(
ctx context.Context,
timeframe *commonconfig.Duration,
) (latestEvent *types.Log, err error) {
if timeframe == nil {
return nil, nil
}
//var foundAt *time.Time
latestBlock, err := sourceCCIP.Common.ChainClient.LatestBlockNumber(ctx)
if err != nil {
return nil, fmt.Errorf("error while getting latest source block number. Error: %w", err)
latestBlock, err1 := sourceCCIP.Common.ChainClient.LatestBlockNumber(ctx)
if err1 != nil {
return nil, fmt.Errorf("error while getting latest source block number. Error: %w", err1)
}
avgBlockTime, err := sourceCCIP.Common.ChainClient.AvgBlockTime(ctx)
if err != nil {
return nil, fmt.Errorf("error while getting average source block time. Error: %w", err)
avgBlockTime, err1 := sourceCCIP.Common.ChainClient.AvgBlockTime(ctx)
if err1 != nil {
return nil, fmt.Errorf("error while getting average source block time. Error: %w", err1)
}
filterFromBlock := latestBlock - uint64(timeframe.Duration()/avgBlockTime)

onRampContract, err := evm_2_evm_onramp.NewEVM2EVMOnRamp(common.HexToAddress(sourceCCIP.OnRamp.EthAddress.Hex()),
onRampContract, err1 := evm_2_evm_onramp.NewEVM2EVMOnRamp(common.HexToAddress(sourceCCIP.OnRamp.EthAddress.Hex()),
sourceCCIP.Common.ChainClient.Backend())
if err != nil {
return nil, fmt.Errorf("error while on ramp contract. Error: %w", err)
if err1 != nil {
return nil, fmt.Errorf("error while on ramp contract. Error: %w", err1)
}
iterator, err := onRampContract.FilterCCIPSendRequested(&bind.FilterOpts{
iterator, err1 := onRampContract.FilterCCIPSendRequested(&bind.FilterOpts{
Start: filterFromBlock,
})
if err != nil {
return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err)
if err1 != nil {
return nil, fmt.Errorf("error while filtering CCIP send requested starting block number: %d. Error: %w", filterFromBlock, err1)
}
defer func() {
_ = iterator.Close()
err = iterator.Close()
}()
if iterator.Next() {
hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(iterator.Event.Raw.BlockNumber)))
if err != nil {
return nil, fmt.Errorf("error getting header for block: %d, Error: %w", iterator.Event.Raw.BlockNumber, err)
lastBlockNumber := uint64(0)
latestEvent = nil
for iterator.Next() {
blockNum := iterator.Event.Raw.BlockNumber
if blockNum > lastBlockNumber {
lastBlockNumber = blockNum
latestEvent = &iterator.Event.Raw
}
return pointer.ToTime(hdr.Timestamp), nil
}

return nil, nil
return latestEvent, nil
}

func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested(
Expand Down
8 changes: 7 additions & 1 deletion integration-tests/ccip-tests/load/ccip_loadgen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,13 +226,19 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response {
res := &wasp.Response{}
sourceCCIP := c.Lane.Source
var recentRequestFoundAt *time.Time
var latestEvent *types.Log
var err error
// Use IsPastRequestTriggeredWithinTimeframe to check for any historical CCIP send request events
// within the specified timeframe for the first message. Subsequently, use the watcher method to monitor
// and detect any new events as they occur.
if c.CurrentMsgSerialNo.Load() == int64(1) {
recentRequestFoundAt, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
latestEvent, err = sourceCCIP.IsPastRequestTriggeredWithinTimeframe(testcontext.Get(c.t), c.SkipRequestIfAnotherRequestTriggeredWithin)
require.NoError(c.t, err, "error while filtering past requests")
if latestEvent != nil {
hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(latestEvent.BlockNumber)))
require.NoError(c.t, err, "error while getting header by block number")
recentRequestFoundAt = pointer.ToTime(hdr.Timestamp)
}
} else {
recentRequestFoundAt = sourceCCIP.IsRequestTriggeredWithinTimeframe(c.SkipRequestIfAnotherRequestTriggeredWithin)
}
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/ccip-tests/load/ccip_multicall_loadgen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func NewMultiCallLoadGenerator(testCfg *testsetups.CCIPTestConfig, lanes []*acti
testCfg.Test, lane, testCfg.TestGroupInput.PhaseTimeout.Duration(),
100000,
testCfg.TestGroupInput.LoadProfile.MsgProfile, 0,
testCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
testCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
)
ccipLoad.BeforeAllCall()
m.E2ELoads[fmt.Sprintf("%s-%s", lane.SourceNetworkName, lane.DestNetworkName)] = ccipLoad
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/ccip-tests/load/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ func (l *LoadArgs) TriggerLoadByLane() {
ccipLoad := NewCCIPLoad(
l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(),
100000, l.TestCfg.TestGroupInput.LoadProfile.MsgProfile, sendMaxData,
l.TestCfg.TestGroupInput.LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin,
l.TestCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin,
)
ccipLoad.BeforeAllCall()
// if it's not multicall set the tokens to nil to free up some space,
Expand Down
42 changes: 32 additions & 10 deletions integration-tests/ccip-tests/smoke/ccip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"
"time"

"github.com/ethereum/go-ethereum/core/types"

"github.com/AlekSi/pointer"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -54,20 +56,40 @@ func TestSmokeCCIPForBidirectionalLane(t *testing.T) {
require.NoError(t, setUpOutput.TearDown())
})

// Create test definitions for each lane.
var tests []testDefinition
for _, lane := range setUpOutput.Lanes {
tests = append(tests, testDefinition{
testName: fmt.Sprintf("CCIP message transfer from network %s to network %s",
lane.ForwardLane.SourceNetworkName, lane.ForwardLane.DestNetworkName),
lane: lane.ForwardLane,
})
if lane.ReverseLane != nil {
lookBackDuration := TestCfg.TestGroupInput.SkipRequestIfAnotherRequestTriggeredWithin
var recentTxFound *types.Log
var err error

addLanesToTest := func(lane *actions.CCIPLane) {
// Create test definitions for given lane if no previous request has been triggered within the specified timeframe.
// By default, the timeframe is set to nil. To define a timeframe, assign a duration to the variable
// SkipRequestIfAnotherRequestTriggeredWithin.
if lookBackDuration != nil {
recentTxFound, err = lane.Source.IsPastRequestTriggeredWithinTimeframe(lane.Context, lookBackDuration)
require.NoError(t, err, "error while finding recent request for lane network %s to network %s",
lane.SourceNetworkName, lane.DestNetworkName)
}
if recentTxFound == nil {
tests = append(tests, testDefinition{
testName: fmt.Sprintf("CCIP message transfer from network %s to network %s",
lane.ReverseLane.SourceNetworkName, lane.ReverseLane.DestNetworkName),
lane: lane.ReverseLane,
lane.SourceNetworkName, lane.DestNetworkName),
lane: lane,
})
} else {
log.Info().
Str("TX", recentTxFound.TxHash.Hex()).
Uint64("Block Number", recentTxFound.BlockNumber).
Str("Source", lane.SourceNetworkName).
Str("Dest", lane.DestNetworkName).
Msgf("Lane Skipped. Recent request found within %v minutes.", lookBackDuration.Duration().Minutes())
}
}
for _, lane := range setUpOutput.Lanes {
addLanesToTest(lane.ForwardLane)
if lane.ReverseLane != nil {
recentTxFound = nil
addLanesToTest(lane.ReverseLane)
}
}

Expand Down
12 changes: 7 additions & 5 deletions integration-tests/ccip-tests/testconfig/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -657,6 +657,13 @@ Specifies the OCR parameters for the execute job. This is only valid if the test

Specifies the value for the `InflightExpiry` in commit job's offchain config. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment).

### CCIP.Groups.[testgroup].SkipRequestIfAnotherRequestTriggeredWithin

If there is CCIP Send requested event present within this duration, the test will skip sending another
request during load run or avoid sending request in smoke test in that lane. For Example,
if `SkipRequestIfAnotherRequestTriggeredWithin` is set to `40m`, and a request is triggered at 0th second, the test will skip sending another request for another 40m.
This particular field is used to avoid sending transaction when there is traffic already in that lane.

### CCIP.Groups.[testgroup].OffRampConfig

Specifies the offramp configuration for the execution job. This is only valid if the test is not run on [existing deployments](#ccipgroupstestgroupexistingdeployment).
Expand Down Expand Up @@ -736,11 +743,6 @@ Specifies the duration network delay used for `NetworkChaos` experiment. This is

If there are multiple chaos experiments, this specifies the duration to wait between each chaos experiment. This is only valid if the test is run on k8s and not on [existing deployments](#ccipgroupstestgroupexistingdeployment).

#### CCIP.Groups.[testgroup].LoadProfile.SkipRequestIfAnotherRequestTriggeredWithin

If a request is triggered within this duration, the test will skip sending another request during load run. For Example, if `SkipRequestIfAnotherRequestTriggeredWithin` is set to `40m`, and a request is triggered at 0th second, the test will skip sending another request for another 40m.
This particular field is used to avoid sending multiple requests in a short duration during load run.

#### CCIP.Groups.[testgroup].LoadProfile.OptimizeSpace

This is used internally to optimize memory usage during load run. If set to true, after the initial lane set up is over the test will discard the lane config to save memory.
Expand Down
Loading

0 comments on commit 293fca4

Please sign in to comment.