Merge branch 'develop' into AUTO-11176/fix-chaos-test

.github/workflows/automation-ondemand-tests.yml

@@ -152,15 +152,42 @@ jobs:
             pyroscope_env: ci-automation-on-demand-chaos
             network: SIMULATED
             command: -run ^TestAutomationChaos$ ./chaos
-          - name: reorg
-            id: reorg
+          - name: reorg 2.0
+            id: reorg-2.0
             suite: reorg
-            nodes: 7
+            nodes: 1
+            os: ubuntu-latest
+            enabled: ${{ inputs.enableReorg }}
+            pyroscope_env: ci-automation-on-demand-reorg
+            network: SIMULATED
+            command: -run ^TestAutomationReorg/registry_2_0 ./reorg
+          - name: reorg 2.1
+            id: reorg-2.1
+            suite: reorg
+            nodes: 2
+            os: ubuntu-latest
+            enabled: ${{ inputs.enableReorg }}
+            pyroscope_env: ci-automation-on-demand-reorg
+            network: SIMULATED
+            command: -run ^TestAutomationReorg/registry_2_1 ./reorg
+          - name: reorg 2.2
+            id: reorg-2.2
+            suite: reorg
+            nodes: 2
+            os: ubuntu-latest
+            enabled: ${{ inputs.enableReorg }}
+            pyroscope_env: ci-automation-on-demand-reorg
+            network: SIMULATED
+            command: -run ^TestAutomationReorg/registry_2_2 ./reorg
+          - name: reorg 2.3
+            id: reorg-2.3
+            suite: reorg
+            nodes: 2
             os: ubuntu-latest
             enabled: ${{ inputs.enableReorg }}
             pyroscope_env: ci-automation-on-demand-reorg
             network: SIMULATED
-            command: -run ^TestAutomationReorg$ ./reorg
+            command: -run ^TestAutomationReorg/registry_2_3 ./reorg
           - name: upgrade 2.0
             id: upgrade-2.0
             type: upgrade

.github/workflows/crib-integration-test.yml

@@ -0,0 +1,74 @@
+# this is disabled because of GAP limitations, should be re-enabled when github-actions-controller will be installed
+
+#name: CRIB Integration Tests
+#on:
+#  push:
+#  workflow_call:
+#concurrency:
+#  group: ${{ github.workflow }}-${{ github.ref }}
+#  cancel-in-progress: true
+#jobs:
+#  test:
+#    runs-on: ubuntu-latest
+#    environment: integration
+#    permissions:
+#      id-token: write
+#      contents: read
+#      actions: read
+#    steps:
+#      - name: Checkout repository
+#        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#
+#      - name: Setup Nix + GATI environment
+#        uses: smartcontractkit/.github/actions/setup-nix-gati@514fe346780e2eddf7ea8b9f48120c2fba120d94
+#        with:
+#          aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_AUTO_PR_TOKEN_ISSUER_ROLE_ARN }}
+#          aws-lambda-url: ${{ secrets.AWS_CORE_TOKEN_ISSUER_LAMBDA_URL }} # see https://github.com/smartcontractkit/  infra/blob/a79bcfb48315c4411023c182e98eb80ff9e9cda6/accounts/production/us-west-2/lambda/  github-app-token-issuer-production/teams/releng/config.json#L9
+#          aws-region: ${{ secrets.AWS_REGION }}
+#          aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }}
+#          enable-magic-cache: true
+#
+#      - name: Nix Develop Action
+#        uses: nicknovitski/nix-develop@v1
+#        with:
+#          arguments: "--accept-flake-config"
+#      - name: setup-gap
+#        uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
+#        with:
+#          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_STAGE }}
+#          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_STAGE }}
+#          aws-region: ${{ secrets.AWS_REGION }}
+#          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+#          k8s-cluster-name: ${{ secrets.AWS_K8S_CLUSTER_NAME_STAGE }}
+#          use-private-ecr-registry: true
+#          use-k8s: true
+#          metrics-job-name: "k8s"
+#          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+#          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+#          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#        name: Checkout CRIB repository
+#        with:
+#          repository: 'smartcontractkit/crib'
+#          ref: 'main'
+#      - name: Generate Short UUID
+#        id: uuid
+#        run: echo "CRIB_NAMESPACE=$(uuidgen | cut -c1-5)" >> $GITHUB_ENV
+#      - name: Create a new CRIB environment
+#        run: |-
+#          devspace use namespace $CRIB_NAMESPACE
+#          devspace deploy --profile local-dev-simulated-core-ocr1
+#      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+#      - name: Setup go
+#        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+#        with:
+#          go-version-file: "go.mod"
+#      - name: Run CRIB integration test
+#        working-directory: integration-tests/crib
+#        env:
+#          K8S_STAGING_INGRESS_SUFFIX: ${{ secrets.K8S_STAGING_INGRESS_SUFFIX }}
+#          CRIB_NAMESPACE: ${{ env.CRIB_NAMESPACE }}
+#          CRIB_NETWORK: geth
+#          CRIB_NODES: 5
+#        run: |-
+#          go test -v -run TestCRIB

integration-tests/actions/automation_ocr_helpers_local.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/lib/pq"
 	"github.com/rs/zerolog"
@@ -26,6 +28,24 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
 
+func AutomationDefaultRegistryConfig(c tc.AutomationTestConfig) contracts.KeeperRegistrySettings {
+	registrySettings := c.GetAutomationConfig().AutomationConfig.RegistrySettings
+	return contracts.KeeperRegistrySettings{
+		PaymentPremiumPPB:    *registrySettings.PaymentPremiumPPB,
+		FlatFeeMicroLINK:     *registrySettings.FlatFeeMicroLINK,
+		CheckGasLimit:        *registrySettings.CheckGasLimit,
+		StalenessSeconds:     registrySettings.StalenessSeconds,
+		GasCeilingMultiplier: *registrySettings.GasCeilingMultiplier,
+		MinUpkeepSpend:       registrySettings.MinUpkeepSpend,
+		MaxPerformGas:        *registrySettings.MaxPerformGas,
+		FallbackGasPrice:     registrySettings.FallbackGasPrice,
+		FallbackLinkPrice:    registrySettings.FallbackLinkPrice,
+		MaxCheckDataSize:     *registrySettings.MaxCheckDataSize,
+		MaxPerformDataSize:   *registrySettings.MaxPerformDataSize,
+		MaxRevertDataSize:    *registrySettings.MaxRevertDataSize,
+	}
+}
+
 func BuildAutoOCR2ConfigVarsLocal(
 	l zerolog.Logger,
 	chainlinkNodes []*client.ChainlinkClient,

integration-tests/actions/ocr_helpers.go

@@ -2,8 +2,15 @@ package actions
 
 import (
 	"fmt"
+	"math/big"
+	"math/rand"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/rs/zerolog"
+	"github.com/smartcontractkit/seth"
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
@@ -217,3 +224,55 @@ func BuildNodeContractPairID(node contracts.ChainlinkNodeWithKeysAndAddress, ocr
 	shortOCRAddr := ocrInstance.Address()[2:12]
 	return strings.ToLower(fmt.Sprintf("node_%s_contract_%s", shortNodeAddr, shortOCRAddr)), nil
 }
+
+func SetupOCRv1Cluster(
+	l zerolog.Logger,
+	seth *seth.Client,
+	workerNodes []*client.ChainlinkK8sClient,
+) (common.Address, error) {
+	err := FundChainlinkNodesFromRootAddress(l, seth, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes), big.NewFloat(3))
+	if err != nil {
+		return common.Address{}, err
+	}
+	linkContract, err := contracts.DeployLinkTokenContract(l, seth)
+	if err != nil {
+		return common.Address{}, err
+	}
+	return common.HexToAddress(linkContract.Address()), nil
+}
+
+func SetupOCRv1Feed(
+	l zerolog.Logger,
+	seth *seth.Client,
+	lta common.Address,
+	msClient *ctfClient.MockserverClient,
+	bootstrapNode *client.ChainlinkK8sClient,
+	workerNodes []*client.ChainlinkK8sClient,
+) ([]contracts.OffchainAggregator, error) {
+	ocrInstances, err := DeployOCRv1Contracts(l, seth, 1, lta, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes))
+	if err != nil {
+		return nil, err
+	}
+	err = CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, fmt.Sprint(seth.ChainID))
+	if err != nil {
+		return nil, err
+	}
+	return ocrInstances, nil
+}
+
+func SimulateOCRv1EAActivity(
+	l zerolog.Logger,
+	eaChangeInterval time.Duration,
+	ocrInstances []contracts.OffchainAggregator,
+	workerNodes []*client.ChainlinkK8sClient,
+	msClient *ctfClient.MockserverClient,
+) {
+	go func() {
+		for {
+			time.Sleep(eaChangeInterval)
+			if err := SetAllAdapterResponsesToTheSameValue(rand.Intn(1000), ocrInstances, workerNodes, msClient); err != nil {
+				l.Error().Err(err).Msg("failed to update mockserver responses")
+			}
+		}
+	}()
+}

integration-tests/crib/README.md

@@ -0,0 +1,19 @@
+### CRIB Health Check Test
+
+## Setup CRIB
+This is a simple smoke + chaos test for CRIB deployment.
+It runs OCRv1 and reboots the environment confirming integration with environment is working and data is properly saved even after reboots.
+Go to the [CRIB](https://github.com/smartcontractkit/crib) repository and spin up a cluster.
+
+```shell
+./scripts/cribbit.sh crib-oh-my-crib
+devspace deploy --debug --profile local-dev-simulated-core-ocr1
+```
+
+## Run the tests
+```shell
+CRIB_NAMESPACE=crib-oh-my-crib
+CRIB_NETWORK=geth # only "geth" is supported for now
+CRIB_NODES=5 # min 5 nodes
+go test -v -run TestCRIB
+```

integration-tests/crib/chaos.go

@@ -0,0 +1,45 @@
+package crib
+
+import (
+	"time"
+
+	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+	"github.com/smartcontractkit/havoc/k8schaos"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func rebootCLNamespace(delay time.Duration, namespace string) (*k8schaos.Chaos, error) {
+	k8sClient, err := k8schaos.NewChaosMeshClient()
+	if err != nil {
+		return nil, err
+	}
+	return k8schaos.NewChaos(k8schaos.ChaosOpts{
+		Description: "Reboot CRIB",
+		DelayCreate: delay,
+		Object: &v1alpha1.PodChaos{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "PodChaos",
+				APIVersion: "chaos-mesh.org/v1alpha1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "reboot-crib",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.PodChaosSpec{
+				ContainerSelector: v1alpha1.ContainerSelector{
+					PodSelector: v1alpha1.PodSelector{
+						Mode: v1alpha1.AllMode,
+						Selector: v1alpha1.PodSelectorSpec{
+							GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
+								Namespaces: []string{namespace},
+							},
+						},
+					},
+				},
+				Action: v1alpha1.PodKillAction,
+			},
+		},
+		Client: k8sClient,
+		Logger: &k8schaos.Logger,
+	})
+}

integration-tests/crib/connect.go

@@ -3,6 +3,7 @@ package crib
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,7 +40,7 @@ func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) {
 
 // ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core
 // connects to default CRIB network if simulated = true
-func ConnectRemote(simulated bool) (
+func ConnectRemote() (
 	*seth.Client,
 	*msClient.MockserverClient,
 	*client.ChainlinkK8sClient,
@@ -50,22 +51,35 @@ func ConnectRemote(simulated bool) (
 	if ingressSuffix == "" {
 		return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses")
 	}
+	cribNamespace := os.Getenv("CRIB_NAMESPACE")
+	if cribNamespace == "" {
+		return nil, nil, nil, nil, errors.New("CRIB_NAMESPACE must be set to connect")
+	}
+	cribNetwork := os.Getenv("CRIB_NETWORK")
+	if cribNetwork == "" {
+		return nil, nil, nil, nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now")
+	}
+	cribNodes := os.Getenv("CRIB_NODES")
+	nodes, err := strconv.Atoi(cribNodes)
+	if err != nil {
+		return nil, nil, nil, nil, errors.New("CRIB_NODES must be a number, 5-19 nodes")
+	}
 	config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	if config.CRIB.CLNodesNum < 2 {
+	if nodes < 2 {
 		return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]")
 	}
-	cfg := config.CRIB
-	mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix)
+	mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cribNamespace, ingressSuffix)
 	var sethClient *seth.Client
-	if simulated {
-		netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix)
-		netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix)
+	switch cribNetwork {
+	case "geth":
+		netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cribNamespace, ingressSuffix)
+		netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cribNamespace, ingressSuffix)
 		setSethConfig(config, netWSURL, netHTTPURL)
 		net := blockchain.EVMNetwork{
-			Name:                 cfg.NetworkName,
+			Name:                 cribNetwork,
 			Simulated:            true,
 			SupportsEIP1559:      true,
 			ClientImplementation: blockchain.EthereumClientImplementation,
@@ -84,27 +98,29 @@ func ConnectRemote(simulated bool) (
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
+	default:
+		return nil, nil, nil, nil, errors.New("CRIB network is not supported")
 	}
 	// bootstrap node
 	clClients := make([]*client.ChainlinkK8sClient, 0)
 	c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
-		URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix),
+		URL:        fmt.Sprintf("https://%s-node%d%s", cribNamespace, 1, ingressSuffix),
 		Email:      client.CLNodeTestEmail,
 		InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1),
 		Password:   client.CLNodeTestPassword,
-	}, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace)
+	}, fmt.Sprintf(internalNodeDNSTemplate, 1), cribNamespace)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	clClients = append(clClients, c)
 	// all the other nodes, indices of nodes in CRIB starts with 1
-	for i := 2; i <= cfg.CLNodesNum; i++ {
+	for i := 2; i <= nodes; i++ {
 		cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
-			URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix),
+			URL:        fmt.Sprintf("https://%s-node%d%s", cribNamespace, i, ingressSuffix),
 			Email:      client.CLNodeTestEmail,
 			InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i),
 			Password:   client.CLNodeTestPassword,
-		}, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace)
+		}, fmt.Sprintf(internalNodeDNSTemplate, i), cribNamespace)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}