fix integration issues

charts/chainlink-cluster/devspace.yaml

@@ -68,7 +68,6 @@ deployments:
       values:
         podSecurityContext:
           fsGroup: 999
-
         chainlink:
           securityContext:
             capabilities:
@@ -291,14 +290,13 @@ dev:
       instance: node-1
     # Sync files between the local filesystem and the development container
     sync:
-      - path: ../../core/services/chainlink:/home/chainlink/core/services/chainlink
-        printLogs: true
-        disableDownload: true
       - path: ../..:/home/chainlink
         printLogs: true
         disableDownload: true
         uploadExcludePaths:
+          - .git/
           - integration-tests/
+          - docs/
           - .github/
           - belt/
           - charts/
@@ -308,16 +306,6 @@ dev:
           - integration-scripts/
           - testdata/
           - evm-test-helpers/
-    # Open a terminal and use the following command
+#    # Open a terminal and use the following command
     terminal:
-      command: bash
-    ssh:
-      enabled: true
-    proxyCommands:
-#       TODO: access issues
-#      - command: devspace
-#      - command: kubectl
-#      - command: helm
-      - gitCredentials: true
-    ports:
-      - port: "2345"
+      command: bash

charts/chainlink-cluster/templates/chainlink-node-deployment.yaml

@@ -55,15 +55,17 @@ spec:
               path: /health
               port: {{ $.Values.chainlink.web_port }}
             initialDelaySeconds: 1
-            periodSeconds: 5
-            timeoutSeconds: 10
+            periodSeconds: 10
+            failureThreshold: 10
+            timeoutSeconds: 20
           readinessProbe:
             httpGet:
               path: /health
               port: {{ $.Values.chainlink.web_port }}
             initialDelaySeconds: 1
-            periodSeconds: 5
-            timeoutSeconds: 10
+            periodSeconds: 10
+            failureThreshold: 10
+            timeoutSeconds: 20
           startupProbe:
             httpGet:
               path: /

integration-tests/go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/smartcontractkit/chainlink-testing-framework v1.22.1
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
+	github.com/smartcontractkit/havoc v0.0.7
 	github.com/smartcontractkit/libocr v0.0.0-20231130143053-c5102a9c0fb7
 	github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1
 	github.com/smartcontractkit/wasp v0.4.0
@@ -371,7 +372,6 @@ require (
 	github.com/smartcontractkit/chainlink-feeds v0.0.0-20231127231053-2232d3a6766d // indirect
 	github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231219140448-151a4725f312 // indirect
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231219014050-0c4a7831293a // indirect
-	github.com/smartcontractkit/havoc v0.0.4 // indirect
 	github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 // indirect
 	github.com/smartcontractkit/wsrpc v0.7.2 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect

integration-tests/go.sum

@@ -365,7 +365,6 @@ github.com/cosmos/rosetta-sdk-go v0.10.0 h1:E5RhTruuoA7KTIXUcMicL76cffyeoyvNybzU
 github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFgWl/ENIznEoYQI4=
 github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
 github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
-github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
@@ -622,8 +621,6 @@ github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7N
 github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
-github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
 github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
 github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
@@ -802,8 +799,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
@@ -1440,8 +1435,6 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
-github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -1517,12 +1510,8 @@ github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306 h1:ko88
 github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
-github.com/smartcontractkit/havoc v0.0.1 h1:zjzGre/dzhzQVBL+VNkE0llNLMgGIqB7hAURxsULR9Y=
-github.com/smartcontractkit/havoc v0.0.1/go.mod h1:zeVImDBJdBoW8VPsWbYGgPRlHjQg9mATUZe/t6FQ7l8=
-github.com/smartcontractkit/havoc v0.0.3 h1:TpOCmiyE41C+M1YyvAsKPx1Xe4K15iA3UrhDK81J87w=
-github.com/smartcontractkit/havoc v0.0.3/go.mod h1:zeVImDBJdBoW8VPsWbYGgPRlHjQg9mATUZe/t6FQ7l8=
-github.com/smartcontractkit/havoc v0.0.4 h1:hHGWujFYTHWUvk70RhqXWGSg50FAHwrZ7+6Af/ovw3o=
-github.com/smartcontractkit/havoc v0.0.4/go.mod h1:zeVImDBJdBoW8VPsWbYGgPRlHjQg9mATUZe/t6FQ7l8=
+github.com/smartcontractkit/havoc v0.0.7 h1:FM3/X0SxASVLhg8Kb0ea672ZZHhe2oAl0rhgqQPFw84=
+github.com/smartcontractkit/havoc v0.0.7/go.mod h1:zeVImDBJdBoW8VPsWbYGgPRlHjQg9mATUZe/t6FQ7l8=
 github.com/smartcontractkit/libocr v0.0.0-20231130143053-c5102a9c0fb7 h1:AA7vf29c6lFsZm+MtIEtXtg6VUOQV6waJo5MUuHfRjQ=
 github.com/smartcontractkit/libocr v0.0.0-20231130143053-c5102a9c0fb7/go.mod h1:WcuWFMskcGK0MYZuH5hEhGJOzdJRUFeNEM4PAKlejI4=
 github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 h1:yiKnypAqP8l0OX0P3klzZ7SCcBUxy5KqTAKZmQOvSQE=
@@ -1650,8 +1639,6 @@ github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFs
 github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
 github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
 github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
-github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
-github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
@@ -1923,7 +1910,6 @@ golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=

integration-tests/load/ocr/config.go

@@ -18,15 +18,20 @@ const (
 )
 
 type PerformanceConfig struct {
-	Load   *Load   `toml:"Load"`
-	Volume *Volume `toml:"Volume"`
-	Common *Common `toml:"Common"`
+	Env    *Env    `toml:"env"`
+	Load   *Load   `toml:"load"`
+	Volume *Volume `toml:"volume"`
+	Common *Common `toml:"common"`
 }
 
 type Common struct {
 	ETHFunds int `toml:"eth_funds"`
 }
 
+type Env struct {
+	Namespace string `toml:"namespace"`
+}
+
 type Load struct {
 	TestDuration          *models.Duration `toml:"test_duration"`
 	Rate                  int64            `toml:"rate"`

integration-tests/load/ocr/config.toml

@@ -1,4 +1,7 @@
-[Load]
+[env]
+namespace = "cl-cluster"
+
+[load]
 # overall load test duration
 test_duration = "3m"
 # rate limit unit duration, 1m = 1 request per minute
@@ -12,7 +15,7 @@ verification_timeout = "3m"
 # how frequently we change external adapter value
 ea_change_interval = "5s"
 
-[Volume]
+[volume]
 # overall volume test duration
 test_duration = "3m"
 # rate limit unit duration, 1m = 1 request per minute
@@ -28,7 +31,7 @@ verification_timeout = "3m"
 # how frequently we change external adapter value
 ea_change_interval = "5s"
 
-[Common]
+[common]
 # each CL node receives 3 ETH
 eth_funds = 3
 
@@ -77,7 +80,7 @@ group_percentage = "30"
 # rand - runs random experiments from all dirs
 mode = "rand"
 # duration of havoc monkey
-duration = "3m"
+duration = "1h"
 # dir is a custom dir you can select, if null monkey will create a new dir
 dir = "havoc-test-dir"
 # cooldown between experiments

integration-tests/load/ocr/helper.go

@@ -15,7 +15,7 @@ import (
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 )
 
-func SetupCluster(
+func SetupClusterContracts(
 	cc blockchain.EVMClient,
 	cd contracts.ContractDeployer,
 	workerNodes []*client.ChainlinkK8sClient,

integration-tests/load/ocr/ocr_test.go

@@ -1,7 +1,6 @@
 package ocr
 
 import (
-	"context"
 	"github.com/rs/zerolog"
 	"github.com/smartcontractkit/havoc"
 	"testing"
@@ -22,31 +21,21 @@ var (
 	}
 )
 
-func runHavocMonkey(t *testing.T, l zerolog.Logger, ctx context.Context) {
+func createMonkey(t *testing.T, l zerolog.Logger, namespace string) *havoc.Controller {
 	havoc.SetGlobalLogger(l)
-	cfg, err := havoc.ReadConfig("config.toml")
+	cfg, err := havoc.ReadConfig(DefaultConfigFilename)
 	require.NoError(t, err)
-	err = havoc.GenerateSpecs(
-		"skudasov-crib",
-		cfg.Havoc.Monkey.Dir,
-		cfg,
-	)
-	require.NoError(t, err)
-	m, err := havoc.NewMonkey(cfg)
-	require.NoError(t, err)
-	err = m.Run(ctx)
+	c, err := havoc.NewController(cfg)
+	err = c.GenerateSpecs(namespace)
 	require.NoError(t, err)
+	return c
 }
 
 func TestOCRLoad(t *testing.T) {
-	testCtx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(func() {
-		cancel()
-	})
 	l := logging.GetTestLogger(t)
 	cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l)
 	require.NoError(t, err)
-	lt, err := SetupCluster(cc, cd, workerNodes)
+	lt, err := SetupClusterContracts(cc, cd, workerNodes)
 	require.NoError(t, err)
 	ocrInstances, err := SetupFeed(cc, msClient, cd, bootstrapNode, workerNodes, lt)
 	require.NoError(t, err)
@@ -66,20 +55,19 @@ func TestOCRLoad(t *testing.T) {
 		Labels:                CommonTestLabels,
 		LokiConfig:            wasp.NewEnvLokiConfig(),
 	}))
-	go runHavocMonkey(t, l, testCtx)
+	monkey := createMonkey(t, l, cfg.Env.Namespace)
+	go monkey.Run()
 	_, err = p.Run(true)
 	require.NoError(t, err)
+	errs := monkey.Stop()
+	require.Len(t, errs, 0)
 }
 
 func TestOCRVolume(t *testing.T) {
-	testCtx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(func() {
-		cancel()
-	})
 	l := logging.GetTestLogger(t)
 	cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l)
 	require.NoError(t, err)
-	lt, err := SetupCluster(cc, cd, workerNodes)
+	lt, err := SetupClusterContracts(cc, cd, workerNodes)
 	require.NoError(t, err)
 	cfg, err := ReadConfig()
 	require.NoError(t, err)
@@ -95,7 +83,10 @@ func TestOCRVolume(t *testing.T) {
 		Labels:      CommonTestLabels,
 		LokiConfig:  wasp.NewEnvLokiConfig(),
 	}))
-	go runHavocMonkey(t, l, testCtx)
+	monkey := createMonkey(t, l, cfg.Env.Namespace)
+	go monkey.Run()
 	_, err = p.Run(true)
 	require.NoError(t, err)
+	errs := monkey.Stop()
+	require.Len(t, errs, 0)
 }