Support devspace local-dev profile for CRIB (#12938)

* Support devspace local-dev profile for CRIB

* Remove testing section in README

charts/chainlink-cluster/.gitignore

@@ -1,3 +1,7 @@
 # Helm
 charts/
 requirements.lock
+.rendered
+values-profiles/*
+!values-profiles/values-dev.yaml.example
+!values-profiles/README.md

charts/chainlink-cluster/Chart.lock

@@ -12,4 +12,4 @@ dependencies:
   repository: https://grafana.github.io/helm-charts
   version: 7.3.2
 digest: sha256:37722063f68689c42ac1d6549ddfae4756370c1659b8ac1c0d7da8916c6fad3d
-generated: "2024-02-27T11:04:29.920915-06:00"
+generated: "2024-04-23T17:49:50.347667-04:00"

charts/chainlink-cluster/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: chainlink-cluster
 description: Chainlink nodes cluster
-version: 0.4.0
+version: 0.4.2
 appVersion: "2.6.0"
 dependencies:
   - name: mockserver

charts/chainlink-cluster/README.md

@@ -1,44 +1,174 @@
 # Chainlink cluster
 
 Example CL nodes cluster for system level tests
+Install `kubefwd` (no nixpkg for it yet, planned)
+
+```sh
+brew install txn2/tap/kubefwd
+```
+
+If you want to build images you need [docker](https://docs.docker.com/engine/install/) service running
+
+Enter the shell (from the root project dir)
+
+```sh
+nix develop
+```
 
 # Develop
-## Install from release
-Note: The setup below doesn't work at the moment.
 
-Add the repository
+## New cluster
+
+We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3)
+
+Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details
 
+Set up your K8s access
+
+```sh
+export DEVSPACE_IMAGE="..."
+./setup.sh ${my-personal-namespace-name-crib}
 ```
-helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/
-helm repo update
+
+Create a .env file based on the .env.sample file
+
+```sh
+cp .env.sample .env
+# Fill in the required values in .env
 ```
 
-Set default namespace
+Build and deploy the current state of your repository
 
+```sh
+devspace deploy
 ```
-kubectl create ns cl-cluster
-kubectl config set-context --current --namespace cl-cluster
+
+Default `ttl` is `72h`, use `ttl` command to update if you need more time
+
+Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s`
+
+```sh
+devspace run ttl ${namespace} 120h
+```
+
+If you want to deploy an image tag that is already available in ECR, use:
+
+```sh
+devspace deploy --override-image-tag "<image-tag>"
 ```
 
-Install
+If you want to deploy an image tag from a public ECR repo, use:
 
+```sh
+export DEVSPACE_IMAGE=public.ecr.aws/chainlink/chainlink
+devspace deploy --override-image-tag 2.9.0
 ```
+
+To apply custom TOML configuration specific for your nodes, create a `values-dev.yaml` file in the `./values-profiles` directory. Start by copying the example file:
+
+```sh
+cp values-profiles/values-dev.yaml.example values-profiles/values-dev.yaml
+
+```
+
+Then customize the values-dev.yaml file as needed. To use this configuration during deployment, pass the --profile local-dev flag:
+
+```sh
+devspace deploy --profile local-dev
+```
+
+Forward ports to check UI or run tests
+
+```sh
+devspace run connect ${my-personal-namespace-name-crib}
+```
+
+List ingress hostnames
+
+```sh
+devspace run ingress-hosts
+```
+
+Destroy the cluster
+
+```sh
+devspace purge
+```
+
+## Running load tests
+
+Check this [doc](../../integration-tests/load/ocr/README.md)
+
+If you used `devspace dev ...` always use `devspace reset pods` to switch the pods back
+
+# Helm
+
+If you would like to use `helm` directly, please uncomment data in `values.yaml`
+
+## Install from local files
+
+```sh
 helm install -f values.yaml cl-cluster .
 ```
 
+Forward all apps (in another terminal)
+
+```sh
+sudo kubefwd svc -n cl-cluster
+```
+
+Then you can connect and run your tests
+
+## Install from release
+
+Add the repository
+
+```sh
+helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/
+helm repo update
+```
+
+Set default namespace
+
+```sh
+kubectl create ns cl-cluster
+kubectl config set-context --current --namespace cl-cluster
+```
+
 ## Create a new release
-Note: The setup below doesn't work at the moment.
 
 Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release
 
 ## Helm Test
 
-```
+```sh
 helm test cl-cluster
 ```
 
 ## Uninstall
 
-```
+```sh
 helm uninstall cl-cluster
-```
+```
+
+# Grafana dashboard
+
+We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboards programmatically
+
+You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker`
+
+```sh
+export LOKI_TENANT_ID=promtail
+export LOKI_URL=...
+export GRAFANA_URL=...
+export GRAFANA_TOKEN=...
+export PROMETHEUS_DATA_SOURCE_NAME=Thanos
+export LOKI_DATA_SOURCE_NAME=Loki
+export INFRA_PLATFORM=kubernetes
+export GRAFANA_FOLDER=CRIB
+export DASHBOARD_NAME=Core-Cluster-Load
+
+devspace run dashboard_deploy
+```
+
+Open Grafana folder `DashboardCoreDebug` and find dashboard `ChainlinkClusterDebug`

charts/chainlink-cluster/templates/chainlink-cm.yaml

@@ -16,9 +16,11 @@ data:
     CREATE
     EXTENSION pg_stat_statements;
   default.toml: |
-  {{ if (hasKey $cfg "toml") }}
-    {{ $cfg.toml | nindent 4 }}
-  {{ else }}
+  {{- if (hasKey $cfg "toml") }}
+  {{ $cfg.toml | nindent 4 }}
+  {{- else if and (hasKey $.Values.chainlink.global "toml") $.Values.chainlink.global.toml }}
+  {{ $.Values.chainlink.global.toml | nindent 4 }}
+  {{- else }}
     RootDir = './clroot'
     [Log]
     JSONConsole = true
@@ -57,12 +59,14 @@ data:
     {{- end }}
     [WebServer.TLS]
     HTTPSPort = 0
-    {{ end }}
+    {{- end }}
   overrides.toml: |
-  {{ if (hasKey $cfg "overridesToml") }}
-    {{ $cfg.overridesToml | nindent 4 }}
+  {{- if (hasKey $cfg "overridesToml") }}
+  {{- $cfg.overridesToml | nindent 4 }}
+  {{- else if and (hasKey $.Values.chainlink.global "overridesToml") $.Values.chainlink.global.overridesToml }}
+  {{- $.Values.chainlink.global.overridesToml | nindent 4 }}
   {{ else }}
-  {{ end }}
+  {{- end }}
   secrets.toml: |
   {{ if (hasKey $cfg "secretsToml") }}
     {{ $cfg.secretsToml | nindent 4 }}

charts/chainlink-cluster/values-profiles/README.md

@@ -0,0 +1,3 @@
+# values-profiles
+
+Store helm values files for different [DevSpace profiles](https://www.devspace.sh/docs/configuration/profiles/) here. The convention is to make the yaml file name in here match the DevSpace profile name.

charts/chainlink-cluster/values-profiles/values-dev.yaml.example

@@ -0,0 +1,95 @@
+# Disable geth if using existing testnets.
+geth:
+  enabled: false
+chainlink:
+  global:
+    toml: |-
+      RootDir = './clroot'
+      [Log]
+      JSONConsole = true
+      Level = 'debug'
+      [WebServer]
+      AllowOrigins = '*'
+      SecureCookies = false
+      SessionTimeout = '999h0m0s'
+      [Feature]
+      FeedsManager = true
+      LogPoller = true
+      UICSAKeys = true
+      [OCR]
+      Enabled = true
+      DefaultTransactionQueueDepth = 0
+      [P2P]
+      [P2P.V2]
+      Enabled = true
+      ListenAddresses = ['0.0.0.0:6690']
+      AnnounceAddresses = ['0.0.0.0:6690']
+      DeltaDial = '500ms'
+      DeltaReconcile = '5s'
+      [[EVM]]
+      # Avax Fuji Testnet
+      ChainID = "43113"
+      MinContractPayment = '0'
+      AutoCreateKey = true
+      FinalityDepth = 1
+      [[EVM.Nodes]]
+      Name = 'avax-fuji-testnet-official'
+      WSURL = 'wss://api.avax-test.network/ext/bc/C/ws'
+      HTTPURL = 'https://api.avax-test.network/ext/bc/C/rpc'
+      [[EVM]]
+      # BSC Testnet
+      ChainID = "97"
+      MinContractPayment = '0'
+      AutoCreateKey = true
+      FinalityDepth = 1
+      [[EVM.Nodes]]
+      Name = 'bsc-testnet-official'
+      HTTPURL = 'https://data-seed-prebsc-1-s.bnbchain.org:8545'
+      WSURL = 'wss://data-seed-prebsc-1-s1.binance.org:8545/'
+      [WebServer.TLS]
+      HTTPSPort = 0
+
+    overridesToml: |-
+      [Feature]
+      LogPoller = true
+
+      [Log]
+      Level = 'debug'
+      JSONConsole = true
+
+      [Log.File]
+      MaxSize = '0b'
+
+      [WebServer]
+      AllowOrigins = '*'
+      HTTPPort = 6688
+      SecureCookies = false
+      HTTPWriteTimeout = '1m'
+
+      [WebServer.RateLimit]
+      Authenticated = 2000
+      Unauthenticated = 1000
+
+      [WebServer.TLS]
+      HTTPSPort = 0
+
+      [Database]
+      MaxIdleConns = 50
+      MaxOpenConns = 50
+      MigrateOnStartup = true
+
+      [OCR2]
+      Enabled = true
+      DefaultTransactionQueueDepth = 0
+
+      [OCR]
+      Enabled = false
+      DefaultTransactionQueueDepth = 0
+
+      [P2P]
+      [P2P.V2]
+      Enabled = true
+      ListenAddresses = ['0.0.0.0:6690']
+      AnnounceAddresses = ['0.0.0.0:6690']
+      DeltaDial = '500ms'
+      DeltaReconcile = '5s'

charts/chainlink-cluster/values.yaml

@@ -11,6 +11,23 @@
 #   version: stable
 chainlink:
   enabled: true
+  global:
+    # `toml` is initially set to null. Uncomment and modify the following example as needed.
+    # Example configuration parameters for all Chainlink nodes. Can be overridden by individual node configurations and via `overridesToml`.
+    # toml: |-
+    #   [Log]
+    #   Level = 'info'
+    #   [WebServer]
+    #   HTTPPort = 6688
+    toml: null
+    # `overridesToml` is initially set to null. Uncomment and modify the following example as needed.
+    # Example configuration parameters that override the main configuration.
+    # overridesToml: |-
+    #   [Log]
+    #   Level = 'debug'
+    #   [WebServer]
+    #   HTTPPort = 6690
+    overridesToml: null
   podSecurityContext:
     fsGroup: 14933
   securityContext:
@@ -73,13 +90,6 @@ chainlink:
     - name: node-4
     - name: node-5
     - name: node-6
-  resources:
-    requests:
-      cpu: 350m
-      memory: 1024Mi
-    limits:
-      cpu: 350m
-      memory: 1024Mi
 
 # each CL node have a dedicated PostgreSQL 11.15
 # use StatefulSet by setting: