Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…

…docker-tests

.changeset/lovely-jeans-confess.md

@@ -0,0 +1,6 @@
+---
+"chainlink": minor
+---
+
+#nops : Enable configurable client error regexes for error classification
+#added : New toml configuration options for [EVM.NodePool.Errors] to pass regexes on NonceTooLow, NonceTooHigh, ReplacementTransactionUnderpriced, LimitReached, TransactionAlreadyInMempool, TerminallyUnderpriced, InsufficientEth, TxFeeExceedsCap, L2FeeTooLow, L2FeeTooHigh, L2Full, TransactionAlreadyMined, Fatal, and ServiceUnavailable.

.changeset/new-forks-grab.md

@@ -1,5 +1,5 @@
 ---
-"chainlink": removed
+"chainlink": patch
 ---
 
 Drop unused queryTimeout config from TXM strategy #internal

.changeset/sour-needles-collect.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+cleanup set #bugfix

.github/workflows/build-publish-develop.yml

@@ -56,44 +56,6 @@ jobs:
           dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
           git-commit-sha: ${{ steps.git-ref.outputs.checked-out || github.sha }}
 
-      - name: Setup GAP
-        # Don't run for plugins.
-        if: matrix.image.name == ''
-        uses: smartcontractkit/.github/actions/setup-gap@1bc7ce34fa81fffcb4a6eb0e4e12e59d94d0fc8f # [email protected]
-        with:
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_SAND }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_SAND }}
-          use-k8s: "true"
-          k8s-cluster-name: ${{ secrets.AWS_EKS_CLUSTER_NAME_SAND }}
-          use-private-ecr-registry: true
-          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
-          metrics-job-name: push-chainlink-develop ${{ matrix.image.name }}
-          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-
-      # A mutable image tag is used for these CRIBs and it was just built/published
-      # from this workflow. The deployment has an `imagePullPolicy: Always` set, so
-      # we need to restart the deployments to pick up the new image.
-      - name: Restart K8s Deployments for CRIBs
-        # Don't run for plugins.
-        if: matrix.image.name == ''
-        shell: bash
-        run: |
-          set -euo pipefail
-          # Removes the "smartcontractkit/" (org name) prefix.
-          REPO_NAME_ONLY="${GITHUB_REPOSITORY##*/}"
-          K8S_NAMESPACE="crib-${REPO_NAME_ONLY}-develop"
-
-          deployment_node_names=$(kubectl --namespace "${K8S_NAMESPACE}" \
-              get deployments \
-              -l "app=${K8S_NAMESPACE}" \
-              -o custom-columns=:metadata.name --no-headers)
-
-          echo "Restarting deployments: $deployment_node_names"
-          kubectl rollout restart deployment $deployment_node_names --namespace "${K8S_NAMESPACE}"
-
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics

.github/workflows/ci-core.yml

@@ -171,8 +171,16 @@ jobs:
         run: |
           cat output.txt | gotestloghelper -ci
       - name: Print Races
+        id: print-races
         if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && needs.filter.outputs.changes == 'true' }}
-        run: find race.* | xargs cat
+        run: |
+          find race.* | xargs cat > race.txt
+          if [[ -s race.txt ]]; then
+            cat race.txt
+            echo "post_to_slack=true" >> $GITHUB_OUTPUT
+          else
+            echo "post_to_slack=false" >> $GITHUB_OUTPUT
+          fi
       - name: Print postgres logs
         if: ${{ always() &&  needs.filter.outputs.changes == 'true' }}
         run: docker compose logs postgres | tee ../../../postgres_logs.txt
@@ -189,7 +197,7 @@ jobs:
             ./coverage.txt
             ./postgres_logs.txt
       - name: Notify Slack
-        if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && (github.event_name == 'merge_group' || github.event.branch == 'develop') &&  needs.filter.outputs.changes == 'true' }}
+        if: ${{ failure() && steps.print-races.outputs.post_to_slack == 'true' && matrix.type.cmd == 'go_core_race_tests' && (github.event_name == 'merge_group' || github.event.branch == 'develop') &&  needs.filter.outputs.changes == 'true' }}
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

.github/workflows/solidity.yml

@@ -212,8 +212,8 @@ jobs:
           echo "Bumping version to ${{ needs.tag-check.outputs.pre-release-version }}"
           pnpm version ${{ needs.tag-check.outputs.pre-release-version }} --no-git-tag-version --no-commit-hooks --no-git-checks
 
-      - name: Publish to NPM (Dry Run)
-        uses: smartcontractkit/.github/actions/ci-publish-npm@e1c9d45fc66369d6be5d3863c65af1750797a7f5 # ci-publish-npm@0.3.0     
+      - name: Publish to NPM (beta)
+        uses: smartcontractkit/.github/actions/ci-publish-npm@4b0ab756abcb1760cb82e1e87b94ff431905bffc # ci-publish-npm@0.4.0
         with:
           npm-token: ${{ secrets.NPM_TOKEN }}
           create-github-release: false
@@ -254,8 +254,8 @@ jobs:
             exit 1
           fi
 
-      - name: Publish to NPM (Dry Run)
-        uses: smartcontractkit/.github/actions/ci-publish-npm@e1c9d45fc66369d6be5d3863c65af1750797a7f5 # ci-publish-npm@0.3.0     
+      - name: Publish to NPM (latest)
+        uses: smartcontractkit/.github/actions/ci-publish-npm@4b0ab756abcb1760cb82e1e87b94ff431905bffc # ci-publish-npm@0.4.0
         with:
           npm-token: ${{ secrets.NPM_TOKEN }}
           github-token: ${{ secrets.GITHUB_TOKEN }}

charts/chainlink-cluster/.gitignore

@@ -1,3 +1,7 @@
 # Helm
 charts/
 requirements.lock
+.rendered
+values-profiles/*
+!values-profiles/values-dev.yaml.example
+!values-profiles/README.md

charts/chainlink-cluster/Chart.lock

@@ -12,4 +12,4 @@ dependencies:
   repository: https://grafana.github.io/helm-charts
   version: 7.3.2
 digest: sha256:37722063f68689c42ac1d6549ddfae4756370c1659b8ac1c0d7da8916c6fad3d
-generated: "2024-02-27T11:04:29.920915-06:00"
+generated: "2024-04-23T17:49:50.347667-04:00"

charts/chainlink-cluster/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: chainlink-cluster
 description: Chainlink nodes cluster
-version: 0.4.0
+version: 0.4.2
 appVersion: "2.6.0"
 dependencies:
   - name: mockserver

charts/chainlink-cluster/README.md

@@ -1,44 +1,174 @@
 # Chainlink cluster
 
 Example CL nodes cluster for system level tests
+Install `kubefwd` (no nixpkg for it yet, planned)
+
+```sh
+brew install txn2/tap/kubefwd
+```
+
+If you want to build images you need [docker](https://docs.docker.com/engine/install/) service running
+
+Enter the shell (from the root project dir)
+
+```sh
+nix develop
+```
 
 # Develop
-## Install from release
-Note: The setup below doesn't work at the moment.
 
-Add the repository
+## New cluster
+
+We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3)
+
+Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details
 
+Set up your K8s access
+
+```sh
+export DEVSPACE_IMAGE="..."
+./setup.sh ${my-personal-namespace-name-crib}
 ```
-helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/
-helm repo update
+
+Create a .env file based on the .env.sample file
+
+```sh
+cp .env.sample .env
+# Fill in the required values in .env
 ```
 
-Set default namespace
+Build and deploy the current state of your repository
 
+```sh
+devspace deploy
 ```
-kubectl create ns cl-cluster
-kubectl config set-context --current --namespace cl-cluster
+
+Default `ttl` is `72h`, use `ttl` command to update if you need more time
+
+Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s`
+
+```sh
+devspace run ttl ${namespace} 120h
+```
+
+If you want to deploy an image tag that is already available in ECR, use:
+
+```sh
+devspace deploy --override-image-tag "<image-tag>"
 ```
 
-Install
+If you want to deploy an image tag from a public ECR repo, use:
 
+```sh
+export DEVSPACE_IMAGE=public.ecr.aws/chainlink/chainlink
+devspace deploy --override-image-tag 2.9.0
 ```
+
+To apply custom TOML configuration specific for your nodes, create a `values-dev.yaml` file in the `./values-profiles` directory. Start by copying the example file:
+
+```sh
+cp values-profiles/values-dev.yaml.example values-profiles/values-dev.yaml
+
+```
+
+Then customize the values-dev.yaml file as needed. To use this configuration during deployment, pass the --profile local-dev flag:
+
+```sh
+devspace deploy --profile local-dev
+```
+
+Forward ports to check UI or run tests
+
+```sh
+devspace run connect ${my-personal-namespace-name-crib}
+```
+
+List ingress hostnames
+
+```sh
+devspace run ingress-hosts
+```
+
+Destroy the cluster
+
+```sh
+devspace purge
+```
+
+## Running load tests
+
+Check this [doc](../../integration-tests/load/ocr/README.md)
+
+If you used `devspace dev ...` always use `devspace reset pods` to switch the pods back
+
+# Helm
+
+If you would like to use `helm` directly, please uncomment data in `values.yaml`
+
+## Install from local files
+
+```sh
 helm install -f values.yaml cl-cluster .
 ```
 
+Forward all apps (in another terminal)
+
+```sh
+sudo kubefwd svc -n cl-cluster
+```
+
+Then you can connect and run your tests
+
+## Install from release
+
+Add the repository
+
+```sh
+helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/
+helm repo update
+```
+
+Set default namespace
+
+```sh
+kubectl create ns cl-cluster
+kubectl config set-context --current --namespace cl-cluster
+```
+
 ## Create a new release
-Note: The setup below doesn't work at the moment.
 
 Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release
 
 ## Helm Test
 
-```
+```sh
 helm test cl-cluster
 ```
 
 ## Uninstall
 
-```
+```sh
 helm uninstall cl-cluster
-```
+```
+
+# Grafana dashboard
+
+We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboards programmatically
+
+You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker`
+
+```sh
+export LOKI_TENANT_ID=promtail
+export LOKI_URL=...
+export GRAFANA_URL=...
+export GRAFANA_TOKEN=...
+export PROMETHEUS_DATA_SOURCE_NAME=Thanos
+export LOKI_DATA_SOURCE_NAME=Loki
+export INFRA_PLATFORM=kubernetes
+export GRAFANA_FOLDER=CRIB
+export DASHBOARD_NAME=Core-Cluster-Load
+
+devspace run dashboard_deploy
+```
+
+Open Grafana folder `DashboardCoreDebug` and find dashboard `ChainlinkClusterDebug`

charts/chainlink-cluster/templates/chainlink-cm.yaml

@@ -16,9 +16,11 @@ data:
     CREATE
     EXTENSION pg_stat_statements;
   default.toml: |
-  {{ if (hasKey $cfg "toml") }}
-    {{ $cfg.toml | nindent 4 }}
-  {{ else }}
+  {{- if (hasKey $cfg "toml") }}
+  {{ $cfg.toml | nindent 4 }}
+  {{- else if and (hasKey $.Values.chainlink.global "toml") $.Values.chainlink.global.toml }}
+  {{ $.Values.chainlink.global.toml | nindent 4 }}
+  {{- else }}
     RootDir = './clroot'
     [Log]
     JSONConsole = true
@@ -57,12 +59,14 @@ data:
     {{- end }}
     [WebServer.TLS]
     HTTPSPort = 0
-    {{ end }}
+    {{- end }}
   overrides.toml: |
-  {{ if (hasKey $cfg "overridesToml") }}
-    {{ $cfg.overridesToml | nindent 4 }}
+  {{- if (hasKey $cfg "overridesToml") }}
+  {{- $cfg.overridesToml | nindent 4 }}
+  {{- else if and (hasKey $.Values.chainlink.global "overridesToml") $.Values.chainlink.global.overridesToml }}
+  {{- $.Values.chainlink.global.overridesToml | nindent 4 }}
   {{ else }}
-  {{ end }}
+  {{- end }}
   secrets.toml: |
   {{ if (hasKey $cfg "secretsToml") }}
     {{ $cfg.secretsToml | nindent 4 }}

charts/chainlink-cluster/values-profiles/README.md

@@ -0,0 +1,3 @@
+# values-profiles
+
+Store helm values files for different [DevSpace profiles](https://www.devspace.sh/docs/configuration/profiles/) here. The convention is to make the yaml file name in here match the DevSpace profile name.