From fc47d0716787b82c5417aa7944d0bbdf5f9894c8 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:44:17 -0400 Subject: [PATCH 1/2] Support devspace local-dev profile for CRIB --- charts/chainlink-cluster/.gitignore | 4 + charts/chainlink-cluster/Chart.lock | 2 +- charts/chainlink-cluster/Chart.yaml | 2 +- charts/chainlink-cluster/README.md | 179 ++++++++++++++++-- .../templates/chainlink-cm.yaml | 18 +- .../values-profiles/README.md | 3 + .../values-profiles/values-dev.yaml.example | 95 ++++++++++ charts/chainlink-cluster/values.yaml | 24 ++- crib/devspace.yaml | 48 ++++- 9 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 charts/chainlink-cluster/values-profiles/README.md create mode 100644 charts/chainlink-cluster/values-profiles/values-dev.yaml.example diff --git a/charts/chainlink-cluster/.gitignore b/charts/chainlink-cluster/.gitignore index 3ee791f740a..9e704020530 100644 --- a/charts/chainlink-cluster/.gitignore +++ b/charts/chainlink-cluster/.gitignore @@ -1,3 +1,7 @@ # Helm charts/ requirements.lock +.rendered +values-profiles/* +!values-profiles/values-dev.yaml.example +!values-profiles/README.md \ No newline at end of file diff --git a/charts/chainlink-cluster/Chart.lock b/charts/chainlink-cluster/Chart.lock index 38e7f09705e..11ebdfdf691 100644 --- a/charts/chainlink-cluster/Chart.lock +++ b/charts/chainlink-cluster/Chart.lock @@ -12,4 +12,4 @@ dependencies: repository: https://grafana.github.io/helm-charts version: 7.3.2 digest: sha256:37722063f68689c42ac1d6549ddfae4756370c1659b8ac1c0d7da8916c6fad3d -generated: "2024-02-27T11:04:29.920915-06:00" +generated: "2024-04-23T17:49:50.347667-04:00" diff --git a/charts/chainlink-cluster/Chart.yaml b/charts/chainlink-cluster/Chart.yaml index fa45ea9bd77..d8d3dd98b22 100644 --- a/charts/chainlink-cluster/Chart.yaml +++ b/charts/chainlink-cluster/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: chainlink-cluster description: Chainlink nodes cluster -version: 0.4.0 +version: 0.4.2 appVersion: "2.6.0" dependencies: - name: mockserver diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index b76d55090dd..2c59a3312a8 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -1,44 +1,197 @@ # Chainlink cluster Example CL nodes cluster for system level tests +Install `kubefwd` (no nixpkg for it yet, planned) + +```sh +brew install txn2/tap/kubefwd +``` + +If you want to build images you need [docker](https://docs.docker.com/engine/install/) service running + +Enter the shell (from the root project dir) + +```sh +nix develop +``` # Develop -## Install from release -Note: The setup below doesn't work at the moment. -Add the repository +## New cluster + +We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3) + +Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details + +Set up your K8s access +```sh +export DEVSPACE_IMAGE="..." +./setup.sh ${my-personal-namespace-name-crib} ``` -helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/ -helm repo update + +Create a .env file based on the .env.sample file + +```sh +cp .env.sample .env +# Fill in the required values in .env ``` -Set default namespace +Build and deploy the current state of your repository +```sh +devspace deploy ``` -kubectl create ns cl-cluster -kubectl config set-context --current --namespace cl-cluster + +Default `ttl` is `72h`, use `ttl` command to update if you need more time + +Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s` + +```sh +devspace run ttl ${namespace} 120h ``` -Install +If you want to deploy an image tag that is already available in ECR, use: +```sh +devspace deploy --override-image-tag "" ``` + +If you want to deploy an image tag from a public ECR repo, use: + +```sh +export DEVSPACE_IMAGE=public.ecr.aws/chainlink/chainlink +devspace deploy --override-image-tag 2.9.0 +``` + +To apply custom TOML configuration specific for your nodes, create a `values-dev.yaml` file in the `./values-profiles` directory. Start by copying the example file: + +```sh +cp values-profiles/values-dev.yaml.example values-profiles/values-dev.yaml + +``` + +Then customize the values-dev.yaml file as needed. To use this configuration during deployment, pass the --profile local-dev flag: + +```sh +devspace deploy --profile local-dev +``` + +Forward ports to check UI or run tests + +```sh +devspace run connect ${my-personal-namespace-name-crib} +``` + +List ingress hostnames + +```sh +devspace run ingress-hosts +``` + +Destroy the cluster + +```sh +devspace purge +``` + +## Running load tests + +Check this [doc](../../integration-tests/load/ocr/README.md) + +If you used `devspace dev ...` always use `devspace reset pods` to switch the pods back + +# Helm + +If you would like to use `helm` directly, please uncomment data in `values.yaml` + +## Install from local files + +```sh helm install -f values.yaml cl-cluster . ``` +Forward all apps (in another terminal) + +```sh +sudo kubefwd svc -n cl-cluster +``` + +Then you can connect and run your tests + +## Install from release + +Add the repository + +```sh +helm repo add chainlink-cluster https://raw.githubusercontent.com/smartcontractkit/chainlink/helm-release/ +helm repo update +``` + +Set default namespace + +```sh +kubectl create ns cl-cluster +kubectl config set-context --current --namespace cl-cluster +``` + ## Create a new release -Note: The setup below doesn't work at the moment. Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release ## Helm Test -``` +```sh helm test cl-cluster ``` ## Uninstall -``` +```sh helm uninstall cl-cluster -``` \ No newline at end of file +``` + +# Grafana dashboard + +We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboards programmatically + +You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker` + +```sh +export LOKI_TENANT_ID=promtail +export LOKI_URL=... +export GRAFANA_URL=... +export GRAFANA_TOKEN=... +export PROMETHEUS_DATA_SOURCE_NAME=Thanos +export LOKI_DATA_SOURCE_NAME=Loki +export INFRA_PLATFORM=kubernetes +export GRAFANA_FOLDER=CRIB +export DASHBOARD_NAME=Core-Cluster-Load + +devspace run dashboard_deploy +``` + +Open Grafana folder `DashboardCoreDebug` and find dashboard `ChainlinkClusterDebug` + +# Testing + +Deploy your dashboard and run soak/load [tests](../../integration-tests/load/), check [README](../../integration-tests/README.md) for further explanations + +```sh +devspace run dashboard_deploy +devspace run workload +devspace run dashboard_test +``` + +# Local Testing + +Go to [dashboard-lib](../../dashboard) and link the modules locally + +```sh +cd dashboard +pnpm link --global +cd charts/chainlink-cluster/dashboard/tests +pnpm link --global dashboard-tests +``` + +Then run the tests with commands mentioned above diff --git a/charts/chainlink-cluster/templates/chainlink-cm.yaml b/charts/chainlink-cluster/templates/chainlink-cm.yaml index 9a99e2a5128..bd0c911331c 100644 --- a/charts/chainlink-cluster/templates/chainlink-cm.yaml +++ b/charts/chainlink-cluster/templates/chainlink-cm.yaml @@ -16,9 +16,11 @@ data: CREATE EXTENSION pg_stat_statements; default.toml: | - {{ if (hasKey $cfg "toml") }} - {{ $cfg.toml | nindent 4 }} - {{ else }} + {{- if (hasKey $cfg "toml") }} + {{ $cfg.toml | nindent 4 }} + {{- else if and (hasKey $.Values.chainlink.global "toml") $.Values.chainlink.global.toml }} + {{ $.Values.chainlink.global.toml | nindent 4 }} + {{- else }} RootDir = './clroot' [Log] JSONConsole = true @@ -57,12 +59,14 @@ data: {{- end }} [WebServer.TLS] HTTPSPort = 0 - {{ end }} + {{- end }} overrides.toml: | - {{ if (hasKey $cfg "overridesToml") }} - {{ $cfg.overridesToml | nindent 4 }} + {{- if (hasKey $cfg "overridesToml") }} + {{- $cfg.overridesToml | nindent 4 }} + {{- else if and (hasKey $.Values.chainlink.global "overridesToml") $.Values.chainlink.global.overridesToml }} + {{- $.Values.chainlink.global.overridesToml | nindent 4 }} {{ else }} - {{ end }} + {{- end }} secrets.toml: | {{ if (hasKey $cfg "secretsToml") }} {{ $cfg.secretsToml | nindent 4 }} diff --git a/charts/chainlink-cluster/values-profiles/README.md b/charts/chainlink-cluster/values-profiles/README.md new file mode 100644 index 00000000000..7f5b6df4904 --- /dev/null +++ b/charts/chainlink-cluster/values-profiles/README.md @@ -0,0 +1,3 @@ +# values-profiles + +Store helm values files for different [DevSpace profiles](https://www.devspace.sh/docs/configuration/profiles/) here. The convention is to make the yaml file name in here match the DevSpace profile name. diff --git a/charts/chainlink-cluster/values-profiles/values-dev.yaml.example b/charts/chainlink-cluster/values-profiles/values-dev.yaml.example new file mode 100644 index 00000000000..99151149182 --- /dev/null +++ b/charts/chainlink-cluster/values-profiles/values-dev.yaml.example @@ -0,0 +1,95 @@ +# Disable geth if using existing testnets. +geth: + enabled: false +chainlink: + global: + toml: |- + RootDir = './clroot' + [Log] + JSONConsole = true + Level = 'debug' + [WebServer] + AllowOrigins = '*' + SecureCookies = false + SessionTimeout = '999h0m0s' + [Feature] + FeedsManager = true + LogPoller = true + UICSAKeys = true + [OCR] + Enabled = true + DefaultTransactionQueueDepth = 0 + [P2P] + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:6690'] + AnnounceAddresses = ['0.0.0.0:6690'] + DeltaDial = '500ms' + DeltaReconcile = '5s' + [[EVM]] + # Avax Fuji Testnet + ChainID = "43113" + MinContractPayment = '0' + AutoCreateKey = true + FinalityDepth = 1 + [[EVM.Nodes]] + Name = 'avax-fuji-testnet-official' + WSURL = 'wss://api.avax-test.network/ext/bc/C/ws' + HTTPURL = 'https://api.avax-test.network/ext/bc/C/rpc' + [[EVM]] + # BSC Testnet + ChainID = "97" + MinContractPayment = '0' + AutoCreateKey = true + FinalityDepth = 1 + [[EVM.Nodes]] + Name = 'bsc-testnet-official' + HTTPURL = 'https://data-seed-prebsc-1-s.bnbchain.org:8545' + WSURL = 'wss://data-seed-prebsc-1-s1.binance.org:8545/' + [WebServer.TLS] + HTTPSPort = 0 + + overridesToml: |- + [Feature] + LogPoller = true + + [Log] + Level = 'debug' + JSONConsole = true + + [Log.File] + MaxSize = '0b' + + [WebServer] + AllowOrigins = '*' + HTTPPort = 6688 + SecureCookies = false + HTTPWriteTimeout = '1m' + + [WebServer.RateLimit] + Authenticated = 2000 + Unauthenticated = 1000 + + [WebServer.TLS] + HTTPSPort = 0 + + [Database] + MaxIdleConns = 50 + MaxOpenConns = 50 + MigrateOnStartup = true + + [OCR2] + Enabled = true + DefaultTransactionQueueDepth = 0 + + [OCR] + Enabled = false + DefaultTransactionQueueDepth = 0 + + [P2P] + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:6690'] + AnnounceAddresses = ['0.0.0.0:6690'] + DeltaDial = '500ms' + DeltaReconcile = '5s' \ No newline at end of file diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index ed497a65ff9..d6315f488fd 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -11,6 +11,23 @@ # version: stable chainlink: enabled: true + global: + # `toml` is initially set to null. Uncomment and modify the following example as needed. + # Example configuration parameters for all Chainlink nodes. Can be overridden by individual node configurations and via `overridesToml`. + # toml: |- + # [Log] + # Level = 'info' + # [WebServer] + # HTTPPort = 6688 + toml: null + # `overridesToml` is initially set to null. Uncomment and modify the following example as needed. + # Example configuration parameters that override the main configuration. + # overridesToml: |- + # [Log] + # Level = 'debug' + # [WebServer] + # HTTPPort = 6690 + overridesToml: null podSecurityContext: fsGroup: 14933 securityContext: @@ -73,13 +90,6 @@ chainlink: - name: node-4 - name: node-5 - name: node-6 - resources: - requests: - cpu: 350m - memory: 1024Mi - limits: - cpu: 350m - memory: 1024Mi # each CL node have a dedicated PostgreSQL 11.15 # use StatefulSet by setting: diff --git a/crib/devspace.yaml b/crib/devspace.yaml index ca12e1ed205..cc82c6a464c 100644 --- a/crib/devspace.yaml +++ b/crib/devspace.yaml @@ -107,6 +107,42 @@ deployments: fsGroup: 999 chainlink: + global: + overridesToml: |- + [Feature] + LogPoller = true + [Log] + Level = 'debug' + JSONConsole = true + [Log.File] + MaxSize = '0b' + [WebServer] + AllowOrigins = '*' + HTTPPort = 6688 + SecureCookies = false + HTTPWriteTimeout = '1m' + [WebServer.RateLimit] + Authenticated = 2000 + Unauthenticated = 1000 + [WebServer.TLS] + HTTPSPort = 0 + [Database] + MaxIdleConns = 50 + MaxOpenConns = 50 + MigrateOnStartup = true + [OCR2] + Enabled = true + DefaultTransactionQueueDepth = 0 + [OCR] + Enabled = false + DefaultTransactionQueueDepth = 0 + [P2P] + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:6690'] + AnnounceAddresses = ['0.0.0.0:6690'] + DeltaDial = '500ms' + DeltaReconcile = '5s' securityContext: capabilities: drop: @@ -118,7 +154,7 @@ deployments: web_port: 6688 p2p_port: 6690 # extraEnvVars: - # "CL_MEDIAN_CMD": "chainlink-feeds" + # "CL_MEDIAN_CMD": "chainlink-feeds" nodes: - name: node-1 image: ${runtime.images.app} @@ -385,3 +421,13 @@ deployments: nodeSelector: tolerations: affinity: +profiles: + - name: local-dev + patches: + # Remove the global overridesToml field. + # This will be configured via a values file. + - op: remove + path: deployments.app.helm.values.chainlink.global.overridesToml + - op: add + path: deployments.app.helm.valuesFiles + value: ["../charts/chainlink-cluster/values-profiles/values-dev.yaml"] From 2d79ac559022301a526358d57ae1c1a075638516 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:21:59 -0400 Subject: [PATCH 2/2] Remove testing section in README --- charts/chainlink-cluster/README.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 2c59a3312a8..403ab9b876a 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -172,26 +172,3 @@ devspace run dashboard_deploy ``` Open Grafana folder `DashboardCoreDebug` and find dashboard `ChainlinkClusterDebug` - -# Testing - -Deploy your dashboard and run soak/load [tests](../../integration-tests/load/), check [README](../../integration-tests/README.md) for further explanations - -```sh -devspace run dashboard_deploy -devspace run workload -devspace run dashboard_test -``` - -# Local Testing - -Go to [dashboard-lib](../../dashboard) and link the modules locally - -```sh -cd dashboard -pnpm link --global -cd charts/chainlink-cluster/dashboard/tests -pnpm link --global dashboard-tests -``` - -Then run the tests with commands mentioned above