From 554807a93f72d85b8a45e930cf1cf3d1d2e16ddb Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:58:06 -0500 Subject: [PATCH 1/6] Make CRIB demo work (#11927) * temp CRIB workshop * tested script + build/deploy separation * check images in ECR, fail if no image built * another try with ecr * force login to registry --------- Co-authored-by: skudasov --- charts/chainlink-cluster/README.md | 23 ++++++- charts/chainlink-cluster/devspace.yaml | 64 ++++++++++++++----- charts/chainlink-cluster/setup.sh | 37 +++++++++++ .../templates/chainlink-db-networkpolicy.yaml | 23 ------- .../chainlink-node-networkpolicy.yaml | 19 ------ .../templates/geth-networkpolicy.yaml | 25 -------- .../templates/mockserver-networkpolicy.yaml | 23 ------- .../templates/networkpolicy-default.yaml | 41 ------------ .../templates/runner-networkpolicy.yaml | 19 ------ shell.nix | 1 + 10 files changed, 106 insertions(+), 169 deletions(-) create mode 100755 charts/chainlink-cluster/setup.sh delete mode 100644 charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml delete mode 100644 charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml delete mode 100644 charts/chainlink-cluster/templates/geth-networkpolicy.yaml delete mode 100644 charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml delete mode 100644 charts/chainlink-cluster/templates/networkpolicy-default.yaml delete mode 100644 charts/chainlink-cluster/templates/runner-networkpolicy.yaml diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 3deb37794a6..7f2736034a7 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -5,6 +5,7 @@ Install `kubefwd` (no nixpkg for it yet, planned) ``` brew install txn2/tap/kubefwd ``` +If you want to build images you need [docker](https://docs.docker.com/engine/install/) service running Enter the shell (from the root project dir) ``` @@ -18,17 +19,33 @@ We are using [devspace](https://www.devspace.sh/docs/getting-started/installatio Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details -Enter the shell and deploy +Configure your `cluster` setup (one time setup, internal usage only) +``` +export DEVSPACE_IMAGE="..." +cd charts/chainlink-cluster +./setup.sh ${my-personal-namespace-name-crib} +``` + +Build and deploy current commit ``` -# set your unique namespace if it's a new cluster -devspace use namespace cl-cluster devspace deploy ``` + If you don't need a build use ``` devspace deploy --skip-build ``` +To deploy particular commit (must be in registry) use +``` +devspace deploy --skip-build ${short_sha_of_image} +``` + +Forward ports to check UI or run tests +``` +devspace run connect ${my-personal-namespace-name-crib} +``` + Connect to your environment, by replacing container with label `node-1` with your local repository files ``` devspace dev -p node diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 0157ce7451c..545923d4124 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -4,26 +4,64 @@ name: chainlink vars: NS_TTL: 72h DEVSPACE_IMAGE: + noCache: true source: env # This is a list of `pipelines` that DevSpace can execute (you can define your own) pipelines: - # create_deployments --all can be used to apply manifests again - # This is the pipeline for the main command: `devspace dev` (or `devspace run-pipeline dev`) dev: run: |- run_dependencies --all # 1. Deploy any projects this project needs (see "dependencies") ensure_pull_secrets --all # 2. Ensure pull secrets start_dev app # 3. Start dev mode "app" (see "dev" section) - # You can run this pipeline via `devspace deploy` (or `devspace run-pipeline deploy`) deploy: run: |- + set -o pipefail + echo "Removing .devspace cache!" + rm -rf .devspace/ || true + registry_id=$(echo "$DEVSPACE_IMAGE" | cut -d'.' -f1) + + # Login into registry + echo "Authorizing into ECR registry" + aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${registry_id}.dkr.ecr.us-west-2.amazonaws.com + run_dependencies --all ensure_pull_secrets --all build_images ---var DOCKER_DEFAULT_PLATFORM=linux/amd64 --all -t $(git rev-parse --short HEAD) kubectl annotate namespace ${DEVSPACE_NAMESPACE} janitor/ttl=${NS_TTL} || true kubectl label namespace/${DEVSPACE_NAMESPACE} network=crib || true - create_deployments --all + if [ -n "$1" ]; then + echo "Deploying tag $1" + tag=$1 + image=${DEVSPACE_IMAGE}:$tag + else + echo "Deploying current commit tag: $(git rev-parse --short HEAD)" + tag=$(git rev-parse --short HEAD) + image=${DEVSPACE_IMAGE}:$tag + fi + + echo "Checking tag: $tag" + repository_name="chainlink-devspace" + desired_tag=$tag + + # Check if the desired tag is present in the repository + image_list=$(aws ecr list-images --repository-name "$repository_name") + tag_exists=$(echo "$image_list" | jq -e '.imageIds[] | select(.imageTag == "'"${desired_tag}"'")' >/dev/null && echo true || echo false) + + # Check the value of the tag_exists variable + if [ "$tag_exists" = "true" ]; then + echo "Image tag '$tag' found." + else + echo "Image tag '$tag' not found. Please build the image using 'devspace deploy'" + exit 1 + fi + create_deployments app \ + --set=helm.values.chainlink.nodes[0].image=$image \ + --set=helm.values.chainlink.nodes[1].image=$image \ + --set=helm.values.chainlink.nodes[2].image=$image \ + --set=helm.values.chainlink.nodes[3].image=$image \ + --set=helm.values.chainlink.nodes[4].image=$image \ + --set=helm.values.chainlink.nodes[5].image=$image echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" purge: run: |- @@ -31,7 +69,7 @@ pipelines: commands: connect: |- - sudo kubefwd svc -n ${DEVSPACE_NAMESPACE} + sudo kubefwd svc -n $1 images: app: @@ -50,7 +88,7 @@ hooks: labelSelector: # vars don't work here, = releaseName release: "app" - events: ["after:deploy:app"] + events: [ "after:deploy:app" ] name: "wait-for-pod-hook" # This is a list of `deployments` that DevSpace can create for this project @@ -83,7 +121,6 @@ deployments: nodes: - name: node-1 image: ${DEVSPACE_IMAGE} - version: latest # default resources are 300m/1Gi # first node need more resources to build faster inside container # at least 2Gi of memory is required otherwise build will fail (OOM) @@ -128,19 +165,14 @@ deployments: # overridesToml: | - name: node-2 image: ${DEVSPACE_IMAGE} - version: latest - name: node-3 image: ${DEVSPACE_IMAGE} - version: latest - name: node-4 image: ${DEVSPACE_IMAGE} - version: latest - name: node-5 image: ${DEVSPACE_IMAGE} - version: latest - name: node-6 image: ${DEVSPACE_IMAGE} - version: latest # each CL node have a dedicated PostgreSQL 11.15 # use StatefulSet by setting: @@ -314,10 +346,10 @@ dev: ssh: enabled: true proxyCommands: -# TODO: access issues -# - command: devspace -# - command: kubectl -# - command: helm + # TODO: access issues + # - command: devspace + # - command: kubectl + # - command: helm - gitCredentials: true ports: - port: "2345" diff --git a/charts/chainlink-cluster/setup.sh b/charts/chainlink-cluster/setup.sh new file mode 100755 index 00000000000..536bef57c75 --- /dev/null +++ b/charts/chainlink-cluster/setup.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e +user_home="$HOME" +file_path="$user_home/.aws/config" +image="" +registry_id=$(echo "$DEVSPACE_IMAGE" | cut -d'.' -f1) + +if grep -q "staging-crib" "$file_path"; then + echo "Staging AWS config is already applied, role is 'staging-crib'" +else + cat <> "$file_path" +[profile staging-crib] +region=us-west-2 +sso_start_url=https://smartcontract.awsapps.com/start +sso_region=us-west-2 +sso_account_id=${registry_id} +sso_role_name=CRIB-ECR-Power +EOF + echo "~/.aws/config modified, added 'staging-crib" +fi + +# Login through SSO +aws sso login --profile staging-crib +# Update kubeconfig and switch context +export AWS_PROFILE=staging-crib +aws eks update-kubeconfig --name main-stage-cluster --alias main-stage-cluster-crib --profile staging-crib + +# Check if the Docker daemon is running +if docker info > /dev/null 2>&1; then + echo "Docker daemon is running, authorizing registry" +else + echo "Docker daemon is not running, exiting" + exit 1 +fi +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${registry_id}.dkr.ecr.us-west-2.amazonaws.com +devspace use namespace $1 diff --git a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml deleted file mode 100644 index e5d029b7865..00000000000 --- a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ $.Release.Name }}-db -spec: - podSelector: - matchLabels: - app: {{ $.Release.Name }}-db - policyTypes: - - Ingress - ingress: - - from: - # Allow all node pods to access the database pods. - - podSelector: - matchLabels: - app: {{ $.Release.Name }} - # Allow all runner pods to access the database pods. - - podSelector: - matchLabels: - app: runner - ports: - - protocol: TCP - port: 5432 diff --git a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml deleted file mode 100644 index 321bc531626..00000000000 --- a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ $.Release.Name }}-node -spec: - podSelector: - matchLabels: - app: {{ $.Release.Name }} - policyTypes: - - Ingress - ingress: - # Allow all ingress traffic between the node pods and from runner pod. - - from: - - podSelector: - matchLabels: - app: {{ $.Release.Name }} - - podSelector: - matchLabels: - app: runner diff --git a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml deleted file mode 100644 index 5be59136251..00000000000 --- a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ $.Release.Name }}-geth -spec: - podSelector: - matchLabels: - app: geth - policyTypes: - - Ingress - ingress: - - from: - # Allow http and websocket connections from the node pods. - - podSelector: - matchLabels: - app: {{ $.Release.Name }} - # Allow http and websocket connections from the runner pods. - - podSelector: - matchLabels: - app: runner - ports: - - protocol: TCP - port: 8544 - - protocol: TCP - port: 8546 diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml deleted file mode 100644 index 074b1ab089a..00000000000 --- a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ $.Release.Name }}-mockserver -spec: - podSelector: - matchLabels: - app: mockserver - policyTypes: - - Ingress - ingress: - - from: - # Allow http traffic from the node pods. - - podSelector: - matchLabels: - app: {{ $.Release.Name }} - # Allow http traffic from the runner pods. - - podSelector: - matchLabels: - app: runner - ports: - - protocol: TCP - port: 1080 diff --git a/charts/chainlink-cluster/templates/networkpolicy-default.yaml b/charts/chainlink-cluster/templates/networkpolicy-default.yaml deleted file mode 100644 index f2d9416cf15..00000000000 --- a/charts/chainlink-cluster/templates/networkpolicy-default.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default -spec: - podSelector: - matchLabels: {} - policyTypes: - - Ingress - - Egress - ingress: - {{- if and .Values.networkPolicyDefault.ingress.allowCustomCidrs (not (empty .Values.networkPolicyDefault.ingress.customCidrs)) }} - # Using a comma separated list to make it easy to pass in with: - # `helm template ... --set networkPolicyDefault.ingress.customCidrs=...` - {{- $cidrs := splitList "," .Values.networkPolicyDefault.ingress.customCidrs }} - - from: - {{- range $cidr := $cidrs }} - - ipBlock: - cidr: {{ $cidr | quote }} - {{- end }} - {{- else }} - # Deny all ingress if no rules are specified. Rules can still be specified in other templates. - - {} - {{- end }} - egress: - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: "{{ $.Release.Namespace }}" - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - protocol: TCP - port: 53 - - protocol: UDP - port: 53 diff --git a/charts/chainlink-cluster/templates/runner-networkpolicy.yaml b/charts/chainlink-cluster/templates/runner-networkpolicy.yaml deleted file mode 100644 index 2bb6ac98625..00000000000 --- a/charts/chainlink-cluster/templates/runner-networkpolicy.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ $.Release.Name }}-runner -spec: - podSelector: - matchLabels: - app: runner - policyTypes: - - Ingress - ingress: - # Allow all ingress traffic between the node pods and from runner pod. - - from: - - podSelector: - matchLabels: - app: {{ $.Release.Name }} - - podSelector: - matchLabels: - app: runner diff --git a/shell.nix b/shell.nix index ff4381b49c1..7881af59ba2 100644 --- a/shell.nix +++ b/shell.nix @@ -30,6 +30,7 @@ mkShell { delve golangci-lint github-cli + jq # deployment devspace From 08c37dba681e70bbe4f8ad5d54e5dd12609988ba Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:21:59 -0500 Subject: [PATCH 2/6] Add devspace helm values to create ingress in K8s (#11938) * Add devspace helm values to create ingress in K8s * fix devspace ingress defs * Fix geth port * Fix print out of URLs --------- Co-authored-by: skudasov --- charts/chainlink-cluster/devspace.yaml | 140 ++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 14 deletions(-) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 545923d4124..8549f1165a2 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -6,6 +6,15 @@ vars: DEVSPACE_IMAGE: noCache: true source: env + # This is the base domain in AWS Route 53 that our ingress subdomains will use. + DEVSPACE_INGRESS_BASE_DOMAIN: + source: env + # This is the ARN of the AWS ACM certificate that will be used for the ingress. + DEVSPACE_INGRESS_CERT_ARN: + source: env + # This is a comma separated list of CIDR blocks that will be allowed to access the ingress. + DEVSPACE_INGRESS_CIDRS: + source: env # This is a list of `pipelines` that DevSpace can execute (you can define your own) pipelines: @@ -20,11 +29,11 @@ pipelines: echo "Removing .devspace cache!" rm -rf .devspace/ || true registry_id=$(echo "$DEVSPACE_IMAGE" | cut -d'.' -f1) - + # Login into registry echo "Authorizing into ECR registry" aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${registry_id}.dkr.ecr.us-west-2.amazonaws.com - + run_dependencies --all ensure_pull_secrets --all build_images ---var DOCKER_DEFAULT_PLATFORM=linux/amd64 --all -t $(git rev-parse --short HEAD) @@ -39,15 +48,15 @@ pipelines: tag=$(git rev-parse --short HEAD) image=${DEVSPACE_IMAGE}:$tag fi - + echo "Checking tag: $tag" repository_name="chainlink-devspace" desired_tag=$tag - + # Check if the desired tag is present in the repository image_list=$(aws ecr list-images --repository-name "$repository_name") tag_exists=$(echo "$image_list" | jq -e '.imageIds[] | select(.imageTag == "'"${desired_tag}"'")' >/dev/null && echo true || echo false) - + # Check the value of the tag_exists variable if [ "$tag_exists" = "true" ]; then echo "Image tag '$tag' found." @@ -63,6 +72,16 @@ pipelines: --set=helm.values.chainlink.nodes[4].image=$image \ --set=helm.values.chainlink.nodes[5].image=$image echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" + + echo + echo "############################################" + echo "Ingress Domains" + echo "############################################" + ingress_names="node1 node2 node3 node4 node5 node6 geth-http geth-ws" + for ingress in ${ingress_names}; do + echo "https://${DEVSPACE_NAMESPACE}-${ingress}.${DEVSPACE_INGRESS_BASE_DOMAIN}" + done + purge: run: |- kubectl delete ns ${DEVSPACE_NAMESPACE} @@ -88,7 +107,7 @@ hooks: labelSelector: # vars don't work here, = releaseName release: "app" - events: [ "after:deploy:app" ] + events: ["after:deploy:app"] name: "wait-for-pod-hook" # This is a list of `deployments` that DevSpace can create for this project @@ -102,7 +121,7 @@ deployments: path: . # for simplicity, we define all the values here # they can be defined the same way in values.yml - # devspace merging this "values" and "values.yml" before deploy + # devspace merges these "values" with the "values.yaml" before deploy values: podSecurityContext: fsGroup: 999 @@ -263,23 +282,116 @@ deployments: limits: cpu: 1 memory: 512Mi - affinity: { } - tolerations: [ ] - nodeSelector: { } + affinity: {} + tolerations: [] + nodeSelector: {} ingress: enabled: false className: "" - hosts: [ ] - tls: [ ] - annotations: { } + hosts: [] + tls: [] + annotations: {} service: type: NodePort port: 8080 - # monitoring.coreos.com/v1 PodMonitor for each node prometheusMonitor: true + networkPolicy: + ingress: + # Should be a comma separated list of CIDR blocks. To include + # AWS ALB private CIDRs and optionally other custom CIDRs. + # Example format: 10.0.0.0/16,192.168.0.1/24 + allowCustomCidrs: ${DEVSPACE_INGRESS_CIDRS} + # These ingresses create AWS ALB resources and Route 53 Records. + ingress: + enabled: true + annotation_certificate_arn: ${DEVSPACE_INGRESS_CERT_ARN} + annotation_group_name: ${DEVSPACE_NAMESPACE} + hosts: + - host: ${DEVSPACE_NAMESPACE}-node1.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-1 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-node2.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-2 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-node3.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-3 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-node4.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-4 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-node5.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-5 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-node6.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: app-node-6 + port: + number: 6688 + - host: ${DEVSPACE_NAMESPACE}-geth-http.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: geth + port: + number: 8544 + - host: ${DEVSPACE_NAMESPACE}-geth-ws.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: geth + port: + number: 8546 + - host: ${DEVSPACE_NAMESPACE}-mockserver.${DEVSPACE_INGRESS_BASE_DOMAIN} + http: + paths: + - path: / + backend: + service: + name: mockserver + port: + number: 1080 + # deployment placement, standard helm stuff podAnnotations: nodeSelector: From 8b17f459a1371d1dfefbfaee87f31954857d4a2f Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:06:54 -0500 Subject: [PATCH 3/6] Add helm network policies back (#11940) * Re-add back network policies. From commit: dc9a073ab7e67a66eed49dd74e6ac0da1787b24c * Create network policies conditionally --- .../templates/chainlink-db-networkpolicy.yaml | 25 +++++++++++ .../chainlink-node-networkpolicy.yaml | 21 +++++++++ .../templates/geth-networkpolicy.yaml | 27 ++++++++++++ .../templates/mockserver-networkpolicy.yaml | 25 +++++++++++ .../templates/networkpolicy-default.yaml | 43 +++++++++++++++++++ .../templates/runner-networkpolicy.yaml | 21 +++++++++ charts/chainlink-cluster/values.yaml | 3 ++ 7 files changed, 165 insertions(+) create mode 100644 charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml create mode 100644 charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml create mode 100644 charts/chainlink-cluster/templates/geth-networkpolicy.yaml create mode 100644 charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml create mode 100644 charts/chainlink-cluster/templates/networkpolicy-default.yaml create mode 100644 charts/chainlink-cluster/templates/runner-networkpolicy.yaml diff --git a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml new file mode 100644 index 00000000000..5f7e7706ced --- /dev/null +++ b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml @@ -0,0 +1,25 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $.Release.Name }}-db +spec: + podSelector: + matchLabels: + app: {{ $.Release.Name }}-db + policyTypes: + - Ingress + ingress: + - from: + # Allow all node pods to access the database pods. + - podSelector: + matchLabels: + app: {{ $.Release.Name }} + # Allow all runner pods to access the database pods. + - podSelector: + matchLabels: + app: runner + ports: + - protocol: TCP + port: 5432 +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml new file mode 100644 index 00000000000..e63759a994f --- /dev/null +++ b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml @@ -0,0 +1,21 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $.Release.Name }}-node +spec: + podSelector: + matchLabels: + app: {{ $.Release.Name }} + policyTypes: + - Ingress + ingress: + # Allow all ingress traffic between the node pods and from runner pod. + - from: + - podSelector: + matchLabels: + app: {{ $.Release.Name }} + - podSelector: + matchLabels: + app: runner +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml new file mode 100644 index 00000000000..025d6184501 --- /dev/null +++ b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $.Release.Name }}-geth +spec: + podSelector: + matchLabels: + app: geth + policyTypes: + - Ingress + ingress: + - from: + # Allow http and websocket connections from the node pods. + - podSelector: + matchLabels: + app: {{ $.Release.Name }} + # Allow http and websocket connections from the runner pods. + - podSelector: + matchLabels: + app: runner + ports: + - protocol: TCP + port: 8544 + - protocol: TCP + port: 8546 +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml new file mode 100644 index 00000000000..6ac4f658e37 --- /dev/null +++ b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml @@ -0,0 +1,25 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $.Release.Name }}-mockserver +spec: + podSelector: + matchLabels: + app: mockserver + policyTypes: + - Ingress + ingress: + - from: + # Allow http traffic from the node pods. + - podSelector: + matchLabels: + app: {{ $.Release.Name }} + # Allow http traffic from the runner pods. + - podSelector: + matchLabels: + app: runner + ports: + - protocol: TCP + port: 1080 +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/networkpolicy-default.yaml b/charts/chainlink-cluster/templates/networkpolicy-default.yaml new file mode 100644 index 00000000000..a2cc23ed7f9 --- /dev/null +++ b/charts/chainlink-cluster/templates/networkpolicy-default.yaml @@ -0,0 +1,43 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default +spec: + podSelector: + matchLabels: {} + policyTypes: + - Ingress + - Egress + ingress: + {{- if and .Values.networkPolicyDefault.ingress.allowCustomCidrs (not (empty .Values.networkPolicyDefault.ingress.customCidrs)) }} + # Using a comma separated list to make it easy to pass in with: + # `helm template ... --set networkPolicyDefault.ingress.customCidrs=...` + {{- $cidrs := splitList "," .Values.networkPolicyDefault.ingress.customCidrs }} + - from: + {{- range $cidr := $cidrs }} + - ipBlock: + cidr: {{ $cidr | quote }} + {{- end }} + {{- else }} + # Deny all ingress if no rules are specified. Rules can still be specified in other templates. + - {} + {{- end }} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: "{{ $.Release.Namespace }}" + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: TCP + port: 53 + - protocol: UDP + port: 53 +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/runner-networkpolicy.yaml b/charts/chainlink-cluster/templates/runner-networkpolicy.yaml new file mode 100644 index 00000000000..b75a2ffa772 --- /dev/null +++ b/charts/chainlink-cluster/templates/runner-networkpolicy.yaml @@ -0,0 +1,21 @@ +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $.Release.Name }}-runner +spec: + podSelector: + matchLabels: + app: runner + policyTypes: + - Ingress + ingress: + # Allow all ingress traffic between the node pods and from runner pod. + - from: + - podSelector: + matchLabels: + app: {{ $.Release.Name }} + - podSelector: + matchLabels: + app: runner +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index 24914a40a91..3e58cbaea24 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -284,6 +284,9 @@ nodeSelector: tolerations: affinity: +networkPolicies: + enabled: true + # Configure the default network policy. networkPolicyDefault: ingress: From b3ec54a83df3a56d6aa399d2f783c64dabe1920c Mon Sep 17 00:00:00 2001 From: Bolek <1416262+bolekk@users.noreply.github.com> Date: Mon, 5 Feb 2024 20:14:54 -0800 Subject: [PATCH 4/6] [Gateway] Better logging around node connectivity (#11929) 1. Rename "heartbeat" -> "keepalive" to avoid confusion with other types of heartbeats. 2. Log node name alongside failed pings for easier debugging. --- core/services/gateway/connectionmanager.go | 37 +++++++++++-------- .../gateway/connectionmanager_test.go | 16 ++++++++ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/core/services/gateway/connectionmanager.go b/core/services/gateway/connectionmanager.go index e5f7fb13afb..a3c39211c6e 100644 --- a/core/services/gateway/connectionmanager.go +++ b/core/services/gateway/connectionmanager.go @@ -27,9 +27,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/utils" ) -var promHeartbeatsSent = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "gateway_heartbeats_sent", - Help: "Metric to track the number of successful node heartbeates per DON", +var promKeepalivesSent = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gateway_keepalives_sent", + Help: "Metric to track the number of successful keepalive ping messages per DON", }, []string{"don_id"}) // ConnectionManager holds all connections between Gateway and Nodes. @@ -77,6 +77,7 @@ type donConnectionManager struct { } type nodeState struct { + name string conn network.WSConnectionWrapper } @@ -107,10 +108,14 @@ func NewConnectionManager(gwConfig *config.GatewayConfig, clock utils.Clock, lgg if ok { return nil, fmt.Errorf("duplicate node address %s in DON %s", nodeAddress, donConfig.DonId) } - nodes[nodeAddress] = &nodeState{conn: network.NewWSConnectionWrapper(lggr)} - if nodes[nodeAddress].conn == nil { + connWrapper := network.NewWSConnectionWrapper(lggr) + if connWrapper == nil { return nil, fmt.Errorf("error creating WSConnectionWrapper for node %s", nodeAddress) } + nodes[nodeAddress] = &nodeState{ + name: nodeConfig.Name, + conn: connWrapper, + } } dons[donConfig.DonId] = &donConnectionManager{ donConfig: &donConfig, @@ -148,7 +153,7 @@ func (m *connectionManager) Start(ctx context.Context) error { go donConnMgr.readLoop(nodeAddress, nodeState) } donConnMgr.closeWait.Add(1) - go donConnMgr.heartbeatLoop(m.config.HeartbeatIntervalSec) + go donConnMgr.keepaliveLoop(m.config.HeartbeatIntervalSec) } return m.wsServer.Start(ctx) }) @@ -231,7 +236,7 @@ func (m *connectionManager) FinalizeHandshake(attemptId string, response []byte, } if conn != nil { conn.SetPongHandler(func(data string) error { - m.lggr.Debugw("received heartbeat pong from node", "nodeAddress", attempt.nodeAddress) + m.lggr.Debugw("received keepalive pong from node", "nodeAddress", attempt.nodeAddress) return nil }) } @@ -299,34 +304,34 @@ func (m *donConnectionManager) readLoop(nodeAddress string, nodeState *nodeState } } -func (m *donConnectionManager) heartbeatLoop(intervalSec uint32) { +func (m *donConnectionManager) keepaliveLoop(intervalSec uint32) { ctx, _ := m.shutdownCh.NewCtx() defer m.closeWait.Done() if intervalSec == 0 { - m.lggr.Error("heartbeat interval is 0, heartbeat disabled") + m.lggr.Errorw("keepalive interval is 0, keepalive disabled", "donID", m.donConfig.DonId) return } - m.lggr.Info("starting heartbeat loop") + m.lggr.Infow("starting keepalive loop", "donID", m.donConfig.DonId) - ticker := time.NewTicker(time.Duration(intervalSec) * time.Second) - defer ticker.Stop() + keepaliveTicker := time.NewTicker(time.Duration(intervalSec) * time.Second) + defer keepaliveTicker.Stop() for { select { case <-m.shutdownCh: return - case <-ticker.C: + case <-keepaliveTicker.C: errorCount := 0 for nodeAddress, nodeState := range m.nodes { err := nodeState.conn.Write(ctx, websocket.PingMessage, []byte{}) if err != nil { - m.lggr.Debugw("unable to send heartbeat to node", "nodeAddress", nodeAddress, "err", err) + m.lggr.Debugw("unable to send keepalive ping to node", "nodeAddress", nodeAddress, "name", nodeState.name, "donID", m.donConfig.DonId, "err", err) errorCount++ } } - promHeartbeatsSent.WithLabelValues(m.donConfig.DonId).Set(float64(len(m.nodes) - errorCount)) - m.lggr.Infow("sent heartbeat to nodes", "donID", m.donConfig.DonId, "errCount", errorCount) + promKeepalivesSent.WithLabelValues(m.donConfig.DonId).Set(float64(len(m.nodes) - errorCount)) + m.lggr.Infow("sent keepalive pings to nodes", "donID", m.donConfig.DonId, "errCount", errorCount) } } } diff --git a/core/services/gateway/connectionmanager_test.go b/core/services/gateway/connectionmanager_test.go index d198ef67295..b176837d9ca 100644 --- a/core/services/gateway/connectionmanager_test.go +++ b/core/services/gateway/connectionmanager_test.go @@ -227,3 +227,19 @@ func TestConnectionManager_SendToNode_Failures(t *testing.T) { err = donMgr.SendToNode(testutils.Context(t), "some_other_node", message) require.Error(t, err) } + +func TestConnectionManager_CleanStartClose(t *testing.T) { + t.Parallel() + + config, _ := newTestConfig(t, 2) + config.ConnectionManagerConfig.HeartbeatIntervalSec = 1 + clock := utils.NewFixedClock(time.Now()) + mgr, err := gateway.NewConnectionManager(config, clock, logger.TestLogger(t)) + require.NoError(t, err) + + err = mgr.Start(testutils.Context(t)) + require.NoError(t, err) + + err = mgr.Close() + require.NoError(t, err) +} From e0da2ca89b9d78951661298d052c6e2390b4a76e Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 6 Feb 2024 11:49:30 -0300 Subject: [PATCH 5/6] modify devspace Dockerfile to build linux/amd64 images on ARM chips + remove some whitespaces (#11941) --- charts/chainlink-cluster/devspace.yaml | 6 +++--- charts/chainlink-cluster/devspace_start.sh | 6 +++--- core/chainlink.devspace.Dockerfile | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 8549f1165a2..7148f271516 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -34,8 +34,8 @@ pipelines: echo "Authorizing into ECR registry" aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${registry_id}.dkr.ecr.us-west-2.amazonaws.com - run_dependencies --all - ensure_pull_secrets --all + run_dependencies --all + ensure_pull_secrets --all build_images ---var DOCKER_DEFAULT_PLATFORM=linux/amd64 --all -t $(git rev-parse --short HEAD) kubectl annotate namespace ${DEVSPACE_NAMESPACE} janitor/ttl=${NS_TTL} || true kubectl label namespace/${DEVSPACE_NAMESPACE} network=crib || true @@ -63,7 +63,7 @@ pipelines: else echo "Image tag '$tag' not found. Please build the image using 'devspace deploy'" exit 1 - fi + fi create_deployments app \ --set=helm.values.chainlink.nodes[0].image=$image \ --set=helm.values.chainlink.nodes[1].image=$image \ diff --git a/charts/chainlink-cluster/devspace_start.sh b/charts/chainlink-cluster/devspace_start.sh index cebf9e346e2..368b5dd9384 100755 --- a/charts/chainlink-cluster/devspace_start.sh +++ b/charts/chainlink-cluster/devspace_start.sh @@ -7,9 +7,9 @@ COLOR_RESET="\033[0m" # Print useful output for user echo -e "${COLOR_BLUE} - %########% - %###########% ____ _____ - %#########% | _ \ ___ __ __ / ___/ ____ ____ ____ ___ + %########% + %###########% ____ _____ + %#########% | _ \ ___ __ __ / ___/ ____ ____ ____ ___ %#########% | | | | / _ \\\\\ \ / / \___ \ | _ \ / _ | / __// _ \\ %#############% | |_| |( __/ \ V / ____) )| |_) )( (_| |( (__( __/ %#############% |____/ \___| \_/ \____/ | __/ \__,_| \___\\\\\___| diff --git a/core/chainlink.devspace.Dockerfile b/core/chainlink.devspace.Dockerfile index c639190a80f..28504b40d7a 100644 --- a/core/chainlink.devspace.Dockerfile +++ b/core/chainlink.devspace.Dockerfile @@ -1,5 +1,5 @@ # Build image: Chainlink binary -FROM golang:1.21-bullseye as buildgo +FROM golang:1.21-bullseye AS buildgo RUN go version WORKDIR /chainlink @@ -22,7 +22,7 @@ RUN go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds | xargs RUN go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-solana | xargs -I % ln -s % /chainlink-solana # Build image: Plugins -FROM golang:1.21-bullseye as buildplugins +FROM golang:1.21-bullseye AS buildplugins RUN go version WORKDIR /chainlink-feeds @@ -34,7 +34,7 @@ COPY --from=buildgo /chainlink-solana . RUN go install ./pkg/solana/cmd/chainlink-solana # Final image: ubuntu with chainlink binary -FROM golang:1.21-bullseye +FROM --platform=linux/amd64 golang:1.21-bullseye ARG CHAINLINK_USER=chainlink ENV DEBIAN_FRONTEND noninteractive From 2635ce3b425eae8ffd789cad9f1af5fda83ed616 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 6 Feb 2024 10:45:46 -0500 Subject: [PATCH 6/6] Cleaning up CODEOWNERS (#11868) * WIP: cleanup * further cleanup * Update CODEOWNERS Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> * further cleanup * further cleanup * Update CODEOWNERS Co-authored-by: Jordan Krage * further cleanup * further cleanup * Update CODEOWNERS Co-authored-by: Jordan Krage * Update CODEOWNERS Co-authored-by: Jordan Krage * Update CODEOWNERS Co-authored-by: Jordan Krage --------- Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> Co-authored-by: Jordan Krage --- CODEOWNERS | 92 +++++++++++++++++++----------------------------------- 1 file changed, 32 insertions(+), 60 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 83505b0ed8c..74a7208cc0a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,37 +2,29 @@ # 1. Per Github docs: "Order is important; the last matching pattern takes the most precedence." # Please define less specific codeowner paths before more specific codeowner paths in order for the more specific rule to have priority -# Misc -/core/store @samsondav @jmank88 -/core/chains @samsondav -/core/logger @jmank88 -/core/internal @samsondav @jmank88 +# Core +/core @smartcontractkit/foundations # Chains /common @smartcontractkit/integrations -/core/chains/evm @samsondav @prashantkumar1982 -/core/chains/evm/headtracker @pinebit @samsondav @prashantkumar1982 -/core/chains/evm/client @samsondav @prashantkumar1982 -/core/chains/evm/log @jmank88 @prashantkumar1982 @connorwstein -/core/chains/evm/logpoller @reductionista @connorwstein -/core/chains/cosmos @archseer @cfal +/core/chains/ @smartcontractkit/integrations # Services -/core/services/cron @samsondav -/core/services/directrequest @connorwstein @samsondav -/core/services/feeds @jkongie -/core/services/fluxmonitorv2 @jkongie @PiotrTrzpil @connorwstein -/core/services/health @samsondav -/core/services/job @connorwstein -/core/services/keystore @RyanRHall -/core/services/ocr* @connorwstein @samsondav -/core/services/ocr2 @jmank88 @krehermann -/core/services/periodicbackup @PiotrTrzpil @samsondav -/core/services/pg @samsondav -/core/services/pipeline @connorwstein @prashantkumar1982 -/core/services/synchronization -/core/services/telemetry -/core/services/relay/evm/mercury @samsondav @sdrug @martin-cll +/core/services/directrequest @smartcontractkit/keepers +/core/services/feeds @smartcontractkit/FMS + +# To be deprecated in Chainlink V3 +/core/services/fluxmonitorv2 @smartcontractkit/foundations +/core/services/job @smartcontractkit/ccip +/core/services/keystore @smartcontractkit/keepers +/core/services/ocr* @smartcontractkit/foundations +/core/services/periodicbackup @smartcontractkit/foundations +/core/services/pg @smartcontractkit/foundations @samsondav +/core/services/pipeline @smartcontractkit/foundations @smartcontractkit/integrations +/core/services/telemetry @smartcontractkit/realtime +/core/services/relay/evm/mercury @smartcontractkit/mercury-team +/core/services/webhook @smartcontractkit/foundations @smartcontractkit/integrations +/core/services/llo @smartcontractkit/mercury-team # VRF-related services /core/services/vrf @smartcontractkit/vrf-team @@ -48,27 +40,16 @@ /core/services/keeper @smartcontractkit/keepers /core/services/ocr2/plugins/ocr2keeper @smartcontractkit/keepers -/core/services/webhook @samsondav - # Chainlink Functions core/services/functions @smartcontractkit/functions core/services/ocr2/plugins/functions @smartcontractkit/functions -core/services/s4 @pinebit @bolekk -core/service/ocr2/plugins/s4 @pinebit @bolekk -core/services/ocr2/plugins/threshold @KuphJr @bolekk -core/services/relay/evm/functions.go @bolekk @justinkaseman @KuphJr -core/services/relay/evm/functions @bolekk @justinkaseman @KuphJr +core/services/s4 @smartcontractkit/functions +core/service/ocr2/plugins/s4 @smartcontractkit/functions +core/services/ocr2/plugins/threshold @smartcontractkit/functions +core/services/relay/evm/functions @smartcontractkit/functions +core/services/relay/evm/functions @smartcontractkit/functions core/scripts/functions @smartcontractkit/functions -core/scripts/gateway @bolekk @pinebit - -# API -/core/web @jkongie - -# CLI -/core/cmd @jkongie @jmank88 @samsondav - -# UI -/operator-ui/ @DeividasK @jkongie +core/scripts/gateway @smartcontractkit/functions # Contracts /contracts/ @RensR @@ -78,23 +59,21 @@ core/scripts/gateway @bolekk @pinebit # This could give some false positives, so afterwards we match on the project directories # to ensure the entire directory is always owned by the correct team. -/contracts/**/*shared* @RensR /contracts/**/*keeper* @smartcontractkit/keepers /contracts/**/*upkeep* @smartcontractkit/keepers /contracts/**/*automation* @smartcontractkit/keepers /contracts/**/*functions* @smartcontractkit/functions -/contracts/**/*llo-feeds* @austinborn @Fletch153 +/contracts/**/*llo-feeds* @smartcontrackit/mercury-team /contracts/**/*vrf* @smartcontractkit/vrf-team -/contracts/**/*l2ep* @simsonraj -/contracts/**/*operatorforwarder* @essamhassan +/contracts/**/*l2ep* @smartcontractkit/integrations /contracts/src/v0.8/automation @smartcontractkit/keepers /contracts/src/v0.8/functions @smartcontractkit/functions # TODO: interfaces folder, folder should be removed and files moved to the correct folders -/contracts/src/v0.8/l2ep @simsonraj -/contracts/src/v0.8/llo-feeds @austinborn @Fletch153 +/contracts/src/v0.8/l2ep @chris-de-leon-cll +/contracts/src/v0.8/llo-feeds @smartcontractkit/mercury-team # TODO: mocks folder, folder should be removed and files moved to the correct folders -/contracts/src/v0.8/operatorforwarder @essamhassan +/contracts/src/v0.8/operatorforwarder @smartcontractkit/foundations /contracts/src/v0.8/shared @RensR # TODO: tests folder, folder should be removed and files moved to the correct folders # TODO: transmission folder, owner should be found @@ -112,10 +91,7 @@ core/scripts/gateway @bolekk @pinebit /integration-tests/**/*automation* @smartcontractkit/keepers # CI/CD -/.github/** @chainchad @javuto @jkongie @jmank88 @samsondav -/.github/workflows/integration-tests.yml @smartcontractkit/test-tooling-team -/.github/workflows/integration-chaos-tests.yml @smartcontractkit/test-tooling-team -/.github/workflows/integration-tests-publish.yml @smartcontractkit/test-tooling-team +/.github/** @smartcontractkit/releng @smartcontractkit/test-tooling-team /.github/workflows/performance-tests.yml @smartcontractkit/test-tooling-team /.github/workflows/automation-ondemand-tests.yml @smartcontractkit/keepers /.github/workflows/automation-benchmark-tests.yml @smartcontractkit/keepers @@ -138,9 +114,5 @@ flake.nix @smartcontractkit/prodsec-public flake.lock @smartcontractkit/prodsec-public # Config -./core/config @samsondav @jmank88 -./docs/CONFIG.md @samsondav @jmank88 @dwightjl -./internal/config/docs.toml @samsondav @jmank88 @dwightjl - -# LOOP Plugins -/plugins @jmank88 @krehermann +./docs/CONFIG.md @smartcontractkit/foundations @smartcontractkit/devrel +./internal/config/docs.toml @smartcontractkit/foundations @smartcontractkit/devrel