Merge branch 'develop' into f_better_logstream_start

smartcontractkit · Jan 18, 2024 · 32615d3 · 32615d3
2 parents 6c33fe0 + 286c8c0
commit 32615d3
Show file tree

Hide file tree

Showing 108 changed files with 1,071 additions and 745 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -13,7 +13,7 @@ on:
 jobs:
   analyze:
     name: Analyze ${{ matrix.language }}
-    runs-on: ubuntu20.04-4cores-16GB
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -129,7 +129,7 @@ jobs:
     needs: [changes, enforce-ctf-version]
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -142,7 +142,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
       - name: Build Chainlink Image
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: ./.github/actions/build-chainlink-image
         with:
           tag_suffix: ${{ matrix.image.tag-suffix }}
@@ -164,7 +164,7 @@ jobs:
     needs: [changes]
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -177,7 +177,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
       - name: Build Test Image
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: ./.github/actions/build-test-image
         with:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
@@ -228,7 +228,7 @@ jobs:
           fi
 
   eth-smoke-tests-matrix-automation:
-    if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
+    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
     environment: integration
     permissions:
       checks: write
@@ -250,7 +250,7 @@ jobs:
     name: ETH Smoke Tests ${{ matrix.product.name }}
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -274,7 +274,7 @@ jobs:
           fi
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
         env:
           PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
@@ -297,7 +297,7 @@ jobs:
           QA_KUBECONFIG: ""
 
   eth-smoke-tests-matrix:
-    if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
+    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
     environment: integration
     permissions:
       checks: write
@@ -315,25 +315,25 @@ jobs:
       matrix:
         product:
           - name: cron
-            nodes: 1
+            nodes: 2
             os: ubuntu-latest
             pyroscope_env: ""
           - name: flux
             nodes: 1
             os: ubuntu-latest
             pyroscope_env: ""
           - name: ocr
-            nodes: 1
+            nodes: 2
             os: ubuntu-latest
             file: ocr
             pyroscope_env: ci-smoke-ocr-evm-simulated
           - name: ocr2
-            nodes: 1
+            nodes: 4
             os: ubuntu-latest
             file: ocr2
             pyroscope_env: ci-smoke-ocr2-evm-simulated
           - name: ocr2
-            nodes: 1
+            nodes: 4
             os: ubuntu-latest
             pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated
             tag_suffix: "-plugins"
@@ -342,15 +342,15 @@ jobs:
             os: ubuntu-latest
             pyroscope_env: ""
           - name: vrf
-            nodes: 1
+            nodes: 2
             os: ubuntu-latest
             pyroscope_env: ci-smoke-vrf-evm-simulated
           - name: vrfv2
-            nodes: 1
+            nodes: 2
             os: ubuntu-latest
             pyroscope_env: ci-smoke-vrf2-evm-simulated
           - name: vrfv2plus
-            nodes: 1
+            nodes: 3
             os: ubuntu-latest
             pyroscope_env: ci-smoke-vrf2plus-evm-simulated
           - name: forwarder_ocr
@@ -365,7 +365,7 @@ jobs:
     name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -441,7 +441,7 @@ jobs:
           docker logs otel-collector
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
         env:
           PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
@@ -772,7 +772,7 @@ jobs:
         FORCE_COLOR: 1
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -788,7 +788,7 @@ jobs:
           repository: smartcontractkit/chainlink-solana
           ref: ${{ needs.get_solana_sha.outputs.sha }}
       - name: Build contracts
-        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
+        if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
         uses: smartcontractkit/chainlink-solana/.github/actions/build_contract_artifacts@21675b3a7dcdff8e790391708d4763020cace21e # stable action on December 18 2023
         with:
           ref: ${{ needs.get_solana_sha.outputs.sha }}
@@ -813,7 +813,7 @@ jobs:
       CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
+        if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -822,13 +822,13 @@ jobs:
           this-job-name: Solana Build Test Image
         continue-on-error: true
       - name: Checkout the repo
-        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
+        if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           repository: smartcontractkit/chainlink-solana
           ref: ${{ needs.get_solana_sha.outputs.sha }}
       - name: Build Test Image
-        if: needs.changes.outputs.src == 'true' && needs.solana-test-image-exists.outputs.exists == 'false'
+        if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
         uses: ./.github/actions/build-test-image
         with:
           tag: ${{ needs.get_solana_sha.outputs.sha }}
@@ -864,7 +864,7 @@ jobs:
       CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
     steps:
       - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
         with:
@@ -879,7 +879,7 @@ jobs:
           repository: smartcontractkit/chainlink-solana
           ref: ${{ needs.get_solana_sha.outputs.sha }}
       - name: Run Setup
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
         with:
           go_mod_path: ./integration-tests/go.mod
@@ -892,7 +892,7 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
       - name: Pull Artfacts
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         run: |
           IMAGE_NAME=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }}
           # Pull the Docker image
@@ -908,7 +908,7 @@ jobs:
           # Remove the created container
           docker rm "$CONTAINER_ID"
       - name: Run Tests
-        if: needs.changes.outputs.src == 'true'
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16
         with:
           test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke

diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml
@@ -21,6 +21,13 @@ env:
   INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
   MOD_CACHE_VERSION: 2
   CHAINLINK_NODE_FUNDING: .5
+  PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+  LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+  LOKI_URL: ${{ secrets.LOKI_URL }}
+  LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+  LOGSTREAM_LOG_TARGETS: loki
+  GRAFANA_URL: ${{ vars.GRAFANA_URL }}
+  RUN_ID: ${{ github.run_id }}
 
   CHAINLINK_COMMIT_SHA: ${{ github.sha }}
   CHAINLINK_ENV_USER: ${{ github.actor }}

diff --git a/.github/workflows/solidity-hardhat.yml b/.github/workflows/solidity-hardhat.yml
@@ -63,7 +63,7 @@ jobs:
       fail-fast: false
       matrix:
         split: ${{ fromJson(needs.split-tests.outputs.splits) }}
-    runs-on: ubuntu20.04-4cores-16GB
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -128,7 +128,7 @@ jobs:
       fail-fast: false
       matrix:
         split: ${{ fromJson(needs.split-tests.outputs.splits) }}
-    runs-on: ubuntu20.04-4cores-16GB
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml
@@ -21,8 +21,8 @@ pipelines:
       run_dependencies --all                           
       ensure_pull_secrets --all                         
       build_images ---var DOCKER_DEFAULT_PLATFORM=linux/amd64  --all -t $(git rev-parse --short HEAD)
-      kubectl annotate namespace ${DEVSPACE_NAMESPACE} janitor/ttl=${NS_TTL}
-      kubectl label namespace/${DEVSPACE_NAMESPACE} network=crib
+      kubectl annotate namespace ${DEVSPACE_NAMESPACE} janitor/ttl=${NS_TTL} || true
+      kubectl label namespace/${DEVSPACE_NAMESPACE} network=crib || true
       create_deployments --all                          
       echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}"
   purge:

diff --git a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ $.Release.Name }}-db
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ $.Release.Name }}-db
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        # Allow all node pods to access the database pods.
+        - podSelector:
+            matchLabels:
+              app: {{ $.Release.Name }}
+        # Allow all runner pods to access the database pods.
+        - podSelector:
+            matchLabels:
+              app: runner
+      ports:
+        - protocol: TCP
+          port: 5432
diff --git a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml
@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ $.Release.Name }}-node
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ $.Release.Name }}
+  policyTypes:
+    - Ingress
+  ingress:
+    # Allow all ingress traffic between the node pods and from runner pod.
+    - from:
+      - podSelector:
+          matchLabels:
+            app: {{ $.Release.Name }}
+      - podSelector:
+          matchLabels:
+            app: runner
diff --git a/charts/chainlink-cluster/templates/geth-deployment.yaml b/charts/chainlink-cluster/templates/geth-deployment.yaml
@@ -65,6 +65,8 @@ spec:
           - '--miner.etherbase'
           - '0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266'
           - '--ipcdisable'
+          - '--http.api'
+          - 'admin,debug,web3,eth,txpool,personal,miner,net'
           - '--http'
           - '--http.vhosts'
           - '*'

diff --git a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml
@@ -0,0 +1,25 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ $.Release.Name }}-geth
+spec:
+  podSelector:
+    matchLabels:
+      app: geth
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        # Allow http and websocket connections from the node pods.
+        - podSelector:
+            matchLabels:
+              app: {{ $.Release.Name }}
+        # Allow http and websocket connections from the runner pods.
+        - podSelector:
+            matchLabels:
+              app: runner
+      ports:
+        - protocol: TCP
+          port: 8544
+        - protocol: TCP
+          port: 8546
diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ $.Release.Name }}-mockserver
+spec:
+  podSelector:
+    matchLabels:
+      app: mockserver
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        # Allow http traffic from the node pods.
+        - podSelector:
+            matchLabels:
+              app: {{ $.Release.Name }}
+        # Allow http traffic from the runner pods.
+        - podSelector:
+            matchLabels:
+              app: runner
+      ports:
+        - protocol: TCP
+          port: 1080