[TT-1396] [CCIP-2804] Migrate CCIP to test secrets (#1189)

We're updating how we manage test secrets on GitHub CI to enhance security. Instead of passing secrets like the Chainlink image name or wallet key through secrets.toml, we will now load them from the `~/.testsecrets file in your home directory. This change helps us handle secrets more securely. To set test secrets, see instructions integration-tests/ccip-tests/testconfig/examples/.testsecrets.example --- TODO: - [x] @kalverra Run tests locally and see if all works fine - [ ] @kalverra Run all CI workflows and see if it all works fine - [x] @kalverra Update all documentation (docs in this repo are already updated by lukaszcl) - [x] @kalverra Notify CCIP developers that this change will be merged soon - [x] @lukaszcl Merge CTF PR which is a dependecy here https://github.com/smartcontractkit/chainlink-testing-framework/pull/1028/files - [x] @lukaszcl Update all CI workflows for CCIP - [x] @lukaszcl Update setup-create-base64-config-ccip in all workflows to correctly use output - [x] @lukaszcl Allow CI workflows to run with custom test secrets ([guide how to run it](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/README.md#run-github-workflow-with-your-test-secrets)) --------- Co-authored-by: Adam Hamrick <[email protected]>
smartcontractkit · Aug 28, 2024 · caa0304 · caa0304
1 parent 8572a7a
commit caa0304
Show file tree

Hide file tree

Showing 18 changed files with 580 additions and 359 deletions.
diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml
@@ -11,62 +11,39 @@ inputs:
     default: "false"
   selectedNetworks:
     description: The networks to run tests against
-  chainlinkImage:
-    description: The chainlink image to use
-    default: "public.ecr.aws/chainlink/chainlink"
   chainlinkVersion:
     description: The git commit sha to use for the image tag
-  upgradeImage:
-    description: The chainlink image to upgrade to
-    default: ""
   upgradeVersion:
-    description: The git commit sha to use for the image tag
-  lokiEndpoint:
-    description: Loki push endpoint
-  lokiTenantId:
-    description: Loki tenant id
-  lokiBasicAuth:
-    description: Loki basic auth
+    description: The git commit sha to use for the image tag    
   logstreamLogTargets:
     description: Where to send logs (e.g. file, loki)
-  grafanaUrl:
-    description: Grafana URL
-  grafanaDashboardUrl:
-    description: Grafana dashboard URL
-  grafanaBearerToken:
-    description: Grafana bearer token
   customEvmNodes:
     description: Custom EVM nodes to use  in key=value format, where key is chain id and value is docker image to use. If they are provided the number of networksSelected must be equal to the number of customEvmNodes
   evmNodeLogLevel:
     description: Log level for the custom EVM nodes
     default: "info"
+outputs:
+  base64_config:
+    description: The base64-encoded config
+    value: ${{ steps.base64_config_override.outputs.base64_config }}
 
 runs:
   using: composite
   steps:
     - name: Prepare Base64 TOML override
       shell: bash
-      id: base64-config-override
+      id: base64_config_override
       env:
         RUN_ID: ${{ inputs.runId }}
         SELECTED_NETWORKS: ${{ inputs.selectedNetworks }}
         EXISTING_NAMESPACE: ${{ inputs.existingNamespace }}
         TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
-        CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
         CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
-        UPGRADE_IMAGE: ${{ inputs.upgradeImage }}
         UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
-        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
-        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
-        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
         LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
-        GRAFANA_URL: ${{ inputs.grafanaUrl }}
-        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
-        GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }}
         CUSTOM_EVM_NODES: ${{ inputs.customEvmNodes }}
         EVM_NODE_LOG_LEVEL: ${{ inputs.evmNodeLogLevel }}
       run: |
-        echo ::add-mask::$CHAINLINK_IMAGE
         function convert_to_toml_array() {
           local IFS=','
           local input_array=($1)
@@ -133,11 +110,6 @@ runs:
           fi
         fi
 
-        grafana_bearer_token=""
-        if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
-          grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
-        fi
-
         cat << EOF > config.toml
         [CCIP]
         [CCIP.Env]
@@ -147,13 +119,8 @@ runs:
         [CCIP.Env.NewCLCluster]
         [CCIP.Env.NewCLCluster.Common]
         [CCIP.Env.NewCLCluster.Common.ChainlinkImage]
-        image="$CHAINLINK_IMAGE"
         version="$CHAINLINK_VERSION"
 
-        [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage]
-        image="$UPGRADE_IMAGE"
-        version="$UPGRADE_VERSION"
-
         $custom_nodes_toml
 
         [CCIP.Env.Logging]
@@ -163,16 +130,6 @@ runs:
         [CCIP.Env.Logging.LogStream]
         log_targets=$log_targets
 
-        [CCIP.Env.Logging.Loki]
-        tenant_id="$LOKI_TENANT_ID"
-        endpoint="$LOKI_ENDPOINT"
-        basic_auth_secret="$LOKI_BASIC_AUTH"
-
-        [CCIP.Env.Logging.Grafana]
-        base_url="$GRAFANA_URL"
-        dashboard_url="$GRAFANA_DASHBOARD_URL"
-        $grafana_bearer_token
-
         [CCIP.Groups.load]
         TestRunName = '$EXISTING_NAMESPACE'
 
@@ -181,7 +138,14 @@ runs:
 
         EOF
 
-        BASE64_CCIP_SECRETS_CONFIG=$(cat config.toml | base64 -w 0)
-        echo ::add-mask::$BASE64_CCIP_SECRETS_CONFIG
-        echo "BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV
-        echo "TEST_BASE64_CCIP_SECRETS_CONFIG=$BASE64_CCIP_SECRETS_CONFIG" >> $GITHUB_ENV
+        # Check if UPGRADE_VERSION is not empty and append to config.toml
+        if [ -n "$UPGRADE_VERSION" ]; then
+          cat << EOF >> config.toml
+        [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage]
+        version="$UPGRADE_VERSION"
+        EOF
+        fi     
+
+        BASE64_CONFIG=$(cat config.toml | base64 -w 0)
+        echo ::add-mask::$BASE64_CONFIG
+        echo "base64_config=$BASE64_CONFIG" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml
@@ -123,23 +123,20 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Prepare Base64 TOML override for CCIP secrets
         uses: ./.github/actions/setup-create-base64-config-ccip
+        id: setup_create_base64_config_ccip
         with:
           runId: ${{ github.run_id }}
           testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
           chainlinkVersion: ${{ github.sha }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
+        env:
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
           test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          cl_image_tag: ${{ github.sha }}
           artifacts_location: ./integration-tests/chaos/logs
           publish_check_name: CCIP Chaos Test Results
           publish_report_paths: ./tests-chaos-report.xml
@@ -153,6 +150,13 @@ jobs:
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
+          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+          DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+
       ## Notify in slack if the job fails
       - name: Notify Slack
         if: failure() && github.event_name != 'workflow_dispatch'
@@ -205,23 +209,20 @@ jobs:
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       - name: Prepare Base64 TOML override for CCIP secrests
         uses: ./.github/actions/setup-create-base64-config-ccip
+        id: setup_create_base64_config_ccip
         with:
           runId: ${{ github.run_id }}
           testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
           chainlinkVersion: ${{ github.sha }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests"
       - name: Run Load With Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
+        env:
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
         with:
           test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: make gomod
-          cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-          cl_image_tag: ${{ github.sha }}
           artifacts_location: ./integration-tests/load/logs
           publish_check_name: CCIP Chaos With Load Test Results
           publish_report_paths: ./tests-chaos-with-load-report.xml
@@ -235,6 +236,12 @@ jobs:
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
+          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+          DEFAULT_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
       ## Notify in slack if the job fails
       - name: Notify Slack
         if: failure() && github.event_name != 'workflow_dispatch'

diff --git a/.github/workflows/ccip-client-compatibility-tests.yml b/.github/workflows/ccip-client-compatibility-tests.yml
@@ -492,33 +492,18 @@ jobs:
           runId: ${{ github.run_id }}
           testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
           selectedNetworks: ${{ matrix.evm_node.networks }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
           chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          pyroscopeServer: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          pyroscopeEnvironment: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }}
-          pyroscopeKey: ${{ secrets.QA_PYROSCOPE_KEY }}
-          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Prepare Base64 TOML override for CCIP secrets
         uses: ./.github/actions/setup-create-base64-config-ccip
+        id: setup_create_base64_config_ccip
         with:
           runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
           selectedNetworks: ${{ matrix.evm_node.networks }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
+          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
           chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }}
-          lokiEndpoint: ${{ secrets.LOKI_URL_CI }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }}
           logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
           customEvmNodes: ${{ matrix.evm_node.docker_image }}
-          evmNodeLogLevel: "trace"
       - name: Prepare test log name
         run: |
           replace_special_chars() {
@@ -547,12 +532,13 @@ jobs:
           echo "EVM Implementation Networks: ${{ matrix.evm_node.networks }}"
           echo "Test identifier: ${{ matrix.evm_node.name }}"
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26
+        env:
+          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
+          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          cl_repo: ${{ env.CHAINLINK_IMAGE }}
-          cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name:  ${{ env.TEST_LOG_NAME }}
           artifacts_location: |
@@ -568,6 +554,17 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+          DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL_CI }}
+          DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+          DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
+          DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
+          DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
+          DEFAULT_PYROSCOPE_SERVER_URL: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
+          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          DEFAULT_PYROSCOPE_ENVIRONMENT: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }}
+          DEFAULT_PYROSCOPE_ENABLED: 'true'
+
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@1587f59bfd626b668d303abbc90fee41b12397e6 # v2.3.23