stream automation tests logs to grafana (AUTO-10273) (#12953)

* stream nightly tests logs to loki

* stream ondemand tests logs to loki

.github/actions/setup-create-base64-upgrade-config/action.yml

@@ -17,6 +17,23 @@ inputs:
     default: "public.ecr.aws/chainlink/chainlink"
   upgradeVersion:
     description: The git commit sha to use for the image tag
+  runId:
+    description: The run id
+  testLogCollect:
+    description: Whether to always collect logs, even for passing tests
+    default: "false"
+  lokiEndpoint:
+    description: Loki push endpoint
+  lokiTenantId:
+    description: Loki tenant id
+  lokiBasicAuth:
+    description: Loki basic auth
+  logstreamLogTargets:
+    description: Where to send logs (e.g. file, loki)
+  grafanaUrl:
+    description: Grafana URL
+  grafanaDashboardUrl:
+    description: Grafana dashboard URL
 
 runs:
   using: composite
@@ -31,6 +48,14 @@ runs:
         CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
         UPGRADE_IMAGE: ${{ inputs.upgradeImage }}
         UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
+        RUN_ID: ${{ inputs.runId }}
+        TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
+        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
+        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
+        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
+        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
+        GRAFANA_URL: ${{ inputs.grafanaUrl }}
+        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
       run: |
         function convert_to_toml_array() {
           local IFS=','
@@ -47,6 +72,14 @@ runs:
 
         selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
 
+        if [ -n "$TEST_LOG_COLLECT" ]; then
+          test_log_collect=true
+        else
+          test_log_collect=false
+        fi
+
+        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
+
         cat << EOF > config.toml
         [Network]
         selected_networks=$selected_networks
@@ -59,6 +92,22 @@ runs:
         [ChainlinkUpgradeImage]
         image="$UPGRADE_IMAGE"
         version="$UPGRADE_VERSION"
+
+        [Logging]
+        test_log_collect=$test_log_collect
+        run_id="$RUN_ID"
+
+        [Logging.LogStream]
+        log_targets=$log_targets
+
+        [Logging.Loki]
+        tenant_id="$LOKI_TENANT_ID"
+        endpoint="$LOKI_ENDPOINT"
+        basic_auth_secret="$LOKI_BASIC_AUTH"
+
+        [Logging.Grafana]
+        base_url="$GRAFANA_URL"
+        dashboard_url="$GRAFANA_DASHBOARD_URL"
         EOF
 
         BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)

.github/workflows/automation-nightly-tests.yml

@@ -95,6 +95,14 @@ jobs:
           chainlinkVersion: "latest"
           upgradeImage: ${{ env.CHAINLINK_IMAGE }}
           upgradeVersion: ${{ github.sha }}
+          runId: ${{ github.run_id }}
+          testLogCollect: "true"
+          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: ${{ vars.GRAFANA_URL }}
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
         env:

.github/workflows/automation-ondemand-tests.yml

@@ -226,7 +226,15 @@ jobs:
           UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }}
           PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
           PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}          
+          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          RUN_ID: ${{ github.run_id }}
+          TEST_LOG_COLLECT: "true"
+          LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          LOKI_BASIC_AUTH:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
+          GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
         run: |
           echo ::add-mask::$UPGRADE_IMAGE
           echo ::add-mask::$OLD_IMAGE
@@ -242,6 +250,14 @@ jobs:
             pyroscope_enabled=false
           fi          
 
+          if [ -n "$TEST_LOG_COLLECT" ]; then
+          test_log_collect=true
+          else
+          test_log_collect=false
+          fi
+
+          log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
+
           cat << EOF > config.toml
           [Network]
           selected_networks=$selected_networks
@@ -254,6 +270,22 @@ jobs:
           image="$UPGRADE_IMAGE"
           version="$UPGRADE_VERSION"
 
+          [Logging]
+          test_log_collect=$test_log_collect
+          run_id="$RUN_ID"
+
+          [Logging.LogStream]
+          log_targets=$log_targets
+
+          [Logging.Loki]
+          tenant_id="$LOKI_TENANT_ID"
+          endpoint="$LOKI_ENDPOINT"
+          basic_auth_secret="$LOKI_BASIC_AUTH"
+
+          [Logging.Grafana]
+          base_url="$GRAFANA_URL"
+          dashboard_url="$GRAFANA_DASHBOARD_URL"
+
           [Pyroscope]
           enabled=$pyroscope_enabled
           server_url="$PYROSCOPE_SERVER"