From 7b09076a101b47b731fa1b0ee9ce61f4a98254ab Mon Sep 17 00:00:00 2001 From: Anirudh Warrier <12178754+anirudhwarrier@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:35:52 +0530 Subject: [PATCH] stream automation tests logs to grafana (AUTO-10273) (#12953) * stream nightly tests logs to loki * stream ondemand tests logs to loki --- .../action.yml | 49 +++++++++++++++++++ .../workflows/automation-nightly-tests.yml | 8 +++ .../workflows/automation-ondemand-tests.yml | 34 ++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-create-base64-upgrade-config/action.yml b/.github/actions/setup-create-base64-upgrade-config/action.yml index daa2d400262..ed25fd6375f 100644 --- a/.github/actions/setup-create-base64-upgrade-config/action.yml +++ b/.github/actions/setup-create-base64-upgrade-config/action.yml @@ -17,6 +17,23 @@ inputs: default: "public.ecr.aws/chainlink/chainlink" upgradeVersion: description: The git commit sha to use for the image tag + runId: + description: The run id + testLogCollect: + description: Whether to always collect logs, even for passing tests + default: "false" + lokiEndpoint: + description: Loki push endpoint + lokiTenantId: + description: Loki tenant id + lokiBasicAuth: + description: Loki basic auth + logstreamLogTargets: + description: Where to send logs (e.g. file, loki) + grafanaUrl: + description: Grafana URL + grafanaDashboardUrl: + description: Grafana dashboard URL runs: using: composite @@ -31,6 +48,14 @@ runs: CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }} UPGRADE_IMAGE: ${{ inputs.upgradeImage }} UPGRADE_VERSION: ${{ inputs.upgradeVersion }} + RUN_ID: ${{ inputs.runId }} + TEST_LOG_COLLECT: ${{ inputs.testLogCollect }} + LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }} + LOKI_TENANT_ID: ${{ inputs.lokiTenantId }} + LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }} + LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} + GRAFANA_URL: ${{ inputs.grafanaUrl }} + GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} run: | function convert_to_toml_array() { local IFS=',' @@ -47,6 +72,14 @@ runs: selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") + if [ -n "$TEST_LOG_COLLECT" ]; then + test_log_collect=true + else + test_log_collect=false + fi + + log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") + cat << EOF > config.toml [Network] selected_networks=$selected_networks @@ -59,6 +92,22 @@ runs: [ChainlinkUpgradeImage] image="$UPGRADE_IMAGE" version="$UPGRADE_VERSION" + + [Logging] + test_log_collect=$test_log_collect + run_id="$RUN_ID" + + [Logging.LogStream] + log_targets=$log_targets + + [Logging.Loki] + tenant_id="$LOKI_TENANT_ID" + endpoint="$LOKI_ENDPOINT" + basic_auth_secret="$LOKI_BASIC_AUTH" + + [Logging.Grafana] + base_url="$GRAFANA_URL" + dashboard_url="$GRAFANA_DASHBOARD_URL" EOF BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml index c3b874548ac..b44a7f56e65 100644 --- a/.github/workflows/automation-nightly-tests.yml +++ b/.github/workflows/automation-nightly-tests.yml @@ -95,6 +95,14 @@ jobs: chainlinkVersion: "latest" upgradeImage: ${{ env.CHAINLINK_IMAGE }} upgradeVersion: ${{ github.sha }} + runId: ${{ github.run_id }} + testLogCollect: "true" + lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 env: diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index 167dccf3bbf..dade0f331b9 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -226,7 +226,15 @@ jobs: UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} - PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + RUN_ID: ${{ github.run_id }} + TEST_LOG_COLLECT: "true" + LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} + GRAFANA_URL: ${{ vars.GRAFANA_URL }} + GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" run: | echo ::add-mask::$UPGRADE_IMAGE echo ::add-mask::$OLD_IMAGE @@ -242,6 +250,14 @@ jobs: pyroscope_enabled=false fi + if [ -n "$TEST_LOG_COLLECT" ]; then + test_log_collect=true + else + test_log_collect=false + fi + + log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") + cat << EOF > config.toml [Network] selected_networks=$selected_networks @@ -254,6 +270,22 @@ jobs: image="$UPGRADE_IMAGE" version="$UPGRADE_VERSION" + [Logging] + test_log_collect=$test_log_collect + run_id="$RUN_ID" + + [Logging.LogStream] + log_targets=$log_targets + + [Logging.Loki] + tenant_id="$LOKI_TENANT_ID" + endpoint="$LOKI_ENDPOINT" + basic_auth_secret="$LOKI_BASIC_AUTH" + + [Logging.Grafana] + base_url="$GRAFANA_URL" + dashboard_url="$GRAFANA_DASHBOARD_URL" + [Pyroscope] enabled=$pyroscope_enabled server_url="$PYROSCOPE_SERVER"