From b5bfb47874ac608f50cbd3756f1d8b8e6c7f4b4e Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:24:28 +0200 Subject: [PATCH] migrate otel traces steps to reusable workflow --- .github/e2e-tests.yml | 3 +- .github/workflows/integration-tests.yml | 466 +----------------- .../run-e2e-tests-reusable-workflow.yml | 57 ++- 3 files changed, 59 insertions(+), 467 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index f96468dcb8f..4ed75a6bcf8 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -177,13 +177,14 @@ runner-test-matrix: test_env_type: docker runs_on: ubuntu22.04-16cores-64GB workflows: - - Run PR E2E Tests + - Run PR E2E Tests 3 - Run Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated test_inputs: # chainlink_version: '{{ env.GITHUB_SHA_PLUGINS }}' # This is the chainlink version that has the plugins chainlink_version: develop-plugins + enable_otel_traces: true # END: OCR tests diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1616091b871..97e55edcfba 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -250,9 +250,7 @@ jobs: test_workflow: Run PR E2E Tests 3 upload_cl_node_coverage_artifact: true upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_ - # slack_notification_after_tests: true - # slack_notification_after_tests_channel_id: "#team-test-tooling-internal" - # slack_notification_after_tests_name: Nightly E2E Tests + enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }} secrets: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} @@ -295,468 +293,6 @@ jobs: channel-id: "#team-test-tooling-internal" slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying ', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}" - # eth-smoke-tests-matrix-automation: - # if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} - # environment: integration - # permissions: - # checks: write - # pull-requests: write - # id-token: write - # contents: read - # needs: - # [build-chainlink, changes, compare-tests, build-lint-integration-tests] - # env: - # SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - # CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - # CHAINLINK_ENV_USER: ${{ github.actor }} - # TEST_LOG_LEVEL: debug - # strategy: - # fail-fast: false - # matrix: - # product: ${{fromJson(needs.compare-tests.outputs.automation-matrix)}} - # runs-on: ${{ matrix.product.os }} - # name: ETH Smoke Tests ${{ matrix.product.name }} - # steps: - # - name: Collect Metrics - # if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - # id: collect-gha-metrics - # uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - # with: - # id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.name }} - # basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - # org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # this-job-name: ETH Smoke Tests ${{ matrix.product.name }} - # test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - # continue-on-error: true - # - name: Checkout the repo - # uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - # with: - # repository: smartcontractkit/chainlink - # ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - # - name: Build Go Test Command - # id: build-go-test-command - # run: | - # # if the matrix.product.run is set, use it for a different command - # if [ "${{ matrix.product.run }}" != "" ]; then - # echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - # else - # echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - # fi - # - name: Setup GAP for Grafana - # uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - # id: setup-gap - # with: - # # aws inputs - # aws-region: ${{ secrets.AWS_REGION }} - # aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - # api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # # other inputs - # duplicate-authorization-header: "true" - - # ## Run this step when changes that require tests to be run are made - # - name: Run Tests - # if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 - # with: - # test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - # test_download_vendor_packages_command: cd ./integration-tests && go mod download - # test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - # test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - # test_config_logging_run_id: ${{ github.run_id }} - # test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - # test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - # cl_repo: ${{ env.CHAINLINK_IMAGE }} - # cl_image_tag: ${{ inputs.evm-ref || github.sha }} - # aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - # artifacts_name: ${{ matrix.product.name }}-test-artifacts - # artifacts_location: | - # ./integration-tests/smoke/logs/ - # ./integration-tests/smoke/db_dumps/ - # ./integration-tests/smoke/seth_artifacts/ - # /tmp/gotest.log - # publish_check_name: ${{ matrix.product.name }} - # token: ${{ secrets.GITHUB_TOKEN }} - # go_mod_path: ./integration-tests/go.mod - # cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - # cache_restore_only: "true" - # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - # QA_KUBECONFIG: "" - # should_tidy: "false" - # go_coverage_src_dir: /var/tmp/go-coverage - # go_coverage_dest_dir: ${{ github.workspace }}/.covdata - # DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - # DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - # DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - # DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - # DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - # DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - # DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - # DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - # DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - # eth-smoke-tests-matrix-log-poller: - # if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') || inputs.distinct_run_name != '' }} - # environment: integration - # permissions: - # checks: write - # pull-requests: write - # id-token: write - # contents: read - # needs: - # [build-chainlink, changes, compare-tests, build-lint-integration-tests] - # env: - # SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - # CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - # CHAINLINK_ENV_USER: ${{ github.actor }} - # TEST_LOG_LEVEL: debug - # strategy: - # fail-fast: false - # matrix: - # product: ${{fromJson(needs.compare-tests.outputs.lp-matrix)}} - # runs-on: ${{ matrix.product.os }} - # name: ETH Smoke Tests ${{ matrix.product.name }} - # steps: - # - name: Collect Metrics - # if: needs.changes.outputs.src == 'true' - # id: collect-gha-metrics - # uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - # with: - # id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.name }} - # org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - # this-job-name: ETH Smoke Tests ${{ matrix.product.name }} - # test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - # continue-on-error: true - # - name: Checkout the repo - # uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - # with: - # repository: smartcontractkit/chainlink - # ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - # - name: Build Go Test Command - # id: build-go-test-command - # run: | - # # if the matrix.product.run is set, use it for a different command - # if [ "${{ matrix.product.run }}" != "" ]; then - # echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - # else - # echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - # fi - # - name: Setup GAP for Grafana - # uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - # id: setup-gap - # with: - # # aws inputs - # aws-region: ${{ secrets.AWS_REGION }} - # aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - # api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # # other inputs - # duplicate-authorization-header: "true" - # ## Run this step when changes that require tests to be run are made - # - name: Run Tests - # if: needs.changes.outputs.src == 'true' - # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 - # with: - # test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - # test_download_vendor_packages_command: cd ./integration-tests && go mod download - # test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - # test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - # test_config_logging_run_id: ${{ github.run_id }} - # test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - # test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - # cl_repo: ${{ env.CHAINLINK_IMAGE }} - # cl_image_tag: ${{ inputs.evm-ref || github.sha }} - # aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - # artifacts_name: ${{ matrix.product.name }}-test-artifacts - # artifacts_location: | - # ./integration-tests/smoke/logs/ - # ./integration-tests/smoke/db_dumps/ - # ./integration-tests/smoke/seth_artifacts/ - # /tmp/gotest.log - # publish_check_name: ${{ matrix.product.name }} - # token: ${{ secrets.GITHUB_TOKEN }} - # go_mod_path: ./integration-tests/go.mod - # cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - # cache_restore_only: "true" - # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - # QA_KUBECONFIG: "" - # should_tidy: "false" - # go_coverage_src_dir: /var/tmp/go-coverage - # go_coverage_dest_dir: ${{ github.workspace }}/.covdata - # DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - # DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - # DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - # DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - # DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - # DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - # DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - # DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - # DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - # eth-smoke-tests-matrix: - # if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} - # environment: integration - # permissions: - # actions: read - # checks: write - # pull-requests: write - # id-token: write - # contents: read - # needs: [build-chainlink, changes, build-lint-integration-tests] - # env: - # SELECTED_NETWORKS: SIMULATED - # CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - # CHAINLINK_ENV_USER: ${{ github.actor }} - # TEST_LOG_LEVEL: debug - # strategy: - # fail-fast: false - # matrix: - # product: - # - name: runlog - # id: runlog - # nodes: 2 - # os: ubuntu-latest - # pyroscope_env: "ci-smoke-runlog-evm-simulated" - # - name: cron - # id: cron - # nodes: 2 - # os: ubuntu-latest - # pyroscope_env: "ci-smoke-cron-evm-simulated" - # - name: flux - # id: flux - # nodes: 1 - # os: ubuntu-latest - # pyroscope_env: "ci-smoke-flux-evm-simulated" - # - name: ocr - # id: ocr - # nodes: 2 - # os: ubuntu-latest - # file: ocr - # pyroscope_env: ci-smoke-ocr-evm-simulated - # - name: reorg_above_finality - # id: reorg_above_finality - # nodes: 1 - # os: ubuntu-latest - # file: reorg_above_finality - # pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated - # - name: ocr2 - # id: ocr2 - # nodes: 6 - # os: ubuntu22.04-16cores-64GB - # file: ocr2 - # pyroscope_env: ci-smoke-ocr2-evm-simulated - # - name: ocr2 - # id: ocr2-plugins - # nodes: 6 - # os: ubuntu22.04-16cores-64GB - # pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated - # tag_suffix: "-plugins" - # - name: vrf - # id: vrf - # nodes: 2 - # os: ubuntu-latest - # pyroscope_env: ci-smoke-vrf-evm-simulated - # - name: vrfv2 - # id: vrfv2 - # nodes: 6 - # os: ubuntu-latest - # pyroscope_env: ci-smoke-vrf2-evm-simulated - # - name: vrfv2plus - # id: vrfv2plus - # nodes: 9 - # os: ubuntu-latest - # pyroscope_env: ci-smoke-vrf2plus-evm-simulated - # - name: forwarder_ocr - # id: forwarder_ocr - # nodes: 2 - # os: ubuntu-latest - # pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - # - name: forwarders_ocr2 - # id: forwarders_ocr2 - # nodes: 2 - # os: ubuntu-latest - # pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - # runs-on: ${{ matrix.product.os }} - # name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - # steps: - # # Handy for debugging resource usage - # # - name: Collect Workflow Telemetry - # # uses: catchpoint/workflow-telemetry-action@v2 - # - name: Collect Metrics - # if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - # id: collect-gha-metrics - # uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - # with: - # id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }} - # org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - # this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - # test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - # continue-on-error: true - # - name: Checkout the repo - # uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - # with: - # repository: smartcontractkit/chainlink - # ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - # - name: Build Go Test Command - # id: build-go-test-command - # run: | - # # if the matrix.product.run is set, use it for a different command - # if [ "${{ matrix.product.run }}" != "" ]; then - # echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - # else - # echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - # fi - # - name: Check for "enable tracing" label - # id: check-label - # run: | - # label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") - - # if [[ -n "$label" ]]; then - # if [[ "$label" == "enable tracing" ]]; then - # echo "Enable tracing label found." - # echo "trace=true" >> $GITHUB_OUTPUT - # else - # echo "Enable tracing label not found." - # echo "trace=false" >> $GITHUB_OUTPUT - # fi - # else - # echo "No labels present or labels are null." - # echo "trace=false" >> $GITHUB_OUTPUT - # fi - - # - name: Setup Grafana and OpenTelemetry - # id: docker-setup - # if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - # run: | - # # Create network - # docker network create --driver bridge tracing - - # # Make trace directory - # cd integration-tests/smoke/ - # mkdir ./traces - # chmod -R 777 ./traces - - # # Switch directory - # cd ../../.github/tracing - - # # Create a Docker volume for traces - # # docker volume create otel-traces - - # # Start OpenTelemetry Collector - # # Note the user must be set to the same user as the runner for the trace data to be accessible - # docker run -d --network=tracing --name=otel-collector \ - # -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ - # -v $PWD/../../integration-tests/smoke/traces:/tracing \ - # --user "$(id -u):$(id -g)" \ - # -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml - - # - name: Locate Docker Volume - # id: locate-volume - # if: false - # run: | - # echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT - - # - name: Show Otel-Collector Logs - # if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - # run: | - # docker logs otel-collector - - # - name: Setup GAP for Grafana - # uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - # id: setup-gap - # with: - # # aws inputs - # aws-region: ${{ secrets.AWS_REGION }} - # aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - # api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # # other inputs - # duplicate-authorization-header: "true" - - # ## Run this step when changes that require tests to be run are made - # - name: Run Tests - # if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 - # with: - # test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - # test_download_vendor_packages_command: cd ./integration-tests && go mod download - # test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - # test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - # test_config_logging_run_id: ${{ github.run_id }} - # test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - # test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - # cl_repo: ${{ env.CHAINLINK_IMAGE }} - # cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} - # aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - # artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-artifacts - # artifacts_location: | - # ./integration-tests/smoke/logs/ - # ./integration-tests/smoke/db_dumps/ - # ./integration-tests/smoke/seth_artifacts/ - # /tmp/gotest.log - # publish_check_name: ${{ matrix.product.name }} - # token: ${{ secrets.GITHUB_TOKEN }} - # go_mod_path: ./integration-tests/go.mod - # cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - # cache_restore_only: "true" - # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - # QA_KUBECONFIG: "" - # should_tidy: "false" - # go_coverage_src_dir: /var/tmp/go-coverage - # go_coverage_dest_dir: ${{ github.workspace }}/.covdata - # DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - # DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - # DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - # DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - # DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - # DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - # DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - # DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - # DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - # DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - # DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - - # # Run this step when changes that do not need the test to run are made - # - name: Run Setup - # if: needs.changes.outputs.src == 'false' - # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - # with: - # test_download_vendor_packages_command: cd ./integration-tests && go mod download - # go_mod_path: ./integration-tests/go.mod - # cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - # cache_restore_only: "true" - # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - # QA_KUBECONFIG: "" - # should_tidy: "false" - - # - name: Show Otel-Collector Logs - # if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - # run: | - # docker logs otel-collector - - # - name: Permissions on traces - # if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - # run: | - # ls -l ./integration-tests/smoke/traces - - # - name: Upload Trace Data - # if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - # uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - # with: - # name: trace-data - # path: ./integration-tests/smoke/traces/trace-data.json - cleanup: name: Clean up integration environment deployments if: always() diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml index 6a77452a2a4..baa32a0050f 100644 --- a/.github/workflows/run-e2e-tests-reusable-workflow.yml +++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml @@ -82,7 +82,12 @@ on: upload_cl_node_coverage_artifact_prefix: description: 'Prefix for the Chainlink node coverage artifact' required: false - type: string + type: string + enable_otel_traces_for_ocr2_plugins: + description: 'Set to "true" to enable OpenTelemetry traces for OCR2 plugins tests' + required: false + type: boolean + default: false outputs: test_results: description: 'Test results from all executed tests' @@ -424,6 +429,7 @@ jobs: run: echo '${{ toJson(matrix.tests) }}' | jq . - name: Setup Go uses: ./.github/actions/setup-go + - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 id: setup-gap @@ -433,6 +439,38 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} duplicate-authorization-header: "true" + - name: Setup Grafana and OpenTelemetry + id: docker-setup + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_inputs.enable_otel_traces == 'true' + run: | + # Create network + docker network create --driver bridge tracing + + # Make trace directory + cd integration-tests/smoke/ + mkdir ./traces + chmod -R 777 ./traces + + # Switch directory + cd ../../.github/tracing + + # Create a Docker volume for traces + # docker volume create otel-traces + + # Start OpenTelemetry Collector + # Note the user must be set to the same user as the runner for the trace data to be accessible + docker run -d --network=tracing --name=otel-collector \ + -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ + -v $PWD/../../integration-tests/smoke/traces:/tracing \ + --user "$(id -u):$(id -g)" \ + -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml + + - name: Show Otel-Collector Logs + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_inputs.enable_otel_traces == 'true' + run: | + echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT + docker logs otel-collector + - name: Run tests id: run_tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fdaf56b1df7248d18e30ad09982d03ec67d6b71c # v2.3.29 @@ -481,6 +519,23 @@ jobs: DEFAULT_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} + - name: Show Otel-Collector logs + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_inputs.enable_otel_traces == 'true' + run: | + docker logs otel-collector + + - name: Permissions on traces + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_inputs.enable_otel_traces == 'true' + run: | + ls -l ./integration-tests/smoke/traces + + - name: Upload trace data as Github artifact + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_inputs.enable_otel_traces == 'true' + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: trace-data + path: ./integration-tests/smoke/traces/trace-data.json + - name: Upload test log as Github artifact uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: inputs.test_log_upload_on_failure && failure()