diff --git a/.changeset/funny-monkeys-heal.md b/.changeset/funny-monkeys-heal.md new file mode 100644 index 00000000000..6267569432e --- /dev/null +++ b/.changeset/funny-monkeys-heal.md @@ -0,0 +1,6 @@ +--- +"chainlink": patch +--- + +#changed: +AUTO-10539: adjust logging for offchain config and gas control diff --git a/.changeset/nine-plants-crash.md b/.changeset/nine-plants-crash.md new file mode 100644 index 00000000000..06a7c9bb309 --- /dev/null +++ b/.changeset/nine-plants-crash.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#internal improve mercury tranmission debugging diff --git a/.changeset/sweet-avocados-do.md b/.changeset/sweet-avocados-do.md new file mode 100644 index 00000000000..4b8bb33a7e1 --- /dev/null +++ b/.changeset/sweet-avocados-do.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#internal Use Aggregator factory for OCR capability diff --git a/.changeset/twelve-wolves-clean.md b/.changeset/twelve-wolves-clean.md new file mode 100644 index 00000000000..c38fbe3fd83 --- /dev/null +++ b/.changeset/twelve-wolves-clean.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Performance improvements for mercury single insert for multiple mercury servers #internal diff --git a/.github/actions/setup-create-base64-config-live-testnets/action.yml b/.github/actions/setup-create-base64-config-live-testnets/action.yml index 879dcce6df3..0dac6e9d55b 100644 --- a/.github/actions/setup-create-base64-config-live-testnets/action.yml +++ b/.github/actions/setup-create-base64-config-live-testnets/action.yml @@ -33,6 +33,8 @@ inputs: description: Grafana URL grafanaDashboardUrl: description: Grafana dashboard URL + grafanaBearerToken: + description: Grafana bearer token network: description: Network to run tests on httpEndpoints: @@ -62,6 +64,7 @@ runs: LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} GRAFANA_URL: ${{ inputs.grafanaUrl }} GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} + GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} NETWORK: ${{ inputs.network }} HTTP_ENDPOINTS: ${{ inputs.httpEndpoints }} WS_ENDPOINTS: ${{ inputs.wsEndpoints }} @@ -115,6 +118,7 @@ runs: [Logging.Grafana] base_url="$GRAFANA_URL" dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" [Network] selected_networks=["$NETWORK"] diff --git a/.github/actions/setup-create-base64-config/action.yml b/.github/actions/setup-create-base64-config/action.yml index 447f5be42cb..46de20b6cd6 100644 --- a/.github/actions/setup-create-base64-config/action.yml +++ b/.github/actions/setup-create-base64-config/action.yml @@ -35,6 +35,8 @@ inputs: description: Grafana URL grafanaDashboardUrl: description: Grafana dashboard URL + grafanaBearerToken: + description: Grafana bearer token ethExecutionClient: description: Ethereum execution client to use (geth, besu, nethermind or erigon) customEthClientDockerImage: @@ -62,6 +64,7 @@ runs: LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} GRAFANA_URL: ${{ inputs.grafanaUrl }} GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} + GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }} CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }} run: | @@ -139,6 +142,7 @@ runs: [Logging.Grafana] base_url="$GRAFANA_URL" dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" [PrivateEthereumNetwork] execution_layer="$execution_layer" diff --git a/.github/actions/setup-create-base64-upgrade-config/action.yml b/.github/actions/setup-create-base64-upgrade-config/action.yml index 8f514784725..a7cfcafde4c 100644 --- a/.github/actions/setup-create-base64-upgrade-config/action.yml +++ b/.github/actions/setup-create-base64-upgrade-config/action.yml @@ -34,6 +34,8 @@ inputs: description: Grafana URL grafanaDashboardUrl: description: Grafana dashboard URL + grafanaBearerToken: + description: Grafana bearer token runs: using: composite @@ -56,6 +58,7 @@ runs: LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} GRAFANA_URL: ${{ inputs.grafanaUrl }} GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }} + GRAFANA_BEARER_TOKEN: ${{ inputs.grafanaBearerToken }} run: | function convert_to_toml_array() { local IFS=',' @@ -109,6 +112,7 @@ runs: [Logging.Grafana] base_url="$GRAFANA_URL" dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" EOF BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) diff --git a/.github/actions/setup-merge-base64-config/action.yml b/.github/actions/setup-merge-base64-config/action.yml index 43dcab940ab..48ca96bf948 100644 --- a/.github/actions/setup-merge-base64-config/action.yml +++ b/.github/actions/setup-merge-base64-config/action.yml @@ -53,6 +53,11 @@ runs: basic_auth_secret="$LOKI_BASIC_AUTH" # legacy, you only need this to access the cloud version # bearer_token_secret="bearer_token" + + [Logging.Grafana] + base_url="$GRAFANA_URL" + dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" EOF echo "$decoded_toml" >> final_config.toml diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml index b44a7f56e65..0aeaf1ae47a 100644 --- a/.github/workflows/automation-nightly-tests.yml +++ b/.github/workflows/automation-nightly-tests.yml @@ -87,6 +87,20 @@ jobs: uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ github.head_ref || github.ref_name }} + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-upgrade-config with: @@ -101,8 +115,9 @@ jobs: lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 env: diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index dade0f331b9..c232a6ba9e6 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -217,6 +217,20 @@ jobs: echo "image=$READ_CL_UPGR_IMAGE" >>$GITHUB_OUTPUT echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT fi + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML config env: SELECTED_NETWORKS: ${{ matrix.tests.network }} @@ -233,8 +247,9 @@ jobs: LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} - GRAFANA_URL: ${{ vars.GRAFANA_URL }} + GRAFANA_URL: "http://localhost:8080/primary" GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + $GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} run: | echo ::add-mask::$UPGRADE_IMAGE echo ::add-mask::$OLD_IMAGE @@ -285,6 +300,7 @@ jobs: [Logging.Grafana] base_url="$GRAFANA_URL" dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" [Pyroscope] enabled=$pyroscope_enabled diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index 0ca81c422fe..6b327824fb6 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -164,6 +164,9 @@ jobs: PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} ETH2_EL_CLIENT: ${{matrix.client}} CHAINLINK_VERSION: ${{ github.sha }} + GRAFANA_URL: "http://localhost:8080/primary" + GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} run: | convert_to_toml_array() { local IFS=',' @@ -205,6 +208,11 @@ jobs: server_url="$PYROSCOPE_SERVER" environment="$PYROSCOPE_ENVIRONMENT" key_secret="$PYROSCOPE_KEY" + + [Logging.Grafana] + base_url="$GRAFANA_URL" + dashboard_url="$GRAFANA_DASHBOARD_URL" + bearer_token_secret="$GRAFANA_BEARER_TOKEN" [PrivateEthereumNetwork] ethereum_version="eth2" @@ -242,6 +250,9 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 start-slack-thread: name: Start Slack Thread diff --git a/.github/workflows/evm-version-compatibility-tests.yml b/.github/workflows/evm-version-compatibility-tests.yml index 8d8240b474b..db13b241053 100644 --- a/.github/workflows/evm-version-compatibility-tests.yml +++ b/.github/workflows/evm-version-compatibility-tests.yml @@ -209,6 +209,20 @@ jobs: else echo "run_command=./smoke/${{ matrix.evm_node.product }}_test.go" >> "$GITHUB_OUTPUT" fi + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config with: @@ -221,8 +235,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ethExecutionClient: ${{ matrix.evm_node.eth_client }} customEthClientDockerImage: ${{ matrix.evm_node.docker_image }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 992a7ad1f50..6d365b6d801 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -317,6 +317,20 @@ jobs: else echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" fi + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config with: @@ -332,8 +346,9 @@ jobs: lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" # This is GAP's address grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests @@ -420,6 +435,20 @@ jobs: else echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" fi + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config with: @@ -435,8 +464,9 @@ jobs: lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" # This is GAP's address grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' @@ -631,6 +661,20 @@ jobs: if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | docker logs otel-collector + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config with: @@ -646,8 +690,9 @@ jobs: lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index 0f081bedaf1..470dd8aa1cd 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -247,6 +247,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -261,8 +275,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "sepolia" httpEndpoints: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_SEPOLIA_URLS }} @@ -320,6 +335,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -334,8 +363,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "bsc_testnet" httpEndpoints: ${{ secrets.QA_BSC_TESTNET_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_BSC_TESTNET_URLS }} @@ -393,6 +423,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -407,8 +451,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "optimism_sepolia" httpEndpoints: ${{ secrets.QA_OPTIMISM_SEPOLIA_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_OPTIMISM_SEPOLIA_URLS }} @@ -466,6 +511,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -480,8 +539,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "arbitrum_sepolia" httpEndpoints: ${{ secrets.QA_ARBITRUM_SEPOLIA_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_ARBITRUM_SEPOLIA_URLS }} @@ -535,6 +595,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -549,8 +623,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "base_sepolia" httpEndpoints: ${{ secrets.QA_BASE_SEPOLIA_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_BASE_SEPOLIA_URLS }} @@ -608,6 +683,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -622,8 +711,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "polygon_mumbai" httpEndpoints: ${{ secrets.QA_POLYGON_MUMBAI_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_POLYGON_MUMBAI_URLS }} @@ -681,6 +771,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -695,8 +799,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "avalanche_fuji" httpEndpoints: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_AVALANCHE_FUJI_URLS }} @@ -754,6 +859,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -768,8 +887,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "fantom_testnet" httpEndpoints: ${{ secrets.QA_FANTOM_TESTNET_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_FANTOM_TESTNET_URLS }} @@ -823,6 +943,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -837,8 +971,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "celo_alfajores" httpEndpoints: ${{ secrets.QA_CELO_ALFAJORES_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_CELO_ALFAJORES_URLS }} @@ -892,6 +1027,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -906,8 +1055,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "scroll_sepolia" httpEndpoints: ${{ secrets.QA_SCROLL_SEPOLIA_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_SCROLL_SEPOLIA_URLS }} @@ -961,6 +1111,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -975,8 +1139,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: "linea_goerli" httpEndpoints: ${{ secrets.QA_LINEA_GOERLI_HTTP_URLS }} wsEndpoints: ${{ secrets.QA_LINEA_GOERLI_URLS }} diff --git a/.github/workflows/live-vrf-tests.yml b/.github/workflows/live-vrf-tests.yml index 9b52af70ff0..a4242926008 100644 --- a/.github/workflows/live-vrf-tests.yml +++ b/.github/workflows/live-vrf-tests.yml @@ -138,6 +138,20 @@ jobs: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@main + with: + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" + # metrics inputs + metrics-job-name: "grafana" + gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} + gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - name: Prepare Base64 TOML override uses: ./.github/actions/setup-create-base64-config-live-testnets with: @@ -149,8 +163,9 @@ jobs: lokiTenantId: ${{ vars.LOKI_TENANT_ID }} lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaUrl: "http://localhost:8080/primary" grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} network: ${{ matrix.network }} httpEndpoints: ${{ secrets[env.HTTP_URLS_SECRET_NAME] }} wsEndpoints: ${{ secrets[env.URLS_SECRET_NAME] }} diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml index f520e2307d9..9695da8d6da 100644 --- a/.github/workflows/on-demand-vrfv2-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2-performance-test.yml @@ -38,6 +38,9 @@ jobs: REF_NAME: ${{ github.head_ref || github.ref_name }} SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }} + GRAFANA_URL: "http://localhost:8080/primary" + GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} WASP_LOG_LEVEL: info steps: - name: Collect Metrics diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml index 16d37617a68..1d3e6a06148 100644 --- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml @@ -38,6 +38,9 @@ jobs: REF_NAME: ${{ github.head_ref || github.ref_name }} SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }} + GRAFANA_URL: "http://localhost:8080/primary" + GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} WASP_LOG_LEVEL: info steps: - name: Collect Metrics diff --git a/core/capabilities/aggregator_factory.go b/core/capabilities/aggregator_factory.go new file mode 100644 index 00000000000..bd0f0ceb237 --- /dev/null +++ b/core/capabilities/aggregator_factory.go @@ -0,0 +1,21 @@ +package capabilities + +import ( + "fmt" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/datafeeds" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/values" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/streams" +) + +func NewAggregator(name string, config values.Map, lggr logger.Logger) (types.Aggregator, error) { + switch name { + case "data_feeds": + mc := streams.NewCodec() + return datafeeds.NewDataFeedsAggregator(config, mc, lggr) + default: + return nil, fmt.Errorf("aggregator %s not supported", name) + } +} diff --git a/core/capabilities/streams/codec.go b/core/capabilities/streams/codec.go new file mode 100644 index 00000000000..c04d7ee6a0c --- /dev/null +++ b/core/capabilities/streams/codec.go @@ -0,0 +1,24 @@ +package streams + +import ( + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/mercury" + "github.com/smartcontractkit/chainlink-common/pkg/values" +) + +type Codec struct { +} + +func (c Codec) Unwrap(raw values.Value) ([]mercury.FeedReport, error) { + dest := []mercury.FeedReport{} + err := raw.UnwrapTo(&dest) + // TODO (KS-196): validate reports + return dest, err +} + +func (c Codec) Wrap(reports []mercury.FeedReport) (values.Value, error) { + return values.Wrap(reports) +} + +func NewCodec() Codec { + return Codec{} +} diff --git a/core/capabilities/syncer.go b/core/capabilities/syncer.go index 455582c4598..67e069c831d 100644 --- a/core/capabilities/syncer.go +++ b/core/capabilities/syncer.go @@ -2,12 +2,12 @@ package capabilities import ( "context" + "math/big" "slices" "sync" "time" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/mercury" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/triggers" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -18,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/streams" "github.com/smartcontractkit/chainlink/v2/core/logger" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" ) @@ -97,9 +98,9 @@ func (s *registrySyncer) Start(ctx context.Context) error { } // NOTE: temporary hard-coded capabilities capId := "mercury-trigger" - triggerInfo := commoncap.CapabilityInfo{ + triggerInfo := capabilities.CapabilityInfo{ ID: capId, - CapabilityType: commoncap.CapabilityTypeTrigger, + CapabilityType: capabilities.CapabilityTypeTrigger, Description: "Remote Trigger", Version: "0.0.1", DON: &triggerCapabilityDonInfo, @@ -111,7 +112,8 @@ func (s *registrySyncer) Start(ctx context.Context) error { } if slices.Contains(workflowDONPeers, myId) { s.lggr.Info("member of a workflow DON - starting remote subscribers") - aggregator := triggers.NewMercuryRemoteAggregator(s.lggr) + codec := streams.NewCodec() + aggregator := triggers.NewMercuryRemoteAggregator(codec, s.lggr) triggerCap := remote.NewTriggerSubscriber(config, triggerInfo, triggerCapabilityDonInfo, workflowDonInfo, s.dispatcher, aggregator, s.lggr) err = s.registry.Add(ctx, triggerCap) if err != nil { @@ -224,30 +226,30 @@ func (m *mockMercuryDataProducer) loop() { ticker := time.NewTicker(time.Duration(sleepSec) * time.Second) defer ticker.Stop() - prices := []int64{300000, 40000, 5000000} + prices := []*big.Int{big.NewInt(300000), big.NewInt(40000), big.NewInt(5000000)} for range ticker.C { for i := range prices { - prices[i] = prices[i] + 1 + prices[i].Add(prices[i], big.NewInt(1)) } reports := []mercury.FeedReport{ { FeedID: "0x1111111111111111111100000000000000000000000000000000000000000000", FullReport: []byte{0x11, 0xaa, 0xbb, 0xcc}, - BenchmarkPrice: prices[0], + BenchmarkPrice: prices[0].Bytes(), ObservationTimestamp: time.Now().Unix(), }, { FeedID: "0x2222222222222222222200000000000000000000000000000000000000000000", FullReport: []byte{0x22, 0xaa, 0xbb, 0xcc}, - BenchmarkPrice: prices[1], + BenchmarkPrice: prices[1].Bytes(), ObservationTimestamp: time.Now().Unix(), }, { FeedID: "0x3333333333333333333300000000000000000000000000000000000000000000", FullReport: []byte{0x33, 0xaa, 0xbb, 0xcc}, - BenchmarkPrice: prices[2], + BenchmarkPrice: prices[2].Bytes(), ObservationTimestamp: time.Now().Unix(), }, } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 2135c6833c9..542bdece430 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -21,7 +21,7 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/shopspring/decimal v1.3.1 github.com/smartcontractkit/chainlink-automation v1.0.3 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 47254fb049d..a20ad0cce4f 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1185,8 +1185,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs= github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 h1:e/qJZHPDVcgv/bnydjyYBk3JYbDnxPaZ2LvTlfDZeXA= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c h1:KiG8PAwUrdYn/AGBQ+B4p6erEUbEB+g6LJKhAaDjJ2s= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69 h1:Sec/GpBpUVaTEax1kSHlTvkzF/+d3w5roAQXaj5+SLA= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69/go.mod h1:ZQKf+0OLzCLYIisH/OdOIQuFRI6bDuw+jPBTATyHfFM= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo= diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice.go index 2c376443fa5..f84a48c1ff8 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice.go @@ -4,6 +4,8 @@ import ( "context" "math/big" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/smartcontractkit/chainlink/v2/core/cbor" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" @@ -27,17 +29,18 @@ type UpkeepOffchainConfig struct { // CheckGasPrice retrieves the current gas price and compare against the max gas price configured in upkeep's offchain config // any errors in offchain config decoding will result in max gas price check disabled func CheckGasPrice(ctx context.Context, upkeepId *big.Int, offchainConfigBytes []byte, ge gas.EvmFeeEstimator, lggr logger.Logger) encoding.UpkeepFailureReason { + // check for empty offchain config if len(offchainConfigBytes) == 0 { return encoding.UpkeepFailureReasonNone } var offchainConfig UpkeepOffchainConfig if err := cbor.ParseDietCBORToStruct(offchainConfigBytes, &offchainConfig); err != nil { - lggr.Errorw("failed to parse upkeep offchain config, gas price check is disabled", "upkeepId", upkeepId.String(), "err", err) + lggr.Warnw("failed to parse upkeep offchain config, gas price check is disabled", "offchainconfig", hexutil.Encode(offchainConfigBytes), "upkeepId", upkeepId.String(), "err", err) return encoding.UpkeepFailureReasonNone } if offchainConfig.MaxGasPrice == nil || offchainConfig.MaxGasPrice.Int64() <= 0 { - lggr.Warnw("maxGasPrice is not configured or incorrectly configured in upkeep offchain config, gas price check is disabled", "upkeepId", upkeepId.String()) + lggr.Debugw("maxGasPrice is not configured or incorrectly configured in upkeep offchain config, gas price check is disabled", "offchainconfig", hexutil.Encode(offchainConfigBytes), "upkeepId", upkeepId.String()) return encoding.UpkeepFailureReasonNone } lggr.Debugf("successfully decode offchain config for %s, max gas price is %s", upkeepId.String(), offchainConfig.MaxGasPrice.String()) diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry_check_pipeline.go index e341730c794..491099496cb 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry_check_pipeline.go @@ -313,7 +313,7 @@ func (r *EvmRegistry) simulatePerformUpkeeps(ctx context.Context, checkResults [ } fr := gasprice.CheckGasPrice(ctx, upkeepId, oc, r.ge, r.lggr) if uint8(fr) == uint8(encoding.UpkeepFailureReasonGasPriceTooHigh) { - r.lggr.Infof("upkeep %s upkeep failure reason is %d", upkeepId, fr) + r.lggr.Debugf("upkeep %s upkeep failure reason is %d", upkeepId, fr) checkResults[i].Eligible = false checkResults[i].Retryable = false checkResults[i].IneligibilityReason = uint8(fr) diff --git a/core/services/relay/evm/mercury/orm.go b/core/services/relay/evm/mercury/orm.go index 6426ef54a5d..65df9ab4cc6 100644 --- a/core/services/relay/evm/mercury/orm.go +++ b/core/services/relay/evm/mercury/orm.go @@ -5,6 +5,8 @@ import ( "crypto/sha256" "database/sql" "errors" + "fmt" + "strings" "sync" "github.com/ethereum/go-ethereum/common" @@ -19,7 +21,7 @@ import ( ) type ORM interface { - InsertTransmitRequest(ctx context.Context, serverURL string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error + InsertTransmitRequest(ctx context.Context, serverURLs []string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error DeleteTransmitRequests(ctx context.Context, serverURL string, reqs []*pb.TransmitRequest) error GetTransmitRequests(ctx context.Context, serverURL string, jobID int32) ([]*Transmission, error) PruneTransmitRequests(ctx context.Context, serverURL string, jobID int32, maxSize int) error @@ -42,11 +44,14 @@ func NewORM(ds sqlutil.DataSource) ORM { } // InsertTransmitRequest inserts one transmit request if the payload does not exist already. -func (o *orm) InsertTransmitRequest(ctx context.Context, serverURL string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error { +func (o *orm) InsertTransmitRequest(ctx context.Context, serverURLs []string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error { feedID, err := FeedIDFromReport(req.Payload) if err != nil { return err } + if len(serverURLs) == 0 { + return errors.New("no server URLs provided") + } var wg sync.WaitGroup wg.Add(2) @@ -54,11 +59,30 @@ func (o *orm) InsertTransmitRequest(ctx context.Context, serverURL string, req * go func() { defer wg.Done() - _, err1 = o.ds.ExecContext(ctx, ` - INSERT INTO mercury_transmit_requests (server_url, payload, payload_hash, config_digest, epoch, round, extra_hash, job_id, feed_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + + values := make([]string, len(serverURLs)) + args := []interface{}{ + req.Payload, + hashPayload(req.Payload), + reportCtx.ConfigDigest[:], + reportCtx.Epoch, + reportCtx.Round, + reportCtx.ExtraHash[:], + jobID, + feedID[:], + } + for i, serverURL := range serverURLs { + // server url is the only thing that changes, might as well re-use + // the same parameters for each insert + values[i] = fmt.Sprintf("($1, $2, $3, $4, $5, $6, $7, $8, $%d)", i+9) + args = append(args, serverURL) + } + + _, err1 = o.ds.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO mercury_transmit_requests (payload, payload_hash, config_digest, epoch, round, extra_hash, job_id, feed_id, server_url) + VALUES %s ON CONFLICT (server_url, payload_hash) DO NOTHING - `, serverURL, req.Payload, hashPayload(req.Payload), reportCtx.ConfigDigest[:], reportCtx.Epoch, reportCtx.Round, reportCtx.ExtraHash[:], jobID, feedID[:]) + `, strings.Join(values, ",")), args...) }() go func() { diff --git a/core/services/relay/evm/mercury/orm_test.go b/core/services/relay/evm/mercury/orm_test.go index f928acdb538..f3ff70cdced 100644 --- a/core/services/relay/evm/mercury/orm_test.go +++ b/core/services/relay/evm/mercury/orm_test.go @@ -48,15 +48,15 @@ func TestORM(t *testing.T) { // Test insert and get requests. // s1 - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[0]}, jobID, reportContexts[0]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, reportContexts[0]) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[1]}, jobID, reportContexts[1]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[1]}, jobID, reportContexts[1]) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[2]}, jobID, reportContexts[2]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, reportContexts[2]) require.NoError(t, err) // s2 - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[0]) + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[0]) require.NoError(t, err) transmissions, err := orm.GetTransmitRequests(ctx, sURL, jobID) @@ -119,7 +119,7 @@ func TestORM(t *testing.T) { require.Empty(t, transmissions) // More inserts. - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) require.NoError(t, err) transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) @@ -129,9 +129,9 @@ func TestORM(t *testing.T) { }) // Duplicate requests are ignored. - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) require.NoError(t, err) transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) @@ -150,6 +150,51 @@ func TestORM(t *testing.T) { require.Len(t, transmissions, 1) } +func TestORM_InsertTransmitRequest_MultipleServerURLs(t *testing.T) { + ctx := testutils.Context(t) + db := pgtest.NewSqlxDB(t) + + jobID := rand.Int32() // foreign key constraints disabled so value doesn't matter + pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) + pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) + orm := NewORM(db) + feedID := sampleFeedID + + reports := sampleReports + reportContexts := make([]ocrtypes.ReportContext, 4) + for i := range reportContexts { + reportContexts[i] = ocrtypes.ReportContext{ + ReportTimestamp: ocrtypes.ReportTimestamp{ + ConfigDigest: ocrtypes.ConfigDigest{'1'}, + Epoch: 10, + Round: uint8(i), + }, + ExtraHash: [32]byte{'2'}, + } + } + err := orm.InsertTransmitRequest(ctx, []string{sURL, sURL2, sURL3}, &pb.TransmitRequest{Payload: reports[0]}, jobID, reportContexts[0]) + require.NoError(t, err) + + transmissions, err := orm.GetTransmitRequests(ctx, sURL, jobID) + require.NoError(t, err) + require.Len(t, transmissions, 1) + assert.Equal(t, transmissions[0], &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}) + + transmissions, err = orm.GetTransmitRequests(ctx, sURL2, jobID) + require.NoError(t, err) + require.Len(t, transmissions, 1) + assert.Equal(t, transmissions[0], &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}) + + transmissions, err = orm.GetTransmitRequests(ctx, sURL3, jobID) + require.NoError(t, err) + require.Len(t, transmissions, 1) + assert.Equal(t, transmissions[0], &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}) + + l, err := orm.LatestReport(testutils.Context(t), feedID) + require.NoError(t, err) + assert.Equal(t, reports[0], l) +} + func TestORM_PruneTransmitRequests(t *testing.T) { ctx := testutils.Context(t) db := pgtest.NewSqlxDB(t) @@ -173,18 +218,18 @@ func TestORM_PruneTransmitRequests(t *testing.T) { } // s1 - err := orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) + err := orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) require.NoError(t, err) // s2 - should not be touched - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 0)) + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 0)) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 3)) + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 3)) require.NoError(t, err) // Max size greater than number of records, expect no-op @@ -220,9 +265,9 @@ func TestORM_PruneTransmitRequests(t *testing.T) { {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: makeReportContext(1, 1)}, }, transmissions) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(2, 1)) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(2, 1)) require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 2)) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 2)) require.NoError(t, err) // Max size is table size - 1, expect the oldest row to be pruned. @@ -266,13 +311,13 @@ func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { } } - err := orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext( + err := orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext( 0, 0, )) require.NoError(t, err) // this should be ignored, because report context is the same - err = orm.InsertTransmitRequest(ctx, sURL2, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext( + err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext( 0, 0, )) require.NoError(t, err) @@ -282,7 +327,7 @@ func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { assert.Equal(t, reports[0], l) t.Run("replaces if epoch and round are larger", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, "foo", &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 1)) + err = orm.InsertTransmitRequest(ctx, []string{"foo"}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 1)) require.NoError(t, err) l, err = orm.LatestReport(testutils.Context(t), feedID) @@ -290,7 +335,7 @@ func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { assert.Equal(t, reports[1], l) }) t.Run("replaces if epoch is the same but round is greater", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 2)) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 2)) require.NoError(t, err) l, err = orm.LatestReport(testutils.Context(t), feedID) @@ -298,7 +343,7 @@ func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { assert.Equal(t, reports[2], l) }) t.Run("replaces if epoch is larger but round is smaller", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, "bar", &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 1)) + err = orm.InsertTransmitRequest(ctx, []string{"bar"}, &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 1)) require.NoError(t, err) l, err = orm.LatestReport(testutils.Context(t), feedID) @@ -306,7 +351,7 @@ func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { assert.Equal(t, reports[3], l) }) t.Run("does not overwrite if epoch/round is the same", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, sURL, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(2, 1)) + err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(2, 1)) require.NoError(t, err) l, err = orm.LatestReport(testutils.Context(t), feedID) diff --git a/core/services/relay/evm/mercury/persistence_manager.go b/core/services/relay/evm/mercury/persistence_manager.go index d49d0d4ed01..38576174423 100644 --- a/core/services/relay/evm/mercury/persistence_manager.go +++ b/core/services/relay/evm/mercury/persistence_manager.go @@ -69,7 +69,7 @@ func (pm *PersistenceManager) Close() error { } func (pm *PersistenceManager) Insert(ctx context.Context, req *pb.TransmitRequest, reportCtx ocrtypes.ReportContext) error { - return pm.orm.InsertTransmitRequest(ctx, pm.serverURL, req, pm.jobID, reportCtx) + return pm.orm.InsertTransmitRequest(ctx, []string{pm.serverURL}, req, pm.jobID, reportCtx) } func (pm *PersistenceManager) Delete(ctx context.Context, req *pb.TransmitRequest) error { diff --git a/core/services/relay/evm/mercury/transmitter.go b/core/services/relay/evm/mercury/transmitter.go index 7ba5f5df606..415ecbb1787 100644 --- a/core/services/relay/evm/mercury/transmitter.go +++ b/core/services/relay/evm/mercury/transmitter.go @@ -18,6 +18,7 @@ import ( pkgerrors "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" @@ -96,6 +97,7 @@ type ConfigTracker interface { type TransmitterReportDecoder interface { BenchmarkPriceFromReport(report ocrtypes.Report) (*big.Int, error) + ObservationTimestampFromReport(report ocrtypes.Report) (uint32, error) } var _ Transmitter = (*mercuryTransmitter)(nil) @@ -110,6 +112,7 @@ type mercuryTransmitter struct { lggr logger.Logger cfg TransmitterConfig + orm ORM servers map[string]*server codec TransmitterReportDecoder @@ -253,7 +256,7 @@ func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, feed b.Reset() if res.Error == "" { s.transmitSuccessCount.Inc() - s.lggr.Debugw("Transmit report success", "payload", hexutil.Encode(t.Req.Payload), "response", res, "reportCtx", t.ReportCtx) + s.lggr.Debugw("Transmit report success", "payload", hexutil.Encode(t.Req.Payload), "response", res, "repts", t.ReportCtx.ReportTimestamp) } else { // We don't need to retry here because the mercury server // has confirmed it received the report. We only need to retry @@ -262,7 +265,7 @@ func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, feed case DuplicateReport: s.transmitSuccessCount.Inc() s.transmitDuplicateCount.Inc() - s.lggr.Debugw("Transmit report success; duplicate report", "payload", hexutil.Encode(t.Req.Payload), "response", res, "reportCtx", t.ReportCtx) + s.lggr.Debugw("Transmit report success; duplicate report", "payload", hexutil.Encode(t.Req.Payload), "response", res, "repts", t.ReportCtx.ReportTimestamp) default: transmitServerErrorCount.WithLabelValues(feedIDHex, fmt.Sprintf("%d", res.Code)).Inc() s.lggr.Errorw("Transmit report failed; mercury server returned error", "response", res, "reportCtx", t.ReportCtx, "err", res.Error, "code", res.Code) @@ -302,6 +305,7 @@ func NewTransmitter(lggr logger.Logger, cfg TransmitterConfig, clients map[strin services.StateMachine{}, lggr.Named("MercuryTransmitter").With("feedID", feedIDHex), cfg, + orm, servers, codec, triggerCapability, @@ -402,16 +406,20 @@ func (mt *mercuryTransmitter) Transmit(ctx context.Context, reportCtx ocrtypes.R Payload: payload, } - mt.lggr.Tracew("Transmit enqueue", "req.Payload", req.Payload, "report", report, "reportCtx", reportCtx, "signatures", signatures) + ts, err := mt.codec.ObservationTimestampFromReport(report) + if err != nil { + mt.lggr.Warnw("Failed to get observation timestamp from report", "err", err) + } + mt.lggr.Debugw("Transmit enqueue", "req.Payload", hexutil.Encode(req.Payload), "report", report, "repts", reportCtx.ReportTimestamp, "signatures", signatures, "observationsTimestamp", ts) + + if err := mt.orm.InsertTransmitRequest(ctx, maps.Keys(mt.servers), req, mt.jobID, reportCtx); err != nil { + return err + } g := new(errgroup.Group) for _, s := range mt.servers { s := s // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { - if err := s.pm.Insert(ctx, req, reportCtx); err != nil { - s.transmitQueueInsertErrorCount.Inc() - return err - } if ok := s.q.Push(req, reportCtx); !ok { s.transmitQueuePushErrorCount.Inc() return errors.New("transmit queue is closed") diff --git a/core/services/relay/evm/mercury/transmitter_test.go b/core/services/relay/evm/mercury/transmitter_test.go index b286509e0ab..38d1eb621d5 100644 --- a/core/services/relay/evm/mercury/transmitter_test.go +++ b/core/services/relay/evm/mercury/transmitter_test.go @@ -216,6 +216,10 @@ func (m *mockCodec) BenchmarkPriceFromReport(_ ocrtypes.Report) (*big.Int, error return m.val, m.err } +func (m *mockCodec) ObservationTimestampFromReport(report ocrtypes.Report) (uint32, error) { + return 0, nil +} + func Test_MercuryTransmitter_LatestPrice(t *testing.T) { t.Parallel() lggr := logger.TestLogger(t) diff --git a/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go b/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go index 8f2eac59c33..f4c6af32b8e 100644 --- a/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go +++ b/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go @@ -98,3 +98,11 @@ func (r *ReportCodec) BenchmarkPriceFromReport(report ocrtypes.Report) (*big.Int } return decoded.BenchmarkPrice, nil } + +func (r *ReportCodec) ObservationTimestampFromReport(report ocrtypes.Report) (uint32, error) { + decoded, err := r.Decode(report) + if err != nil { + return 0, err + } + return decoded.ObservationsTimestamp, nil +} diff --git a/go.mod b/go.mod index 1c44fac9da9..4250c0fef43 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/shopspring/decimal v1.3.1 github.com/smartcontractkit/chain-selectors v1.0.10 github.com/smartcontractkit/chainlink-automation v1.0.3 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69 github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 github.com/smartcontractkit/chainlink-feeds v0.0.0-20240422130241-13c17a91b2ab diff --git a/go.sum b/go.sum index 7cc88d7c0f4..5fa3897fc21 100644 --- a/go.sum +++ b/go.sum @@ -1171,8 +1171,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs= github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 h1:e/qJZHPDVcgv/bnydjyYBk3JYbDnxPaZ2LvTlfDZeXA= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c h1:KiG8PAwUrdYn/AGBQ+B4p6erEUbEB+g6LJKhAaDjJ2s= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69 h1:Sec/GpBpUVaTEax1kSHlTvkzF/+d3w5roAQXaj5+SLA= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69/go.mod h1:ZQKf+0OLzCLYIisH/OdOIQuFRI6bDuw+jPBTATyHfFM= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo= diff --git a/integration-tests/actions/vrf/vrfv2/setup_steps.go b/integration-tests/actions/vrf/vrfv2/setup_steps.go index 5c441bf811e..c21ff0a11e6 100644 --- a/integration-tests/actions/vrf/vrfv2/setup_steps.go +++ b/integration-tests/actions/vrf/vrfv2/setup_steps.go @@ -320,7 +320,16 @@ func SetupVRFV2WrapperEnvironment( return wrapperContracts, &wrapperSubID, nil } -func SetupVRFV2Universe(ctx context.Context, t *testing.T, testConfig tc.TestConfig, chainID int64, cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { +func SetupVRFV2Universe( + ctx context.Context, + t *testing.T, + testConfig tc.TestConfig, + chainID int64, + cleanupFn func(), + newEnvConfig vrfcommon.NewEnvConfig, + l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, +) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { var ( env *test_env.CLClusterTestEnv vrfContracts *vrfcommon.VRFContracts @@ -334,7 +343,7 @@ func SetupVRFV2Universe(ctx context.Context, t *testing.T, testConfig tc.TestCon return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 for Existing env", err) } } else { - vrfContracts, vrfKey, env, nodeTypeToNodeMap, err = SetupVRFV2ForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l) + vrfContracts, vrfKey, env, nodeTypeToNodeMap, err = SetupVRFV2ForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 for New env", err) } @@ -350,17 +359,20 @@ func SetupVRFV2ForNewEnv( cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) (*vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, *test_env.CLClusterTestEnv, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { network, err := actions.EthereumNetworkConfigFromConfig(l, &testConfig) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error building ethereum network config", err) } + env, err := test_env.NewCLTestEnvBuilder(). WithTestInstance(t). WithTestConfig(&testConfig). WithPrivateEthereumNetwork(network.EthereumNetworkConfig). WithCLNodes(len(newEnvConfig.NodesToCreate)). WithFunding(big.NewFloat(*testConfig.Common.ChainlinkNodeFunding)). + WithChainlinkNodeLogScanner(chainlinkNodeLogScannerSettings). WithCustomCleanup(cleanupFn). Build() diff --git a/integration-tests/actions/vrf/vrfv2plus/setup_steps.go b/integration-tests/actions/vrf/vrfv2plus/setup_steps.go index ab973ffe110..fdb5592767a 100644 --- a/integration-tests/actions/vrf/vrfv2plus/setup_steps.go +++ b/integration-tests/actions/vrf/vrfv2plus/setup_steps.go @@ -371,7 +371,16 @@ func SetupVRFV2PlusWrapperEnvironment( return wrapperContracts, wrapperSubID, nil } -func SetupVRFV2PlusUniverse(ctx context.Context, t *testing.T, testConfig tc.TestConfig, chainID int64, cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { +func SetupVRFV2PlusUniverse( + ctx context.Context, + t *testing.T, + testConfig tc.TestConfig, + chainID int64, + cleanupFn func(), + newEnvConfig vrfcommon.NewEnvConfig, + l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, +) (*test_env.CLClusterTestEnv, *vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { var ( env *test_env.CLClusterTestEnv vrfContracts *vrfcommon.VRFContracts @@ -385,7 +394,7 @@ func SetupVRFV2PlusUniverse(ctx context.Context, t *testing.T, testConfig tc.Tes return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 Plus for Existing env", err) } } else { - vrfContracts, vrfKey, env, nodeTypeToNode, err = SetupVRFV2PlusForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l) + vrfContracts, vrfKey, env, nodeTypeToNode, err = SetupVRFV2PlusForNewEnv(ctx, t, testConfig, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) if err != nil { return nil, nil, nil, nil, fmt.Errorf("%s, err: %w", "Error setting up VRF V2 Plus for New env", err) } @@ -401,6 +410,7 @@ func SetupVRFV2PlusForNewEnv( cleanupFn func(), newEnvConfig vrfcommon.NewEnvConfig, l zerolog.Logger, + chainlinkNodeLogScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) (*vrfcommon.VRFContracts, *vrfcommon.VRFKeyData, *test_env.CLClusterTestEnv, map[vrfcommon.VRFNodeType]*vrfcommon.VRFNode, error) { network, err := actions.EthereumNetworkConfigFromConfig(l, &testConfig) if err != nil { @@ -412,6 +422,7 @@ func SetupVRFV2PlusForNewEnv( WithPrivateEthereumNetwork(network.EthereumNetworkConfig). WithCLNodes(len(newEnvConfig.NodesToCreate)). WithFunding(big.NewFloat(*testConfig.Common.ChainlinkNodeFunding)). + WithChainlinkNodeLogScanner(chainlinkNodeLogScannerSettings). WithCustomCleanup(cleanupFn). Build() if err != nil { diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index 19617b0c5f7..0089ce0778a 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -1,15 +1,19 @@ package test_env import ( - "errors" + "context" "fmt" "math/big" "os" + "slices" "testing" + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/smartcontractkit/seth" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" @@ -17,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/logstream" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/osutil" evmcfg "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" @@ -35,38 +40,65 @@ const ( CleanUpTypeCustom CleanUpType = "custom" ) +type ChainlinkNodeLogScannerSettings struct { + FailingLogLevel zapcore.Level + Threshold uint + AllowedMessages []testreporters.AllowedLogMessage +} + type CLTestEnvBuilder struct { - hasLogStream bool - hasKillgrave bool - hasForwarders bool - hasSeth bool - hasEVMClient bool - clNodeConfig *chainlink.Config - secretsConfig string - clNodesCount int - clNodesOpts []func(*ClNode) - customNodeCsaKeys []string - defaultNodeCsaKeys []string - l zerolog.Logger - t *testing.T - te *CLClusterTestEnv - isNonEVM bool - cleanUpType CleanUpType - cleanUpCustomFn func() - chainOptionsFn []ChainOption - evmNetworkOption []EVMNetworkOption - privateEthereumNetworks []*ctf_config.EthereumNetworkConfig - testConfig ctf_config.GlobalTestConfig + hasLogStream bool + hasKillgrave bool + hasForwarders bool + hasSeth bool + hasEVMClient bool + clNodeConfig *chainlink.Config + secretsConfig string + clNodesCount int + clNodesOpts []func(*ClNode) + customNodeCsaKeys []string + defaultNodeCsaKeys []string + l zerolog.Logger + t *testing.T + te *CLClusterTestEnv + isNonEVM bool + cleanUpType CleanUpType + cleanUpCustomFn func() + chainOptionsFn []ChainOption + evmNetworkOption []EVMNetworkOption + privateEthereumNetworks []*ctf_config.EthereumNetworkConfig + testConfig ctf_config.GlobalTestConfig + chainlinkNodeLogScannerSettings *ChainlinkNodeLogScannerSettings /* funding */ ETHFunds *big.Float } +var DefaultAllowedMessages = []testreporters.AllowedLogMessage{ + testreporters.NewAllowedLogMessage("Failed to get LINK balance", "Happens only when we deploy LINK token for test purposes. Harmless.", zapcore.ErrorLevel, testreporters.WarnAboutAllowedMsgs_No), +} + +var DefaultChainlinkNodeLogScannerSettings = ChainlinkNodeLogScannerSettings{ + FailingLogLevel: zapcore.DPanicLevel, + Threshold: 1, // we want to fail on the first concerning log + AllowedMessages: DefaultAllowedMessages, +} + +func GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(extraAllowedMessages ...testreporters.AllowedLogMessage) ChainlinkNodeLogScannerSettings { + allowedMessages := append(DefaultAllowedMessages, extraAllowedMessages...) + return ChainlinkNodeLogScannerSettings{ + FailingLogLevel: zapcore.DPanicLevel, + Threshold: 1, + AllowedMessages: allowedMessages, + } +} + func NewCLTestEnvBuilder() *CLTestEnvBuilder { return &CLTestEnvBuilder{ - l: log.Logger, - hasLogStream: true, - hasEVMClient: true, + l: log.Logger, + hasLogStream: true, + hasEVMClient: true, + chainlinkNodeLogScannerSettings: &DefaultChainlinkNodeLogScannerSettings, } } @@ -114,6 +146,16 @@ func (b *CLTestEnvBuilder) WithoutLogStream() *CLTestEnvBuilder { return b } +func (b *CLTestEnvBuilder) WithoutChainlinkNodeLogScanner() *CLTestEnvBuilder { + b.chainlinkNodeLogScannerSettings = &ChainlinkNodeLogScannerSettings{} + return b +} + +func (b *CLTestEnvBuilder) WithChainlinkNodeLogScanner(settings ChainlinkNodeLogScannerSettings) *CLTestEnvBuilder { + b.chainlinkNodeLogScannerSettings = &settings + return b +} + func (b *CLTestEnvBuilder) WithCLNodes(clNodesCount int) *CLTestEnvBuilder { b.clNodesCount = clNodesCount return b @@ -231,6 +273,12 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } if b.hasLogStream { + loggingConfig := b.testConfig.GetLoggingConfig() + // we need to enable logging to file if we want to scan logs + if b.chainlinkNodeLogScannerSettings != nil && !slices.Contains(loggingConfig.LogStream.LogTargets, string(logstream.File)) { + b.l.Debug().Msg("Enabling logging to file in order to support Chainlink node log scanning") + loggingConfig.LogStream.LogTargets = append(loggingConfig.LogStream.LogTargets, string(logstream.File)) + } b.te.LogStream, err = logstream.NewLogStream(b.te.t, b.testConfig.GetLoggingConfig()) if err != nil { return nil, err @@ -279,11 +327,55 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { b.l.Info().Str("Absolute path", logPath).Msg("LogStream logs folder location") } + var scanClNodeLogs = func() { + //filter out non-cl logs + logLocation := b.te.LogStream.GetLogLocation() + logFiles, err := testreporters.FindAllLogFilesToScan(logLocation, "cl-node") + if err != nil { + b.l.Warn().Err(err).Msg("Error looking for Chainlink Node log files to scan") + } else { + // we ignore the context returned by errgroup here, since we have no way of interrupting ongoing scanning of logs + verifyLogsGroup, _ := errgroup.WithContext(context.Background()) + for _, f := range logFiles { + file := f + verifyLogsGroup.Go(func() error { + logErr := testreporters.VerifyLogFile(file, b.chainlinkNodeLogScannerSettings.FailingLogLevel, b.chainlinkNodeLogScannerSettings.Threshold, b.chainlinkNodeLogScannerSettings.AllowedMessages...) + if logErr != nil { + return errors.Wrapf(logErr, "Found a concerning log in %s", file.Name()) + } + return nil + }) + } + if err := verifyLogsGroup.Wait(); err != nil { + b.l.Error().Err(err).Msg("Found a concerning log. Failing test.") + b.t.Fatalf("Found a concerning log in Chainklink Node logs: %v", err) + } + } + b.l.Info().Msg("Finished scanning Chainlink Node logs for concerning errors") + } + if b.t.Failed() || *b.testConfig.GetLoggingConfig().TestLogCollect { // we can't do much if this fails, so we just log the error in logstream - _ = b.te.LogStream.FlushAndShutdown() + flushErr := b.te.LogStream.FlushAndShutdown() + if flushErr != nil { + b.l.Error().Err(flushErr).Msg("Error flushing and shutting down LogStream") + return + } b.te.LogStream.PrintLogTargetsLocations() b.te.LogStream.SaveLogLocationInTestSummary() + + // if test hasn't failed, but we have chainlinkNodeLogScannerSettings, we should check the logs + if !b.t.Failed() && b.chainlinkNodeLogScannerSettings != nil { + scanClNodeLogs() + } + } else if b.chainlinkNodeLogScannerSettings != nil { + flushErr := b.te.LogStream.FlushAndShutdown() + if flushErr != nil { + b.l.Error().Err(flushErr).Msg("Error flushing and shutting down LogStream") + return + } + + scanClNodeLogs() } }) } @@ -303,6 +395,10 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } } + if b.te.LogStream == nil && b.chainlinkNodeLogScannerSettings != nil { + log.Warn().Msg("Chainlink node log scanner settings provided, but LogStream is not enabled. Ignoring Chainlink node log scanner settings, as no logs will be available.") + } + // in this case we will use the builder only to start chains, not the cluster, because currently we support only 1 network config per cluster if len(b.privateEthereumNetworks) > 1 { b.te.rpcProviders = make(map[int64]*test_env.RpcProvider) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 17b62c4d844..c831f3f181d 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -27,8 +27,8 @@ require ( github.com/shopspring/decimal v1.3.1 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 - github.com/smartcontractkit/chainlink-testing-framework v1.28.11 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c + github.com/smartcontractkit/chainlink-testing-framework v1.28.12 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 893dff72823..dfacb294fd9 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1512,8 +1512,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs= github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 h1:e/qJZHPDVcgv/bnydjyYBk3JYbDnxPaZ2LvTlfDZeXA= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c h1:KiG8PAwUrdYn/AGBQ+B4p6erEUbEB+g6LJKhAaDjJ2s= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69 h1:Sec/GpBpUVaTEax1kSHlTvkzF/+d3w5roAQXaj5+SLA= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69/go.mod h1:ZQKf+0OLzCLYIisH/OdOIQuFRI6bDuw+jPBTATyHfFM= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo= @@ -1524,8 +1524,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11 h1:6IcgmK/6UAfmcOYSiXudVlOa60MkbObTxA68hSJT4O4= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 83ef49e1682..782408dff3c 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -16,8 +16,8 @@ require ( github.com/rs/zerolog v1.30.0 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 - github.com/smartcontractkit/chainlink-testing-framework v1.28.11 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c + github.com/smartcontractkit/chainlink-testing-framework v1.28.12 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 34d954a1604..00112420705 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1502,8 +1502,8 @@ github.com/smartcontractkit/chain-selectors v1.0.10 h1:t9kJeE6B6G+hKD0GYR4kGJSCq github.com/smartcontractkit/chain-selectors v1.0.10/go.mod h1:d4Hi+E1zqjy9HqMkjBE5q1vcG9VGgxf5VxiRHfzi2kE= github.com/smartcontractkit/chainlink-automation v1.0.3 h1:h/ijT0NiyV06VxYVgcNfsE3+8OEzT3Q0Z9au0z1BPWs= github.com/smartcontractkit/chainlink-automation v1.0.3/go.mod h1:RjboV0Qd7YP+To+OrzHGXaxUxoSONveCoAK2TQ1INLU= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65 h1:e/qJZHPDVcgv/bnydjyYBk3JYbDnxPaZ2LvTlfDZeXA= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20240509130051-b54aae6a8b65/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c h1:KiG8PAwUrdYn/AGBQ+B4p6erEUbEB+g6LJKhAaDjJ2s= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20240514153505-0ddba5aa4d2c/go.mod h1:sj0pjL+METqeYL9ibp0T8SXquymlaQsofa6bdfLgXX8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69 h1:Sec/GpBpUVaTEax1kSHlTvkzF/+d3w5roAQXaj5+SLA= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240508101745-af1ed7bc8a69/go.mod h1:ZQKf+0OLzCLYIisH/OdOIQuFRI6bDuw+jPBTATyHfFM= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240220203239-09be0ea34540 h1:xFSv8561jsLtF6gYZr/zW2z5qUUAkcFkApin2mnbYTo= @@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a8 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240510181707-46b1311a5a83/go.mod h1:RdAtOeBUWq2zByw2kEbwPlXaPIb7YlaDOmnn+nVUBJI= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69 h1:ssh/w3oXWu+C6bE88GuFRC1+0Bx/4ihsbc80XMLrl2k= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240508155030-1024f2b55c69/go.mod h1:VsfjhvWgjxqWja4q+FlXEtX5lu8BSxn10xRo6gi948g= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11 h1:6IcgmK/6UAfmcOYSiXudVlOa60MkbObTxA68hSJT4O4= -github.com/smartcontractkit/chainlink-testing-framework v1.28.11/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12 h1:15ssos9DvWekvj6JjmiPjTYsj/uw12HvTWlm1FHdYaA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.12/go.mod h1:x1zDOz8zcLjEvs9fNA9y/DMguLam/2+CJdpxX0+rM8A= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go index ca41dd40ea9..a62e4cdd552 100644 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ b/integration-tests/load/vrfv2/vrfv2_test.go @@ -99,7 +99,7 @@ func TestVRFV2Performance(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -243,7 +243,7 @@ func TestVRFV2BHSPerformance(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index e24b2ff17d8..2496e2b692b 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -97,7 +97,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -240,7 +240,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := testEnv.GetEVMClient(chainID) diff --git a/integration-tests/smoke/log_poller_test.go b/integration-tests/smoke/log_poller_test.go index f75dfe96c7a..04327471e7f 100644 --- a/integration-tests/smoke/log_poller_test.go +++ b/integration-tests/smoke/log_poller_test.go @@ -11,9 +11,11 @@ import ( "github.com/onsi/gomega" "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth" "github.com/smartcontractkit/chainlink/integration-tests/contracts" @@ -25,45 +27,52 @@ import ( core_logger "github.com/smartcontractkit/chainlink/v2/core/logger" ) +var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( + "SLOW SQL QUERY", + "It is expected, because we are pausing the Postgres container", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No, +)) + // consistency test with no network disruptions with approximate emission of 1500-1600 logs per second for ~110-120 seconds // 6 filters are registered func TestLogPollerFewFiltersFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } func TestLogPollerFewFiltersFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } // consistency test with no network disruptions with approximate emission of 1000-1100 logs per second for ~110-120 seconds // 900 filters are registered func TestLogPollerManyFiltersFixedDepth(t *testing.T) { t.Skip("Execute manually, when needed as it runs for a long time") - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } func TestLogPollerManyFiltersFinalityTag(t *testing.T) { t.Skip("Execute manually, when needed as it runs for a long time") - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, test_env.DefaultChainlinkNodeLogScannerSettings) } -// consistency test that introduces random distruptions by pausing either Chainlink or Postgres containers for random interval of 5-20 seconds +// consistency test that introduces random disruptions by pausing either Chainlink or Postgres containers for random interval of 5-20 seconds // with approximate emission of 520-550 logs per second for ~110 seconds // 6 filters are registered func TestLogPollerWithChaosFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosPostgresFixedDepth(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } func TestLogPollerWithChaosPostgresFinalityTag(t *testing.T) { - executeBasicLogPollerTest(t) + executeBasicLogPollerTest(t, logScannerSettings) } // consistency test that registers filters after events were emitted and then triggers replay via API @@ -80,7 +89,7 @@ func TestLogPollerReplayFinalityTag(t *testing.T) { } // HELPER FUNCTIONS -func executeBasicLogPollerTest(t *testing.T) { +func executeBasicLogPollerTest(t *testing.T, logScannerSettings test_env.ChainlinkNodeLogScannerSettings) { testConfig, err := tc.GetConfig(t.Name(), tc.LogPoller) require.NoError(t, err, "Error getting config") overrideEphemeralAddressesCount(&testConfig) @@ -96,7 +105,7 @@ func executeBasicLogPollerTest(t *testing.T) { l := logging.GetTestLogger(t) coreLogger := core_logger.TestLogger(t) //needed by ORM ¯\_(ツ)_/¯ - lpTestEnv := prepareEnvironment(l, t, &testConfig) + lpTestEnv := prepareEnvironment(l, t, &testConfig, logScannerSettings) testEnv := lpTestEnv.testEnv ctx := testcontext.Get(t) @@ -177,7 +186,7 @@ func executeLogPollerReplay(t *testing.T, consistencyTimeout string) { l := logging.GetTestLogger(t) coreLogger := core_logger.TestLogger(t) //needed by ORM ¯\_(ツ)_/¯ - lpTestEnv := prepareEnvironment(l, t, &testConfig) + lpTestEnv := prepareEnvironment(l, t, &testConfig, test_env.DefaultChainlinkNodeLogScannerSettings) testEnv := lpTestEnv.testEnv ctx := testcontext.Get(t) @@ -262,7 +271,7 @@ type logPollerEnvironment struct { // prepareEnvironment prepares environment for log poller tests by starting DON, private Ethereum network, // deploying registry and log emitter contracts and registering log triggered upkeeps -func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfig) logPollerEnvironment { +func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfig, logScannerSettings test_env.ChainlinkNodeLogScannerSettings) logPollerEnvironment { cfg := testConfig.LogPoller if cfg.General.EventsToEmit == nil || len(cfg.General.EventsToEmit) == 0 { l.Warn().Msg("No events to emit specified, using all events from log emitter contract") @@ -287,6 +296,7 @@ func prepareEnvironment(l zerolog.Logger, t *testing.T, testConfig *tc.TestConfi *cfg.General.BackupLogPollerBlockDelay, *cfg.General.UseFinalityTag, testConfig, + logScannerSettings, ) _, upkeepIDs := actions_seth.DeployConsumers( diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index ce9f448ee66..946a73f0e18 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -14,10 +14,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" commonassets "github.com/smartcontractkit/chainlink-common/pkg/assets" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/conversions" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" @@ -75,7 +77,7 @@ func TestVRFv2Basic(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) require.NoError(t, err, "Getting EVM client shouldn't fail") @@ -610,7 +612,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -718,7 +720,7 @@ func TestVRFOwner(t *testing.T) { UseTestCoordinator: true, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -866,7 +868,7 @@ func TestVRFV2WithBHS(t *testing.T) { UseTestCoordinator: false, } - testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFV2 universe") evmClient, err := testEnv.GetEVMClient(chainID) @@ -1070,7 +1072,20 @@ func TestVRFV2NodeReorg(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( + testreporters.NewAllowedLogMessage( + "This is a problem and either means a very deep re-org occurred", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + testreporters.NewAllowedLogMessage( + "Reorg greater than finality depth detected", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + ) + + env, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2 universe") evmClient, err := env.GetEVMClient(chainID) @@ -1232,7 +1247,7 @@ func TestVRFv2BatchFulfillmentEnabledDisabled(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2 universe") evmClient, err := env.GetEVMClient(chainID) diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index 0ef4e716733..6a64d08569a 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -13,10 +13,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" + "github.com/smartcontractkit/chainlink-testing-framework/testreporters" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -76,7 +78,7 @@ func TestVRFv2Plus(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -775,7 +777,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -881,7 +883,7 @@ func TestVRFv2PlusMigration(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1309,7 +1311,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1532,8 +1534,14 @@ func TestVRFV2PlusWithBHF(t *testing.T) { UseTestCoordinator: false, } + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( + "Pipeline error", + "Test is expecting this error to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No)) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse( - testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err) evmClient, err := env.GetEVMClient(chainID) require.NoError(t, err, "Getting EVM client shouldn't fail") @@ -1679,7 +1687,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) { config.VRFv2Plus.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) config.VRFv2Plus.General.SubscriptionFundingAmountNative = ptr.Ptr(float64(0)) - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1876,7 +1884,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) config.VRFv2Plus.General.MinimumConfirmations = ptr.Ptr[uint16](0) config.VRFv2Plus.General.VRFJobSimulationBlock = ptr.Ptr[string]("pending") - env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "error setting up VRFV2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -1965,7 +1973,20 @@ func TestVRFv2PlusNodeReorg(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( + testreporters.NewAllowedLogMessage( + "This is a problem and either means a very deep re-org occurred", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + testreporters.NewAllowedLogMessage( + "Reorg greater than finality depth detected", + "Test is expecting a reorg to occur", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), + ) + + env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, chainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) @@ -2125,7 +2146,7 @@ func TestVRFv2PlusBatchFulfillmentEnabledDisabled(t *testing.T) { UseTestCoordinator: false, } - env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l, test_env.DefaultChainlinkNodeLogScannerSettings) require.NoError(t, err, "Error setting up VRFv2Plus universe") evmClient, err := env.GetEVMClient(chainID) diff --git a/integration-tests/universal/log_poller/helpers.go b/integration-tests/universal/log_poller/helpers.go index 07334aa79ae..d732ad4af5d 100644 --- a/integration-tests/universal/log_poller/helpers.go +++ b/integration-tests/universal/log_poller/helpers.go @@ -1013,7 +1013,7 @@ func ExecuteChaosExperiment(l zerolog.Logger, testEnv *test_env.CLClusterTestEnv <-guardChan wg.Done() current := i + 1 - l.Info().Str("Current/Total", fmt.Sprintf("%d/%d", current, testConfig.LogPoller.ChaosConfig.ExperimentCount)).Msg("Done with experiment") + l.Info().Str("Current/Total", fmt.Sprintf("%d/%d", current, *testConfig.LogPoller.ChaosConfig.ExperimentCount)).Msg("Done with experiment") }() chaosChan <- chaosPauseSyncFn(l, sethClient, testEnv.ClCluster, *testConfig.LogPoller.ChaosConfig.TargetComponent) time.Sleep(10 * time.Second) @@ -1115,6 +1115,7 @@ func SetupLogPollerTestDocker( backupPollingInterval uint64, finalityTagEnabled bool, testConfig *tc.TestConfig, + logScannerSettings test_env.ChainlinkNodeLogScannerSettings, ) ( *seth.Client, []*client.ChainlinkClient, @@ -1179,6 +1180,7 @@ func SetupLogPollerTestDocker( WithFunding(big.NewFloat(chainlinkNodeFunding)). WithChainOptions(logPolllerSettingsFn). EVMNetworkOptions(evmNetworkSettingsFn). + WithChainlinkNodeLogScanner(logScannerSettings). WithStandardCleanup(). WithSeth(). Build() @@ -1344,6 +1346,7 @@ func FluentlyCheckIfAllNodesHaveLogCount(duration string, startBlock, endBlock i } l.Warn(). Msg("At least one CL node did not have expected log count. Retrying...") + time.Sleep(10 * time.Second) } return allNodesLogCountMatches, nil diff --git a/plugins/cmd/chainlink-ocr3-capability/main.go b/plugins/cmd/chainlink-ocr3-capability/main.go index 85767554a1c..c70a5a6f2ad 100644 --- a/plugins/cmd/chainlink-ocr3-capability/main.go +++ b/plugins/cmd/chainlink-ocr3-capability/main.go @@ -10,6 +10,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/loop/reportingplugins" ocr3rp "github.com/smartcontractkit/chainlink-common/pkg/loop/reportingplugins/ocr3" "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" ) @@ -22,8 +23,9 @@ func main() { defer s.Stop() c := ocr3.Config{ - Logger: s.Logger, - EncoderFactory: evm.NewEVMEncoder, + Logger: s.Logger, + EncoderFactory: evm.NewEVMEncoder, + AggregatorFactory: capabilities.NewAggregator, } p := ocr3.NewOCR3(c) if err := p.Start(context.Background()); err != nil {