diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index b0f75fbffa1..5db59dd7c49 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -48,7 +48,7 @@ runner-test-matrix: workflows: - Run Automation Product Nightly E2E Tests test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json - test-config-override: + test-config-overrides: chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }} chainlink-upgrade-version: develop chainlink-image: public.ecr.aws/chainlink/chainlink @@ -62,7 +62,7 @@ runner-test-matrix: workflows: - Run Automation Product Nightly E2E Tests test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json - test-config-override: + test-config-overrides: chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }} chainlink-upgrade-version: develop chainlink-image: public.ecr.aws/chainlink/chainlink @@ -76,7 +76,7 @@ runner-test-matrix: workflows: - Run Automation Product Nightly E2E Tests test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json - test-config-override: + test-config-overrides: chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }} chainlink-upgrade-version: develop chainlink-image: public.ecr.aws/chainlink/chainlink diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml new file mode 100644 index 00000000000..7ee6a27b930 --- /dev/null +++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml @@ -0,0 +1,116 @@ +name: Run Automation On Demand Tests (TEST WORKFLOW) + +on: + workflow_dispatch: + inputs: + chainlinkVersion: + description: Chainlink image version to use + required: false + type: string + chainlinkImage: + description: Chainlink image repo to use (Leave empty to build from head/ref) + required: false + type: string + chainlinkVersionUpdate: + description: Chainlink image version to use initially for upgrade test + default: latest + required: true + type: string + chainlinkImageUpdate: + description: Chainlink image repo to use initially for upgrade test + required: true + default: public.ecr.aws/chainlink/chainlink + type: string + enableChaos: + description: Check to enable chaos tests + type: boolean + default: false + required: true + enableReorg: + description: Check to enable reorg tests + type: boolean + default: false + required: true + with_existing_remote_runner_version: + description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"' + required: false + type: string + +jobs: + # Set tests to run based on the workflow inputs + set-tests-to-run: + name: Set tests to run + runs-on: ubuntu-latest + outputs: + test_map: ${{ steps.set-tests.outputs.test_map }} + steps: + - name: Set tests to run + id: set-tests + run: | + if [ "${{ inputs.chainlinkImage }}" != "" ]; then + CHAINLINK_IMAGE=${{ inputs.chainlinkImage }} + else + CHAINLINK_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + fi + CHAINLINK_VERSION="${{ inputs.chainlinkVersion || github.sha }}" + CHAINLINK_UPGRADE_IMAGE="${{ inputs.chainlinkImageUpdate }}" + CHAINLINK_UPGRADE_VERSION="${{ inputs.chainlinkVersionUpdate }}" + + cat > test_map.yaml <> test_map.yaml <> $GITHUB_OUTPUT + + call-run-e2e-tests-workflow: + name: Run E2E Tests + needs: set-tests-to-run + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + test_map: ${{ needs.set-tests-to-run.outputs.test_map }} + with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }} + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml index dce0ff19e77..3fa41670d1a 100644 --- a/.github/workflows/run-e2e-tests-reusable-workflow.yml +++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml @@ -6,12 +6,16 @@ on: inputs: chainlink_version: description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha' - required: true - type: string + required: false + type: string test_ids: description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml' required: false type: string + test_map: + description: 'Run tests by test map.' + required: true + type: string test_workflow: description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"' required: false @@ -112,7 +116,7 @@ jobs: id: set-docker-matrix run: | cd integration-tests/e2e_tests_ci_tool/ - MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') + MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') echo "Docker tests:" echo "$MATRIX_JSON" | jq echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT @@ -120,7 +124,7 @@ jobs: id: set-k8s-runner-matrix run: | cd integration-tests/e2e_tests_ci_tool/ - MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') + MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') echo "K8s tests:" echo "$MATRIX_JSON" | jq echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT @@ -189,11 +193,11 @@ jobs: # if: ${{ inputs.test_config_base64_override == '' }} uses: ./.github/actions/create-default-e2e-config-override with: - chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }} - chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }} - chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }} - chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }} - selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}} + chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }} + chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }} + chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }} + chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }} + selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}} pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }} pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }} pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }} @@ -312,11 +316,11 @@ jobs: # if: ${{ inputs.test_config_base64_override == '' }} uses: ./.github/actions/create-default-e2e-config-override with: - chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }} - chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }} - chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }} - chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }} - selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}} + chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }} + chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }} + chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }} + chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }} + selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}} pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }} pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }} pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }} diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go index 56a88702738..d3fd7eb7e7b 100644 --- a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go +++ b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go @@ -1,6 +1,7 @@ package cmd import ( + "encoding/base64" "encoding/json" "fmt" "log" @@ -13,14 +14,14 @@ import ( ) // Filter tests based on workflow, test type, and test IDs. -func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestConf { +func filterTests(allTests []CITestConf, workflow, testType, ids string) []CITestConf { workflowFilter := workflow typeFilter := testType idFilter := strings.Split(ids, ",") var filteredTests []CITestConf - for _, test := range tests { + for _, test := range allTests { workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter) typeMatch := testType == "" || test.TestType == typeFilter idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID) @@ -34,6 +35,40 @@ func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestCon return filteredTests } +func filterAndMergeTests(allTests []CITestConf, base64Tests string) ([]CITestConf, error) { + decodedBytes, err := base64.StdEncoding.DecodeString(base64Tests) + if err != nil { + return nil, err + } + var decodedTests []CITestConf + err = yaml.Unmarshal(decodedBytes, &decodedTests) + if err != nil { + log.Fatalf("Error parsing YAML data: %v", err) + } + + idFilter := make(map[string]CITestConf) + for _, dt := range decodedTests { + idFilter[dt.ID] = dt + } + + var filteredTests []CITestConf + for _, test := range allTests { + if decodedTest, exists := idFilter[test.ID]; exists { + // Apply config overrides + for k, v := range decodedTest.TestConfigOverrides { + if test.TestConfigOverrides == nil { + test.TestConfigOverrides = make(map[string]string) + } + test.TestConfigOverrides[k] = v + } + test.IDSanitized = sanitizeTestID(test.ID) + filteredTests = append(filteredTests, test) + } + } + + return filteredTests, nil +} + func sanitizeTestID(id string) string { // Define a regular expression that matches any character not a letter, digit, hyphen re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`) @@ -63,6 +98,7 @@ Example usage: workflow, _ := cmd.Flags().GetString("workflow") testType, _ := cmd.Flags().GetString("test-type") testIDs, _ := cmd.Flags().GetString("test-ids") + testMap, _ := cmd.Flags().GetString("test-map") data, err := os.ReadFile(yamlFile) if err != nil { @@ -75,7 +111,15 @@ Example usage: log.Fatalf("Error parsing YAML data: %v", err) } - filteredTests := filterTests(config.Tests, workflow, testType, testIDs) + var filteredTests []CITestConf + if testMap == "" { + filteredTests = filterTests(config.Tests, workflow, testType, testIDs) + } else { + filteredTests, err = filterAndMergeTests(config.Tests, testMap) + if err != nil { + log.Fatalf("Error filtering and merging tests: %v", err) + } + } matrix := map[string][]CITestConf{"tests": filteredTests} matrixJSON, err := json.Marshal(matrix) if err != nil { @@ -88,9 +132,10 @@ Example usage: func init() { filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file") - filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter") - filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by") + filterCmd.Flags().String("test-map", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test-config-overrides for each test.") filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by") + filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by") + filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter") err := filterCmd.MarkFlagRequired("file") if err != nil { fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err) diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go index c41012a70ec..a85815835db 100644 --- a/integration-tests/e2e_tests_ci_tool/cmd/types.go +++ b/integration-tests/e2e_tests_ci_tool/cmd/types.go @@ -7,24 +7,17 @@ type Test struct { // CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows. type CITestConf struct { - ID string `yaml:"id" json:"id"` - IDSanitized string `json:"idSanitized"` - Path string `yaml:"path" json:"path"` - TestType string `yaml:"test-type" json:"testType"` - RunsOn string `yaml:"runs-on" json:"runsOn"` - TestCmd string `yaml:"test-cmd" json:"testCmd"` - TestConfigOverride TestConfigOverride `yaml:"test-config-override" json:"testConfigOverride"` - RemoteRunnerTestSuite string `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"` - RemoteRunnerMemory string `yaml:"remote-runner-memory" json:"remoteRunnerMemory"` - PyroscopeEnv string `yaml:"pyroscope-env" json:"pyroscopeEnv"` - Workflows []string `yaml:"workflows" json:"workflows"` -} - -type TestConfigOverride struct { - ChainlinkUpgradeImage string `yaml:"chainlink-upgrade-image" json:"chainlinkUpgradeImage"` - ChainlinkUpgradeVersion string `yaml:"chainlink-upgrade-version" json:"chainlinkUpgradeVersion"` - ChainlinkImage string `yaml:"chainlink-image" json:"chainlinkImage"` - ChainlinkVersion string `yaml:"chainlink-version" json:"chainlinkVersion"` + ID string `yaml:"id" json:"id"` + IDSanitized string `json:"idSanitized"` + Path string `yaml:"path" json:"path"` + TestType string `yaml:"test-type" json:"testType"` + RunsOn string `yaml:"runs-on" json:"runsOn"` + TestCmd string `yaml:"test-cmd" json:"testCmd"` + TestConfigOverrides map[string]string `yaml:"test-config-overrides" json:"testConfigOverrides"` + RemoteRunnerTestSuite string `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"` + RemoteRunnerMemory string `yaml:"remote-runner-memory" json:"remoteRunnerMemory"` + PyroscopeEnv string `yaml:"pyroscope-env" json:"pyroscopeEnv"` + Workflows []string `yaml:"workflows" json:"workflows"` } type Config struct {