Add a test workflow with overrides

smartcontractkit · Jul 1, 2024 · 2d0ae2b · 2d0ae2b
1 parent 7a0466b
commit 2d0ae2b
Show file tree

Hide file tree

Showing 5 changed files with 198 additions and 40 deletions.
diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
@@ -48,7 +48,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json
-    test-config-override:
+    test-config-overrides:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink
@@ -62,7 +62,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests    
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override:
+    test-config-overrides:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink
@@ -76,7 +76,7 @@ runner-test-matrix:
     workflows:
       - Run Automation Product Nightly E2E Tests
     test-cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json
-    test-config-override:
+    test-config-overrides:
       chainlink-upgrade-image: ${{ env.CHAINLINK_IMAGE }}
       chainlink-upgrade-version: develop
       chainlink-image: public.ecr.aws/chainlink/chainlink

diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml
@@ -0,0 +1,116 @@
+name: Run Automation On Demand Tests (TEST WORKFLOW)
+
+on:
+  workflow_dispatch:
+    inputs:
+      chainlinkVersion:
+        description: Chainlink image version to use
+        required: false
+        type: string
+      chainlinkImage:
+        description: Chainlink image repo to use (Leave empty to build from head/ref)
+        required: false
+        type: string
+      chainlinkVersionUpdate:
+        description: Chainlink image version to use initially for upgrade test
+        default: latest
+        required: true
+        type: string
+      chainlinkImageUpdate:
+        description: Chainlink image repo to use initially for upgrade test
+        required: true
+        default: public.ecr.aws/chainlink/chainlink
+        type: string
+      enableChaos:
+        description: Check to enable chaos tests
+        type: boolean
+        default: false
+        required: true
+      enableReorg:
+        description: Check to enable reorg tests
+        type: boolean
+        default: false
+        required: true
+      with_existing_remote_runner_version:
+        description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"'
+        required: false
+        type: string                 
+
+jobs:
+  # Set tests to run based on the workflow inputs
+  set-tests-to-run:
+    name: Set tests to run
+    runs-on: ubuntu-latest
+    outputs:
+      test_map: ${{ steps.set-tests.outputs.test_map }}
+    steps:
+      - name: Set tests to run
+        id: set-tests
+        run: |
+          if [ "${{ inputs.chainlinkImage }}" != "" ]; then
+            CHAINLINK_IMAGE=${{ inputs.chainlinkImage }}
+          else
+            CHAINLINK_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
+          fi
+          CHAINLINK_VERSION="${{ inputs.chainlinkVersion || github.sha }}"
+          CHAINLINK_UPGRADE_IMAGE="${{ inputs.chainlinkImageUpdate }}"
+          CHAINLINK_UPGRADE_VERSION="${{ inputs.chainlinkVersionUpdate }}"
+
+          cat > test_map.yaml <<EOF
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0
+            test-config-overrides:
+              chainlink-image: $CHAINLINK_IMAGE
+              chainlink-version: $CHAINLINK_VERSION
+              chainlink-upgrade-image: $CHAINLINK_UPGRADE_IMAGE
+              chainlink-upgrade-version: $CHAINLINK_UPGRADE_VERSION
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
+            test-config-overrides:
+              chainlink-image: $CHAINLINK_IMAGE
+              chainlink-version: $CHAINLINK_VERSION
+              chainlink-upgrade-image: $CHAINLINK_UPGRADE_IMAGE
+              chainlink-upgrade-version: $CHAINLINK_UPGRADE_VERSION              
+          - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
+            test-config-overrides:
+              chainlink-image: $CHAINLINK_IMAGE
+              chainlink-version: $CHAINLINK_VERSION
+              chainlink-upgrade-image: $CHAINLINK_UPGRADE_IMAGE
+              chainlink-upgrade-version: $CHAINLINK_UPGRADE_VERSION              
+          EOF
+
+          if [[ "${{ github.event.inputs.enableChaos }}" == 'true' ]]; then
+            cat >> test_map.yaml <<EOF
+            - id: ^TestChaos$
+              test-config-overrides:
+                chainlink-image: $CHAINLINK_IMAGE
+                chainlink-version: $CHAINLINK_VERSION
+                chaos-setting: enabled
+          EOF
+          fi
+
+          cat test_map.yaml
+          cat test_map.yaml | base64 -w 0
+          echo "test_map=$(cat test_map.yaml | base64 -w 0)" >> $GITHUB_OUTPUT
+
+  call-run-e2e-tests-workflow:
+    name: Run E2E Tests
+    needs: set-tests-to-run
+    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
+    with:
+      test_map: ${{ needs.set-tests-to-run.outputs.test_map }}
+      with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}        
+
diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml
@@ -6,12 +6,16 @@ on:
     inputs:
       chainlink_version:
         description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha'
-        required: true
-        type: string
+        required: false
+        type: string    
       test_ids:
         description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml'
         required: false
         type: string
+      test_map:
+        description: 'Run tests by test map.'
+        required: true
+        type: string            
       test_workflow:
         description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"'
         required: false
@@ -112,15 +116,15 @@ jobs:
         id: set-docker-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'docker' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "Docker tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
       - name: Generate K8s Tests Matrix
         id: set-k8s-runner-matrix
         run: |
           cd integration-tests/e2e_tests_ci_tool/
-          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
+          MATRIX_JSON=$(go run main.go filter --file ../../.github/e2e-tests.yml --test-type 'k8s-remote-runner' --test-map '${{ inputs.test_map }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}')
           echo "K8s tests:"
           echo "$MATRIX_JSON" | jq
           echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
@@ -189,11 +193,11 @@ jobs:
         # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
-          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
+          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}
@@ -312,11 +316,11 @@ jobs:
         # if: ${{ inputs.test_config_base64_override == '' }}
         uses: ./.github/actions/create-default-e2e-config-override
         with:
-          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverride.chainlinkVersion || github.sha }}
-          chainlink_image: ${{ matrix.tests.testConfigOverride.chainlinkImage || env.CHAINLINK_IMAGE }}
-          chainlink_postgres_version: ${{ matrix.tests.testConfigOverride.chainlinkPostgresVersion }}
-          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverride.chainlinkUpgradeVersion }}
-          selected_networks: ${{ matrix.tests.testConfigOverride.selectedNetworks || env.SELECTED_NETWORKS}}
+          chainlink_version: ${{ inputs.chainlink_version || matrix.tests.testConfigOverrides.chainlinkVersion || github.sha }}
+          chainlink_image: ${{ matrix.tests.testConfigOverrides.chainlinkImage || env.CHAINLINK_IMAGE }}
+          chainlink_postgres_version: ${{ matrix.tests.testConfigOverrides.chainlinkPostgresVersion }}
+          chainlink_upgrade_version: ${{ matrix.tests.testConfigOverrides.chainlinkUpgradeVersion }}
+          selected_networks: ${{ matrix.tests.testConfigOverrides.selectedNetworks || env.SELECTED_NETWORKS}}
           pyroscope_server: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
           pyroscope_environment: ${{ matrix.tests.pyroscopeEnv }}
           pyroscope_key: ${{ secrets.QA_PYROSCOPE_KEY }}

diff --git a/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go b/integration-tests/e2e_tests_ci_tool/cmd/filter_cmd.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -13,14 +14,14 @@ import (
 )
 
 // Filter tests based on workflow, test type, and test IDs.
-func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestConf {
+func filterTests(allTests []CITestConf, workflow, testType, ids string) []CITestConf {
 	workflowFilter := workflow
 	typeFilter := testType
 	idFilter := strings.Split(ids, ",")
 
 	var filteredTests []CITestConf
 
-	for _, test := range tests {
+	for _, test := range allTests {
 		workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter)
 		typeMatch := testType == "" || test.TestType == typeFilter
 		idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID)
@@ -34,6 +35,40 @@ func filterTests(tests []CITestConf, workflow, testType, ids string) []CITestCon
 	return filteredTests
 }
 
+func filterAndMergeTests(allTests []CITestConf, base64Tests string) ([]CITestConf, error) {
+	decodedBytes, err := base64.StdEncoding.DecodeString(base64Tests)
+	if err != nil {
+		return nil, err
+	}
+	var decodedTests []CITestConf
+	err = yaml.Unmarshal(decodedBytes, &decodedTests)
+	if err != nil {
+		log.Fatalf("Error parsing YAML data: %v", err)
+	}
+
+	idFilter := make(map[string]CITestConf)
+	for _, dt := range decodedTests {
+		idFilter[dt.ID] = dt
+	}
+
+	var filteredTests []CITestConf
+	for _, test := range allTests {
+		if decodedTest, exists := idFilter[test.ID]; exists {
+			// Apply config overrides
+			for k, v := range decodedTest.TestConfigOverrides {
+				if test.TestConfigOverrides == nil {
+					test.TestConfigOverrides = make(map[string]string)
+				}
+				test.TestConfigOverrides[k] = v
+			}
+			test.IDSanitized = sanitizeTestID(test.ID)
+			filteredTests = append(filteredTests, test)
+		}
+	}
+
+	return filteredTests, nil
+}
+
 func sanitizeTestID(id string) string {
 	// Define a regular expression that matches any character not a letter, digit, hyphen
 	re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
@@ -63,6 +98,7 @@ Example usage:
 		workflow, _ := cmd.Flags().GetString("workflow")
 		testType, _ := cmd.Flags().GetString("test-type")
 		testIDs, _ := cmd.Flags().GetString("test-ids")
+		testMap, _ := cmd.Flags().GetString("test-map")
 
 		data, err := os.ReadFile(yamlFile)
 		if err != nil {
@@ -75,7 +111,15 @@ Example usage:
 			log.Fatalf("Error parsing YAML data: %v", err)
 		}
 
-		filteredTests := filterTests(config.Tests, workflow, testType, testIDs)
+		var filteredTests []CITestConf
+		if testMap == "" {
+			filteredTests = filterTests(config.Tests, workflow, testType, testIDs)
+		} else {
+			filteredTests, err = filterAndMergeTests(config.Tests, testMap)
+			if err != nil {
+				log.Fatalf("Error filtering and merging tests: %v", err)
+			}
+		}
 		matrix := map[string][]CITestConf{"tests": filteredTests}
 		matrixJSON, err := json.Marshal(matrix)
 		if err != nil {
@@ -88,9 +132,10 @@ Example usage:
 
 func init() {
 	filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file")
-	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
-	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
+	filterCmd.Flags().String("test-map", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test-config-overrides for each test.")
 	filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by")
+	filterCmd.Flags().StringP("test-type", "y", "", "Type of test to filter by")
+	filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter")
 	err := filterCmd.MarkFlagRequired("file")
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err)

diff --git a/integration-tests/e2e_tests_ci_tool/cmd/types.go b/integration-tests/e2e_tests_ci_tool/cmd/types.go
@@ -7,24 +7,17 @@ type Test struct {
 
 // CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows.
 type CITestConf struct {
-	ID                    string             `yaml:"id" json:"id"`
-	IDSanitized           string             `json:"idSanitized"`
-	Path                  string             `yaml:"path" json:"path"`
-	TestType              string             `yaml:"test-type" json:"testType"`
-	RunsOn                string             `yaml:"runs-on" json:"runsOn"`
-	TestCmd               string             `yaml:"test-cmd" json:"testCmd"`
-	TestConfigOverride    TestConfigOverride `yaml:"test-config-override" json:"testConfigOverride"`
-	RemoteRunnerTestSuite string             `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
-	RemoteRunnerMemory    string             `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
-	PyroscopeEnv          string             `yaml:"pyroscope-env" json:"pyroscopeEnv"`
-	Workflows             []string           `yaml:"workflows" json:"workflows"`
-}
-
-type TestConfigOverride struct {
-	ChainlinkUpgradeImage   string `yaml:"chainlink-upgrade-image" json:"chainlinkUpgradeImage"`
-	ChainlinkUpgradeVersion string `yaml:"chainlink-upgrade-version" json:"chainlinkUpgradeVersion"`
-	ChainlinkImage          string `yaml:"chainlink-image" json:"chainlinkImage"`
-	ChainlinkVersion        string `yaml:"chainlink-version" json:"chainlinkVersion"`
+	ID                    string            `yaml:"id" json:"id"`
+	IDSanitized           string            `json:"idSanitized"`
+	Path                  string            `yaml:"path" json:"path"`
+	TestType              string            `yaml:"test-type" json:"testType"`
+	RunsOn                string            `yaml:"runs-on" json:"runsOn"`
+	TestCmd               string            `yaml:"test-cmd" json:"testCmd"`
+	TestConfigOverrides   map[string]string `yaml:"test-config-overrides" json:"testConfigOverrides"`
+	RemoteRunnerTestSuite string            `yaml:"remote-runner-test-suite" json:"remoteRunnerTestSuite"`
+	RemoteRunnerMemory    string            `yaml:"remote-runner-memory" json:"remoteRunnerMemory"`
+	PyroscopeEnv          string            `yaml:"pyroscope-env" json:"pyroscopeEnv"`
+	Workflows             []string          `yaml:"workflows" json:"workflows"`
 }
 
 type Config struct {