fix: ensure obsolete tests are not run on 4.0

Add support for masking out tests from 3.x that are not applicable to 4.0.
juju · Nov 25, 2024 · d02fb25 · d02fb25
1 parent 4e40ea9
commit d02fb25
Show file tree

Hide file tree

Showing 24 changed files with 1,199 additions and 118 deletions.
diff --git a/go.mod b/go.mod
@@ -1,8 +1,18 @@
 module github.com/juju/juju-qa-jenkins
 
-go 1.20
+go 1.23
+
+toolchain go1.23.3
 
 require (
+	github.com/schollz/progressbar/v3 v3.17.1
 	gopkg.in/yaml.v2 v2.3.0
 	mvdan.cc/sh/v3 v3.6.0
 )
+
+require (
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/term v0.26.0 // indirect
+)
diff --git a/go.sum b/go.sum
@@ -1,11 +1,38 @@
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U=
+github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 mvdan.cc/sh/v3 v3.6.0 h1:gtva4EXJ0dFNvl5bHjcUEvws+KRcDslT8VKheTYkbGU=
 mvdan.cc/sh/v3 v3.6.0/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA=
diff --git a/jobs/ci-run/integration/gen/test-actions.yml b/jobs/ci-run/integration/gen/test-actions.yml
@@ -107,6 +107,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -186,6 +189,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -265,6 +271,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -340,6 +349,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"

diff --git a/jobs/ci-run/integration/gen/test-authorized_keys.yml b/jobs/ci-run/integration/gen/test-authorized_keys.yml
@@ -131,6 +131,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -210,6 +213,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -289,6 +295,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -364,6 +373,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -443,6 +455,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -522,6 +537,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -601,6 +619,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -676,6 +697,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -755,6 +779,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -834,6 +861,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -913,6 +943,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -988,6 +1021,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -1067,6 +1103,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -1146,6 +1185,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -1225,6 +1267,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"
@@ -1300,6 +1345,9 @@
       - wait-for-cloud-init
       - prepare-integration-test
       - conditional-step:
+          # Only run on regexp version match.
+          # Accounts for tests which do not exist
+          # until a given Juju version.
           condition-kind: regex-match
           regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
           label: "${{JUJU_VERSION}}"

diff --git a/jobs/ci-run/integration/gen/test-backup.yml b/jobs/ci-run/integration/gen/test-backup.yml
@@ -102,11 +102,22 @@
       - select-oci-registry
       - wait-for-cloud-init
       - prepare-integration-test
-      - run-integration-test:
-            test_name: 'backup'
-            setup_steps: ''
-            task_name: ''
-            skip_tasks: ''
+      - conditional-step:
+          # Do not run on regexp version match.
+          # Accounts for tests which do not exist
+          # in later Juju versions.
+          condition-kind: not
+          condition-operand:
+            condition-kind: regex-match
+            regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
+            label: "${{JUJU_VERSION}}"
+            on-evaluation-failure: "dont-run"
+          steps:
+            - run-integration-test:
+                test_name: 'backup'
+                setup_steps: ''
+                task_name: ''
+                skip_tasks: ''
     publishers:
       - integration-artifacts
 
@@ -171,10 +182,21 @@
       - select-oci-registry
       - wait-for-cloud-init
       - prepare-integration-test
-      - run-integration-test:
-            test_name: 'backup'
-            setup_steps: ''
-            task_name: ''
-            skip_tasks: ''
+      - conditional-step:
+          # Do not run on regexp version match.
+          # Accounts for tests which do not exist
+          # in later Juju versions.
+          condition-kind: not
+          condition-operand:
+            condition-kind: regex-match
+            regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*"
+            label: "${{JUJU_VERSION}}"
+            on-evaluation-failure: "dont-run"
+          steps:
+            - run-integration-test:
+                test_name: 'backup'
+                setup_steps: ''
+                task_name: ''
+                skip_tasks: ''
     publishers:
       - integration-artifacts