Merge pull request #123 from coreycb/add-upgrade-tests

Enable upgrade tests for charms and openstack

README.rst

@@ -40,7 +40,7 @@ Deploy and test a specific bundle:
 
 .. code-block:: bash
 
-  tox -e func-target keystone_v3_smoke_focal:jammy-yoga
+  tox -e func-target jammy-yoga
 
 Deploy and test all smoke bundles:
 
@@ -67,9 +67,9 @@ Specifying which tests to run can be done with the following keys: smoke, whitel
 
   tests_options:
     tempest:
-      keystone_v3_smoke:
+      model_alias_smoke:
         smoke: True
-      keystone_v3_full:
+      model_alias_targeted:
         whitelist:
            - "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers"
            - "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details"
@@ -88,7 +88,7 @@ re-running tests, set keep-workspace to True in tests.yaml:
 
   tests_options:
     tempest:
-      keystone_v3_smoke:
+      model_alias:
         smoke: True
         keep-workspace: True
 

osci.yaml

@@ -22,6 +22,8 @@
       - cot_distro-regression_focal-ussuri-security
       - cot_distro-regression_jammy-yoga-security
       - cot_distro-regression_lunar-antelope-security
+      - cot_distro-regression_focal-ussuri-to-yoga-upgrades
+      - cot_distro-regression_jammy-yoga-to-caracal-upgrades
 - job:
     name: cot-func-target
     parent: func-target
@@ -117,3 +119,13 @@
     parent: cot-func-target
     vars:
       tox_extra_args: '-- lunar-antelope-security'
+- job:
+    name: cot_distro-regression_focal-ussuri-to-yoga-upgrades
+    parent: cot-func-target
+    vars:
+      tox_extra_args: '-- focal-ussuri-to-yoga-upgrades'
+- job:
+    name: cot_distro-regression_jammy-yoga-to-caracal-upgrades
+    parent: cot-func-target
+    vars:
+      tox_extra_args: '-- jammy-yoga-to-caracal-upgrades'

tests/distro-regression/tests/bundles/focal-ussuri-to-yoga-upgrades.yaml

@@ -0,0 +1 @@
+focal-ussuri.yaml

tests/distro-regression/tests/bundles/focal-xena.yaml

@@ -2,7 +2,7 @@ variables:
   source: &source cloud:focal-xena/proposed
   openstack-origin: &openstack-origin cloud:focal-xena/proposed
   retrofit-uca-pocket: &retrofit-uca-pocket xena
-  openstack-channel: &openstack-channel wallaby/edge
+  openstack-channel: &openstack-channel xena/edge
   ceph-channel: &ceph-channel pacific/edge
   ovn-channel: &ovn-channel 21.09/edge
   mysql-channel: &mysql-channel 8.0/edge

tests/distro-regression/tests/bundles/jammy-yoga-to-caracal-upgrades.yaml

@@ -0,0 +1 @@
+jammy-yoga.yaml

tests/distro-regression/tests/tests.yaml

@@ -1,25 +1,27 @@
 smoke_bundles:
-  - keystone_bionic_queens: bionic-queens
-  - keystone_bionic_ussuri: bionic-ussuri
-  - keystone_focal_ussuri: focal-ussuri
-  - keystone_focal_ussuri: focal-ussuri-ovn-22.03
-  - keystone_focal_ussuri: focal-victoria
-  - keystone_focal_wallaby: focal-wallaby
-  - keystone_focal_wallaby: focal-xena
-  - keystone_focal_wallaby: focal-yoga
-  - keystone_focal_wallaby: jammy-yoga
-  - keystone_focal_wallaby: jammy-zed
-  - keystone_focal_wallaby: jammy-antelope
-  - keystone_focal_wallaby: jammy-bobcat
-  - keystone_focal_wallaby: lunar-antelope
-  - keystone_focal_wallaby: mantic-bobcat
-  - keystone_bionic_queens_security: bionic-queens-security
-  - keystone_focal_ussuri_security: focal-ussuri-security
-  - keystone_focal_ussuri_security: jammy-yoga-security
-  - keystone_focal_ussuri_security: lunar-antelope-security
-  - keystone_focal_ussuri_security: mantic-bobcat-security
+  - bionic_queens: bionic-queens
+  - bionic_ussuri: bionic-ussuri
+  - focal_ussuri: focal-ussuri
+  - focal_ussuri: focal-ussuri-ovn-22.03
+  - focal_ussuri: focal-victoria
+  - focal_wallaby: focal-wallaby
+  - focal_wallaby: focal-xena
+  - focal_wallaby: focal-yoga
+  - focal_wallaby: jammy-yoga
+  - focal_wallaby: jammy-zed
+  - focal_wallaby: jammy-antelope
+  - focal_wallaby: jammy-bobcat
+  - focal_wallaby: lunar-antelope
+  - focal_wallaby: mantic-bobcat
+  - bionic_queens_security: bionic-queens-security
+  - focal_ussuri_security: focal-ussuri-security
+  - focal_ussuri_security: jammy-yoga-security
+  - focal_ussuri_security: lunar-antelope-security
+  - focal_ussuri_security: mantic-bobcat-security
+  - focal_upgrades: focal-ussuri-to-yoga-upgrades
+  - jammy_upgrades: jammy-yoga-to-caracal-upgrades
 configure:
-  - keystone_bionic_queens: &keystone_bionic_queens
+  - bionic_queens: &bionic_queens
     - zaza.openstack.charm_tests.ceilometer.setup.basic_setup
     - zaza.openstack.charm_tests.glance.setup.add_lts_image
     - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
@@ -29,8 +31,8 @@ configure:
     - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
     - zaza.openstack.charm_tests.glance.setup.add_cirros_image
     - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
-  - keystone_bionic_queens_security: *keystone_bionic_queens
-  - keystone_bionic_ussuri:
+  - bionic_queens_security: *bionic_queens
+  - bionic_ussuri:
     - zaza.openstack.charm_tests.ceilometer.setup.basic_setup
     - zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images
     - zaza.openstack.charm_tests.glance.setup.add_lts_image
@@ -45,7 +47,7 @@ configure:
     - zaza.openstack.charm_tests.glance.setup.add_cirros_image
     - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
     - zaza.openstack.charm_tests.octavia.setup.centralized_fip_network
-  - keystone_focal_ussuri: &keystone_focal_ussuri
+  - focal_ussuri: &focal_ussuri
     - zaza.openstack.charm_tests.vault.setup.auto_initialize
     - zaza.openstack.charm_tests.ceilometer.setup.basic_setup
     - zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images
@@ -63,30 +65,61 @@ configure:
     - zaza.openstack.charm_tests.octavia.setup.centralized_fip_network
     - zaza.openstack.charm_tests.magnum.setup.domain_setup
     - zaza.openstack.charm_tests.magnum.setup.add_image
-  - keystone_focal_ussuri_security: *keystone_focal_ussuri
-  - keystone_focal_wallaby: *keystone_focal_ussuri
+  - focal_ussuri_security: *focal_ussuri
+  - focal_upgrades: *focal_ussuri
+  - focal_wallaby: *focal_ussuri
+  - jammy_upgrades: *focal_ussuri
 tests:
-  - keystone_bionic_queens:
+  - bionic_queens:
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
-  - keystone_bionic_queens_security:
+  - bionic_queens_security:
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
-  - keystone_bionic_ussuri:
+  - bionic_ussuri:
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
-  - keystone_focal_ussuri:
+  - focal_ussuri:
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
-  - keystone_focal_ussuri_security:
+  - focal_ussuri_security:
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
-  - keystone_focal_wallaby:
+  - focal_wallaby:
+    - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+  - focal_upgrades:
+    # ussuri->victoria
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # victoria->wallaby
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # wallaby->xena
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # xena->yoga
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+  - jammy_upgrades:
+    # yoga->zed
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # zed->antelope
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # antelope->bobcat
+    - zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # bobcat->caracal
+    # Note(coreycb): Consider skip level upgrade from antelope->caracal.
+    #- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
+    #- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
     - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
 tests_options:
-  keystone_bionic_queens_security:
+  bionic_queens_security:
     overlay_ppas:
       - ppa:ubuntu-security-proposed/ppa
-  keystone_focal_ussuri_security:
+  focal_ussuri_security:
     overlay_ppas:
       - ppa:ubuntu-security-proposed/ppa
   tempest:
-    keystone_bionic_queens:
+    bionic_queens:
       smoke: True
       serial: True
       exclude-list:
@@ -96,7 +129,7 @@ tests_options:
         - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest"
         # octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found'
         - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
-    keystone_bionic_queens_security:
+    bionic_queens_security:
       smoke: True
       serial: True
       exclude-list:
@@ -106,10 +139,10 @@ tests_options:
         - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest"
         # octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found'
         - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
-    keystone_bionic_ussuri:
+    bionic_ussuri:
       smoke: True
       serial: True
-    keystone_focal_ussuri:
+    focal_ussuri:
       smoke: True
       serial: True
       include-list:
@@ -144,7 +177,7 @@ tests_options:
         - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
         # Note(coreycb): Disable watcher tests until all the failures can be debugged.
         - "watcher_tempest_plugin.*"
-    keystone_focal_ussuri_security:
+    focal_ussuri_security:
       smoke: True
       serial: True
       include-list:
@@ -179,7 +212,74 @@ tests_options:
         - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
         # Note(coreycb): Disable watcher tests until all the failures can be debugged.
         - "watcher_tempest_plugin.*"
-    keystone_focal_wallaby:
+    focal_upgrades:
+      smoke: True
+      serial: True
+      include-list:
+        - "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster_template"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin"
+        - "magnum_tempest_plugin.tests.api.v1.test_magnum_service"
+        # Note(coreycb): Disable watcher tests until all the failures can be debugged.
+        # - "watcher_tempest_plugin.tests.api"
+        # - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance"
+        # - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation"
+      exclude-list:
+        # designate failures due to check_list_show_RBAC_enforcement returning "Unauthorized"
+        - "designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_show_zone_export"
+        - "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_show_zone_import"
+        # Exclude the known failures due to issues with octavia/manila policy
+        - "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest"
+        - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest"
+        # Implemented on container-infra 1.10 which is available in >=Xena
+        # https://opendev.org/openstack/magnum/commit/0e6d17893
+        # https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes"
+        # The test expects a 400 error while the server returns a 401 error due to glance
+        # See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor"
+        - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
+        # Note(coreycb): Disable watcher tests until all the failures can be debugged.
+        - "watcher_tempest_plugin.*"
+    focal_wallaby:
+      smoke: True
+      serial: True
+      include-list:
+        - "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*"
+        - "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster_template"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin"
+        - "magnum_tempest_plugin.tests.api.v1.test_magnum_service"
+        # Note(coreycb): Disable watcher tests until all the failures can be debugged.
+        # - "watcher_tempest_plugin.tests.api"
+        # - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance"
+        # - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation"
+      exclude-list:
+        # Exclude the known failures due to issues with octavia/manila policy
+        - "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest"
+        - "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest"
+        # Implemented on container-infra 1.10 which is available in >=Xena
+        # https://opendev.org/openstack/magnum/commit/0e6d17893
+        # https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters"
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes"
+        # The test expects a 400 error while the server returns a 401 error due to glance
+        # See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/
+        - "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor"
+        - "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
+        # Note(coreycb): Disable watcher tests until all the failures can be debugged.
+        - "watcher_tempest_plugin.*"
+    jammy_upgrades:
       smoke: True
       serial: True
       include-list: