Skip to content

Commit

Permalink
Enable upgrade tests for charms and openstack
Browse files Browse the repository at this point in the history
Add charm and openstack upgrade tests for focal-ussuri->focal-yoga
and jammy-yoga->jammy-caracal.

Also some general cleanup to the README and tests.yaml to clean up
model aliases.
  • Loading branch information
Corey Bryant committed Oct 25, 2023
1 parent 8300fda commit 65653a5
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 44 deletions.
8 changes: 4 additions & 4 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Deploy and test a specific bundle:

.. code-block:: bash
tox -e func-target keystone_v3_smoke_focal:jammy-yoga
tox -e func-target jammy-yoga
Deploy and test all smoke bundles:

Expand All @@ -67,9 +67,9 @@ Specifying which tests to run can be done with the following keys: smoke, whitel
tests_options:
tempest:
keystone_v3_smoke:
model_alias_smoke:
smoke: True
keystone_v3_full:
model_alias_targeted:
whitelist:
- "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers"
- "tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details"
Expand All @@ -88,7 +88,7 @@ re-running tests, set keep-workspace to True in tests.yaml:
tests_options:
tempest:
keystone_v3_smoke:
model_alias:
smoke: True
keep-workspace: True
Expand Down
12 changes: 12 additions & 0 deletions osci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
- cot_distro-regression_focal-ussuri-security
- cot_distro-regression_jammy-yoga-security
- cot_distro-regression_lunar-antelope-security
- cot_distro-regression_focal-ussuri-to-yoga-upgrades
- cot_distro-regression_jammy-yoga-to-caracal-upgrades
- job:
name: cot-func-target
parent: func-target
Expand Down Expand Up @@ -117,3 +119,13 @@
parent: cot-func-target
vars:
tox_extra_args: '-- lunar-antelope-security'
- job:
name: cot_distro-regression_focal-ussuri-to-yoga-upgrades
parent: cot-func-target
vars:
tox_extra_args: '-- focal-ussuri-to-yoga-upgrades'
- job:
name: cot_distro-regression_jammy-yoga-to-caracal-upgrades
parent: cot-func-target
vars:
tox_extra_args: '-- jammy-yoga-to-caracal-upgrades'
2 changes: 1 addition & 1 deletion tests/distro-regression/tests/bundles/focal-xena.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ variables:
source: &source cloud:focal-xena/proposed
openstack-origin: &openstack-origin cloud:focal-xena/proposed
retrofit-uca-pocket: &retrofit-uca-pocket xena
openstack-channel: &openstack-channel wallaby/edge
openstack-channel: &openstack-channel xena/edge
ceph-channel: &ceph-channel pacific/edge
ovn-channel: &ovn-channel 21.09/edge
mysql-channel: &mysql-channel 8.0/edge
Expand Down
178 changes: 139 additions & 39 deletions tests/distro-regression/tests/tests.yaml
Original file line number Diff line number Diff line change
@@ -1,25 +1,27 @@
smoke_bundles:
- keystone_bionic_queens: bionic-queens
- keystone_bionic_ussuri: bionic-ussuri
- keystone_focal_ussuri: focal-ussuri
- keystone_focal_ussuri: focal-ussuri-ovn-22.03
- keystone_focal_ussuri: focal-victoria
- keystone_focal_wallaby: focal-wallaby
- keystone_focal_wallaby: focal-xena
- keystone_focal_wallaby: focal-yoga
- keystone_focal_wallaby: jammy-yoga
- keystone_focal_wallaby: jammy-zed
- keystone_focal_wallaby: jammy-antelope
- keystone_focal_wallaby: jammy-bobcat
- keystone_focal_wallaby: lunar-antelope
- keystone_focal_wallaby: mantic-bobcat
- keystone_bionic_queens_security: bionic-queens-security
- keystone_focal_ussuri_security: focal-ussuri-security
- keystone_focal_ussuri_security: jammy-yoga-security
- keystone_focal_ussuri_security: lunar-antelope-security
- keystone_focal_ussuri_security: mantic-bobcat-security
- bionic_queens: bionic-queens
- bionic_ussuri: bionic-ussuri
- focal_ussuri: focal-ussuri
- focal_ussuri: focal-ussuri-ovn-22.03
- focal_ussuri: focal-victoria
- focal_wallaby: focal-wallaby
- focal_wallaby: focal-xena
- focal_wallaby: focal-yoga
- focal_wallaby: jammy-yoga
- focal_wallaby: jammy-zed
- focal_wallaby: jammy-antelope
- focal_wallaby: jammy-bobcat
- focal_wallaby: lunar-antelope
- focal_wallaby: mantic-bobcat
- bionic_queens_security: bionic-queens-security
- focal_ussuri_security: focal-ussuri-security
- focal_ussuri_security: jammy-yoga-security
- focal_ussuri_security: lunar-antelope-security
- focal_ussuri_security: mantic-bobcat-security
- focal_upgrades: focal-ussuri-to-yoga-upgrades
- jammy_upgrades: jammy-yoga-to-caracal-upgrades
configure:
- keystone_bionic_queens: &keystone_bionic_queens
- bionic_queens: &bionic_queens
- zaza.openstack.charm_tests.ceilometer.setup.basic_setup
- zaza.openstack.charm_tests.glance.setup.add_lts_image
- zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
Expand All @@ -29,8 +31,8 @@ configure:
- zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
- zaza.openstack.charm_tests.glance.setup.add_cirros_image
- zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
- keystone_bionic_queens_security: *keystone_bionic_queens
- keystone_bionic_ussuri:
- bionic_queens_security: *bionic_queens
- bionic_ussuri:
- zaza.openstack.charm_tests.ceilometer.setup.basic_setup
- zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images
- zaza.openstack.charm_tests.glance.setup.add_lts_image
Expand All @@ -45,7 +47,7 @@ configure:
- zaza.openstack.charm_tests.glance.setup.add_cirros_image
- zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
- zaza.openstack.charm_tests.octavia.setup.centralized_fip_network
- keystone_focal_ussuri: &keystone_focal_ussuri
- focal_ussuri: &focal_ussuri
- zaza.openstack.charm_tests.vault.setup.auto_initialize
- zaza.openstack.charm_tests.ceilometer.setup.basic_setup
- zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images
Expand All @@ -63,30 +65,61 @@ configure:
- zaza.openstack.charm_tests.octavia.setup.centralized_fip_network
- zaza.openstack.charm_tests.magnum.setup.domain_setup
- zaza.openstack.charm_tests.magnum.setup.add_image
- keystone_focal_ussuri_security: *keystone_focal_ussuri
- keystone_focal_wallaby: *keystone_focal_ussuri
- focal_ussuri_security: *focal_ussuri
- focal_upgrades: *focal_ussuri
- focal_wallaby: *focal_ussuri
- jammy_upgrades: *focal_ussuri
tests:
- keystone_bionic_queens:
- bionic_queens:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- keystone_bionic_queens_security:
- bionic_queens_security:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- keystone_bionic_ussuri:
- bionic_ussuri:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- keystone_focal_ussuri:
- focal_ussuri:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- keystone_focal_ussuri_security:
- focal_ussuri_security:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- keystone_focal_wallaby:
- focal_wallaby:
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- focal_upgrades:
# ussuri->victoria
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# victoria->wallaby
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# wallaby->xena
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# xena->yoga
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
- jammy_upgrades:
# yoga->zed
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# zed->antelope
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# antelope->bobcat
- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
# bobcat->caracal
# Note(coreycb): Consider skip level upgrade from antelope->caracal.
#- zaza.openstack.charm_tests.charm_upgrade.tests.FullCloudCharmUpgradeTest
#- zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
- zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
tests_options:
keystone_bionic_queens_security:
bionic_queens_security:
overlay_ppas:
- ppa:ubuntu-security-proposed/ppa
keystone_focal_ussuri_security:
focal_ussuri_security:
overlay_ppas:
- ppa:ubuntu-security-proposed/ppa
tempest:
keystone_bionic_queens:
bionic_queens:
smoke: True
serial: True
exclude-list:
Expand All @@ -96,7 +129,7 @@ tests_options:
- "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest"
# octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found'
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
keystone_bionic_queens_security:
bionic_queens_security:
smoke: True
serial: True
exclude-list:
Expand All @@ -106,10 +139,10 @@ tests_options:
- "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest"
# octavia test fails with self.creds_client.assign_user_role 'No "load-balancer_admin" role found'
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
keystone_bionic_ussuri:
bionic_ussuri:
smoke: True
serial: True
keystone_focal_ussuri:
focal_ussuri:
smoke: True
serial: True
include-list:
Expand Down Expand Up @@ -144,7 +177,7 @@ tests_options:
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
- "watcher_tempest_plugin.*"
keystone_focal_ussuri_security:
focal_ussuri_security:
smoke: True
serial: True
include-list:
Expand Down Expand Up @@ -179,7 +212,74 @@ tests_options:
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
- "watcher_tempest_plugin.*"
keystone_focal_wallaby:
focal_upgrades:
smoke: True
serial: True
include-list:
- "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*"
- "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*"
- "magnum_tempest_plugin.tests.api.v1.test_cluster"
- "magnum_tempest_plugin.tests.api.v1.test_cluster_template"
- "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin"
- "magnum_tempest_plugin.tests.api.v1.test_magnum_service"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
# - "watcher_tempest_plugin.tests.api"
# - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance"
# - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation"
exclude-list:
# designate failures due to check_list_show_RBAC_enforcement returning "Unauthorized"
- "designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_show_zone_export"
- "designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_show_zone_import"
# Exclude the known failures due to issues with octavia/manila policy
- "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest"
- "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest"
# Implemented on container-infra 1.10 which is available in >=Xena
# https://opendev.org/openstack/magnum/commit/0e6d17893
# https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters"
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes"
# The test expects a 400 error while the server returns a 401 error due to glance
# See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor"
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
- "watcher_tempest_plugin.*"
focal_wallaby:
smoke: True
serial: True
include-list:
- "manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.*"
- "manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.*"
- "manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.*"
- "magnum_tempest_plugin.tests.api.v1.test_cluster"
- "magnum_tempest_plugin.tests.api.v1.test_cluster_template"
- "magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin"
- "magnum_tempest_plugin.tests.api.v1.test_magnum_service"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
# - "watcher_tempest_plugin.tests.api"
# - "watcher_tempest_plugin.tests.scenario.test_execute_host_maintenance"
# - "watcher_tempest_plugin.tests.scenario.test_execute_vm_workload_consolidation"
exclude-list:
# Exclude the known failures due to issues with octavia/manila policy
- "manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest"
- "manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest"
# Implemented on container-infra 1.10 which is available in >=Xena
# https://opendev.org/openstack/magnum/commit/0e6d17893
# https://opendev.org/openstack/magnum-tempest-plugin/commit/b68a678f37de0a769e7ee8dbefa9bdfe6cf445cc
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters"
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes"
# The test expects a 400 error while the server returns a 401 error due to glance
# See logs at https://pastebin.ubuntu.com/p/V3DMcVmtyF/
- "magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor"
- "octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest"
# Note(coreycb): Disable watcher tests until all the failures can be debugged.
- "watcher_tempest_plugin.*"
jammy_upgrades:
smoke: True
serial: True
include-list:
Expand Down

0 comments on commit 65653a5

Please sign in to comment.