From bb43b72c058b6cb85151419101d0a6fcdff58e9b Mon Sep 17 00:00:00 2001 From: Viicos <65306057+Viicos@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:02:28 +0200 Subject: [PATCH 1/3] [#4267] Fix migration crash --- .../forms/migrations/0101_objecttype_url_to_uuid.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py b/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py index 5665252733..50284ff537 100644 --- a/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py +++ b/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py @@ -16,9 +16,14 @@ def objecttype_url_to_uuid( for registration_backend in FormRegistrationBackend.objects.filter( backend="objects_api" ): - objecttype_url = registration_backend.options["objecttype"] - registration_backend.options["objecttype"] = objecttype_url.rsplit("/", 1)[1] - registration_backend.save() + + objecttype_url = registration_backend.options.get("objecttype") + if objecttype_url is not None: + # If it is `None`, we are dealing with broken confs. and upgrade checks where bypassed. + registration_backend.options["objecttype"] = objecttype_url.rsplit("/", 1)[ + 1 + ] + registration_backend.save() class Migration(migrations.Migration): From 3e3919260d6b8dcc08138f4e2857b55d8c76561f Mon Sep 17 00:00:00 2001 From: Viicos <65306057+Viicos@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:26:06 +0200 Subject: [PATCH 2/3] [#4267] Move migrations to the correct app --- .../0100_add_default_objects_api_group.py | 33 +-------- .../migrations/0101_objecttype_url_to_uuid.py | 28 +------- src/openforms/forms/tests/test_migrations.py | 69 ------------------- .../0019_add_default_objects_api_group.py | 51 ++++++++++++++ .../migrations/0020_objecttype_url_to_uuid.py | 41 +++++++++++ .../objects_api/tests/test_migrations.py | 69 +++++++++++++++++++ 6 files changed, 163 insertions(+), 128 deletions(-) create mode 100644 src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py create mode 100644 src/openforms/registrations/contrib/objects_api/migrations/0020_objecttype_url_to_uuid.py diff --git a/src/openforms/forms/migrations/0100_add_default_objects_api_group.py b/src/openforms/forms/migrations/0100_add_default_objects_api_group.py index d6edab68a1..d1829e14d7 100644 --- a/src/openforms/forms/migrations/0100_add_default_objects_api_group.py +++ b/src/openforms/forms/migrations/0100_add_default_objects_api_group.py @@ -2,44 +2,13 @@ from django.db import migrations -from django.db.migrations.state import StateApps -from django.db.backends.base.schema import BaseDatabaseSchemaEditor - - -def add_default_objects_api_group( - apps: StateApps, schema_editor: BaseDatabaseSchemaEditor -) -> None: - FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") - ObjectsAPIGroupConfig = apps.get_model( - "registrations_objects_api", "ObjectsAPIGroupConfig" - ) - - objects_api_group_config = ObjectsAPIGroupConfig.objects.order_by("pk").first() - if objects_api_group_config is None: - # Because this migration runs after registrations_objects_api/0017_move_singleton_data, - # Having no Objects API Group means we had no solo config in the first place, thus - # it is safe to assume no Objects API registration backend was set up. - return - - for registration_backend in FormRegistrationBackend.objects.filter( - backend="objects_api" - ): - registration_backend.options.setdefault( - "objects_api_group", objects_api_group_config.pk - ) - registration_backend.save() - class Migration(migrations.Migration): dependencies = [ ("forms", "0099_auto_20240613_0654"), - ("registrations_objects_api", "0017_move_singleton_data"), ] operations = [ - migrations.RunPython( - add_default_objects_api_group, - migrations.RunPython.noop, - ), + # Moved to `registrations_objects_api.0019_add_default_objects_api_group`. ] diff --git a/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py b/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py index 50284ff537..59e3f61b21 100644 --- a/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py +++ b/src/openforms/forms/migrations/0101_objecttype_url_to_uuid.py @@ -2,29 +2,6 @@ from django.db import migrations -from django.db.migrations.state import StateApps -from django.db.backends.base.schema import BaseDatabaseSchemaEditor - - -def objecttype_url_to_uuid( - apps: StateApps, schema_editor: BaseDatabaseSchemaEditor -) -> None: - """Change the objects API registration options to reference the objecttype UUID instead of the URL.""" - - FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") - - for registration_backend in FormRegistrationBackend.objects.filter( - backend="objects_api" - ): - - objecttype_url = registration_backend.options.get("objecttype") - if objecttype_url is not None: - # If it is `None`, we are dealing with broken confs. and upgrade checks where bypassed. - registration_backend.options["objecttype"] = objecttype_url.rsplit("/", 1)[ - 1 - ] - registration_backend.save() - class Migration(migrations.Migration): @@ -33,8 +10,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython( - objecttype_url_to_uuid, - migrations.RunPython.noop, - ), + # Moved to `registrations_objects_api.0020_objecttype_url_to_uuid`. ] diff --git a/src/openforms/forms/tests/test_migrations.py b/src/openforms/forms/tests/test_migrations.py index 8c938d88a7..9a6281926e 100644 --- a/src/openforms/forms/tests/test_migrations.py +++ b/src/openforms/forms/tests/test_migrations.py @@ -451,75 +451,6 @@ def test_conditionals_are_fixed(self): self.assertTrue(fixed_components[6]["components"][0]["conditional"]["eq"]) -class AddDefaultObjectsAPIGroupMigrationTests(TestMigrations): - app = "forms" - migrate_from = "0099_auto_20240613_0654" - migrate_to = "0100_add_default_objects_api_group" - - def setUpBeforeMigration(self, apps: StateApps) -> None: - Form = apps.get_model("forms", "Form") - FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") - ObjectsAPIGroupConfig = apps.get_model( - "registrations_objects_api", "ObjectsAPIGroupConfig" - ) - - form = Form.objects.create(name="test form") - ObjectsAPIGroupConfig.objects.create(name="Objects API Group") - - FormRegistrationBackend.objects.create( - form=form, - name="Objects API backend", - key="backend", - backend="objects_api", - ) - - def test_sets_default_objects_api_group(self) -> None: - FormRegistrationBackend = self.apps.get_model( - "forms", "FormRegistrationBackend" - ) - ObjectsAPIGroupConfig = self.apps.get_model( - "registrations_objects_api", "ObjectsAPIGroupConfig" - ) - - backend = FormRegistrationBackend.objects.get() - self.assertEqual( - backend.options["objects_api_group"], ObjectsAPIGroupConfig.objects.get().pk - ) - - -class ObjecttypeUrltoUuidMigrationTests(TestMigrations): - app = "forms" - migrate_from = "0100_add_default_objects_api_group" - migrate_to = "0101_objecttype_url_to_uuid" - - def setUpBeforeMigration(self, apps: StateApps) -> None: - Form = apps.get_model("forms", "Form") - FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") - - form = Form.objects.create(name="test form") - - FormRegistrationBackend.objects.create( - form=form, - name="Objects API backend", - key="backend", - backend="objects_api", - options={ - "objecttype": "http://objecttypen.nl/api/v1/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", - }, - ) - - def test_changes_objecttype_key_name(self) -> None: - FormRegistrationBackend = self.apps.get_model( - "forms", "FormRegistrationBackend" - ) - - backend = FormRegistrationBackend.objects.get() - self.assertEqual( - backend.options["objecttype"], - "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", - ) - - class PrefillIdentifierRoleRename(TestMigrations): app = "forms" migrate_from = "0102_alter_formvariable_prefill_identifier_role" diff --git a/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py b/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py new file mode 100644 index 0000000000..86b1e41954 --- /dev/null +++ b/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.11 on 2024-07-02 15:18 + +from django.db import migrations + +from django.db.migrations.state import StateApps +from django.db.backends.base.schema import BaseDatabaseSchemaEditor + + +def add_default_objects_api_group( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") + ObjectsAPIGroupConfig = apps.get_model( + "registrations_objects_api", "ObjectsAPIGroupConfig" + ) + + objects_api_group_config = ObjectsAPIGroupConfig.objects.order_by("pk").first() + if objects_api_group_config is None: + # Because this migration runs after registrations_objects_api/0017_move_singleton_data, + # Having no Objects API Group means we had no solo config in the first place, thus + # it is safe to assume no Objects API registration backend was set up. + return + + for registration_backend in FormRegistrationBackend.objects.filter( + backend="objects_api" + ): + registration_backend.options.setdefault( + "objects_api_group", objects_api_group_config.pk + ) + registration_backend.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "registrations_objects_api", + "0018_remove_objectsapiconfig_catalogi_service_and_more", + ), + ( + "forms", + "0100_add_default_objects_api_group", + ), + ] + + operations = [ + migrations.RunPython( + add_default_objects_api_group, + migrations.RunPython.noop, + ), + ] diff --git a/src/openforms/registrations/contrib/objects_api/migrations/0020_objecttype_url_to_uuid.py b/src/openforms/registrations/contrib/objects_api/migrations/0020_objecttype_url_to_uuid.py new file mode 100644 index 0000000000..8b27cb376e --- /dev/null +++ b/src/openforms/registrations/contrib/objects_api/migrations/0020_objecttype_url_to_uuid.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2.11 on 2024-07-02 15:21 + +from django.db import migrations + +from django.db.migrations.state import StateApps +from django.db.backends.base.schema import BaseDatabaseSchemaEditor + + +def objecttype_url_to_uuid( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """Change the objects API registration options to reference the objecttype UUID instead of the URL.""" + + FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") + + for registration_backend in FormRegistrationBackend.objects.filter( + backend="objects_api" + ): + + objecttype_url = registration_backend.options.get("objecttype") + if objecttype_url is not None and "/" in objecttype_url: + # If it is `None`, we are dealing with broken confs. and upgrade checks where bypassed. + registration_backend.options["objecttype"] = objecttype_url.rsplit("/", 1)[ + 1 + ] + registration_backend.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrations_objects_api", "0019_add_default_objects_api_group"), + ("forms", "0100_add_default_objects_api_group"), + ] + + operations = [ + migrations.RunPython( + objecttype_url_to_uuid, + migrations.RunPython.noop, + ), + ] diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py b/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py index ec089abd4d..a87604ea24 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py @@ -275,3 +275,72 @@ def test_no_zgw_api_group_created(self): ) self.assertFalse(ObjectsAPIGroupConfig.objects.exists()) + + +class AddDefaultObjectsAPIGroupMigrationTests(TestMigrations): + app = "registrations_objects_api" + migrate_from = "0018_remove_objectsapiconfig_catalogi_service_and_more" + migrate_to = "0019_add_default_objects_api_group" + + def setUpBeforeMigration(self, apps: StateApps) -> None: + Form = apps.get_model("forms", "Form") + FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") + ObjectsAPIGroupConfig = apps.get_model( + "registrations_objects_api", "ObjectsAPIGroupConfig" + ) + + form = Form.objects.create(name="test form") + ObjectsAPIGroupConfig.objects.create(name="Objects API Group") + + FormRegistrationBackend.objects.create( + form=form, + name="Objects API backend", + key="backend", + backend="objects_api", + ) + + def test_sets_default_objects_api_group(self) -> None: + FormRegistrationBackend = self.apps.get_model( + "forms", "FormRegistrationBackend" + ) + ObjectsAPIGroupConfig = self.apps.get_model( + "registrations_objects_api", "ObjectsAPIGroupConfig" + ) + + backend = FormRegistrationBackend.objects.get() + self.assertEqual( + backend.options["objects_api_group"], ObjectsAPIGroupConfig.objects.get().pk + ) + + +class ObjecttypeUrltoUuidMigrationTests(TestMigrations): + app = "registrations_objects_api" + migrate_from = "0019_add_default_objects_api_group" + migrate_to = "0020_objecttype_url_to_uuid" + + def setUpBeforeMigration(self, apps: StateApps) -> None: + Form = apps.get_model("forms", "Form") + FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") + + form = Form.objects.create(name="test form") + + FormRegistrationBackend.objects.create( + form=form, + name="Objects API backend", + key="backend", + backend="objects_api", + options={ + "objecttype": "http://objecttypen.nl/api/v1/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + }, + ) + + def test_changes_objecttype_key_name(self) -> None: + FormRegistrationBackend = self.apps.get_model( + "forms", "FormRegistrationBackend" + ) + + backend = FormRegistrationBackend.objects.get() + self.assertEqual( + backend.options["objecttype"], + "8e46e0a5-b1b4-449b-b9e9-fa3cea655f48", + ) From a7c4e628029d9d5762de9e529163887c3cc2af74 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Tue, 2 Jul 2024 17:48:54 +0200 Subject: [PATCH 3/3] :card_file_box: [#4267] Make migration more robust --- .../0019_add_default_objects_api_group.py | 33 +++++++++++++----- .../objects_api/tests/test_migrations.py | 34 +++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py b/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py index 86b1e41954..f71dc19784 100644 --- a/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py +++ b/src/openforms/registrations/contrib/objects_api/migrations/0019_add_default_objects_api_group.py @@ -1,9 +1,8 @@ # Generated by Django 4.2.11 on 2024-07-02 15:18 from django.db import migrations - -from django.db.migrations.state import StateApps from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps def add_default_objects_api_group( @@ -13,17 +12,33 @@ def add_default_objects_api_group( ObjectsAPIGroupConfig = apps.get_model( "registrations_objects_api", "ObjectsAPIGroupConfig" ) + backends_qs = FormRegistrationBackend.objects.filter(backend="objects_api") + # nothing to do if there are no relevant backends + if not backends_qs.exists(): + return objects_api_group_config = ObjectsAPIGroupConfig.objects.order_by("pk").first() if objects_api_group_config is None: - # Because this migration runs after registrations_objects_api/0017_move_singleton_data, - # Having no Objects API Group means we had no solo config in the first place, thus - # it is safe to assume no Objects API registration backend was set up. - return + # This shouldn't happen because of: + # * upgrade checks + # * operations in registrations_objects_api/0017_move_singleton_data + # + # BUT that doesn't mean it's impossible, like on continuously deployed + # environments... For those cases, we generate a (knowingly broken) + # configuration so that migrations don't crash and options have at least the + # right shape so that we can trust the type annotations. + objects_api_group_config = ObjectsAPIGroupConfig.objects.create( + name="AUTO_GENERATED - FIXME", + # DeprecationWarning + # Open Forms 3.0 will drop the nullable fields, so this data migration needs + # to be gone by then. + objects_service=None, + objecttypes_service=None, + drc_service=None, + catalogi_service=None, + ) - for registration_backend in FormRegistrationBackend.objects.filter( - backend="objects_api" - ): + for registration_backend in backends_qs: registration_backend.options.setdefault( "objects_api_group", objects_api_group_config.pk ) diff --git a/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py b/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py index a87604ea24..be958f5cfd 100644 --- a/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py +++ b/src/openforms/registrations/contrib/objects_api/tests/test_migrations.py @@ -313,6 +313,40 @@ def test_sets_default_objects_api_group(self) -> None: ) +class AddDefaultObjectsAPIGroupWithBrokenStateMigrationTests(TestMigrations): + app = "registrations_objects_api" + migrate_from = "0018_remove_objectsapiconfig_catalogi_service_and_more" + migrate_to = "0019_add_default_objects_api_group" + + def setUpBeforeMigration(self, apps: StateApps) -> None: + Form = apps.get_model("forms", "Form") + FormRegistrationBackend = apps.get_model("forms", "FormRegistrationBackend") + ObjectsAPIGroupConfig = apps.get_model( + "registrations_objects_api", "ObjectsAPIGroupConfig" + ) + + form = Form.objects.create(name="test form") + FormRegistrationBackend.objects.create( + form=form, + name="Objects API backend", + key="backend", + backend="objects_api", + ) + assert not ObjectsAPIGroupConfig.objects.exists() + + def test_sets_default_objects_api_group(self) -> None: + FormRegistrationBackend = self.apps.get_model( + "forms", "FormRegistrationBackend" + ) + ObjectsAPIGroupConfig = self.apps.get_model( + "registrations_objects_api", "ObjectsAPIGroupConfig" + ) + auto_created_config = ObjectsAPIGroupConfig.objects.get() + + backend = FormRegistrationBackend.objects.get() + self.assertEqual(backend.options["objects_api_group"], auto_created_config.pk) + + class ObjecttypeUrltoUuidMigrationTests(TestMigrations): app = "registrations_objects_api" migrate_from = "0019_add_default_objects_api_group"