From c43f1daa89493ce4d111e0e9715520913b1c4313 Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Wed, 20 Nov 2024 14:41:46 -0300 Subject: [PATCH] Replace the DEFAULT_FILE_STORAGE settings with STORAGES The DEFAULT_FILE_STORAGE option was deprecated in django 4.2.0 in favor of another option called STORAGES. fixes #5404 --- .ci/ansible/settings.py.j2 | 6 +++--- .../tests/functional/api/test_download_policies.py | 6 +++--- pulpcore/app/apps.py | 6 +++--- pulpcore/app/checks.py | 2 +- pulpcore/app/importexport.py | 2 +- pulpcore/app/settings.py | 13 +++++++------ pulpcore/app/tasks/export.py | 2 +- pulpcore/app/util.py | 4 ++-- pulpcore/pytest_plugin.py | 2 +- .../functional/api/test_artifact_distribution.py | 2 +- pulpcore/tests/functional/api/test_crd_artifacts.py | 2 +- pulpcore/tests/functional/api/test_crud_domains.py | 2 +- pulpcore/tests/functional/api/test_status.py | 2 +- .../functional/api/using_plugin/test_orphans.py | 8 ++++---- pulpcore/tests/unit/models/test_content.py | 2 +- 15 files changed, 31 insertions(+), 30 deletions(-) diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 4ed28f42ad2..b21b40dd0fd 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -27,7 +27,7 @@ API_ROOT = {{ api_root | repr }} {% endif %} {% if s3_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" +STORAGES__default = "storages.backends.s3boto3.S3Boto3Storage" MEDIA_ROOT = "" AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" @@ -41,7 +41,7 @@ AWS_DEFAULT_ACL = "@none None" {% endif %} {% if azure_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" +STORAGES__default = "storages.backends.azure_storage.AzureStorage" MEDIA_ROOT = "" AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" AZURE_ACCOUNT_NAME = "devstoreaccount1" @@ -53,7 +53,7 @@ AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreacc {% endif %} {% if gcp_test | default(false) %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +STORAGES__default = "storages.backends.gcloud.GoogleCloudStorage" MEDIA_ROOT = "" GS_BUCKET_NAME = "gcppulp" GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" diff --git a/pulp_file/tests/functional/api/test_download_policies.py b/pulp_file/tests/functional/api/test_download_policies.py index eeaa5a1fb72..5a201cb3b04 100644 --- a/pulp_file/tests/functional/api/test_download_policies.py +++ b/pulp_file/tests/functional/api/test_download_policies.py @@ -54,7 +54,7 @@ def test_download_policy( download_policy, ): """Test that "on_demand" and "streamed" download policies work as expected.""" - if download_policy == "on_demand" and "SFTP" in pulp_settings.DEFAULT_FILE_STORAGE: + if download_policy == "on_demand" and "SFTP" in settings.STORAGES["default"]: pytest.skip("This storage technology is not properly supported.") remote = file_remote_ssl_factory( @@ -148,7 +148,7 @@ def test_download_policy( assert expected_checksum == actual_checksum if ( download_policy == "immediate" - and settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem" + and settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem" and settings.REDIRECT_TO_OBJECT_STORAGE ): content_disposition = downloaded_file.response_obj.headers.get("Content-Disposition") @@ -187,7 +187,7 @@ def test_download_policy( content_unit = expected_files_list[4] content_unit_url = urljoin(distribution.base_url, content_unit[0]) # The S3 test API project doesn't handle invalid Range values correctly - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": with pytest.raises(ClientResponseError) as exc: range_header = {"Range": "bytes=-1-11"} download_file(content_unit_url, headers=range_header) diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index a3f32540f42..4764451137f 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -321,11 +321,11 @@ def _ensure_default_domain(sender, **kwargs): if ( settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage - or settings.DEFAULT_FILE_STORAGE != default.storage_class + or settings.STORAGES["default"] != default.storage_class ): default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE - default.storage_class = settings.DEFAULT_FILE_STORAGE + default.storage_class = settings.STORAGES["default"] default.save(skip_hooks=True) @@ -392,7 +392,7 @@ def _get_permission(perm): def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): if ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem" or not settings.REDIRECT_TO_OBJECT_STORAGE ): try: diff --git a/pulpcore/app/checks.py b/pulpcore/app/checks.py index a0129bf56b6..f01b4d50360 100644 --- a/pulpcore/app/checks.py +++ b/pulpcore/app/checks.py @@ -38,7 +38,7 @@ def secret_key_check(app_configs, **kwargs): def storage_paths(app_configs, **kwargs): warnings = [] - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": try: media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev except OSError: diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index 3dab3aa6ee9..b54c22b4127 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -114,7 +114,7 @@ def export_artifacts(export, artifact_pks): with ProgressReport(**data) as pb: pb.BATCH_INTERVAL = 5000 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem": with tempfile.TemporaryDirectory(dir=".") as temp_dir: for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE): batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE] diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index ac426091451..97c564d5978 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -56,7 +56,8 @@ STATIC_URL = "/assets/" STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/") -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +STORAGES = {"default": {"BACKEND": "pulpcore.app.models.storage.FileSystem"}} + REDIRECT_TO_OBJECT_STORAGE = True WORKING_DIRECTORY = DEPLOY_ROOT / "tmp" @@ -373,14 +374,14 @@ # Validators storage_validator = ( Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False) - | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage") + | Validator("STORAGES.default", eq="pulpcore.app.models.storage.FileSystem") + | Validator("STORAGES.default", eq="storages.backends.azure_storage.AzureStorage") + | Validator("STORAGES.default", eq="storages.backends.s3boto3.S3Boto3Storage") + | Validator("STORAGES.default", eq="storages.backends.gcloud.GoogleCloudStorage") ) storage_validator.messages["combined"] = ( "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure" - "storage backend configured in DEFAULT_FILE_STORAGE." + "storage backend configured in STORAGES['default']." ) cache_enabled_validator = Validator("CACHE_ENABLED", eq=True) diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py index 71956d2359f..008fcd536df 100644 --- a/pulpcore/app/tasks/export.py +++ b/pulpcore/app/tasks/export.py @@ -70,7 +70,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting """ using_filesystem_storage = ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem" ) if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage: diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index dd3119737e3..06b8c5e0dc6 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -536,7 +536,7 @@ def get_artifact_url(artifact, headers=None, http_method=None): if settings.DOMAIN_ENABLED: loc = f"domain {artifact_domain.name}.storage_class" else: - loc = "settings.DEFAULT_FILE_STORAGE" + loc = "settings.STORAGES['default']" raise NotImplementedError( f"The value {loc}={artifact_domain.storage_class} does not allow redirecting." @@ -582,7 +582,7 @@ def get_default_domain(): try: default_domain = Domain.objects.get(name="default") except Domain.DoesNotExist: - default_domain = Domain(name="default", storage_class=settings.DEFAULT_FILE_STORAGE) + default_domain = Domain(name="default", storage_class=settings.STORAGES["default"]) default_domain.save(skip_hooks=True) return default_domain diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 813d85c9131..7b3ff027a42 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -621,7 +621,7 @@ def _settings_factory(storage_class=None, storage_settings=None): "AZURE_CONNECTION_STRING", ] settings = storage_settings or dict() - backend = storage_class or pulp_settings.DEFAULT_FILE_STORAGE + backend = storage_class or settings.STORAGES["default"] for key in keys[backend]: if key not in settings: settings[key] = getattr(pulp_settings, key, None) diff --git a/pulpcore/tests/functional/api/test_artifact_distribution.py b/pulpcore/tests/functional/api/test_artifact_distribution.py index d6735347c22..48ed81c7d98 100644 --- a/pulpcore/tests/functional/api/test_artifact_distribution.py +++ b/pulpcore/tests/functional/api/test_artifact_distribution.py @@ -29,7 +29,7 @@ def test_artifact_distribution(random_artifact): hasher = sha256() hasher.update(response.content) assert hasher.hexdigest() == random_artifact.sha256 - if settings.DEFAULT_FILE_STORAGE in OBJECT_STORAGES: + if settings.STORAGES["default"] in OBJECT_STORAGES: content_disposition = response.headers.get("Content-Disposition") assert content_disposition is not None filename = artifact_uuid diff --git a/pulpcore/tests/functional/api/test_crd_artifacts.py b/pulpcore/tests/functional/api/test_crd_artifacts.py index 6a9f13274dc..23d7b2eeea9 100644 --- a/pulpcore/tests/functional/api/test_crd_artifacts.py +++ b/pulpcore/tests/functional/api/test_crd_artifacts.py @@ -148,7 +148,7 @@ def test_upload_mixed_attrs(pulpcore_bindings, pulpcore_random_file): def test_delete_artifact(pulpcore_bindings, pulpcore_random_file, gen_user): """Verify that the deletion of artifacts is prohibited for both regular users and administrators.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("this test only works for filesystem storage") media_root = settings.MEDIA_ROOT diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index f67855ef9f2..f309016f033 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -59,7 +59,7 @@ def test_default_domain(pulpcore_bindings): # Read the default domain, ensure storage is set to default default_domain = domains.results[0] assert default_domain.name == "default" - assert default_domain.storage_class == settings.DEFAULT_FILE_STORAGE + assert default_domain.storage_class == settings.STORAGES["default"] assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 861b431b95d..8ed57551ad1 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -142,7 +142,7 @@ def verify_get_response(status, expected_schema): assert status["content_settings"]["content_path_prefix"] is not None assert status["storage"]["used"] is not None - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem": assert status["storage"]["free"] is None assert status["storage"]["total"] is None else: diff --git a/pulpcore/tests/functional/api/using_plugin/test_orphans.py b/pulpcore/tests/functional/api/using_plugin/test_orphans.py index 986087cc80b..86b426e3e18 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_orphans.py +++ b/pulpcore/tests/functional/api/using_plugin/test_orphans.py @@ -73,7 +73,7 @@ def test_orphans_delete( content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) @@ -89,7 +89,7 @@ def test_orphans_delete( with pytest.raises(file_bindings.ApiException) as exc: file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False @@ -108,7 +108,7 @@ def test_orphans_cleanup( content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) @@ -123,7 +123,7 @@ def test_orphans_cleanup( with pytest.raises(file_bindings.ApiException) as exc: file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 319180d64c2..b415a173d25 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -43,7 +43,7 @@ def test_create_read_delete_content(tmp_path): @pytest.mark.django_db def test_storage_location(tmp_path, settings): - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("Skipping test for nonlocal storage.") tf = tmp_path / "ab"