From 5058fc49f89f9b678464e0ce37cec78bf50be01e Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Sat, 27 Mar 2021 20:17:29 +0100 Subject: [PATCH 1/5] New setting.py parameter: MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE Checking the existence of our target bucket has a toll on overall performance, because the check itself is a call and if it's missing, the creation is another. If the user is using static bucket names, then this action is an overkill and wastes execution time --- DjangoExampleProject/settings.py | 1 + README.md | 1 + django_minio_backend/models.py | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py index 81caca4..d42c520 100644 --- a/DjangoExampleProject/settings.py +++ b/DjangoExampleProject/settings.py @@ -163,3 +163,4 @@ MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [ # ('django-backend-dev-private', dummy_policy) ] +MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE = True # Create bucket if missing, then save diff --git a/README.md b/README.md index f9b3af7..e09ae5c 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ MINIO_PUBLIC_BUCKETS = [ 'django-backend-dev-public', ] MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [] +MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE = True # Default: True // Creates bucket if missing, then save ``` 4. Implement your own Attachment handler and integrate **django-minio-backend**: diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 07e2390..a912a26 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -88,8 +88,9 @@ def _save(self, file_path_name: str, content: InMemoryUploadedFile) -> str: :param content (InMemoryUploadedFile): File object :return: """ - # Check if bucket exists, create if not - self.check_bucket_existence() + if get_setting("MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE", True): + # Check if bucket exists, create if not + self.check_bucket_existence() # Check if object with name already exists; delete if so if self._REPLACE_EXISTING and self.stat(file_path_name): From 4447a3c166d1811e3db60e140c56ceb991521c3c Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Mon, 19 Jul 2021 01:19:23 +0200 Subject: [PATCH 2/5] Support for STATIC and MEDIA files - Updated the __init__ of `MinioBackend` to meet Django's custom storage system requirements "Django must be able to instantiate your storage system without any arguments" The `MinioBackend` class can be used as a value for `DEFAULT_FILE_STORAGE` - MinioBackendStatic: A new subclass of `MinioBackend` for supporting django-minio-backend as a static files storage - Added missing docstrings throughout the project - DjangoExampleApplication extended by `class GenericAttachment` for demonstrating a `FileField` without an explicit storage backend - DjangoExampleProject settings.py has been extended with the following new values: - MINIO_MEDIA_FILES_BUCKET - MINIO_STATIC_FILES_BUCKET - Updated README.md --- DjangoExampleApplication/admin.py | 10 +- .../migrations/0003_genericattachment.py | 21 ++++ DjangoExampleApplication/models.py | 19 +++- DjangoExampleProject/settings.py | 10 +- README.md | 46 ++++++++- django_minio_backend/apps.py | 7 ++ .../management/commands/initialize_buckets.py | 2 +- .../management/commands/is_minio_available.py | 2 +- django_minio_backend/models.py | 97 +++++++++++++++++-- django_minio_backend/utils.py | 3 + 10 files changed, 200 insertions(+), 17 deletions(-) create mode 100644 DjangoExampleApplication/migrations/0003_genericattachment.py diff --git a/DjangoExampleApplication/admin.py b/DjangoExampleApplication/admin.py index b18b47e..95111de 100644 --- a/DjangoExampleApplication/admin.py +++ b/DjangoExampleApplication/admin.py @@ -2,7 +2,7 @@ from django.db.models.query import QuerySet from django.contrib import admin from django.core.handlers.wsgi import WSGIRequest -from .models import PublicAttachment, PrivateAttachment, Image +from .models import PublicAttachment, PrivateAttachment, Image, GenericAttachment # https://docs.djangoproject.com/en/2.2/ref/contrib/admin/actions/#writing-action-functions @@ -32,6 +32,14 @@ class ImageAdmin(admin.ModelAdmin): actions = [delete_everywhere, ] +@admin.register(GenericAttachment) +class GenericAttachmentAdmin(admin.ModelAdmin): + list_display = ('id', 'file',) + readonly_fields = ('id', ) + model = GenericAttachment + actions = [delete_everywhere, ] + + # Register your models here. @admin.register(PublicAttachment) class PublicAttachmentAdmin(admin.ModelAdmin): diff --git a/DjangoExampleApplication/migrations/0003_genericattachment.py b/DjangoExampleApplication/migrations/0003_genericattachment.py new file mode 100644 index 0000000..89dd591 --- /dev/null +++ b/DjangoExampleApplication/migrations/0003_genericattachment.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.3 on 2021-07-18 22:07 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('DjangoExampleApplication', '0002_auto_20210313_1049'), + ] + + operations = [ + migrations.CreateModel( + name='GenericAttachment', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('file', models.FileField(upload_to='', verbose_name='Object Upload (to default storage)')), + ], + ), + ] diff --git a/DjangoExampleApplication/models.py b/DjangoExampleApplication/models.py index 5de4e5a..2f70f61 100644 --- a/DjangoExampleApplication/models.py +++ b/DjangoExampleApplication/models.py @@ -23,13 +23,30 @@ class Image(models.Model): def delete(self, *args, **kwargs): """ - Delete must be overridden because the inherited delete method does not call `self.file.delete()`. + Delete must be overridden because the inherited delete method does not call `self.image.delete()`. """ # noinspection PyUnresolvedReferences self.image.delete() super(Image, self).delete(*args, **kwargs) +class GenericAttachment(models.Model): + """ + This is for demonstrating uploads to the default file storage + """ + objects = models.Manager() + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + file = models.FileField(verbose_name="Object Upload (to default storage)") + + def delete(self, *args, **kwargs): + """ + Delete must be overridden because the inherited delete method does not call `self.image.delete()`. + """ + # noinspection PyUnresolvedReferences + self.image.delete() + super(GenericAttachment, self).delete(*args, **kwargs) + + # Create your models here. class PublicAttachment(models.Model): def set_file_path_name(self, file_name_ext: str) -> str: diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py index d91c3f4..cf1d395 100644 --- a/DjangoExampleProject/settings.py +++ b/DjangoExampleProject/settings.py @@ -116,6 +116,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_URL = '/static/' +STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic' +DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend' # #################### # # django_minio_backend # @@ -154,14 +156,18 @@ MINIO_USE_HTTPS = bool(distutils.util.strtobool(os.getenv("GH_MINIO_USE_HTTPS", "true"))) MINIO_PRIVATE_BUCKETS = [ 'django-backend-dev-private', + 'my-media-files-bucket', ] MINIO_PUBLIC_BUCKETS = [ 'django-backend-dev-public', - "t5p2g08k31", - "7xi7lx9rjh", + 't5p2g08k31', + '7xi7lx9rjh', + 'my-static-files-bucket', ] MINIO_URL_EXPIRY_HOURS = timedelta(days=1) # Default is 7 days (longest) if not defined MINIO_CONSISTENCY_CHECK_ON_START = True MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [ # ('django-backend-dev-private', dummy_policy) ] +MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for STATIC_ROOT +MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for MEDIA_ROOT diff --git a/README.md b/README.md index 2e3eb3a..56e3c02 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ MINIO_PUBLIC_BUCKETS = [ 'django-backend-dev-public', ] MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [] +# MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT +# MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT ``` 4. Implement your own Attachment handler and integrate **django-minio-backend**: @@ -79,6 +81,48 @@ python manage.py initialize_buckets Code reference: [initialize_buckets.py](django_minio_backend/management/commands/initialize_buckets.py). +### Static Files Support +**django-minio-backend** allows serving static files from MinIO. +To learn more about Django static files, see [Managing static files](https://docs.djangoproject.com/en/3.2/howto/static-files/), and [STATICFILES_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#staticfiles-storage). + +To enable static files support, update your `settings.py`: +```python +STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic' +MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT +# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.: +# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +``` + +The value of `STATIC_URL` is ignored, but it must be defined otherwise Django will throw an error. + +**IMPORTANT**
+The value set in `MINIO_STATIC_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, +otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. + +**Note:** If `MINIO_STATIC_FILES_BUCKET` is not set, the default value (`auto-generated-static-media-files`) will be used. + +### Default File Storage Support +**django-minio-backend** can be configured as a default file storage. +To learn more, see [DEFAULT_FILE_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage). + +To configure **django-minio-backend** as the default file storage, update your `settings.py`: +```python +DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend' +MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT +# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.: +# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +``` + +The value of `MEDIA_URL` is ignored, but it must be defined otherwise Django will throw an error. + +**IMPORTANT**
+The value set in `MINIO_MEDIA_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, +otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. + +**Note:** If `MINIO_MEDIA_FILES_BUCKET` is not set, the default value (`auto-generated-bucket-media-files`) will be used. + ### Health Check To check the connection link between Django and MinIO, use the provided `MinioBackend.is_minio_available()` method.
It returns a `MinioServerStatus` instance which can be quickly evaluated as boolean.
@@ -87,7 +131,7 @@ It returns a `MinioServerStatus` instance which can be quickly evaluated as bool ```python from django_minio_backend import MinioBackend -minio_available = MinioBackend('').is_minio_available() # An empty string is fine this time +minio_available = MinioBackend().is_minio_available() # An empty string is fine this time if minio_available: print("OK") else: diff --git a/django_minio_backend/apps.py b/django_minio_backend/apps.py index 6d96eed..d08e539 100644 --- a/django_minio_backend/apps.py +++ b/django_minio_backend/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig from .utils import get_setting, ConfigurationError +from .models import MinioBackendStatic __all__ = ['DjangoMinioBackendConfig', ] @@ -20,3 +21,9 @@ def ready(self): external_use_https = get_setting('MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') if (external_address and external_use_https is None) or (not external_address and external_use_https): raise ConfigurationError('MINIO_EXTERNAL_ENDPOINT must be configured together with MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') + + # Validate static storage and default storage configurations + staticfiles_storage: str = get_setting('STATICFILES_STORAGE') + if staticfiles_storage.endswith(MinioBackendStatic.__name__): + mbs = MinioBackendStatic() + mbs.check_bucket_existence() diff --git a/django_minio_backend/management/commands/initialize_buckets.py b/django_minio_backend/management/commands/initialize_buckets.py index 508fee7..0687006 100644 --- a/django_minio_backend/management/commands/initialize_buckets.py +++ b/django_minio_backend/management/commands/initialize_buckets.py @@ -25,7 +25,7 @@ def handle(self, *args, **options): self.stdout.write( f"Bucket ({m.bucket}) policy has been set to public", ending='\n') if not silenced else None - c = MinioBackend('') # Client + c = MinioBackend() # Client for policy_tuple in get_setting('MINIO_POLICY_HOOKS', []): bucket, policy = policy_tuple c.set_bucket_policy(bucket, policy) diff --git a/django_minio_backend/management/commands/is_minio_available.py b/django_minio_backend/management/commands/is_minio_available.py index ef938d0..4d08ba6 100644 --- a/django_minio_backend/management/commands/is_minio_available.py +++ b/django_minio_backend/management/commands/is_minio_available.py @@ -9,7 +9,7 @@ def add_arguments(self, parser): parser.add_argument('--silenced', action='store_true', default=False, help='No console messages') def handle(self, *args, **options): - m = MinioBackend('') # no configured bucket + m = MinioBackend() # use default storage silenced = options.get('silenced') self.stdout.write(f"Checking the availability of MinIO at {m.base_url}\n") if not silenced else None diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 53455a3..c7f93ff 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -1,3 +1,11 @@ +""" +django-minio-backend +A MinIO-compatible custom storage backend for Django + +References: + * https://github.com/minio/minio-py + * https://docs.djangoproject.com/en/3.2/howto/custom-file-storage/ +""" import io import json import mimetypes @@ -7,13 +15,13 @@ from time import mktime from typing import Union, List -# noinspection PyPackageRequirements minIO_requirement +# noinspection PyPackageRequirements MinIO_requirement import certifi import minio import minio.datatypes import minio.error import minio.helpers -# noinspection PyPackageRequirements minIO_requirement +# noinspection PyPackageRequirements MinIO_requirement import urllib3 from django.core.files import File from django.core.files.storage import Storage @@ -27,11 +35,16 @@ def get_iso_date() -> str: + """Get current date in ISO8601 format [year-month-day] as string""" now = datetime.utcnow().replace(tzinfo=utc) return f"{now.year}-{now.month}-{now.day}" def iso_date_prefix(_, file_name_ext: str) -> str: + """ + Get filename prepended with current date in ISO8601 format [year-month-day] as string + The date prefix will be the folder's name storing the object e.g.: 2020-12-31/cat.png + """ return f"{get_iso_date()}/{file_name_ext}" @@ -46,12 +59,24 @@ class MinioBackend(Storage): for the underlying put_object() MinIO SDK method """ + MINIO_MEDIA_FILES_BUCKET = get_setting("MINIO_MEDIA_FILES_BUCKET", default='auto-generated-bucket-media-files') + MINIO_STATIC_FILES_BUCKET = get_setting("MINIO_STATIC_FILES_BUCKET", default='auto-generated-bucket-static-files') + def __init__(self, - bucket_name: str, + bucket_name: str = '', *args, **kwargs): - self._BUCKET_NAME: str = bucket_name + # If bucket_name is not provided, MinioBackend acts as a DEFAULT_FILE_STORAGE + # The automatically selected bucket is MINIO_MEDIA_FILES_BUCKET from settings.py + # See https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage + if not bucket_name or bucket_name == '': + self.__CONFIGURED_AS_DEFAULT_STORAGE = True + self._BUCKET_NAME: str = self.MINIO_MEDIA_FILES_BUCKET + else: + self.__CONFIGURED_AS_DEFAULT_STORAGE = False + self._BUCKET_NAME: str = bucket_name + self._META_ARGS = args self._META_KWARGS = kwargs @@ -73,8 +98,18 @@ def __init__(self, self.PRIVATE_BUCKETS: List[str] = get_setting("MINIO_PRIVATE_BUCKETS", []) self.PUBLIC_BUCKETS: List[str] = get_setting("MINIO_PUBLIC_BUCKETS", []) + # Configure storage type + self.__STORAGE_TYPE = 'custom' + if self.bucket == self.MINIO_MEDIA_FILES_BUCKET: + self.__STORAGE_TYPE = 'media' + if self.bucket == self.MINIO_STATIC_FILES_BUCKET: + self.__STORAGE_TYPE = 'static' + + if self._BUCKET_NAME not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]: + raise ConfigurationError(f'The configured bucket ({self.bucket}) must be declared either in MINIO_PRIVATE_BUCKETS or MINIO_PUBLIC_BUCKETS') + # https://docs.min.io/docs/python-client-api-reference.html - self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = kwargs.get("http_client", None) + self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = self._META_KWARGS.get("http_client", None) bucket_name_intersection: List[str] = list(set(self.PRIVATE_BUCKETS) & set(self.PUBLIC_BUCKETS)) if bucket_name_intersection: @@ -127,12 +162,12 @@ def _open(self, object_name, mode='rb', **kwargs): """ Implements the Storage._open(name,mode='rb') method :param name (str): object_name [path to file excluding bucket name which is implied] - :kwargs (dict): passed on to the underlying minIO client's get_object() method + :kwargs (dict): passed on to the underlying MinIO client's get_object() method """ resp: urllib3.response.HTTPResponse = urllib3.response.HTTPResponse() if mode != 'rb': - raise ValueError('Files retrieved from minIO are read-only. Use save() method to override contents') + raise ValueError('Files retrieved from MinIO are read-only. Use save() method to override contents') try: resp = self.client.get_object(self._BUCKET_NAME, object_name, kwargs) file = File(file=io.BytesIO(resp.read()), name=object_name) @@ -142,6 +177,7 @@ def _open(self, object_name, mode='rb', **kwargs): return file def stat(self, name: str) -> Union[minio.datatypes.Object, bool]: + """Get object information and metadata of an object""" object_name = Path(name).as_posix() try: obj = self.client.stat_object(self._BUCKET_NAME, object_name=object_name) @@ -163,16 +199,19 @@ def delete(self, name: str): self.client.remove_object(bucket_name=self._BUCKET_NAME, object_name=object_name) def exists(self, name: str) -> bool: + """Check if an object with name already exists""" object_name = Path(name).as_posix() if self.stat(object_name): return True return False def listdir(self, bucket_name: str): + """List all objects in a bucket""" objects = self.client.list_objects(bucket_name=bucket_name, recursive=True) return [(obj.object_name, obj) for obj in objects] def size(self, name: str) -> int: + """Get an object's size""" object_name = Path(name).as_posix() obj = self.stat(object_name) if obj: @@ -208,21 +247,22 @@ def url(self, name: str): raise ConnectionError("Couldn't connect to Minio. Check django_minio_backend parameters in Django-Settings") def path(self, name): - raise NotImplementedError("The minIO storage system doesn't support absolute paths.") + """The MinIO storage system doesn't support absolute paths""" + raise NotImplementedError("The MinIO storage system doesn't support absolute paths.") def get_accessed_time(self, name: str) -> datetime: """ Return the last accessed time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('minIO does not store last accessed time') + raise NotImplementedError('MinIO does not store last accessed time') def get_created_time(self, name: str) -> datetime: """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('minIO does not store creation time') + raise NotImplementedError('MinIO does not store creation time') def get_modified_time(self, name: str) -> datetime: """ @@ -254,13 +294,16 @@ def same_endpoints(self) -> bool: @property def bucket(self) -> str: + """Get the configured bucket's [self.bucket] name""" return self._BUCKET_NAME @property def is_bucket_public(self) -> bool: + """Check if configured bucket [self.bucket] is public""" return True if self._BUCKET_NAME in self.PUBLIC_BUCKETS else False def is_minio_available(self) -> MinioServerStatus: + """Check if configured MinIO server is available""" if not self.__MINIO_ENDPOINT: mss = MinioServerStatus(None) mss.add_message('MINIO_ENDPOINT is not configured in Django settings') @@ -282,6 +325,7 @@ def is_minio_available(self) -> MinioServerStatus: @property def client(self) -> minio.Minio: + """Get handle to an (already) instantiated minio.Minio instance (for internal URL access)""" if not self.__CLIENT: self.new_client() return self.__CLIENT @@ -289,6 +333,7 @@ def client(self) -> minio.Minio: @property def client_external(self) -> minio.Minio: + """Get handle to an (already) instantiated minio.Minio instance (for external URL access)""" if not self.__CLIENT_EXTERNAL: self.new_client(internal=False) return self.__CLIENT_EXTERNAL @@ -296,10 +341,12 @@ def client_external(self) -> minio.Minio: @property def base_url(self) -> str: + """Get internal base URL to MinIO""" return self.__BASE_URL @property def base_url_external(self) -> str: + """Get external base URL to MinIO""" return self.__BASE_URL_EXTERNAL def new_client(self, internal: bool = True): @@ -328,18 +375,22 @@ def new_client(self, internal: bool = True): # MAINTENANCE def check_bucket_existence(self): + """Check if configured bucket [self.bucket] exists""" if not self.client.bucket_exists(self.bucket): self.client.make_bucket(bucket_name=self.bucket) def check_bucket_existences(self): # Execute this handler upon starting Django to make sure buckets exist + """Check if all buckets configured in settings.py do exist. If not, create them""" for bucket in [*self.PUBLIC_BUCKETS, *self.PRIVATE_BUCKETS]: if not self.client.bucket_exists(bucket): self.client.make_bucket(bucket_name=bucket) def set_bucket_policy(self, bucket: str, policy: dict): + """Set a custom bucket policy""" self.client.set_bucket_policy(bucket_name=bucket, policy=json.dumps(policy)) def set_bucket_to_public(self): + """Set bucket policy to be public. It can be then accessed via public URLs""" policy_public_read_only = {"Version": "2012-10-17", "Statement": [ { @@ -365,3 +416,29 @@ def set_bucket_to_public(self): } ]} self.set_bucket_policy(self.bucket, policy_public_read_only) + + +@deconstructible +class MinioBackendStatic(MinioBackend): + """ + MinIO-compatible Django custom storage system for Django static files. + The used bucket can be configured in settings.py through `MINIO_STATIC_FILES_BUCKET` + :arg *args: Should not be used for static files. It's here for compatibility only + :arg **kwargs: Should not be used for static files. It's here for compatibility only + """ + def __init__(self, *args, **kwargs): + super().__init__(self.MINIO_STATIC_FILES_BUCKET, *args, **kwargs) + self.check_bucket_existence() # make sure the `MINIO_STATIC_FILES_BUCKET` exists + self.set_bucket_to_public() # the static files bucket must be publicly available + + def path(self, name): + """The MinIO storage system doesn't support absolute paths""" + raise NotImplementedError("The MinIO storage system doesn't support absolute paths.") + + def get_accessed_time(self, name: str): + """MinIO does not store last accessed time""" + raise NotImplementedError('MinIO does not store last accessed time') + + def get_created_time(self, name: str): + """MinIO does not store creation time""" + raise NotImplementedError('MinIO does not store creation time') diff --git a/django_minio_backend/utils.py b/django_minio_backend/utils.py index 2cee815..bd3811a 100644 --- a/django_minio_backend/utils.py +++ b/django_minio_backend/utils.py @@ -61,12 +61,15 @@ def __repr__(self): class PrivatePublicMixedError(Exception): + """Raised on public|private bucket configuration collisions""" pass class ConfigurationError(Exception): + """Raised on django-minio-backend configuration errors""" pass def get_setting(name, default=None): + """Get setting from settings.py. Return a default value if not defined""" return getattr(settings, name, default) From ce60623a55a068621b6fe4a578dbf2630bd50205 Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Mon, 19 Jul 2021 23:33:48 +0200 Subject: [PATCH 3/5] Support for STATIC and MEDIA files: - Moved default bucket names to variables for convenience - Default buckets are private by default -> no need to declare their privacy in settings.py anymore - Refactored non __init__ occurrences of self._BUCKET_NAME to self.bucket for a cleaner code experience - Corrected a bug in MinioBackend > get_modified_time() - Added note in README.md: "Policy setting for default buckets is private." --- README.md | 4 ++-- django_minio_backend/models.py | 35 +++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 56e3c02..2a79598 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ The value of `STATIC_URL` is ignored, but it must be defined otherwise Django wi The value set in `MINIO_STATIC_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. -**Note:** If `MINIO_STATIC_FILES_BUCKET` is not set, the default value (`auto-generated-static-media-files`) will be used. +**Note:** If `MINIO_STATIC_FILES_BUCKET` is not set, the default value (`auto-generated-static-media-files`) will be used. Policy setting for default buckets is **private**. ### Default File Storage Support **django-minio-backend** can be configured as a default file storage. @@ -121,7 +121,7 @@ The value of `MEDIA_URL` is ignored, but it must be defined otherwise Django wil The value set in `MINIO_MEDIA_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. -**Note:** If `MINIO_MEDIA_FILES_BUCKET` is not set, the default value (`auto-generated-bucket-media-files`) will be used. +**Note:** If `MINIO_MEDIA_FILES_BUCKET` is not set, the default value (`auto-generated-bucket-media-files`) will be used. Policy setting for default buckets is **private**. ### Health Check To check the connection link between Django and MinIO, use the provided `MinioBackend.is_minio_available()` method.
diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index c7f93ff..15e4da7 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -12,7 +12,6 @@ import ssl from datetime import datetime, timedelta from pathlib import Path -from time import mktime from typing import Union, List # noinspection PyPackageRequirements MinIO_requirement @@ -58,9 +57,11 @@ class MinioBackend(Storage): Through self._META_KWARGS, the "metadata", "sse" and "progress" fields can be set for the underlying put_object() MinIO SDK method """ - - MINIO_MEDIA_FILES_BUCKET = get_setting("MINIO_MEDIA_FILES_BUCKET", default='auto-generated-bucket-media-files') - MINIO_STATIC_FILES_BUCKET = get_setting("MINIO_STATIC_FILES_BUCKET", default='auto-generated-bucket-static-files') + DEFAULT_MEDIA_FILES_BUCKET = 'auto-generated-bucket-media-files' + DEFAULT_STATIC_FILES_BUCKET = 'auto-generated-bucket-static-files' + DEFAULT_PRIVATE_BUCKETS = [DEFAULT_MEDIA_FILES_BUCKET, DEFAULT_STATIC_FILES_BUCKET] + MINIO_MEDIA_FILES_BUCKET = get_setting("MINIO_MEDIA_FILES_BUCKET", default=DEFAULT_MEDIA_FILES_BUCKET) + MINIO_STATIC_FILES_BUCKET = get_setting("MINIO_STATIC_FILES_BUCKET", default=DEFAULT_STATIC_FILES_BUCKET) def __init__(self, bucket_name: str = '', @@ -105,7 +106,11 @@ def __init__(self, if self.bucket == self.MINIO_STATIC_FILES_BUCKET: self.__STORAGE_TYPE = 'static' - if self._BUCKET_NAME not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]: + # Enforce good bucket security (private vs public) + if (self.bucket in self.DEFAULT_PRIVATE_BUCKETS) and (self.bucket not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]): + self.PRIVATE_BUCKETS.extend(self.DEFAULT_PRIVATE_BUCKETS) # policy for default buckets is PRIVATE + # Require custom buckets to be declared explicitly + if self.bucket not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]: raise ConfigurationError(f'The configured bucket ({self.bucket}) must be declared either in MINIO_PRIVATE_BUCKETS or MINIO_PUBLIC_BUCKETS') # https://docs.min.io/docs/python-client-api-reference.html @@ -138,7 +143,7 @@ def _save(self, file_path_name: str, content: InMemoryUploadedFile) -> str: # Upload object file_path: Path = Path(file_path_name) # app name + file.suffix self.client.put_object( - bucket_name=self._BUCKET_NAME, + bucket_name=self.bucket, object_name=file_path.as_posix(), data=content, length=content.size, @@ -169,7 +174,7 @@ def _open(self, object_name, mode='rb', **kwargs): if mode != 'rb': raise ValueError('Files retrieved from MinIO are read-only. Use save() method to override contents') try: - resp = self.client.get_object(self._BUCKET_NAME, object_name, kwargs) + resp = self.client.get_object(self.bucket, object_name, kwargs) file = File(file=io.BytesIO(resp.read()), name=object_name) finally: resp.close() @@ -180,7 +185,7 @@ def stat(self, name: str) -> Union[minio.datatypes.Object, bool]: """Get object information and metadata of an object""" object_name = Path(name).as_posix() try: - obj = self.client.stat_object(self._BUCKET_NAME, object_name=object_name) + obj = self.client.stat_object(self.bucket, object_name=object_name) return obj except (minio.error.S3Error, minio.error.ServerError): return False @@ -196,7 +201,7 @@ def delete(self, name: str): :param name: File object name """ object_name = Path(name).as_posix() - self.client.remove_object(bucket_name=self._BUCKET_NAME, object_name=object_name) + self.client.remove_object(bucket_name=self.bucket, object_name=object_name) def exists(self, name: str) -> bool: """Check if an object with name already exists""" @@ -227,18 +232,18 @@ def url(self, name: str): :return: (str) URL to object """ if self.is_bucket_public: - return f'{self.base_url_external}/{self._BUCKET_NAME}/{name}' + return f'{self.base_url_external}/{self.bucket}/{name}' try: if self.same_endpoints: u: str = self.client.presigned_get_object( - bucket_name=self._BUCKET_NAME, + bucket_name=self.bucket, object_name=name.encode('utf-8'), expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days ) else: u: str = self.client_external.presigned_get_object( - bucket_name=self._BUCKET_NAME, + bucket_name=self.bucket, object_name=name.encode('utf-8'), expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days ) @@ -270,7 +275,7 @@ def get_modified_time(self, name: str) -> datetime: name. The datetime will be timezone-aware if USE_TZ=True. """ obj = self.stat(name) - return datetime.fromtimestamp(mktime(obj.last_modified)) + return obj.last_modified @staticmethod def _guess_content_type(file_path_name: str, content: InMemoryUploadedFile): @@ -295,12 +300,12 @@ def same_endpoints(self) -> bool: @property def bucket(self) -> str: """Get the configured bucket's [self.bucket] name""" - return self._BUCKET_NAME + return self.bucket @property def is_bucket_public(self) -> bool: """Check if configured bucket [self.bucket] is public""" - return True if self._BUCKET_NAME in self.PUBLIC_BUCKETS else False + return True if self.bucket in self.PUBLIC_BUCKETS else False def is_minio_available(self) -> MinioServerStatus: """Check if configured MinIO server is available""" From 8514d89e1aca677ad538fa90d4b354069d80b8d5 Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Mon, 19 Jul 2021 23:45:03 +0200 Subject: [PATCH 4/5] Support for STATIC and MEDIA files - Updated the __init__ of `MinioBackend` to meet Django's custom storage system requirements "Django must be able to instantiate your storage system without any arguments" The `MinioBackend` class can be used as a value for `DEFAULT_FILE_STORAGE` - MinioBackendStatic: A new subclass of `MinioBackend` for supporting django-minio-backend as a static files storage - Added missing docstrings throughout the project - DjangoExampleApplication extended by `class GenericAttachment` for demonstrating a `FileField` without an explicit storage backend - DjangoExampleProject settings.py has been extended with the following new values: - MINIO_MEDIA_FILES_BUCKET - MINIO_STATIC_FILES_BUCKET - Updated README.md --- django_minio_backend/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 15e4da7..d2849b9 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -300,7 +300,7 @@ def same_endpoints(self) -> bool: @property def bucket(self) -> str: """Get the configured bucket's [self.bucket] name""" - return self.bucket + return self._BUCKET_NAME @property def is_bucket_public(self) -> bool: From e144016f8024db56157157f3aa512c41bb64773a Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Sat, 24 Jul 2021 02:55:50 +0200 Subject: [PATCH 5/5] Default behaviour improvements and explanations * New settings.py parameter: MINIO_BUCKET_CHECK_ON_SAVE [default=False] * New section in README.md: Behaviour --- DjangoExampleProject/settings.py | 2 +- README.md | 17 ++++++++++++++++- django_minio_backend/models.py | 5 +++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py index dda37aa..3467245 100644 --- a/DjangoExampleProject/settings.py +++ b/DjangoExampleProject/settings.py @@ -171,4 +171,4 @@ ] MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for STATIC_ROOT MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for MEDIA_ROOT -MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE = True # Create bucket if missing, then save +MINIO_BUCKET_CHECK_ON_SAVE = False # Create bucket if missing, then save diff --git a/README.md b/README.md index ad2da9e..200a3e9 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ MINIO_PUBLIC_BUCKETS = [ MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [] # MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT # MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT -MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE = True # Default: True // Creates bucket if missing, then save +MINIO_BUCKET_CHECK_ON_SAVE = True # Default: True // Creates bucket if missing, then save ``` 4. Implement your own Attachment handler and integrate **django-minio-backend**: @@ -157,6 +157,21 @@ In case a bucket is missing or its configuration differs, it gets created and co ### Reference Implementation For a reference implementation, see [Examples](examples). +## Behaviour +The following list summarises the key characteristics of **django-minio-backend**: + * Bucket existence is **not** checked on save by default. + To enable this guard, set `MINIO_BUCKET_CHECK_ON_SAVE = True` in your `settings.py`. + * Bucket existences are **not** checked on Django start by default. + To enable this guard, set `MINIO_CONSISTENCY_CHECK_ON_START = True` in your `settings.py`. + * Many configuration errors are validated through `AppConfig` but not every error can be captured there. + * Files with the same name in the same bucket are **not** replaced on save by default. Django will store the newer file with an altered file name + To allow replacing existing files, pass the `replace_existing=True` kwarg to `MinioBackend`. + For example: `image = models.ImageField(storage=MinioBackend(bucket_name='images-public', replace_existing=True))` + * Depending on your configuration, **django-minio-backend** may communicate over two kind of interfaces: internal and external. + If your `settings.py` defines a different value for `MINIO_ENDPOINT` and `MINIO_EXTERNAL_ENDPOINT`, then the former will be used for internal communication + between Django and MinIO, and the latter for generating URLs for users. This behaviour optimises the network communication. + * The uploaded object's content-type is guessed during save. If `mimetypes.guess_type` fails to determine the correct content-type, then it falls back to `application/octet-stream`. + ## Compatibility * Django 2.2 or later * Python 3.6.0 or later diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index ac01db7..52d059e 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -91,6 +91,7 @@ def __init__(self, self.__MINIO_SECRET_KEY: str = get_setting("MINIO_SECRET_KEY") self.__MINIO_USE_HTTPS: bool = get_setting("MINIO_USE_HTTPS") self.__MINIO_EXTERNAL_ENDPOINT_USE_HTTPS: bool = get_setting("MINIO_EXTERNAL_ENDPOINT_USE_HTTPS", self.__MINIO_USE_HTTPS) + self.__MINIO_BUCKET_CHECK_ON_SAVE: bool = get_setting("MINIO_BUCKET_CHECK_ON_SAVE", False) self.__BASE_URL = ("https://" if self.__MINIO_USE_HTTPS else "http://") + self.__MINIO_ENDPOINT self.__BASE_URL_EXTERNAL = ("https://" if self.__MINIO_EXTERNAL_ENDPOINT_USE_HTTPS else "http://") + self.__MINIO_EXTERNAL_ENDPOINT @@ -133,8 +134,8 @@ def _save(self, file_path_name: str, content: InMemoryUploadedFile) -> str: :param content (InMemoryUploadedFile): File object :return: """ - if get_setting("MINIO_BUCKET_EXISTENCE_CHECK_BEFORE_SAVE", True): - # Check if bucket exists, create if not + if self.__MINIO_BUCKET_CHECK_ON_SAVE: + # Create bucket if not exists self.check_bucket_existence() # Check if object with name already exists; delete if so