diff --git a/DjangoExampleApplication/admin.py b/DjangoExampleApplication/admin.py index b18b47e..95111de 100644 --- a/DjangoExampleApplication/admin.py +++ b/DjangoExampleApplication/admin.py @@ -2,7 +2,7 @@ from django.db.models.query import QuerySet from django.contrib import admin from django.core.handlers.wsgi import WSGIRequest -from .models import PublicAttachment, PrivateAttachment, Image +from .models import PublicAttachment, PrivateAttachment, Image, GenericAttachment # https://docs.djangoproject.com/en/2.2/ref/contrib/admin/actions/#writing-action-functions @@ -32,6 +32,14 @@ class ImageAdmin(admin.ModelAdmin): actions = [delete_everywhere, ] +@admin.register(GenericAttachment) +class GenericAttachmentAdmin(admin.ModelAdmin): + list_display = ('id', 'file',) + readonly_fields = ('id', ) + model = GenericAttachment + actions = [delete_everywhere, ] + + # Register your models here. @admin.register(PublicAttachment) class PublicAttachmentAdmin(admin.ModelAdmin): diff --git a/DjangoExampleApplication/migrations/0003_genericattachment.py b/DjangoExampleApplication/migrations/0003_genericattachment.py new file mode 100644 index 0000000..89dd591 --- /dev/null +++ b/DjangoExampleApplication/migrations/0003_genericattachment.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.3 on 2021-07-18 22:07 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('DjangoExampleApplication', '0002_auto_20210313_1049'), + ] + + operations = [ + migrations.CreateModel( + name='GenericAttachment', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('file', models.FileField(upload_to='', verbose_name='Object Upload (to default storage)')), + ], + ), + ] diff --git a/DjangoExampleApplication/models.py b/DjangoExampleApplication/models.py index 5de4e5a..2f70f61 100644 --- a/DjangoExampleApplication/models.py +++ b/DjangoExampleApplication/models.py @@ -23,13 +23,30 @@ class Image(models.Model): def delete(self, *args, **kwargs): """ - Delete must be overridden because the inherited delete method does not call `self.file.delete()`. + Delete must be overridden because the inherited delete method does not call `self.image.delete()`. """ # noinspection PyUnresolvedReferences self.image.delete() super(Image, self).delete(*args, **kwargs) +class GenericAttachment(models.Model): + """ + This is for demonstrating uploads to the default file storage + """ + objects = models.Manager() + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + file = models.FileField(verbose_name="Object Upload (to default storage)") + + def delete(self, *args, **kwargs): + """ + Delete must be overridden because the inherited delete method does not call `self.image.delete()`. + """ + # noinspection PyUnresolvedReferences + self.image.delete() + super(GenericAttachment, self).delete(*args, **kwargs) + + # Create your models here. class PublicAttachment(models.Model): def set_file_path_name(self, file_name_ext: str) -> str: diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py index d91c3f4..cf1d395 100644 --- a/DjangoExampleProject/settings.py +++ b/DjangoExampleProject/settings.py @@ -116,6 +116,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_URL = '/static/' +STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic' +DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend' # #################### # # django_minio_backend # @@ -154,14 +156,18 @@ MINIO_USE_HTTPS = bool(distutils.util.strtobool(os.getenv("GH_MINIO_USE_HTTPS", "true"))) MINIO_PRIVATE_BUCKETS = [ 'django-backend-dev-private', + 'my-media-files-bucket', ] MINIO_PUBLIC_BUCKETS = [ 'django-backend-dev-public', - "t5p2g08k31", - "7xi7lx9rjh", + 't5p2g08k31', + '7xi7lx9rjh', + 'my-static-files-bucket', ] MINIO_URL_EXPIRY_HOURS = timedelta(days=1) # Default is 7 days (longest) if not defined MINIO_CONSISTENCY_CHECK_ON_START = True MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [ # ('django-backend-dev-private', dummy_policy) ] +MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for STATIC_ROOT +MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for MEDIA_ROOT diff --git a/README.md b/README.md index 2e3eb3a..56e3c02 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ MINIO_PUBLIC_BUCKETS = [ 'django-backend-dev-public', ] MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [] +# MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT +# MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT ``` 4. Implement your own Attachment handler and integrate **django-minio-backend**: @@ -79,6 +81,48 @@ python manage.py initialize_buckets Code reference: [initialize_buckets.py](django_minio_backend/management/commands/initialize_buckets.py). +### Static Files Support +**django-minio-backend** allows serving static files from MinIO. +To learn more about Django static files, see [Managing static files](https://docs.djangoproject.com/en/3.2/howto/static-files/), and [STATICFILES_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#staticfiles-storage). + +To enable static files support, update your `settings.py`: +```python +STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic' +MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT +# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.: +# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +``` + +The value of `STATIC_URL` is ignored, but it must be defined otherwise Django will throw an error. + +**IMPORTANT**
+The value set in `MINIO_STATIC_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, +otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. + +**Note:** If `MINIO_STATIC_FILES_BUCKET` is not set, the default value (`auto-generated-static-media-files`) will be used. + +### Default File Storage Support +**django-minio-backend** can be configured as a default file storage. +To learn more, see [DEFAULT_FILE_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage). + +To configure **django-minio-backend** as the default file storage, update your `settings.py`: +```python +DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend' +MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT +# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.: +# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) +``` + +The value of `MEDIA_URL` is ignored, but it must be defined otherwise Django will throw an error. + +**IMPORTANT**
+The value set in `MINIO_MEDIA_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`, +otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private. + +**Note:** If `MINIO_MEDIA_FILES_BUCKET` is not set, the default value (`auto-generated-bucket-media-files`) will be used. + ### Health Check To check the connection link between Django and MinIO, use the provided `MinioBackend.is_minio_available()` method.
It returns a `MinioServerStatus` instance which can be quickly evaluated as boolean.
@@ -87,7 +131,7 @@ It returns a `MinioServerStatus` instance which can be quickly evaluated as bool ```python from django_minio_backend import MinioBackend -minio_available = MinioBackend('').is_minio_available() # An empty string is fine this time +minio_available = MinioBackend().is_minio_available() # An empty string is fine this time if minio_available: print("OK") else: diff --git a/django_minio_backend/apps.py b/django_minio_backend/apps.py index 6d96eed..d08e539 100644 --- a/django_minio_backend/apps.py +++ b/django_minio_backend/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig from .utils import get_setting, ConfigurationError +from .models import MinioBackendStatic __all__ = ['DjangoMinioBackendConfig', ] @@ -20,3 +21,9 @@ def ready(self): external_use_https = get_setting('MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') if (external_address and external_use_https is None) or (not external_address and external_use_https): raise ConfigurationError('MINIO_EXTERNAL_ENDPOINT must be configured together with MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') + + # Validate static storage and default storage configurations + staticfiles_storage: str = get_setting('STATICFILES_STORAGE') + if staticfiles_storage.endswith(MinioBackendStatic.__name__): + mbs = MinioBackendStatic() + mbs.check_bucket_existence() diff --git a/django_minio_backend/management/commands/initialize_buckets.py b/django_minio_backend/management/commands/initialize_buckets.py index 508fee7..0687006 100644 --- a/django_minio_backend/management/commands/initialize_buckets.py +++ b/django_minio_backend/management/commands/initialize_buckets.py @@ -25,7 +25,7 @@ def handle(self, *args, **options): self.stdout.write( f"Bucket ({m.bucket}) policy has been set to public", ending='\n') if not silenced else None - c = MinioBackend('') # Client + c = MinioBackend() # Client for policy_tuple in get_setting('MINIO_POLICY_HOOKS', []): bucket, policy = policy_tuple c.set_bucket_policy(bucket, policy) diff --git a/django_minio_backend/management/commands/is_minio_available.py b/django_minio_backend/management/commands/is_minio_available.py index ef938d0..4d08ba6 100644 --- a/django_minio_backend/management/commands/is_minio_available.py +++ b/django_minio_backend/management/commands/is_minio_available.py @@ -9,7 +9,7 @@ def add_arguments(self, parser): parser.add_argument('--silenced', action='store_true', default=False, help='No console messages') def handle(self, *args, **options): - m = MinioBackend('') # no configured bucket + m = MinioBackend() # use default storage silenced = options.get('silenced') self.stdout.write(f"Checking the availability of MinIO at {m.base_url}\n") if not silenced else None diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 53455a3..c7f93ff 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -1,3 +1,11 @@ +""" +django-minio-backend +A MinIO-compatible custom storage backend for Django + +References: + * https://github.com/minio/minio-py + * https://docs.djangoproject.com/en/3.2/howto/custom-file-storage/ +""" import io import json import mimetypes @@ -7,13 +15,13 @@ from time import mktime from typing import Union, List -# noinspection PyPackageRequirements minIO_requirement +# noinspection PyPackageRequirements MinIO_requirement import certifi import minio import minio.datatypes import minio.error import minio.helpers -# noinspection PyPackageRequirements minIO_requirement +# noinspection PyPackageRequirements MinIO_requirement import urllib3 from django.core.files import File from django.core.files.storage import Storage @@ -27,11 +35,16 @@ def get_iso_date() -> str: + """Get current date in ISO8601 format [year-month-day] as string""" now = datetime.utcnow().replace(tzinfo=utc) return f"{now.year}-{now.month}-{now.day}" def iso_date_prefix(_, file_name_ext: str) -> str: + """ + Get filename prepended with current date in ISO8601 format [year-month-day] as string + The date prefix will be the folder's name storing the object e.g.: 2020-12-31/cat.png + """ return f"{get_iso_date()}/{file_name_ext}" @@ -46,12 +59,24 @@ class MinioBackend(Storage): for the underlying put_object() MinIO SDK method """ + MINIO_MEDIA_FILES_BUCKET = get_setting("MINIO_MEDIA_FILES_BUCKET", default='auto-generated-bucket-media-files') + MINIO_STATIC_FILES_BUCKET = get_setting("MINIO_STATIC_FILES_BUCKET", default='auto-generated-bucket-static-files') + def __init__(self, - bucket_name: str, + bucket_name: str = '', *args, **kwargs): - self._BUCKET_NAME: str = bucket_name + # If bucket_name is not provided, MinioBackend acts as a DEFAULT_FILE_STORAGE + # The automatically selected bucket is MINIO_MEDIA_FILES_BUCKET from settings.py + # See https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage + if not bucket_name or bucket_name == '': + self.__CONFIGURED_AS_DEFAULT_STORAGE = True + self._BUCKET_NAME: str = self.MINIO_MEDIA_FILES_BUCKET + else: + self.__CONFIGURED_AS_DEFAULT_STORAGE = False + self._BUCKET_NAME: str = bucket_name + self._META_ARGS = args self._META_KWARGS = kwargs @@ -73,8 +98,18 @@ def __init__(self, self.PRIVATE_BUCKETS: List[str] = get_setting("MINIO_PRIVATE_BUCKETS", []) self.PUBLIC_BUCKETS: List[str] = get_setting("MINIO_PUBLIC_BUCKETS", []) + # Configure storage type + self.__STORAGE_TYPE = 'custom' + if self.bucket == self.MINIO_MEDIA_FILES_BUCKET: + self.__STORAGE_TYPE = 'media' + if self.bucket == self.MINIO_STATIC_FILES_BUCKET: + self.__STORAGE_TYPE = 'static' + + if self._BUCKET_NAME not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]: + raise ConfigurationError(f'The configured bucket ({self.bucket}) must be declared either in MINIO_PRIVATE_BUCKETS or MINIO_PUBLIC_BUCKETS') + # https://docs.min.io/docs/python-client-api-reference.html - self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = kwargs.get("http_client", None) + self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = self._META_KWARGS.get("http_client", None) bucket_name_intersection: List[str] = list(set(self.PRIVATE_BUCKETS) & set(self.PUBLIC_BUCKETS)) if bucket_name_intersection: @@ -127,12 +162,12 @@ def _open(self, object_name, mode='rb', **kwargs): """ Implements the Storage._open(name,mode='rb') method :param name (str): object_name [path to file excluding bucket name which is implied] - :kwargs (dict): passed on to the underlying minIO client's get_object() method + :kwargs (dict): passed on to the underlying MinIO client's get_object() method """ resp: urllib3.response.HTTPResponse = urllib3.response.HTTPResponse() if mode != 'rb': - raise ValueError('Files retrieved from minIO are read-only. Use save() method to override contents') + raise ValueError('Files retrieved from MinIO are read-only. Use save() method to override contents') try: resp = self.client.get_object(self._BUCKET_NAME, object_name, kwargs) file = File(file=io.BytesIO(resp.read()), name=object_name) @@ -142,6 +177,7 @@ def _open(self, object_name, mode='rb', **kwargs): return file def stat(self, name: str) -> Union[minio.datatypes.Object, bool]: + """Get object information and metadata of an object""" object_name = Path(name).as_posix() try: obj = self.client.stat_object(self._BUCKET_NAME, object_name=object_name) @@ -163,16 +199,19 @@ def delete(self, name: str): self.client.remove_object(bucket_name=self._BUCKET_NAME, object_name=object_name) def exists(self, name: str) -> bool: + """Check if an object with name already exists""" object_name = Path(name).as_posix() if self.stat(object_name): return True return False def listdir(self, bucket_name: str): + """List all objects in a bucket""" objects = self.client.list_objects(bucket_name=bucket_name, recursive=True) return [(obj.object_name, obj) for obj in objects] def size(self, name: str) -> int: + """Get an object's size""" object_name = Path(name).as_posix() obj = self.stat(object_name) if obj: @@ -208,21 +247,22 @@ def url(self, name: str): raise ConnectionError("Couldn't connect to Minio. Check django_minio_backend parameters in Django-Settings") def path(self, name): - raise NotImplementedError("The minIO storage system doesn't support absolute paths.") + """The MinIO storage system doesn't support absolute paths""" + raise NotImplementedError("The MinIO storage system doesn't support absolute paths.") def get_accessed_time(self, name: str) -> datetime: """ Return the last accessed time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('minIO does not store last accessed time') + raise NotImplementedError('MinIO does not store last accessed time') def get_created_time(self, name: str) -> datetime: """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('minIO does not store creation time') + raise NotImplementedError('MinIO does not store creation time') def get_modified_time(self, name: str) -> datetime: """ @@ -254,13 +294,16 @@ def same_endpoints(self) -> bool: @property def bucket(self) -> str: + """Get the configured bucket's [self.bucket] name""" return self._BUCKET_NAME @property def is_bucket_public(self) -> bool: + """Check if configured bucket [self.bucket] is public""" return True if self._BUCKET_NAME in self.PUBLIC_BUCKETS else False def is_minio_available(self) -> MinioServerStatus: + """Check if configured MinIO server is available""" if not self.__MINIO_ENDPOINT: mss = MinioServerStatus(None) mss.add_message('MINIO_ENDPOINT is not configured in Django settings') @@ -282,6 +325,7 @@ def is_minio_available(self) -> MinioServerStatus: @property def client(self) -> minio.Minio: + """Get handle to an (already) instantiated minio.Minio instance (for internal URL access)""" if not self.__CLIENT: self.new_client() return self.__CLIENT @@ -289,6 +333,7 @@ def client(self) -> minio.Minio: @property def client_external(self) -> minio.Minio: + """Get handle to an (already) instantiated minio.Minio instance (for external URL access)""" if not self.__CLIENT_EXTERNAL: self.new_client(internal=False) return self.__CLIENT_EXTERNAL @@ -296,10 +341,12 @@ def client_external(self) -> minio.Minio: @property def base_url(self) -> str: + """Get internal base URL to MinIO""" return self.__BASE_URL @property def base_url_external(self) -> str: + """Get external base URL to MinIO""" return self.__BASE_URL_EXTERNAL def new_client(self, internal: bool = True): @@ -328,18 +375,22 @@ def new_client(self, internal: bool = True): # MAINTENANCE def check_bucket_existence(self): + """Check if configured bucket [self.bucket] exists""" if not self.client.bucket_exists(self.bucket): self.client.make_bucket(bucket_name=self.bucket) def check_bucket_existences(self): # Execute this handler upon starting Django to make sure buckets exist + """Check if all buckets configured in settings.py do exist. If not, create them""" for bucket in [*self.PUBLIC_BUCKETS, *self.PRIVATE_BUCKETS]: if not self.client.bucket_exists(bucket): self.client.make_bucket(bucket_name=bucket) def set_bucket_policy(self, bucket: str, policy: dict): + """Set a custom bucket policy""" self.client.set_bucket_policy(bucket_name=bucket, policy=json.dumps(policy)) def set_bucket_to_public(self): + """Set bucket policy to be public. It can be then accessed via public URLs""" policy_public_read_only = {"Version": "2012-10-17", "Statement": [ { @@ -365,3 +416,29 @@ def set_bucket_to_public(self): } ]} self.set_bucket_policy(self.bucket, policy_public_read_only) + + +@deconstructible +class MinioBackendStatic(MinioBackend): + """ + MinIO-compatible Django custom storage system for Django static files. + The used bucket can be configured in settings.py through `MINIO_STATIC_FILES_BUCKET` + :arg *args: Should not be used for static files. It's here for compatibility only + :arg **kwargs: Should not be used for static files. It's here for compatibility only + """ + def __init__(self, *args, **kwargs): + super().__init__(self.MINIO_STATIC_FILES_BUCKET, *args, **kwargs) + self.check_bucket_existence() # make sure the `MINIO_STATIC_FILES_BUCKET` exists + self.set_bucket_to_public() # the static files bucket must be publicly available + + def path(self, name): + """The MinIO storage system doesn't support absolute paths""" + raise NotImplementedError("The MinIO storage system doesn't support absolute paths.") + + def get_accessed_time(self, name: str): + """MinIO does not store last accessed time""" + raise NotImplementedError('MinIO does not store last accessed time') + + def get_created_time(self, name: str): + """MinIO does not store creation time""" + raise NotImplementedError('MinIO does not store creation time') diff --git a/django_minio_backend/utils.py b/django_minio_backend/utils.py index 2cee815..bd3811a 100644 --- a/django_minio_backend/utils.py +++ b/django_minio_backend/utils.py @@ -61,12 +61,15 @@ def __repr__(self): class PrivatePublicMixedError(Exception): + """Raised on public|private bucket configuration collisions""" pass class ConfigurationError(Exception): + """Raised on django-minio-backend configuration errors""" pass def get_setting(name, default=None): + """Get setting from settings.py. Return a default value if not defined""" return getattr(settings, name, default)