diff --git a/DjangoExampleApplication/admin.py b/DjangoExampleApplication/admin.py
index b18b47e..95111de 100644
--- a/DjangoExampleApplication/admin.py
+++ b/DjangoExampleApplication/admin.py
@@ -2,7 +2,7 @@
from django.db.models.query import QuerySet
from django.contrib import admin
from django.core.handlers.wsgi import WSGIRequest
-from .models import PublicAttachment, PrivateAttachment, Image
+from .models import PublicAttachment, PrivateAttachment, Image, GenericAttachment
# https://docs.djangoproject.com/en/2.2/ref/contrib/admin/actions/#writing-action-functions
@@ -32,6 +32,14 @@ class ImageAdmin(admin.ModelAdmin):
actions = [delete_everywhere, ]
+@admin.register(GenericAttachment)
+class GenericAttachmentAdmin(admin.ModelAdmin):
+ list_display = ('id', 'file',)
+ readonly_fields = ('id', )
+ model = GenericAttachment
+ actions = [delete_everywhere, ]
+
+
# Register your models here.
@admin.register(PublicAttachment)
class PublicAttachmentAdmin(admin.ModelAdmin):
diff --git a/DjangoExampleApplication/migrations/0003_genericattachment.py b/DjangoExampleApplication/migrations/0003_genericattachment.py
new file mode 100644
index 0000000..89dd591
--- /dev/null
+++ b/DjangoExampleApplication/migrations/0003_genericattachment.py
@@ -0,0 +1,21 @@
+# Generated by Django 3.2.3 on 2021-07-18 22:07
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('DjangoExampleApplication', '0002_auto_20210313_1049'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='GenericAttachment',
+ fields=[
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('file', models.FileField(upload_to='', verbose_name='Object Upload (to default storage)')),
+ ],
+ ),
+ ]
diff --git a/DjangoExampleApplication/models.py b/DjangoExampleApplication/models.py
index 5de4e5a..2f70f61 100644
--- a/DjangoExampleApplication/models.py
+++ b/DjangoExampleApplication/models.py
@@ -23,13 +23,30 @@ class Image(models.Model):
def delete(self, *args, **kwargs):
"""
- Delete must be overridden because the inherited delete method does not call `self.file.delete()`.
+ Delete must be overridden because the inherited delete method does not call `self.image.delete()`.
"""
# noinspection PyUnresolvedReferences
self.image.delete()
super(Image, self).delete(*args, **kwargs)
+class GenericAttachment(models.Model):
+ """
+ This is for demonstrating uploads to the default file storage
+ """
+ objects = models.Manager()
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ file = models.FileField(verbose_name="Object Upload (to default storage)")
+
+ def delete(self, *args, **kwargs):
+ """
+ Delete must be overridden because the inherited delete method does not call `self.image.delete()`.
+ """
+ # noinspection PyUnresolvedReferences
+ self.image.delete()
+ super(GenericAttachment, self).delete(*args, **kwargs)
+
+
# Create your models here.
class PublicAttachment(models.Model):
def set_file_path_name(self, file_name_ext: str) -> str:
diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py
index d91c3f4..cf1d395 100644
--- a/DjangoExampleProject/settings.py
+++ b/DjangoExampleProject/settings.py
@@ -116,6 +116,8 @@
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.0/howto/static-files/
STATIC_URL = '/static/'
+STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic'
+DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend'
# #################### #
# django_minio_backend #
@@ -154,14 +156,18 @@
MINIO_USE_HTTPS = bool(distutils.util.strtobool(os.getenv("GH_MINIO_USE_HTTPS", "true")))
MINIO_PRIVATE_BUCKETS = [
'django-backend-dev-private',
+ 'my-media-files-bucket',
]
MINIO_PUBLIC_BUCKETS = [
'django-backend-dev-public',
- "t5p2g08k31",
- "7xi7lx9rjh",
+ 't5p2g08k31',
+ '7xi7lx9rjh',
+ 'my-static-files-bucket',
]
MINIO_URL_EXPIRY_HOURS = timedelta(days=1) # Default is 7 days (longest) if not defined
MINIO_CONSISTENCY_CHECK_ON_START = True
MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [
# ('django-backend-dev-private', dummy_policy)
]
+MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for STATIC_ROOT
+MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for MEDIA_ROOT
diff --git a/README.md b/README.md
index 2e3eb3a..56e3c02 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,8 @@ MINIO_PUBLIC_BUCKETS = [
'django-backend-dev-public',
]
MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = []
+# MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT
+# MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT
```
4. Implement your own Attachment handler and integrate **django-minio-backend**:
@@ -79,6 +81,48 @@ python manage.py initialize_buckets
Code reference: [initialize_buckets.py](django_minio_backend/management/commands/initialize_buckets.py).
+### Static Files Support
+**django-minio-backend** allows serving static files from MinIO.
+To learn more about Django static files, see [Managing static files](https://docs.djangoproject.com/en/3.2/howto/static-files/), and [STATICFILES_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#staticfiles-storage).
+
+To enable static files support, update your `settings.py`:
+```python
+STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic'
+MINIO_STATIC_FILES_BUCKET = 'my-static-files-bucket' # replacement for STATIC_ROOT
+# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.:
+# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET)
+# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET)
+```
+
+The value of `STATIC_URL` is ignored, but it must be defined otherwise Django will throw an error.
+
+**IMPORTANT**
+The value set in `MINIO_STATIC_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`,
+otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private.
+
+**Note:** If `MINIO_STATIC_FILES_BUCKET` is not set, the default value (`auto-generated-static-media-files`) will be used.
+
+### Default File Storage Support
+**django-minio-backend** can be configured as a default file storage.
+To learn more, see [DEFAULT_FILE_STORAGE](https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage).
+
+To configure **django-minio-backend** as the default file storage, update your `settings.py`:
+```python
+DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend'
+MINIO_MEDIA_FILES_BUCKET = 'my-media-files-bucket' # replacement for MEDIA_ROOT
+# Add the value of MINIO_STATIC_FILES_BUCKET to one of the pre-configured bucket lists. eg.:
+# MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET)
+# MINIO_PUBLIC_BUCKETS.append(MINIO_STATIC_FILES_BUCKET)
+```
+
+The value of `MEDIA_URL` is ignored, but it must be defined otherwise Django will throw an error.
+
+**IMPORTANT**
+The value set in `MINIO_MEDIA_FILES_BUCKET` must be added either to `MINIO_PRIVATE_BUCKETS` or `MINIO_PUBLIC_BUCKETS`,
+otherwise **django-minio-backend** will raise an exception. This setting determines the privacy of generated file URLs which can be unsigned public or signed private.
+
+**Note:** If `MINIO_MEDIA_FILES_BUCKET` is not set, the default value (`auto-generated-bucket-media-files`) will be used.
+
### Health Check
To check the connection link between Django and MinIO, use the provided `MinioBackend.is_minio_available()` method.
It returns a `MinioServerStatus` instance which can be quickly evaluated as boolean.
@@ -87,7 +131,7 @@ It returns a `MinioServerStatus` instance which can be quickly evaluated as bool
```python
from django_minio_backend import MinioBackend
-minio_available = MinioBackend('').is_minio_available() # An empty string is fine this time
+minio_available = MinioBackend().is_minio_available() # An empty string is fine this time
if minio_available:
print("OK")
else:
diff --git a/django_minio_backend/apps.py b/django_minio_backend/apps.py
index 6d96eed..d08e539 100644
--- a/django_minio_backend/apps.py
+++ b/django_minio_backend/apps.py
@@ -1,5 +1,6 @@
from django.apps import AppConfig
from .utils import get_setting, ConfigurationError
+from .models import MinioBackendStatic
__all__ = ['DjangoMinioBackendConfig', ]
@@ -20,3 +21,9 @@ def ready(self):
external_use_https = get_setting('MINIO_EXTERNAL_ENDPOINT_USE_HTTPS')
if (external_address and external_use_https is None) or (not external_address and external_use_https):
raise ConfigurationError('MINIO_EXTERNAL_ENDPOINT must be configured together with MINIO_EXTERNAL_ENDPOINT_USE_HTTPS')
+
+ # Validate static storage and default storage configurations
+ staticfiles_storage: str = get_setting('STATICFILES_STORAGE')
+ if staticfiles_storage.endswith(MinioBackendStatic.__name__):
+ mbs = MinioBackendStatic()
+ mbs.check_bucket_existence()
diff --git a/django_minio_backend/management/commands/initialize_buckets.py b/django_minio_backend/management/commands/initialize_buckets.py
index 508fee7..0687006 100644
--- a/django_minio_backend/management/commands/initialize_buckets.py
+++ b/django_minio_backend/management/commands/initialize_buckets.py
@@ -25,7 +25,7 @@ def handle(self, *args, **options):
self.stdout.write(
f"Bucket ({m.bucket}) policy has been set to public", ending='\n') if not silenced else None
- c = MinioBackend('') # Client
+ c = MinioBackend() # Client
for policy_tuple in get_setting('MINIO_POLICY_HOOKS', []):
bucket, policy = policy_tuple
c.set_bucket_policy(bucket, policy)
diff --git a/django_minio_backend/management/commands/is_minio_available.py b/django_minio_backend/management/commands/is_minio_available.py
index ef938d0..4d08ba6 100644
--- a/django_minio_backend/management/commands/is_minio_available.py
+++ b/django_minio_backend/management/commands/is_minio_available.py
@@ -9,7 +9,7 @@ def add_arguments(self, parser):
parser.add_argument('--silenced', action='store_true', default=False, help='No console messages')
def handle(self, *args, **options):
- m = MinioBackend('') # no configured bucket
+ m = MinioBackend() # use default storage
silenced = options.get('silenced')
self.stdout.write(f"Checking the availability of MinIO at {m.base_url}\n") if not silenced else None
diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py
index 53455a3..c7f93ff 100644
--- a/django_minio_backend/models.py
+++ b/django_minio_backend/models.py
@@ -1,3 +1,11 @@
+"""
+django-minio-backend
+A MinIO-compatible custom storage backend for Django
+
+References:
+ * https://github.com/minio/minio-py
+ * https://docs.djangoproject.com/en/3.2/howto/custom-file-storage/
+"""
import io
import json
import mimetypes
@@ -7,13 +15,13 @@
from time import mktime
from typing import Union, List
-# noinspection PyPackageRequirements minIO_requirement
+# noinspection PyPackageRequirements MinIO_requirement
import certifi
import minio
import minio.datatypes
import minio.error
import minio.helpers
-# noinspection PyPackageRequirements minIO_requirement
+# noinspection PyPackageRequirements MinIO_requirement
import urllib3
from django.core.files import File
from django.core.files.storage import Storage
@@ -27,11 +35,16 @@
def get_iso_date() -> str:
+ """Get current date in ISO8601 format [year-month-day] as string"""
now = datetime.utcnow().replace(tzinfo=utc)
return f"{now.year}-{now.month}-{now.day}"
def iso_date_prefix(_, file_name_ext: str) -> str:
+ """
+ Get filename prepended with current date in ISO8601 format [year-month-day] as string
+ The date prefix will be the folder's name storing the object e.g.: 2020-12-31/cat.png
+ """
return f"{get_iso_date()}/{file_name_ext}"
@@ -46,12 +59,24 @@ class MinioBackend(Storage):
for the underlying put_object() MinIO SDK method
"""
+ MINIO_MEDIA_FILES_BUCKET = get_setting("MINIO_MEDIA_FILES_BUCKET", default='auto-generated-bucket-media-files')
+ MINIO_STATIC_FILES_BUCKET = get_setting("MINIO_STATIC_FILES_BUCKET", default='auto-generated-bucket-static-files')
+
def __init__(self,
- bucket_name: str,
+ bucket_name: str = '',
*args,
**kwargs):
- self._BUCKET_NAME: str = bucket_name
+ # If bucket_name is not provided, MinioBackend acts as a DEFAULT_FILE_STORAGE
+ # The automatically selected bucket is MINIO_MEDIA_FILES_BUCKET from settings.py
+ # See https://docs.djangoproject.com/en/3.2/ref/settings/#default-file-storage
+ if not bucket_name or bucket_name == '':
+ self.__CONFIGURED_AS_DEFAULT_STORAGE = True
+ self._BUCKET_NAME: str = self.MINIO_MEDIA_FILES_BUCKET
+ else:
+ self.__CONFIGURED_AS_DEFAULT_STORAGE = False
+ self._BUCKET_NAME: str = bucket_name
+
self._META_ARGS = args
self._META_KWARGS = kwargs
@@ -73,8 +98,18 @@ def __init__(self,
self.PRIVATE_BUCKETS: List[str] = get_setting("MINIO_PRIVATE_BUCKETS", [])
self.PUBLIC_BUCKETS: List[str] = get_setting("MINIO_PUBLIC_BUCKETS", [])
+ # Configure storage type
+ self.__STORAGE_TYPE = 'custom'
+ if self.bucket == self.MINIO_MEDIA_FILES_BUCKET:
+ self.__STORAGE_TYPE = 'media'
+ if self.bucket == self.MINIO_STATIC_FILES_BUCKET:
+ self.__STORAGE_TYPE = 'static'
+
+ if self._BUCKET_NAME not in [*self.PRIVATE_BUCKETS, *self.PUBLIC_BUCKETS]:
+ raise ConfigurationError(f'The configured bucket ({self.bucket}) must be declared either in MINIO_PRIVATE_BUCKETS or MINIO_PUBLIC_BUCKETS')
+
# https://docs.min.io/docs/python-client-api-reference.html
- self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = kwargs.get("http_client", None)
+ self.HTTP_CLIENT: urllib3.poolmanager.PoolManager = self._META_KWARGS.get("http_client", None)
bucket_name_intersection: List[str] = list(set(self.PRIVATE_BUCKETS) & set(self.PUBLIC_BUCKETS))
if bucket_name_intersection:
@@ -127,12 +162,12 @@ def _open(self, object_name, mode='rb', **kwargs):
"""
Implements the Storage._open(name,mode='rb') method
:param name (str): object_name [path to file excluding bucket name which is implied]
- :kwargs (dict): passed on to the underlying minIO client's get_object() method
+ :kwargs (dict): passed on to the underlying MinIO client's get_object() method
"""
resp: urllib3.response.HTTPResponse = urllib3.response.HTTPResponse()
if mode != 'rb':
- raise ValueError('Files retrieved from minIO are read-only. Use save() method to override contents')
+ raise ValueError('Files retrieved from MinIO are read-only. Use save() method to override contents')
try:
resp = self.client.get_object(self._BUCKET_NAME, object_name, kwargs)
file = File(file=io.BytesIO(resp.read()), name=object_name)
@@ -142,6 +177,7 @@ def _open(self, object_name, mode='rb', **kwargs):
return file
def stat(self, name: str) -> Union[minio.datatypes.Object, bool]:
+ """Get object information and metadata of an object"""
object_name = Path(name).as_posix()
try:
obj = self.client.stat_object(self._BUCKET_NAME, object_name=object_name)
@@ -163,16 +199,19 @@ def delete(self, name: str):
self.client.remove_object(bucket_name=self._BUCKET_NAME, object_name=object_name)
def exists(self, name: str) -> bool:
+ """Check if an object with name already exists"""
object_name = Path(name).as_posix()
if self.stat(object_name):
return True
return False
def listdir(self, bucket_name: str):
+ """List all objects in a bucket"""
objects = self.client.list_objects(bucket_name=bucket_name, recursive=True)
return [(obj.object_name, obj) for obj in objects]
def size(self, name: str) -> int:
+ """Get an object's size"""
object_name = Path(name).as_posix()
obj = self.stat(object_name)
if obj:
@@ -208,21 +247,22 @@ def url(self, name: str):
raise ConnectionError("Couldn't connect to Minio. Check django_minio_backend parameters in Django-Settings")
def path(self, name):
- raise NotImplementedError("The minIO storage system doesn't support absolute paths.")
+ """The MinIO storage system doesn't support absolute paths"""
+ raise NotImplementedError("The MinIO storage system doesn't support absolute paths.")
def get_accessed_time(self, name: str) -> datetime:
"""
Return the last accessed time (as a datetime) of the file specified by
name. The datetime will be timezone-aware if USE_TZ=True.
"""
- raise NotImplementedError('minIO does not store last accessed time')
+ raise NotImplementedError('MinIO does not store last accessed time')
def get_created_time(self, name: str) -> datetime:
"""
Return the creation time (as a datetime) of the file specified by name.
The datetime will be timezone-aware if USE_TZ=True.
"""
- raise NotImplementedError('minIO does not store creation time')
+ raise NotImplementedError('MinIO does not store creation time')
def get_modified_time(self, name: str) -> datetime:
"""
@@ -254,13 +294,16 @@ def same_endpoints(self) -> bool:
@property
def bucket(self) -> str:
+ """Get the configured bucket's [self.bucket] name"""
return self._BUCKET_NAME
@property
def is_bucket_public(self) -> bool:
+ """Check if configured bucket [self.bucket] is public"""
return True if self._BUCKET_NAME in self.PUBLIC_BUCKETS else False
def is_minio_available(self) -> MinioServerStatus:
+ """Check if configured MinIO server is available"""
if not self.__MINIO_ENDPOINT:
mss = MinioServerStatus(None)
mss.add_message('MINIO_ENDPOINT is not configured in Django settings')
@@ -282,6 +325,7 @@ def is_minio_available(self) -> MinioServerStatus:
@property
def client(self) -> minio.Minio:
+ """Get handle to an (already) instantiated minio.Minio instance (for internal URL access)"""
if not self.__CLIENT:
self.new_client()
return self.__CLIENT
@@ -289,6 +333,7 @@ def client(self) -> minio.Minio:
@property
def client_external(self) -> minio.Minio:
+ """Get handle to an (already) instantiated minio.Minio instance (for external URL access)"""
if not self.__CLIENT_EXTERNAL:
self.new_client(internal=False)
return self.__CLIENT_EXTERNAL
@@ -296,10 +341,12 @@ def client_external(self) -> minio.Minio:
@property
def base_url(self) -> str:
+ """Get internal base URL to MinIO"""
return self.__BASE_URL
@property
def base_url_external(self) -> str:
+ """Get external base URL to MinIO"""
return self.__BASE_URL_EXTERNAL
def new_client(self, internal: bool = True):
@@ -328,18 +375,22 @@ def new_client(self, internal: bool = True):
# MAINTENANCE
def check_bucket_existence(self):
+ """Check if configured bucket [self.bucket] exists"""
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(bucket_name=self.bucket)
def check_bucket_existences(self): # Execute this handler upon starting Django to make sure buckets exist
+ """Check if all buckets configured in settings.py do exist. If not, create them"""
for bucket in [*self.PUBLIC_BUCKETS, *self.PRIVATE_BUCKETS]:
if not self.client.bucket_exists(bucket):
self.client.make_bucket(bucket_name=bucket)
def set_bucket_policy(self, bucket: str, policy: dict):
+ """Set a custom bucket policy"""
self.client.set_bucket_policy(bucket_name=bucket, policy=json.dumps(policy))
def set_bucket_to_public(self):
+ """Set bucket policy to be public. It can be then accessed via public URLs"""
policy_public_read_only = {"Version": "2012-10-17",
"Statement": [
{
@@ -365,3 +416,29 @@ def set_bucket_to_public(self):
}
]}
self.set_bucket_policy(self.bucket, policy_public_read_only)
+
+
+@deconstructible
+class MinioBackendStatic(MinioBackend):
+ """
+ MinIO-compatible Django custom storage system for Django static files.
+ The used bucket can be configured in settings.py through `MINIO_STATIC_FILES_BUCKET`
+ :arg *args: Should not be used for static files. It's here for compatibility only
+ :arg **kwargs: Should not be used for static files. It's here for compatibility only
+ """
+ def __init__(self, *args, **kwargs):
+ super().__init__(self.MINIO_STATIC_FILES_BUCKET, *args, **kwargs)
+ self.check_bucket_existence() # make sure the `MINIO_STATIC_FILES_BUCKET` exists
+ self.set_bucket_to_public() # the static files bucket must be publicly available
+
+ def path(self, name):
+ """The MinIO storage system doesn't support absolute paths"""
+ raise NotImplementedError("The MinIO storage system doesn't support absolute paths.")
+
+ def get_accessed_time(self, name: str):
+ """MinIO does not store last accessed time"""
+ raise NotImplementedError('MinIO does not store last accessed time')
+
+ def get_created_time(self, name: str):
+ """MinIO does not store creation time"""
+ raise NotImplementedError('MinIO does not store creation time')
diff --git a/django_minio_backend/utils.py b/django_minio_backend/utils.py
index 2cee815..bd3811a 100644
--- a/django_minio_backend/utils.py
+++ b/django_minio_backend/utils.py
@@ -61,12 +61,15 @@ def __repr__(self):
class PrivatePublicMixedError(Exception):
+ """Raised on public|private bucket configuration collisions"""
pass
class ConfigurationError(Exception):
+ """Raised on django-minio-backend configuration errors"""
pass
def get_setting(name, default=None):
+ """Get setting from settings.py. Return a default value if not defined"""
return getattr(settings, name, default)