From 5bc21057778f343f51d89024faec9318a76ddbec Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Sat, 1 May 2021 14:17:54 +0200 Subject: [PATCH 1/4] Create stale.yml --- .github/workflows/stale.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..30c3dd9 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,22 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: "30 1 * * *" + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Stale issue message' + stale-pr-message: 'Stale pull request message' + stale-issue-label: 'no-issue-activity' + stale-pr-label: 'no-pr-activity' From b201e7b6f0d98b736e56ae6a5c64da58f8eeacf3 Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Thu, 20 May 2021 23:27:36 +0200 Subject: [PATCH 2/4] (BUGFIX) Corrected the example application's AppConfig --- DjangoExampleApplication/apps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DjangoExampleApplication/apps.py b/DjangoExampleApplication/apps.py index 31616fe..4e93482 100644 --- a/DjangoExampleApplication/apps.py +++ b/DjangoExampleApplication/apps.py @@ -1,5 +1,5 @@ from django.apps import AppConfig -class AttachmentsConfig(AppConfig): - name = 'Attachments' +class DjangoExampleApplicationConfig(AppConfig): + name = 'DjangoExampleApplication' From 418fc13e045a47d07fa3f6a95790ffb0fde4140d Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Sat, 5 Jun 2021 00:35:04 +0200 Subject: [PATCH 3/4] Adding external endpoint configuration (#23) - Added better support for container environments - Host URL used for file access URL generation is now configurable - New configuration parameter: MINIO_EXTERNAL_ENDPOINT - New configuration parameter: MINIO_EXTERNAL_ENDPOINT_USE_HTTPS - New MinioBackend property: same_endpoints - New MinioBackend property: base_url_external - Revert "Create stale.yml" (This reverts commit 5bc21057) --- .github/workflows/stale.yml | 22 ---------------------- DjangoExampleProject/settings.py | 2 ++ README.md | 2 ++ django_minio_backend/apps.py | 8 +++++++- django_minio_backend/models.py | 20 ++++++++++++++++++-- 5 files changed, 29 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 30c3dd9..0000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Mark stale issues and pull requests - -on: - schedule: - - cron: "30 1 * * *" - -jobs: - stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Stale issue message' - stale-pr-message: 'Stale pull request message' - stale-issue-label: 'no-issue-activity' - stale-pr-label: 'no-pr-activity' diff --git a/DjangoExampleProject/settings.py b/DjangoExampleProject/settings.py index 81caca4..d91c3f4 100644 --- a/DjangoExampleProject/settings.py +++ b/DjangoExampleProject/settings.py @@ -147,6 +147,8 @@ ]} MINIO_ENDPOINT = os.getenv("GH_MINIO_ENDPOINT", "play.min.io") +MINIO_EXTERNAL_ENDPOINT = os.getenv("GH_MINIO_EXTERNAL_ENDPOINT", "externalplay.min.io") +MINIO_EXTERNAL_ENDPOINT_USE_HTTPS = bool(distutils.util.strtobool(os.getenv("GH_MINIO_EXTERNAL_ENDPOINT_USE_HTTPS", "true"))) MINIO_ACCESS_KEY = os.getenv("GH_MINIO_ACCESS_KEY", "Q3AM3UQ867SPQQA43P2F") MINIO_SECRET_KEY = os.getenv("GH_MINIO_SECRET_KEY", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") MINIO_USE_HTTPS = bool(distutils.util.strtobool(os.getenv("GH_MINIO_USE_HTTPS", "true"))) diff --git a/README.md b/README.md index f9b3af7..2e3eb3a 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ from datetime import timedelta from typing import List, Tuple MINIO_ENDPOINT = 'minio.your-company.co.uk' +MINIO_EXTERNAL_ENDPOINT = "external-minio.your-company.co.uk" # Default is same as MINIO_ENDPOINT +MINIO_EXTERNAL_ENDPOINT_USE_HTTPS = True # Default is same as MINIO_USE_HTTPS MINIO_ACCESS_KEY = 'yourMinioAccessKey' MINIO_SECRET_KEY = 'yourVeryS3cr3tP4ssw0rd' MINIO_USE_HTTPS = True diff --git a/django_minio_backend/apps.py b/django_minio_backend/apps.py index 90082ac..6d96eed 100644 --- a/django_minio_backend/apps.py +++ b/django_minio_backend/apps.py @@ -1,5 +1,5 @@ from django.apps import AppConfig -from .utils import get_setting +from .utils import get_setting, ConfigurationError __all__ = ['DjangoMinioBackendConfig', ] @@ -14,3 +14,9 @@ def ready(self): from django.core.management import call_command print("Executing consistency checks...") call_command('initialize_buckets', silenced=True) + + # Validate configuration combinations for EXTERNAL ENDPOINT + external_address = bool(get_setting('MINIO_EXTERNAL_ENDPOINT')) + external_use_https = get_setting('MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') + if (external_address and external_use_https is None) or (not external_address and external_use_https): + raise ConfigurationError('MINIO_EXTERNAL_ENDPOINT must be configured together with MINIO_EXTERNAL_ENDPOINT_USE_HTTPS') diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 07e2390..8c5d533 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -59,11 +59,15 @@ def __init__(self, self.__CLIENT: Union[minio.Minio, None] = None self.__MINIO_ENDPOINT: str = get_setting("MINIO_ENDPOINT") + self.__MINIO_EXTERNAL_ENDPOINT: str = get_setting("MINIO_EXTERNAL_ENDPOINT", self.__MINIO_ENDPOINT) self.__MINIO_ACCESS_KEY: str = get_setting("MINIO_ACCESS_KEY") self.__MINIO_SECRET_KEY: str = get_setting("MINIO_SECRET_KEY") self.__MINIO_USE_HTTPS: bool = get_setting("MINIO_USE_HTTPS") + self.__MINIO_EXTERNAL_ENDPOINT_USE_HTTPS: bool = get_setting("MINIO_EXTERNAL_ENDPOINT_USE_HTTPS", self.__MINIO_USE_HTTPS) self.__BASE_URL = ("https://" if self.__MINIO_USE_HTTPS else "http://") + self.__MINIO_ENDPOINT + self.__BASE_URL_EXTERNAL = ("https://" if self.__MINIO_EXTERNAL_ENDPOINT_USE_HTTPS else "http://") + self.__MINIO_EXTERNAL_ENDPOINT + self.__SAME_ENDPOINTS = self.__MINIO_ENDPOINT == self.__MINIO_EXTERNAL_ENDPOINT self.PRIVATE_BUCKETS: List[str] = get_setting("MINIO_PRIVATE_BUCKETS", []) self.PUBLIC_BUCKETS: List[str] = get_setting("MINIO_PUBLIC_BUCKETS", []) @@ -183,14 +187,15 @@ def url(self, name: str): :return: (str) URL to object """ if self.is_bucket_public: - return f'{self.__BASE_URL}/{self._BUCKET_NAME}/{name}' + return f'{self.base_url_external}/{self._BUCKET_NAME}/{name}' try: - return self.client.presigned_get_object( + u: str = self.client.presigned_get_object( bucket_name=self._BUCKET_NAME, object_name=name.encode('utf-8'), expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days ) + return u if self.same_endpoints else u.replace(self.base_url, self.base_url_external, 1) except urllib3.exceptions.MaxRetryError: raise ConnectionError("Couldn't connect to Minio. Check django_minio_backend parameters in Django-Settings") @@ -232,6 +237,13 @@ def _guess_content_type(file_path_name: str, content: InMemoryUploadedFile): MinioBackend """ + @property + def same_endpoints(self) -> bool: + """ + Returns True if (self.__MINIO_ENDPOINT == self.__MINIO_EXTERNAL_ENDPOINT) + """ + return self.__SAME_ENDPOINTS + @property def bucket(self) -> str: return self._BUCKET_NAME @@ -271,6 +283,10 @@ def client(self) -> minio.Minio: def base_url(self) -> str: return self.__BASE_URL + @property + def base_url_external(self) -> str: + return self.__BASE_URL_EXTERNAL + def new_client(self): """ Instantiates a new Minio client and From 3fc7e22f2104a73340057eabc91a426344347680 Mon Sep 17 00:00:00 2001 From: Kristof Daja Date: Tue, 8 Jun 2021 01:10:25 +0200 Subject: [PATCH 4/4] (BUGFIX) Follow-up to commit 418fc13 which generates invalid external URLs --- django_minio_backend/models.py | 41 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/django_minio_backend/models.py b/django_minio_backend/models.py index 8c5d533..53455a3 100644 --- a/django_minio_backend/models.py +++ b/django_minio_backend/models.py @@ -58,6 +58,7 @@ def __init__(self, self._REPLACE_EXISTING = kwargs.get('replace_existing', False) self.__CLIENT: Union[minio.Minio, None] = None + self.__CLIENT_EXTERNAL: Union[minio.Minio, None] = None self.__MINIO_ENDPOINT: str = get_setting("MINIO_ENDPOINT") self.__MINIO_EXTERNAL_ENDPOINT: str = get_setting("MINIO_EXTERNAL_ENDPOINT", self.__MINIO_ENDPOINT) self.__MINIO_ACCESS_KEY: str = get_setting("MINIO_ACCESS_KEY") @@ -190,12 +191,19 @@ def url(self, name: str): return f'{self.base_url_external}/{self._BUCKET_NAME}/{name}' try: - u: str = self.client.presigned_get_object( - bucket_name=self._BUCKET_NAME, - object_name=name.encode('utf-8'), - expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days - ) - return u if self.same_endpoints else u.replace(self.base_url, self.base_url_external, 1) + if self.same_endpoints: + u: str = self.client.presigned_get_object( + bucket_name=self._BUCKET_NAME, + object_name=name.encode('utf-8'), + expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days + ) + else: + u: str = self.client_external.presigned_get_object( + bucket_name=self._BUCKET_NAME, + object_name=name.encode('utf-8'), + expires=get_setting("MINIO_URL_EXPIRY_HOURS", timedelta(days=7)) # Default is 7 days + ) + return u except urllib3.exceptions.MaxRetryError: raise ConnectionError("Couldn't connect to Minio. Check django_minio_backend parameters in Django-Settings") @@ -279,6 +287,13 @@ def client(self) -> minio.Minio: return self.__CLIENT return self.__CLIENT + @property + def client_external(self) -> minio.Minio: + if not self.__CLIENT_EXTERNAL: + self.new_client(internal=False) + return self.__CLIENT_EXTERNAL + return self.__CLIENT_EXTERNAL + @property def base_url(self) -> str: return self.__BASE_URL @@ -287,10 +302,9 @@ def base_url(self) -> str: def base_url_external(self) -> str: return self.__BASE_URL_EXTERNAL - def new_client(self): + def new_client(self, internal: bool = True): """ - Instantiates a new Minio client and - :return: + Instantiates a new Minio client and assigns it to their respective class variable """ # Safety Guards if not self.PRIVATE_BUCKETS or not self.PUBLIC_BUCKETS: @@ -301,13 +315,16 @@ def new_client(self): ) mc = minio.Minio( - endpoint=self.__MINIO_ENDPOINT, + endpoint=self.__MINIO_ENDPOINT if internal else self.__MINIO_EXTERNAL_ENDPOINT, access_key=self.__MINIO_ACCESS_KEY, secret_key=self.__MINIO_SECRET_KEY, - secure=self.__MINIO_USE_HTTPS, + secure=self.__MINIO_USE_HTTPS if internal else self.__MINIO_EXTERNAL_ENDPOINT_USE_HTTPS, http_client=self.HTTP_CLIENT, ) - self.__CLIENT = mc + if internal: + self.__CLIENT = mc + else: + self.__CLIENT_EXTERNAL = mc # MAINTENANCE def check_bucket_existence(self):