From ecdf30ad37d6d15c3260f0d879e726ac9e187f3e Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 14:38:29 +0100 Subject: [PATCH 01/18] add django-configurations dependency --- requirements.in | 1 + requirements.txt | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/requirements.in b/requirements.in index 61ab412d..75037b28 100644 --- a/requirements.in +++ b/requirements.in @@ -7,3 +7,4 @@ psycopg2-binary django-registration django-friendly-tag-loader fs +django-configurations diff --git a/requirements.txt b/requirements.txt index 93045299..42f2c742 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=requirements.txt requirements.in +# pip-compile requirements.in # -appdirs==1.4.3 # via fs +appdirs==1.4.4 # via fs confusable-homoglyphs==3.2.0 # via django-registration +django-configurations==2.2 # via -r requirements.in django-friendly-tag-loader==1.3.1 # via -r requirements.in -django-registration==3.1 # via -r requirements.in +django-registration==3.1.1 # via -r requirements.in django-widget-tweaks==1.4.8 # via -r requirements.in -django==2.2.13 # via -r requirements.in, django-registration, djangorestframework -djangorestframework==3.11.0 # via -r requirements.in +django==2.2.17 # via -r requirements.in, django-registration, djangorestframework +djangorestframework==3.12.2 # via -r requirements.in fasteners==0.15 # via -r requirements.in fs==2.4.11 # via -r requirements.in monotonic==1.5 # via fasteners -pillow==7.1.2 # via -r requirements.in -psycopg2-binary==2.8.5 # via -r requirements.in -pytz==2020.1 # via django, fs -six==1.14.0 # via fasteners, fs -sqlparse==0.3.1 # via django +pillow==8.0.1 # via -r requirements.in +psycopg2-binary==2.8.6 # via -r requirements.in +pytz==2020.4 # via django, fs +six==1.15.0 # via django-configurations, fasteners, fs +sqlparse==0.4.1 # via django # The following packages are considered to be unsafe in a requirements file: # setuptools From f00c0ae111d4a483880f692cce17ccef2844acd3 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 15:47:35 +0100 Subject: [PATCH 02/18] switch settings to django-configurations --- .gitignore | 1 - imagetagger/imagetagger/settings.py | 237 ++++++++++++++++++++ imagetagger/imagetagger/settings.py.example | 129 ----------- imagetagger/imagetagger/settings_base.py | 154 ------------- imagetagger/imagetagger/wsgi.py | 3 +- imagetagger/manage.py | 3 +- 6 files changed, 241 insertions(+), 286 deletions(-) create mode 100644 imagetagger/imagetagger/settings.py delete mode 100644 imagetagger/imagetagger/settings.py.example delete mode 100644 imagetagger/imagetagger/settings_base.py diff --git a/.gitignore b/.gitignore index 6fcd1562..71e567fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.swp *.swo -settings.py # ---> Python # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py new file mode 100644 index 00000000..86a12263 --- /dev/null +++ b/imagetagger/imagetagger/settings.py @@ -0,0 +1,237 @@ +import os +import typing +from os.path import join as path_join +from configurations import Configuration, values +from django.contrib import messages +from django.core.exceptions import ImproperlyConfigured + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def is_in_docker() -> bool: + return os.getenv('IN_DOCKER', '') != '' + + +class Base(Configuration): + ######################################################################## + # + # General django settings + # https://docs.djangoproject.com/en/3.1/topics/settings/ + # + ######################################################################## + INSTALLED_APPS = [ + 'imagetagger.annotations', + 'imagetagger.base', + 'imagetagger.images', + 'imagetagger.users', + 'imagetagger.tools', + 'imagetagger.administration', + 'django.contrib.admin', + 'imagetagger.tagger_messages', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'widget_tweaks', + 'friendlytagloader', + ] + + MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.locale.LocaleMiddleware', + ] + + TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'imagetagger.base.context_processors.base_data', + ], + }, + }, + ] + + ROOT_URLCONF = 'imagetagger.urls' + WSGI_APPLICATION = 'imagetagger.wsgi.application' + + FILE_UPLOAD_HANDLERS = [ + "django.core.files.uploadhandler.MemoryFileUploadHandler", + "django.core.files.uploadhandler.TemporaryFileUploadHandler", + ] + + # Password validation + # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators + + AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, + ] + + # Internationalization + # https://docs.djangoproject.com/en/1.10/topics/i18n/ + + USE_I18N = True + + USE_L10N = True + + USE_TZ = True + + AUTH_USER_MODEL = 'users.User' + + PROBLEMS_URL = 'https://github.com/bit-bots/imagetagger/issues' + PROBLEMS_TEXT = '' + + LOGIN_URL = '/user/login/' + LOGIN_REDIRECT_URL = '/images/' + + # Flash Messages + # https://docs.djangoproject.com/en/3.1/ref/contrib/messages/ + + MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' + MESSAGE_TAGS = { + messages.INFO: 'info', + messages.ERROR: 'danger', + messages.WARNING: 'warning', + messages.SUCCESS: 'success', + } + + # Static files (CSS, JavaScript, Images) + # https://docs.djangoproject.com/en/1.10/howto/static-files/ + + STATIC_URL = '/static/' + + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + + ######################################################################## + # + # Custom Imagetagger defined settings + # + ######################################################################## + UPLOAD_NOTICE = 'By uploading images to this tool you accept that the images get published under creative ' \ + 'commons license and confirm that you have the permission to do so.' + + EXPORT_SEPARATOR = '|' + + FS_URL = os.path.dirname(BASE_DIR) + TMP_FS_URL = 'temp://imagetagger' + + IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) + TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) + TOOLS_PATH = 'tools' # the path to the folder with the tools relative to the filesystem root (see FS_URL) + + # filename extension of accepted imagefiles + IMAGE_EXTENSION = { + 'png', + 'jpeg', + } + + # Sets the default expire time for new messages in days + DEFAULT_EXPIRE_TIME = 7 + + # Sets the default number of messages per page + MESSAGES_PER_PAGE = 10 + + ACCOUNT_ACTIVATION_DAYS = 7 + + ######################################################################## + # + # Computed properties and hooks + # + ######################################################################## + @property + def DATABASES(self): + """https://docs.djangoproject.com/en/1.10/ref/settings/#databases""" + return { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'HOST': self.DB_HOST, + 'PORT': self.DB_PORT, + 'NAME': self.DB_NAME, + 'USER': self.DB_USER, + 'PASSWORD': self.DB_PASSWORD + } + } + + @classmethod + def post_setup(cls): + super().post_setup() + + if cls.SENTRY_REPORTING_ENABLED: + try: + import sentry_sdk + from sentry_sdk.integrations.django import DjangoIntegration + sentry_sdk.init( + dsn=cls.SENTRY_DSN, + integrations=[DjangoIntegration()], + # If you wish to associate users to errors you may enable sending PII data. + send_default_pii=cls.SENTRY_SEND_DEFAULT_PII + ) + except ImportError: + raise ImproperlyConfigured("Could not import sentry although the server is configured to use it") + + ######################################################################## + # + # User-adaptable settings + # + ######################################################################## + DEBUG = values.BooleanValue(environ_prefix='IT', default=False) + SECRET_KEY = values.SecretValue(environ_prefix='IT') + DB_HOST = values.Value(environ_prefix='IT', environ_required=True) + DB_PORT = values.PositiveIntegerValue(environ_prefix='IT', default=5432) + DB_NAME = values.Value(environ_prefix='IT', default='imagetagger') + DB_USER = values.Value(environ_prefix='IT', default=DB_NAME) + DB_PASSWORD = values.Value(environ_prefix='IT', default=DB_USER) + ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', environ_required=True) + LANGUAGE_CODE = values.Value(environ_prefix='IT', default='en-us') + TIME_ZONE = values.Value(environ_prefix='IT', default='Europe/Berlin') + STATIC_ROOT = values.Value(environ_prefix='IT', default=path_join(BASE_DIR, 'static')) + USE_IMPRINT = values.BooleanValue(environ_prefix='IT', default=False) + IMPRINT_NAME = values.Value(environ_prefix='IT') + IMPRINT_URL = values.Value(environ_prefix='IT') + # the URL where the ImageTagger is hosted e.g. https://imagetagger.bit-bots.de + DOWNLOAD_BASE_URL = values.Value(environ_prefix='IT', environ_required=True) + TOOLS_ENABLED = values.BooleanValue(environ_prefix='IT', default=True) + TOOL_UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='') + ENABLE_ZIP_DOWNLOAD = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) + USE_NGINX_IMAGE_PROVISION = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) + + SENTRY_REPORTING_ENABLED = values.BooleanValue(environ_prefix='IT', default=False) + SENTRY_DSN = values.Value(environ_prefix='IT', environ_required=SENTRY_REPORTING_ENABLED) + SENTRY_SEND_DEFAULT_PII = values.BooleanValue(environ_prefix='IT', default=False) + + +class Dev(Base): + DEBUG = values.BooleanValue(environ_prefix='IT', default=True) + SECRET_KEY = values.Value(environ_prefix='IT', default='DEV-KEY ONLY! DONT USE IN PRODUCTION!') + DB_HOST = values.Value(environ_prefix='IT', default='localhost') + ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', default=['localhost', '127.0.0.1']) + DOWNLOAD_BASE_URL = values.Value(environ_prefix='IT', default='localhost') + + +class Prod(Base): + pass diff --git a/imagetagger/imagetagger/settings.py.example b/imagetagger/imagetagger/settings.py.example deleted file mode 100644 index 96bc15c7..00000000 --- a/imagetagger/imagetagger/settings.py.example +++ /dev/null @@ -1,129 +0,0 @@ -from .settings_base import * -from fs import path - -# Settings for Imagetagger -# Copy this file as settings.py and customise as necessary - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'DEV KEY PLEASE CHANGE IN PRODUCTION INTO SOMETHING RANDOM' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -# Allowed Host headers this site can server -ALLOWED_HOSTS = ['localhost', '127.0.0.1'] - -# Additional installed apps -INSTALLED_APPS += [ - # 'raven.contrib.django.raven_compat', # uncomment if sentry is used -] - -# Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases - -DATABASES = { - 'default': { - # Imagetagger relies on some Postgres features so other Databses will _not_ work - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': 'imagetagger-postgres', - 'NAME': 'imagetagger', - 'PASSWORD': 'imagetagger', - 'USER': 'imagetagger', - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -# TIME_ZONE = 'Europe/Berlin' #Timezone of your server - -STATIC_ROOT = '/var/www/imagetagger' - -# EXPORT_SEPARATOR = '|' - -# IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) -# TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) - -# filename extension of accepted imagefiles -# IMAGE_EXTENSION = { -# 'png', -# 'jpg', -# } - -# defines if images get provided directly via nginx which generally improves imageset download performance -USE_NGINX_IMAGE_PROVISION = True if os.getenv("IN_DOCKER", "") != "" else False - -# The 'report a problem' page on an internal server error can either be a custom url or a text that can be defined here. -# PROBLEMS_URL = 'https://problems.example.com' -# PROBLEMS_TEXT = 'To report a problem, contact admin@example.com' - -USE_IMPRINT = False -IMPRINT_NAME = '' -IMPRINT_URL = '' -UPLOAD_NOTICE = 'By uploading images to this tool you accept that the images get published under creative commons license and confirm that you have the permission to do so.' - -DOWNLOAD_BASE_URL = '' # the URL where the ImageTagger is hosted e.g. https://imagetagger.bit-bots.de - -ACCOUNT_ACTIVATION_DAYS = 7 - -UPLOAD_FS_GROUP = 33 # www-data on debian - -# If enabled, run manage.py runzipdaemon to create the zip files and keep them up to date -ENABLE_ZIP_DOWNLOAD = True if os.getenv("IN_DOCKER", "") != "" else False - -# Test mail functionality by printing mails to console: -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' -TOOLS_ENABLED = True -TOOL_UPLOAD_NOTICE = '' - -IMAGE_PATH = 'images' -TOOLS_PATH = 'tools' - -# Use ldap login -# -# import ldap -# from django_auth_ldap.config import LDAPSearch -# -# AUTHENTICATION_BACKENDS = ( -# 'django_auth_ldap.backend.LDAPBackend', -# 'django.contrib.auth.backends.ModelBackend', -# ) -# -# AUTH_LDAP_SERVER_URI = "ldap_host" -# AUTH_LDAP_BIND_DN = "ldap_bind_dn" -# AUTH_LDAP_BIND_PASSWORD = "ldap_bind_pw" -# AUTH_LDAP_USER_SEARCH = LDAPSearch("ou=People,dc=mafiasi,dc=de", -# ldap.SCOPE_SUBTREE, "(uid=%(user)s)") -# AUTH_LDAP_ALWAYS_UPDATE_USER = True -# -# from django_auth_ldap.config import LDAPSearch, PosixGroupType -# -# AUTH_LDAP_GROUP_SEARCH = LDAPSearch("ou=groups,dc=mafiasi,dc=de", -# ldap.SCOPE_SUBTREE, "(objectClass=posixGroup)" -# ) -# AUTH_LDAP_GROUP_TYPE = PosixGroupType() -# AUTH_LDAP_FIND_GROUP_PERMS = True -# AUTH_LDAP_MIRROR_GROUPS = True -# -# AUTH_LDAP_USER_ATTR_MAP = {"first_name": "givenName", "last_name": "sn", "email": "mail"} -# -# AUTH_LDAP_USER_FLAGS_BY_GROUP = { -# "is_staff": ["cn=Editoren,ou=groups,dc=mafiasi,dc=de", -# "cn=Server-AG,ou=groups,dc=mafiasi,dc=de"], -# "is_superuser": "cn=Server-AG,ou=groups,dc=mafiasi,dc=de" -# } - - -# Usage of sentry for error reporting -# import sentry_sdk -# from sentry_sdk.integrations.django import DjangoIntegration -# -# sentry_sdk.init( -# dsn="https://...", -# integrations=[DjangoIntegration()], -# -# # If you wish to associate users to errors you may enable sending PII data. -# send_default_pii=True -# ) diff --git a/imagetagger/imagetagger/settings_base.py b/imagetagger/imagetagger/settings_base.py deleted file mode 100644 index 6b0667b8..00000000 --- a/imagetagger/imagetagger/settings_base.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Django settings for imagetagger project. - -Generated by 'django-admin startproject' using Django 1.10.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ -""" - -import os -from django.contrib.messages import constants as messages - -FS_URL = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -TMP_FS_URL = 'temp://imagetagger' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False - -ALLOWED_HOSTS = [] - -# Application definition - -INSTALLED_APPS = [ - 'imagetagger.annotations', - 'imagetagger.base', - 'imagetagger.images', - 'imagetagger.users', - 'imagetagger.tools', - 'imagetagger.administration', - 'django.contrib.admin', - 'imagetagger.tagger_messages', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'widget_tweaks', - 'friendlytagloader', -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.locale.LocaleMiddleware', -] - -ROOT_URLCONF = 'imagetagger.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'imagetagger.base.context_processors.base_data', - ], - }, - }, -] - -WSGI_APPLICATION = 'imagetagger.wsgi.application' - -FILE_UPLOAD_HANDLERS = ( - "django.core.files.uploadhandler.MemoryFileUploadHandler", - "django.core.files.uploadhandler.TemporaryFileUploadHandler", -) - -# Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - -AUTH_USER_MODEL = 'users.User' - - -# Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'Europe/Berlin' # Timezone of your server - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -PROBLEMS_URL = 'https://github.com/bit-bots/imagetagger/issues' -PROBLEMS_TEXT = '' - -LOGIN_URL = '/user/login/' -LOGIN_REDIRECT_URL = '/images/' - -MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' -MESSAGE_TAGS = { - messages.INFO: 'info', - messages.ERROR: 'danger', - messages.WARNING: 'warning', - messages.SUCCESS: 'success', -} - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ - -STATIC_URL = '/static/' -STATIC_ROOT = 'static/' - -EXPORT_SEPARATOR = '|' - -IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) -TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) -TOOLS_PATH = 'tools' # the path to the folder with the tools relative to the filesystem root (see FS_URL) -TOOLS_ENABLED = True -TOOL_UPLOAD_NOTICE = '' - -# filename extension of accepted imagefiles -IMAGE_EXTENSION = { - 'png', - 'jpeg', -} - -# Sets the default expire time for new messages in days -DEFAULT_EXPIRE_TIME = 7 - -# Sets the default number of messages per page -MESSAGES_PER_PAGE = 10 diff --git a/imagetagger/imagetagger/wsgi.py b/imagetagger/imagetagger/wsgi.py index 080b3097..5b04acb6 100644 --- a/imagetagger/imagetagger/wsgi.py +++ b/imagetagger/imagetagger/wsgi.py @@ -9,8 +9,9 @@ import os -from django.core.wsgi import get_wsgi_application +from configurations.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imagetagger.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Prod") application = get_wsgi_application() diff --git a/imagetagger/manage.py b/imagetagger/manage.py index b7689a88..42e41ffe 100755 --- a/imagetagger/manage.py +++ b/imagetagger/manage.py @@ -4,8 +4,9 @@ if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imagetagger.settings") + os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") try: - from django.core.management import execute_from_command_line + from configurations.management import execute_from_command_line except ImportError: # The above import may fail for some other reason. Ensure that the # issue is really that Django is missing to avoid masking other From 6b679fe93306d8a30b2341d44817d221b1e0ef3d Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 15:47:04 +0100 Subject: [PATCH 03/18] add pycharm run configurations --- .run/makemigrations.run.xml | 25 +++++++++++++++++++++++++ .run/migrate.run.xml | 25 +++++++++++++++++++++++++ .run/runzipdaemon.run.xml | 25 +++++++++++++++++++++++++ .run/serve.run.xml | 26 ++++++++++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 .run/makemigrations.run.xml create mode 100644 .run/migrate.run.xml create mode 100644 .run/runzipdaemon.run.xml create mode 100644 .run/serve.run.xml diff --git a/.run/makemigrations.run.xml b/.run/makemigrations.run.xml new file mode 100644 index 00000000..a15b200b --- /dev/null +++ b/.run/makemigrations.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/migrate.run.xml b/.run/migrate.run.xml new file mode 100644 index 00000000..b547f375 --- /dev/null +++ b/.run/migrate.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/runzipdaemon.run.xml b/.run/runzipdaemon.run.xml new file mode 100644 index 00000000..0a525470 --- /dev/null +++ b/.run/runzipdaemon.run.xml @@ -0,0 +1,25 @@ + + + + + \ No newline at end of file diff --git a/.run/serve.run.xml b/.run/serve.run.xml new file mode 100644 index 00000000..305ddf8b --- /dev/null +++ b/.run/serve.run.xml @@ -0,0 +1,26 @@ + + + + + \ No newline at end of file From 884bf48a4e9e1c79fa832b7043cb9578f41e9bf8 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 18:50:33 +0100 Subject: [PATCH 04/18] make FS_URL and TMP_FS_URL also configurable --- imagetagger/imagetagger/settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py index 86a12263..c103d64b 100644 --- a/imagetagger/imagetagger/settings.py +++ b/imagetagger/imagetagger/settings.py @@ -137,9 +137,6 @@ class Base(Configuration): EXPORT_SEPARATOR = '|' - FS_URL = os.path.dirname(BASE_DIR) - TMP_FS_URL = 'temp://imagetagger' - IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) TOOLS_PATH = 'tools' # the path to the folder with the tools relative to the filesystem root (see FS_URL) @@ -219,6 +216,8 @@ def post_setup(cls): TOOL_UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='') ENABLE_ZIP_DOWNLOAD = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) USE_NGINX_IMAGE_PROVISION = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) + FS_URL = values.Value(environ_prefix='IT', default=path_join(os.path.dirname(BASE_DIR), 'data')) + TMP_FS_URL = values.Value(environ_prefix='IT', default='temp://imagetagger') SENTRY_REPORTING_ENABLED = values.BooleanValue(environ_prefix='IT', default=False) SENTRY_DSN = values.Value(environ_prefix='IT', environ_required=SENTRY_REPORTING_ENABLED) From bee1917ca02c3413df3d11621b6fab1f814e8233 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 18:54:35 +0100 Subject: [PATCH 05/18] add django system checks for path configuration --- imagetagger/imagetagger/base/__init__.py | 1 + imagetagger/imagetagger/base/checks.py | 40 ++++++++++++++++++++++ imagetagger/imagetagger/base/filesystem.py | 4 +-- imagetagger/imagetagger/settings.py | 3 +- 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 imagetagger/imagetagger/base/checks.py diff --git a/imagetagger/imagetagger/base/__init__.py b/imagetagger/imagetagger/base/__init__.py index e69de29b..f7930fdf 100644 --- a/imagetagger/imagetagger/base/__init__.py +++ b/imagetagger/imagetagger/base/__init__.py @@ -0,0 +1 @@ +from imagetagger.base.checks import * diff --git a/imagetagger/imagetagger/base/checks.py b/imagetagger/imagetagger/base/checks.py new file mode 100644 index 00000000..f74c5ea8 --- /dev/null +++ b/imagetagger/imagetagger/base/checks.py @@ -0,0 +1,40 @@ +from typing import List +from os.path import join, dirname +from django.core.checks import register, CheckMessage, Error +from django.conf import settings +from fs.base import FS +from . import filesystem + + +def _create_check_file(fs: FS, path: str): + fs.makedirs(dirname(path), recreate=True) + fs.create(path, wipe=True) + fs.writebytes(path, bytes('This is a demo file content', encoding='ASCII')) + fs.remove(path) + + +@register +def check_fs_root_config(app_configs, **kwargs) -> List[CheckMessage]: + try: + _create_check_file(filesystem.root(), join(settings.IMAGE_PATH, 'check.tmp.jpeg')) + _create_check_file(filesystem.root(), join(settings.TOOLS_PATH, 'check.tmp.py')) + return [] + except Exception as e: + return [Error( + 'Persistent filesystem is incorrectly configured. Could not create and delete a temporary check file', + hint=f'Check your FS_URL settings (currently {settings.FS_URL})', + obj=e, + )] + + +@register +def check_tmp_fs_config(app_configs, **kwargs) -> List[CheckMessage]: + try: + _create_check_file(filesystem.tmp(), join(settings.TMP_IMAGE_PATH, 'check.tmp.jpeg')) + return [] + except Exception as e: + return [Error( + 'Persistent filesystem is incorrectly configured. Could not create and delete a temporary check file', + hint=f'Check your TMP_FS_URL settings (currently {settings.TMP_FS_URL})', + obj=e, + )] diff --git a/imagetagger/imagetagger/base/filesystem.py b/imagetagger/imagetagger/base/filesystem.py index a99eabe5..eabc1c91 100644 --- a/imagetagger/imagetagger/base/filesystem.py +++ b/imagetagger/imagetagger/base/filesystem.py @@ -6,7 +6,7 @@ def root() -> fs.base.FS: """Get the root filesystem object or create one by opening `settings.FS_URL`.""" if root._instance is None: - root._instance = fs.open_fs(settings.FS_URL) + root._instance = fs.open_fs(settings.FS_URL, writeable=True) return root._instance root._instance = None @@ -15,7 +15,7 @@ def root() -> fs.base.FS: def tmp() -> fs.base.FS: """Get the tmp filesystem object or create one by opening `settings.TMP_FS_URL`.""" if tmp._instance is None: - tmp._instance = fs.open_fs(settings.TMP_FS_URL) + tmp._instance = fs.open_fs(settings.TMP_FS_URL, writeable=True) return tmp._instance tmp._instance = None \ No newline at end of file diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py index c103d64b..4956d4f6 100644 --- a/imagetagger/imagetagger/settings.py +++ b/imagetagger/imagetagger/settings.py @@ -1,5 +1,4 @@ import os -import typing from os.path import join as path_join from configurations import Configuration, values from django.contrib import messages @@ -216,7 +215,7 @@ def post_setup(cls): TOOL_UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='') ENABLE_ZIP_DOWNLOAD = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) USE_NGINX_IMAGE_PROVISION = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) - FS_URL = values.Value(environ_prefix='IT', default=path_join(os.path.dirname(BASE_DIR), 'data')) + FS_URL = values.Value(environ_prefix='IT', default=path_join(BASE_DIR, 'data')) TMP_FS_URL = values.Value(environ_prefix='IT', default='temp://imagetagger') SENTRY_REPORTING_ENABLED = values.BooleanValue(environ_prefix='IT', default=False) From 7458ae8f19fe5168529de2c53a961fe093c93e24 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 19:20:05 +0100 Subject: [PATCH 06/18] move requirements files closer to the python code --- requirements.in => imagetagger/requirements.in | 0 requirements.txt => imagetagger/requirements.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename requirements.in => imagetagger/requirements.in (100%) rename requirements.txt => imagetagger/requirements.txt (100%) diff --git a/requirements.in b/imagetagger/requirements.in similarity index 100% rename from requirements.in rename to imagetagger/requirements.in diff --git a/requirements.txt b/imagetagger/requirements.txt similarity index 100% rename from requirements.txt rename to imagetagger/requirements.txt From f2611252288cfa1c46cb2f23655478ea67dccb04 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Wed, 9 Dec 2020 19:20:32 +0100 Subject: [PATCH 07/18] adapt Dockerfile to new configuration --- Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index fdf3ae3d..db721a47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && \ # add requirements file WORKDIR /app/src -COPY requirements.txt /app/src/requirements.txt +COPY imagetagger/requirements.txt /app/src/requirements.txt # install python dependencies RUN pip3 install -r /app/src/requirements.txt @@ -22,10 +22,12 @@ RUN apt-get clean # add remaining sources COPY imagetagger /app/src/imagetagger -# confiure runtime environment -RUN mkdir /app/data /app/static /app/config -RUN cp /app/src/imagetagger/imagetagger/settings.py.example /app/config/settings.py -RUN ln -sf /app/config/settings.py /app/src/imagetagger/imagetagger/settings.py +# configure /app/config/imagetagger_settings to be python importable so that one can use their own settings file +RUN mkdir -p /app/data /app/static /app/config/imagetagger_settings +RUN touch /app/config/imagetagger_settings/__init__.py +ENV PYTHONPATH=$PYTHONPATH:/app/config:/app/src/imagetagger + +# configure runtime environment RUN sed -i 's/env python/env python3/g' /app/src/imagetagger/manage.py ARG UID_WWW_DATA=5008 @@ -41,6 +43,7 @@ COPY docker/update_points docker/zip_daemon docker/run /app/bin/ RUN ln -sf /app/bin/* /usr/local/bin ENTRYPOINT ["/usr/local/bin/run"] ENV IN_DOCKER=true +ENV DJANGO_CONFIGURATION=Prod # add image metadata EXPOSE 3008 From b658f8673ddf6bc969c01509f3bc269680272e9f Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 10:30:45 +0100 Subject: [PATCH 08/18] change Dockerfile to be easier 3rd-party configurable --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index db721a47..e7207db9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,10 +22,10 @@ RUN apt-get clean # add remaining sources COPY imagetagger /app/src/imagetagger -# configure /app/config/imagetagger_settings to be python importable so that one can use their own settings file -RUN mkdir -p /app/data /app/static /app/config/imagetagger_settings -RUN touch /app/config/imagetagger_settings/__init__.py -ENV PYTHONPATH=$PYTHONPATH:/app/config:/app/src/imagetagger +# configure imagetagger.settings_local to be importable but 3rd party providable +RUN mkdir -p /app/data /app/static /app/config/ +RUN touch /app/config/settings.py +RUN ln -sf /app/config/settings.py /app/src/imagetagger/imagetagger/settings_local.py # configure runtime environment RUN sed -i 's/env python/env python3/g' /app/src/imagetagger/manage.py @@ -44,6 +44,7 @@ RUN ln -sf /app/bin/* /usr/local/bin ENTRYPOINT ["/usr/local/bin/run"] ENV IN_DOCKER=true ENV DJANGO_CONFIGURATION=Prod +ENV IT_FS_URL=/app/data # add image metadata EXPOSE 3008 From ac6896dc30bb4a0140e16974e6fa334b6fb94bbb Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 10:31:15 +0100 Subject: [PATCH 09/18] fix django configuration selection in wsgi script --- imagetagger/imagetagger/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagetagger/imagetagger/wsgi.py b/imagetagger/imagetagger/wsgi.py index 5b04acb6..595fd039 100644 --- a/imagetagger/imagetagger/wsgi.py +++ b/imagetagger/imagetagger/wsgi.py @@ -12,6 +12,6 @@ from configurations.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "imagetagger.settings") -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Prod") +os.environ.setdefault("DJANGO_CONFIGURATION", "Prod") application = get_wsgi_application() From 821c1b86da06fc2984ec1c1b0c6ed0f57c93623a Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 10:51:50 +0100 Subject: [PATCH 10/18] adapt kustomization to new configuration --- k8s/web_app.yml | 11 +++++------ kustomization.yml | 9 +++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/k8s/web_app.yml b/k8s/web_app.yml index 89a8347a..a5fa6cf5 100644 --- a/k8s/web_app.yml +++ b/k8s/web_app.yml @@ -30,9 +30,6 @@ spec: tier: web spec: volumes: - - name: config - configMap: - name: imagetagger-web - name: data persistentVolumeClaim: claimName: imagetagger-image-data @@ -44,10 +41,12 @@ spec: containerPort: 3008 - name: http containerPort: 80 + envFrom: + - configMapRef: + name: imagetagger-web + - configMapRef: + name: imagetagger-postgres volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - name: data mountPath: /app/data livenessProbe: diff --git a/kustomization.yml b/kustomization.yml index 290734e0..ef203336 100644 --- a/kustomization.yml +++ b/kustomization.yml @@ -12,8 +12,13 @@ resources: configMapGenerator: - name: imagetagger-web - files: - - "settings.py=./imagetagger/imagetagger/settings.py.example" + literals: + # service hostname + - "IT_DB_HOST=imagetagger-postgres" + # references to below postgres configMap which is also mounted in the pod + - "IT_DB_USER=$(POSTGRES_USER)" + - "IT_DB_PASSWORD=$(POSTGRES_PASSWORD)" + - "IT_DB_NAME=$(POSTGRES_DB)" - name: imagetagger-postgres literals: - "POSTGRES_PASSWORD=imagetagger" From e52d1f91505c728572ab191dfce8bf5000f72c53 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 11:27:33 +0100 Subject: [PATCH 11/18] fix staticfile serving in docker --- Dockerfile | 5 ++--- docker/run | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e7207db9..2260ae9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get clean COPY imagetagger /app/src/imagetagger # configure imagetagger.settings_local to be importable but 3rd party providable -RUN mkdir -p /app/data /app/static /app/config/ +RUN mkdir -p /app/data /app/config/ RUN touch /app/config/settings.py RUN ln -sf /app/config/settings.py /app/src/imagetagger/imagetagger/settings_local.py @@ -35,8 +35,6 @@ ARG GID_WWW_DATA=33 RUN usermod -u $UID_WWW_DATA -g $GID_WWW_DATA -d /app/data/ www-data RUN chown -R www-data /app -RUN /app/src/imagetagger/manage.py collectstatic --no-input - COPY docker/uwsgi_imagetagger.ini /etc/uwsgi/imagetagger.ini COPY docker/nginx.conf /etc/nginx/sites-enabled/default COPY docker/update_points docker/zip_daemon docker/run /app/bin/ @@ -45,6 +43,7 @@ ENTRYPOINT ["/usr/local/bin/run"] ENV IN_DOCKER=true ENV DJANGO_CONFIGURATION=Prod ENV IT_FS_URL=/app/data +ENV IT_STATIC_ROOT=/var/www/imagetagger # add image metadata EXPOSE 3008 diff --git a/docker/run b/docker/run index bf12f668..2f8e8b9f 100755 --- a/docker/run +++ b/docker/run @@ -1,6 +1,7 @@ #!/bin/bash set -e +/app/src/imagetagger/manage.py collectstatic --no-input /app/src/imagetagger/manage.py migrate nginx From 192c4505cd4003ac244b35a161314614f937bbad Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 11:47:12 +0100 Subject: [PATCH 12/18] fix email configuration in production --- imagetagger/imagetagger/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py index 4956d4f6..7a708e03 100644 --- a/imagetagger/imagetagger/settings.py +++ b/imagetagger/imagetagger/settings.py @@ -232,4 +232,8 @@ class Dev(Base): class Prod(Base): - pass + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + EMAIL_HOST = values.Value(environ_prefix='IT', environ_required=True) + EMAIL_PORT = values.Value(environ_prefix='IT') + EMAIL_HOST_USER = values.Value(environ_prefix='IT') + EMAIL_HOST_PASSWORD = values.Value(environ_prefix='IT') From 904d89ae1f95f27e0458abe731722bc39e0c3ede Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 10 Dec 2020 13:00:22 +0100 Subject: [PATCH 13/18] update README to inform about new deployment and configuration --- README.md | 259 +++++++++++++++++++++++++++++------------------------- 1 file changed, 140 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index dda60c00..bcc956cc 100644 --- a/README.md +++ b/README.md @@ -40,131 +40,152 @@ FIEDLER, Niklas, et al. [ImageTagger: An Open Source Online Platform for Collabo organization={Springer} } ``` -## Upgrade -``` -pip install -U -r requirements.txt -./manage.py migrate -``` - -for additional steps on some releases see instructions -in [UPGRADE.md](https://github.com/bit-bots/imagetagger/blob/master/UPGRADE.md) - -## Install - -Checkout the latest release: - -``` -git checkout v0.x -``` - -In our production Senty is used for error reporting (pip install sentry-sdk). -django-auth-ldap is used for login via ldap -uwsgi is used to serve the app to nginx - -Install Python Dependencies: - -``` -pip3 install -r requirements.txt -``` - -Copy settings.py.example to settings.py in the imagetagger folder: - -``` -cp imagetagger/settings.py.example imagetagger/settings.py -``` - -and customize the settings.py. - -The following settings should probably be changed: - -+ The secret key -+ The DEBUG setting -+ The ALLOWED\_HOSTS -+ The database settings -+ The UPLOAD\_FS\_GROUP to the id of the group that should access and create the uploaded images - -For the database, postgresql is used. Install it by running `sudo apt install postgresql` on Debian based operating systems. A default database cluster is automatically initialized. - -Other systems may require different commands to install the package and the database cluster may -have to be initialized manually (e.g. using `sudo -iu postgres initdb --locale en_US.UTF-8 -D '/var/lib/postgres/data'`). - -To start the postgresql server, run `sudo systemctl start postgresql.service`. If the server should always be started on boot, run `sudo systemctl enable postgresql.service`. - -Then, create the user and the database by running - -`sudo -iu postgres psql` - -and then, in the postgres environment - -``` -CREATE USER imagetagger PASSWORD 'imagetagger'; -CREATE DATABASE imagetagger WITH OWNER imagetagger ENCODING UTF8; -``` - -where of course the password and the user should be adapted to the ones specified in the database settings in the settings.py. - -To initialize the database, run `./manage.py migrate` - -To create an administrator user, run `./manage.py createsuperuser`. - -`./manage.py runserver` starts the server with the configuration given in the settings.py file. - -To create annotation types, log into the application and click on Administration at the very bottom of the home page. - -For **production** systems it is necessary to run the following commands after each upgrade - -```bash +## Installing and running Imagetagger +Imagetagger can be installed and run locally (best for development), in a docker container or in Kubernetes +(used in our deployment). + +### Locally + +1. #### Install the latest release + + You should probably do this in a [virtual environment](https://virtualenv.pypa.io/en/stable/) + ```shell + git checkout v… + pip3 install -r imagetagger/requirements.txt + ``` + +2. #### Setup a database server + + As a database server [postgresql](https://www.postgresql.org/) is required. + Please seek a guide specific to your operating system on how to install a server and get it running. + + Once postgresql is installed, a user and database need to be set up for imagetagger. + Of course, the user and password can be changed to something else. + ```postgresql + CREATE USER imagetagger PASSWORD 'imagetagger'; + CREATE DATABASE imagetagger WITH OWNER imagetagger; + ``` + +3. #### Configuring ImageTagger to connect to the database + + Please see the lower section about application configuration on how to configure ImageTagger for your specific + database credentials. + +4. #### Initialize the database + + Run the following to create and setup all necessary database tables: + ```shell + ./manage.py migrate + ``` + +5. #### Create a user + + ```shell + export DJANGO_CONFIGURATION=Prod + ./manage.py createsuperuser + ``` + +6. #### Run the server + + ```shell + export DJANGO_CONFIGURATION=Prod + ./manage.py runserver + ``` + +**In a production deployment** it is necessary to run the following commands after each upgrade: +```shell +export DJANGO_CONFIGURATION=Prod ./manage.py migrate ./manage.py compilemessages ./manage.py collectstatic ``` -Our production uwisgi config can be found at https://github.com/fsinfuhh/mafiasi-rkt/blob/master/imagetagger/uwsgi-imagetagger.ini - -Example Nginx Config: - -``` -server { - listen 443; - server_name imagetagger.bit-bots.de; - - ssl_certificate /etc/letsencrypt/certs/imagetagger.bit-bots.de/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/certs/imagetagger.bit-bots.de/privkey.pem; - include /etc/nginx/ssl.conf; - include /etc/nginx/acme.conf; - ssl on; - - client_max_body_size 4G; - - access_log /var/log/nginx/imagetagger.bit-bots.de.access.log; - error_log /var/log/nginx/imagetagger.bit-bots.de.error.log; - - location /static { - expires 1h; - alias /var/www/imagetagger; - } - - location /ngx_static_dn/ { - internal; - alias /srv/data/imagetagger/storage/pictures/; - } - - location / { - include uwsgi_params; - uwsgi_pass 127.0.0.1:4819; - uwsgi_read_timeout 120; - } -} -``` - -If you want to provide zip files of image sets, set `ENABLE_ZIP_DOWNLOAD = True` in your `settings.py`. -A daemon that creates and updates the zip files is necessary, you can start it with `./manage.py runzipdaemon`. -Please take into account that the presence of zip files will double your storage requirement. - -Zip archive download via a script is also possible. The URL is `/images/imageset//download/`. A successful request -returns HTTP 200 OK and the zip file. When the file generation is still in progress, HTTP 202 ACCEPTED is returned. -For an empty image set, HTTP 204 NO CONTENT is returned instead of an empty zip archive. +for additional steps on some releases see instructions in [UPGRADE.md](https://github.com/bit-bots/imagetagger/blob/master/UPGRADE.md) + +### In-Docker + +1. #### Checkout the latest release + + ```shell + git checkout v… + ``` + +2. #### Build the container image + + *Note:* the Dockerfile has been tested with [podman](https://podman.io/) as well. + ```shell + docker build -t imagetagger . + ``` + +3. #### Run the container + + ```shell + docker run -it -p 8000:80 --name imagetagger imagetagger + ``` + + This step will not work out of the box because configuration still needs to be done. + See the lower section about configuring ImageTagger on how to fix this. + +4. #### Create a user + + *Note: This step requires a container running in the background.* + ```shell + docker exec imagetagger /app/src/imagetagger/manage.py createsuperuser + ``` + +#### About the Container + +| Kind | Description | +|---|---| +| Volume | `/app/data` is where persistent data (like images) are stored +| Environment | ImageTagger can mostly be configured via environment variables +| Ports | The container internal webserver listens on port 80 for incoming connections. + +### On Kubernetes + +1. Follow the steps for *In-Docker* on how to build a container image + +2. **Apply kubernetes configuration** + + *Note: This assumes you have access to a kubernetes cluster configured and can use kubectl* + + We use [kustomize](https://kustomize.io/) for management of our kubernetes configuration. + It can be applied by running the following commands: + ```shell + kustomize build . > k8s.yml + kubectl apply -f k8s.yml + ``` + +#### Generated Kubernetes resources + +| Kind | Name | Description +|---|---|--- +| [Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) | imagetagger | The namespace in which each resource resides +| [Deployment](https://kubernetes.io/es/docs/concepts/workloads/controllers/deployment/) + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) | imagetagger-postgres | postgresql server which is required by ImageTagger. The replica count can be dialed down to 0 if an the usage of an external server is desired. +| [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) | imagetagger-database | Where the postgresql server stores its data +| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-postgres | Configuration of the postgresql server. Also available inside the application server deployment so that settings can be [referenced](https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/). +| [Deployment](https://kubernetes.io/es/docs/concepts/workloads/controllers/deployment/) + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) | imagetagger-web | application server. Per default this deployment references the image `imagetagger:local` which is probably not resolvable and should be replaced by a reference to where your previously built container image is available. +| [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) | imagetagger-image-data | Where the application server stores its images (and tools). +| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-web | Configuration of the application server. Mounted as environment variables. + +## Configuration + +ImageTagger is a Django application and uses [django-configurations](https://django-configurations.readthedocs.io/en/stable/) +for better configuration management. + +It is configured to use a *Dev* configuration when running locally and *Prod* when running in a container. +This can be overridden via the environment variable `DJANGO_CONFIGURATION`. + +For a list of available configuration values see [settings.py](https://github.com/bit-bots/imagetagger/blob/master/imagetagger/imagetagger/settings.py). +Towards the bottom is a list of *values*. These are taken from environment variables where the key is the variable name +but with an `IT_` prefix. + +If completely custom configuration is desired, `imagetagger/imagetagger/settings_local.py` can be created in which +a custom configuration class may be created. In docker this file may be located at `/app/config/settings.py` so that +mounting it should be simple. +To use this custom configuration class, the environment variables `DJANGO_SETTINGS_MODULE=imagetagger.settings_local` +and `DJANGO_CONFIGURATION=MyCustomClass` must be set. ## Used dependencies From 696b89fcedc12ce9b3a588363b8c112b9dcd98bf Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Mon, 14 Dec 2020 14:58:10 +0100 Subject: [PATCH 14/18] adapt to PR feedback --- README.md | 60 +++++++++++++++++++++++------ imagetagger/imagetagger/settings.py | 47 +++++++++------------- 2 files changed, 68 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index bcc956cc..f91e1cfa 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,19 @@ If you are participating in RoboCup, you should not install your own instance bu For a short overview of the functions please have a look at the following poster: https://robocup.informatik.uni-hamburg.de/wp-content/uploads/2017/11/imagetagger-poster.pdf +Table of Contents: +- [ImageTagger](#imagetagger) + * [Features](#features) + * [Planned Features](#planned-features) + * [Reference](#reference) + * [Installing and running ImageTagger](#installing-and-running-imagetagger) + + [Locally](#locally) + + [In-Docker](#in-docker) + + [On Kubernetes](#on-kubernetes) + * [Configuration](#configuration) + * [Used dependencies](#used-dependencies) + + ## Features * team creation @@ -27,7 +40,7 @@ For a short overview of the functions please have a look at the following poster ## Reference -This paper describes the Bit-Bots Imagetagger more in depth. Please cite if you use this tool in your research: +This paper describes the Bit-Bots ImageTagger more in depth. Please cite if you use this tool in your research: FIEDLER, Niklas, et al. [ImageTagger: An Open Source Online Platform for Collaborative Image Labeling.](https://robocup.informatik.uni-hamburg.de/wp-content/uploads/2018/11/imagetagger_paper.pdf) In: RoboCup 2018: Robot World Cup XXII. Springer, 2018. @@ -41,12 +54,16 @@ FIEDLER, Niklas, et al. [ImageTagger: An Open Source Online Platform for Collabo } ``` -## Installing and running Imagetagger -Imagetagger can be installed and run locally (best for development), in a docker container or in Kubernetes +## Installing and running ImageTagger +ImageTagger can be installed and run locally (best for development), in a docker container or in Kubernetes (used in our deployment). ### Locally +In some of the following code snippets, the `DJANGO_CONFIGURATION` environment variable is exported. +This defines the type of deployment by selecting one of our predefined configuration presets. +If ImageTagger is running in a development environment, no export is necessary. + 1. #### Install the latest release You should probably do this in a [virtual environment](https://virtualenv.pypa.io/en/stable/) @@ -60,7 +77,7 @@ Imagetagger can be installed and run locally (best for development), in a docker As a database server [postgresql](https://www.postgresql.org/) is required. Please seek a guide specific to your operating system on how to install a server and get it running. - Once postgresql is installed, a user and database need to be set up for imagetagger. + Once postgresql is installed, a user and database need to be set up for ImageTagger. Of course, the user and password can be changed to something else. ```postgresql CREATE USER imagetagger PASSWORD 'imagetagger'; @@ -69,7 +86,7 @@ Imagetagger can be installed and run locally (best for development), in a docker 3. #### Configuring ImageTagger to connect to the database - Please see the lower section about application configuration on how to configure ImageTagger for your specific + Please see the lower [Configuration](#configuration) section on how to configure ImageTagger for your specific database credentials. 4. #### Initialize the database @@ -125,13 +142,13 @@ for additional steps on some releases see instructions in [UPGRADE.md](https://g ``` This step will not work out of the box because configuration still needs to be done. - See the lower section about configuring ImageTagger on how to fix this. + See the lower [section about configuring](#configuration) ImageTagger on how to fix this. 4. #### Create a user *Note: This step requires a container running in the background.* ```shell - docker exec imagetagger /app/src/imagetagger/manage.py createsuperuser + docker exec -it imagetagger /app/src/imagetagger/manage.py createsuperuser ``` #### About the Container @@ -139,12 +156,13 @@ for additional steps on some releases see instructions in [UPGRADE.md](https://g | Kind | Description | |---|---| | Volume | `/app/data` is where persistent data (like images) are stored +| Volume | `/app/config` is where additional custom configuration files can be placed. See the [Configuration section](#configuration) below | Environment | ImageTagger can mostly be configured via environment variables | Ports | The container internal webserver listens on port 80 for incoming connections. ### On Kubernetes -1. Follow the steps for *In-Docker* on how to build a container image +1. Follow the steps for [In-Docker](#in-docker) on how to build a container image 2. **Apply kubernetes configuration** @@ -167,14 +185,14 @@ for additional steps on some releases see instructions in [UPGRADE.md](https://g | [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-postgres | Configuration of the postgresql server. Also available inside the application server deployment so that settings can be [referenced](https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/). | [Deployment](https://kubernetes.io/es/docs/concepts/workloads/controllers/deployment/) + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) | imagetagger-web | application server. Per default this deployment references the image `imagetagger:local` which is probably not resolvable and should be replaced by a reference to where your previously built container image is available. | [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) | imagetagger-image-data | Where the application server stores its images (and tools). -| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-web | Configuration of the application server. Mounted as environment variables. +| [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) | imagetagger-web | Configuration of the application server. Mounted as environment variables. See [Configuration](#configuration) for details. ## Configuration ImageTagger is a Django application and uses [django-configurations](https://django-configurations.readthedocs.io/en/stable/) for better configuration management. -It is configured to use a *Dev* configuration when running locally and *Prod* when running in a container. +ImageTagger is configured to use a *Dev* configuration when running locally and *Prod* when running in a container. This can be overridden via the environment variable `DJANGO_CONFIGURATION`. For a list of available configuration values see [settings.py](https://github.com/bit-bots/imagetagger/blob/master/imagetagger/imagetagger/settings.py). @@ -182,11 +200,31 @@ Towards the bottom is a list of *values*. These are taken from environment varia but with an `IT_` prefix. If completely custom configuration is desired, `imagetagger/imagetagger/settings_local.py` can be created in which -a custom configuration class may be created. In docker this file may be located at `/app/config/settings.py` so that +a custom configuration class may be created. In Docker this file may be located at `/app/config/settings.py` so that mounting it should be simple. To use this custom configuration class, the environment variables `DJANGO_SETTINGS_MODULE=imagetagger.settings_local` and `DJANGO_CONFIGURATION=MyCustomClass` must be set. +If downloading zip files of Imagesets is desired, the feature can be enabled by settings `IT_ENABLE_ZIP_DOWNLOAD` to `true`. +A separate zip generation daemon must then be started via the following command. +This feature is enabled and running automatically in Docker based deployments. +```shell +export DJANGO_CONFIGURATION=Prod +./manage.py runzipdaemon +``` + + +### Minimal production Configuration + +In production, the following configuration values **must** be defined (as environment variables) + +| Key | Description +|---|--- +| `IT_SECRET_KEY` | The [django secret key](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY) used by ImageTagger. It is used for password hashing and other cryptographic operations. +| `IT_ALLOWED_HOSTS` | [django ALLOWED_HOSTS](https://docs.djangoproject.com/en/3.0/ref/settings/#allowed-hosts) as comma separated list of values. It defines the hostnames which this application will allow to be used under. +| `IT_DB_HOST` | Hostname (or IP-address) of the postgresql server. When deploying on kubernetes, the provided Kustomization sets this to reference the database deployment. +| `IT_DOWNLOAD_BASE_URL` | Base-URL under which this application is reachable. It defines the prefix for generated download links. + ## Used dependencies The ImageTagger relies on the following plugins, libraries and frameworks: diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py index 7a708e03..7e556dfc 100644 --- a/imagetagger/imagetagger/settings.py +++ b/imagetagger/imagetagger/settings.py @@ -119,40 +119,18 @@ class Base(Configuration): messages.SUCCESS: 'success', } - # Static files (CSS, JavaScript, Images) - # https://docs.djangoproject.com/en/1.10/howto/static-files/ - - STATIC_URL = '/static/' - - EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - - ######################################################################## - # - # Custom Imagetagger defined settings - # - ######################################################################## - UPLOAD_NOTICE = 'By uploading images to this tool you accept that the images get published under creative ' \ - 'commons license and confirm that you have the permission to do so.' - - EXPORT_SEPARATOR = '|' - - IMAGE_PATH = 'images' # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) - TMP_IMAGE_PATH = 'images' # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) - TOOLS_PATH = 'tools' # the path to the folder with the tools relative to the filesystem root (see FS_URL) - - # filename extension of accepted imagefiles - IMAGE_EXTENSION = { - 'png', - 'jpeg', - } - # Sets the default expire time for new messages in days DEFAULT_EXPIRE_TIME = 7 # Sets the default number of messages per page MESSAGES_PER_PAGE = 10 - ACCOUNT_ACTIVATION_DAYS = 7 + # Static files (CSS, JavaScript, Images) + # https://docs.djangoproject.com/en/1.10/howto/static-files/ + + STATIC_URL = '/static/' + + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' ######################################################################## # @@ -217,6 +195,19 @@ def post_setup(cls): USE_NGINX_IMAGE_PROVISION = values.BooleanValue(environ_prefix='IT', default=is_in_docker()) FS_URL = values.Value(environ_prefix='IT', default=path_join(BASE_DIR, 'data')) TMP_FS_URL = values.Value(environ_prefix='IT', default='temp://imagetagger') + UPLOAD_NOTICE = values.Value(environ_prefix='IT', default='By uploading images to this tool you accept that ' + 'the images get published under creative commons license ' + 'and confirm that you have the permission to do so.') + EXPORT_SEPARATOR = values.Value(environ_prefix='IT', default='|') + # the path to the folder with the imagesets relative to the filesystem root (see FS_URL) + IMAGE_PATH = values.Value(environ_prefix='IT', default='images') + # the path to use for temporary image files relative to the temp filesystem (see TMP_FS_URL) + TMP_IMAGE_PATH = values.Value(environ_prefix='IT', default='images') + # the path to the folder with the tools relative to the filesystem root (see FS_URL) + TOOLS_PATH = values.Value(environ_prefix='IT', default='tools') + # filename extension of accepted imagefiles + IMAGE_EXTENSION = values.ListValue(environ_prefix='IT', default=['png', 'jpeg']) + ACCOUNT_ACTIVATION_DAYS = values.PositiveIntegerValue(environ_prefix='IT', default=7) SENTRY_REPORTING_ENABLED = values.BooleanValue(environ_prefix='IT', default=False) SENTRY_DSN = values.Value(environ_prefix='IT', environ_required=SENTRY_REPORTING_ENABLED) From 9165009ff453dc78fb3fcc4b5ecf1ee3648a228a Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Mon, 14 Dec 2020 15:30:38 +0100 Subject: [PATCH 15/18] add upgrading instructions --- UPGRADE.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/UPGRADE.md b/UPGRADE.md index a38a3ccf..39210e85 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,6 +1,17 @@ Upgrade instructions ==================== +Upgrading 0.5 to 0.6 +-------------------- + +Release 0.6 changed the application configuration to be largely based on environment variables instead of the operator +providing a custom `settings.py` file. + +Most settings which previously had to be defined can now be given as environment variables but with an `IT_` prefix. +See [the README](https://github.com/bit-bots/imagetagger#configuration) on more details. + +If desired, a custom configuration file can still be supplied. This is explained in the README as well. + Upgrading 0.4 to 0.5 -------------------- From 1f855fcde7364bab752b0b153ec66e93a12b18e8 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 18 Dec 2020 12:05:22 +0100 Subject: [PATCH 16/18] adapt to PR feedback --- README.md | 59 +++++++++++++++++++++++------ imagetagger/imagetagger/settings.py | 3 +- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f91e1cfa..a7a1b955 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,22 @@ For a short overview of the functions please have a look at the following poster Table of Contents: - [ImageTagger](#imagetagger) - * [Features](#features) - * [Planned Features](#planned-features) - * [Reference](#reference) - * [Installing and running ImageTagger](#installing-and-running-imagetagger) - + [Locally](#locally) - + [In-Docker](#in-docker) - + [On Kubernetes](#on-kubernetes) - * [Configuration](#configuration) - * [Used dependencies](#used-dependencies) + + - [Features](#features) + + - [Planned Features](#planned-features) + + - [Reference](#reference) + + - [Installing and running ImageTagger](#installing-and-running-imagetagger) + + - [Locally](#locally) + - [In-Docker](#in-docker) + - [On Kubernetes](#on-kubernetes) + + - [Configuration](#configuration) + + - [Used dependencies](#used-dependencies) ## Features @@ -66,9 +73,12 @@ If ImageTagger is running in a development environment, no export is necessary. 1. #### Install the latest release - You should probably do this in a [virtual environment](https://virtualenv.pypa.io/en/stable/) + You should probably do this in a [virtual environment](https://virtualenv.pypa.io/en/stable/) + + Replace `v…` with the latest release tag. ```shell git checkout v… + cd imagetagger pip3 install -r imagetagger/requirements.txt ``` @@ -83,6 +93,18 @@ If ImageTagger is running in a development environment, no export is necessary. CREATE USER imagetagger PASSWORD 'imagetagger'; CREATE DATABASE imagetagger WITH OWNER imagetagger; ``` + +3. ### Select a Configuration preset + + When running ImageTagger as a developer, no step is necessary because a development configuration is used per + default when not running as a docker based deployment. + However if this is supposed to be a production deployment, export the following environment variable. + + Currently available presets are `Dev` and `Prod` + + ```shell + export DJANGO_CONFIGURATION=… + ``` 3. #### Configuring ImageTagger to connect to the database @@ -99,7 +121,6 @@ If ImageTagger is running in a development environment, no export is necessary. 5. #### Create a user ```shell - export DJANGO_CONFIGURATION=Prod ./manage.py createsuperuser ``` @@ -112,7 +133,6 @@ If ImageTagger is running in a development environment, no export is necessary. **In a production deployment** it is necessary to run the following commands after each upgrade: ```shell -export DJANGO_CONFIGURATION=Prod ./manage.py migrate ./manage.py compilemessages ./manage.py collectstatic @@ -213,6 +233,8 @@ export DJANGO_CONFIGURATION=Prod ./manage.py runzipdaemon ``` +To create annotation types, log into the application and click on Administration at the very bottom of the home page. + ### Minimal production Configuration @@ -223,8 +245,21 @@ In production, the following configuration values **must** be defined (as enviro | `IT_SECRET_KEY` | The [django secret key](https://docs.djangoproject.com/en/3.0/ref/settings/#std:setting-SECRET_KEY) used by ImageTagger. It is used for password hashing and other cryptographic operations. | `IT_ALLOWED_HOSTS` | [django ALLOWED_HOSTS](https://docs.djangoproject.com/en/3.0/ref/settings/#allowed-hosts) as comma separated list of values. It defines the hostnames which this application will allow to be used under. | `IT_DB_HOST` | Hostname (or IP-address) of the postgresql server. When deploying on kubernetes, the provided Kustomization sets this to reference the database deployment. +| `IT_DB_PASSWORD ` | Password used for authenticating against the postgresql server. | `IT_DOWNLOAD_BASE_URL` | Base-URL under which this application is reachable. It defines the prefix for generated download links. +### Database Configuration + +The following environment variables configure how the postgresql server is accessed + +| Key | Required | Default | Description +|---|---|---|--- +| `IT_DB_HOST` | yes | | Hostname (or IP-address) of the postgresql server. When deploying on kubernetes, the provided Kustomization sets this to reference the database deployment. +| `IT_DB_PORT` | no | 5432 | Port on which the postgresql server listens. +| `IT_DB_NAME` | no | imagetagger | Database name on the postgresql server. ImageTagger requires full access to it. +| `IT_DB_USER` | no | imagetagger | User as which to authenticate on the postgresql server. +| `IT_DB_PASSWORD` | yes | | Password used for authentication. + ## Used dependencies The ImageTagger relies on the following plugins, libraries and frameworks: diff --git a/imagetagger/imagetagger/settings.py b/imagetagger/imagetagger/settings.py index 7e556dfc..0edd694e 100644 --- a/imagetagger/imagetagger/settings.py +++ b/imagetagger/imagetagger/settings.py @@ -179,7 +179,7 @@ def post_setup(cls): DB_PORT = values.PositiveIntegerValue(environ_prefix='IT', default=5432) DB_NAME = values.Value(environ_prefix='IT', default='imagetagger') DB_USER = values.Value(environ_prefix='IT', default=DB_NAME) - DB_PASSWORD = values.Value(environ_prefix='IT', default=DB_USER) + DB_PASSWORD = values.Value(environ_prefix='IT', environ_required=True) ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', environ_required=True) LANGUAGE_CODE = values.Value(environ_prefix='IT', default='en-us') TIME_ZONE = values.Value(environ_prefix='IT', default='Europe/Berlin') @@ -218,6 +218,7 @@ class Dev(Base): DEBUG = values.BooleanValue(environ_prefix='IT', default=True) SECRET_KEY = values.Value(environ_prefix='IT', default='DEV-KEY ONLY! DONT USE IN PRODUCTION!') DB_HOST = values.Value(environ_prefix='IT', default='localhost') + DB_PASSWORD = values.Value(environ_prefix='IT', default='imagetagger') ALLOWED_HOSTS = values.ListValue(environ_prefix='IT', default=['localhost', '127.0.0.1']) DOWNLOAD_BASE_URL = values.Value(environ_prefix='IT', default='localhost') From 58e8909359ee87dff37e54c520b47d8966555251 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 31 Dec 2020 17:05:55 +0100 Subject: [PATCH 17/18] explicitly define example DJANGO_CONFIGURATION in code snippets Co-authored-by: Timon Engelke --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7a1b955..66eb6304 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ If ImageTagger is running in a development environment, no export is necessary. Currently available presets are `Dev` and `Prod` ```shell - export DJANGO_CONFIGURATION=… + export DJANGO_CONFIGURATION=Prod ``` 3. #### Configuring ImageTagger to connect to the database From 595a809389f563813273280a290f1b526a24b92c Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 31 Dec 2020 17:10:41 +0100 Subject: [PATCH 18/18] fix README code snippets to be consistent --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 66eb6304..23b131e7 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ If ImageTagger is running in a development environment, no export is necessary. ```shell git checkout v… cd imagetagger - pip3 install -r imagetagger/requirements.txt + pip3 install -r requirements.txt ``` 2. #### Setup a database server @@ -127,7 +127,6 @@ If ImageTagger is running in a development environment, no export is necessary. 6. #### Run the server ```shell - export DJANGO_CONFIGURATION=Prod ./manage.py runserver ```