Merge branch 'develop' into feature/privacy-settings-1

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 4.1.4
-date-released: '2023-08-15'
+version: 4.2.0
+date-released: '2023-09-28'

backend/README.md

@@ -65,6 +65,14 @@ If you are overriding the default settings, you may pass `--pythonpath` and `--s
 
 ### Running the application (development server)
 
+If you made any changes to your configured corpora, load them into the database before running the aplication.
+
+```console
+$ python manage.py loadcorpora
+```
+
+Start the development server with:
+
 ```console
 $ python manage.py runserver
 ```

backend/addcorpus/admin.py

@@ -1,7 +1,140 @@
-from django.contrib import admin
-from .models import Corpus
+from django.contrib import admin, messages
+from .models import Corpus, CorpusConfiguration, Field
+
+def show_warning_message(request):
+    '''
+    Message to display when loading a form for a resource based on a python class
+    '''
+
+    messages.add_message(
+        request,
+        messages.WARNING,
+        'Corpus configurations are based on python classes; any changes here will be reset on server startup'
+    )
 
 class CorpusAdmin(admin.ModelAdmin):
-    readonly_fields = ['name', 'description']
+    readonly_fields = ['name', 'configuration']
+    fields = ['name', 'groups', 'configuration']
+
+class InlineFieldAdmin(admin.StackedInline):
+    model = Field
+    fields = ['display_name', 'description']
+    show_change_link = True
+    extra = 0
+
+class CorpusConfigurationAdmin(admin.ModelAdmin):
+    readonly_fields = ['corpus']
+
+    inlines = [
+        InlineFieldAdmin
+    ]
+
+    fieldsets = [
+        (
+            None,
+            {
+                'fields': [
+                    'corpus',
+                    'title',
+                    'description',
+                    'description_page',
+                    'image',
+                ]
+            }
+        ), (
+            'Content',
+            {
+                'fields': [
+                    'category',
+                    'languages',
+                    'min_date',
+                    'max_date',
+                    'document_context',
+                ]
+            }
+        ), (
+            'Elasticsearch',
+            {
+                'fields': [
+                    'es_index',
+                    'es_alias',
+                ]
+            }
+        ), (
+            'Scans',
+            {
+                'fields': [
+                    'scan_image_type',
+                    'allow_image_download',
+                ]
+            }
+        ), (
+            'Word models',
+            {
+                'fields': ['word_models_present']
+            }
+        )
+    ]
+
+    def get_form(self, request, obj=None, **kwargs):
+        show_warning_message(request)
+        return super().get_form(request, obj, **kwargs)
+
+
+class FieldAdmin(admin.ModelAdmin):
+    readonly_fields = ['corpus_configuration']
+
+    fieldsets = [
+        (
+            None,
+            {
+                'fields': [
+                    'name',
+                    'corpus_configuration',
+                    'display_name',
+                    'description',
+                    'hidden',
+                    'downloadable',
+                ]
+            }
+        ),
+        (
+            'Indexing options',
+            {
+                'fields': [
+                    'es_mapping',
+                    'indexed',
+                    'required',
+                ]
+            }
+        ), (
+            'Search interface',
+            {
+                'fields': [
+                    'search_filter',
+                    'results_overview',
+                    'searchable',
+                    'search_field_core',
+                    'sortable',
+                    'primary_sort',
+                ]
+            }
+        ), (
+            'Visualisations',
+            {
+                'fields': [
+                    'visualizations',
+                    'visualization_sort',
+                ]
+            }
+        )
+    ]
+
+    def get_form(self, request, obj=None, **kwargs):
+        show_warning_message(request)
+        return super().get_form(request, obj, **kwargs)
+
 
 admin.site.register(Corpus, CorpusAdmin)
+admin.site.register(CorpusConfiguration, CorpusConfigurationAdmin)
+admin.site.register(Field, FieldAdmin)

backend/addcorpus/constants.py

@@ -1,7 +1,8 @@
+from enum import Enum
+
 CATEGORIES = [
-    ('newspaper', 'Newspapers'),
     ('parliament', 'Parliamentary debates'),
-    ('periodical', 'Periodicals'),
+    ('periodical', 'Newspapers and other periodicals'),
     ('finance', 'Financial reports'),
     ('ruling', 'Court rulings'),
     ('review', 'Online reviews'),
@@ -12,3 +13,44 @@
 '''
 Types of data
 '''
+
+class MappingType(Enum):
+    'Elasticsearch mapping types that are implemented in I-analyzer'
+
+    TEXT = 'text'
+    KEYWORD = 'keyword'
+    DATE = 'date'
+    INTEGER  = 'integer'
+    FLOAT = 'float'
+    BOOLEAN = 'boolean'
+
+
+class VisualizationType(Enum):
+    '''Types of visualisations available'''
+
+    RESULTS_COUNT = 'resultscount'
+    TERM_FREQUENCY = 'termfrequency'
+    NGRAM = 'ngram'
+    WORDCLOUD = 'wordcloud'
+
+FORBIDDEN_FIELD_NAMES = [
+    'query',
+    'fields',
+    'sort',
+    'highlight',
+    'visualize',
+    'visualizedField',
+    'normalize',
+    'size',
+    'positions',
+    'freqCompensation',
+    'analysis',
+    'maxDocuments',
+    'numberOfNgrams',
+    'dateField',
+]
+'''
+Field names that cannot be used because they are also query parameters in frontend routes.
+
+Using them would make routing ambiguous.
+'''