From 06aa5ad033898cde22f544af977437c9ce149795 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 14:52:13 +0200
Subject: [PATCH 01/50] add search history setting to users

---
 .../0004_customuser_enable_search_history.py   | 18 ++++++++++++++++++
 backend/users/models.py                        |  2 ++
 2 files changed, 20 insertions(+)
 create mode 100644 backend/users/migrations/0004_customuser_enable_search_history.py

diff --git a/backend/users/migrations/0004_customuser_enable_search_history.py b/backend/users/migrations/0004_customuser_enable_search_history.py
new file mode 100644
index 000000000..c6964fb27
--- /dev/null
+++ b/backend/users/migrations/0004_customuser_enable_search_history.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.9 on 2023-07-18 12:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0003_sitedomain'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='customuser',
+            name='enable_search_history',
+            field=models.BooleanField(default=True),
+        ),
+    ]
diff --git a/backend/users/models.py b/backend/users/models.py
index 04abbe933..8c1c6878b 100644
--- a/backend/users/models.py
+++ b/backend/users/models.py
@@ -18,3 +18,5 @@ def has_access(self, corpus_name):
         # check if any corpus added to the user's group(s) match the corpus name
         return any(corpus for group in self.groups.all()
                    for corpus in group.corpora.filter(name=corpus_name))
+
+    enable_search_history = models.BooleanField(default=True)

From 0837953569070a1e2ef99aa18b6c351d01f5879a Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 15:00:25 +0200
Subject: [PATCH 02/50] add search history setting to serializer

---
 backend/api/views.py                        |  1 -
 backend/users/serializers.py                |  2 +-
 backend/users/tests/test_user_serializer.py | 19 ++++++++++++++++---
 backend/users/views.py                      |  9 ++++++++-
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/backend/api/views.py b/backend/api/views.py
index b8efc8f2b..6e6f6be54 100644
--- a/backend/api/views.py
+++ b/backend/api/views.py
@@ -5,7 +5,6 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.exceptions import APIException
 import logging
-from rest_framework.permissions import IsAuthenticated
 from api.utils import check_json_keys
 from celery import current_app as celery_app
 
diff --git a/backend/users/serializers.py b/backend/users/serializers.py
index 0715e9d6b..2935cf865 100644
--- a/backend/users/serializers.py
+++ b/backend/users/serializers.py
@@ -9,7 +9,7 @@ class CustomUserDetailsSerializer(UserDetailsSerializer):
 
     class Meta(UserDetailsSerializer.Meta):
         fields = ('id', 'username', 'email', 'saml',
-                  'download_limit', 'is_admin')
+                  'download_limit', 'is_admin', 'enable_search_history')
 
 
 class CustomRegistrationSerializer(RegisterSerializer):
diff --git a/backend/users/tests/test_user_serializer.py b/backend/users/tests/test_user_serializer.py
index 700716486..f46a1a43c 100644
--- a/backend/users/tests/test_user_serializer.py
+++ b/backend/users/tests/test_user_serializer.py
@@ -11,7 +11,8 @@ def test_user_serializer(auth_client,
         'email': user_credentials['email'],
         'download_limit': 10000,
         'is_admin': False,
-        'saml': False
+        'saml': False,
+        'enable_search_history': True
     }
 
 
@@ -24,6 +25,18 @@ def test_admin_serializer(admin_client, admin_credentials):
         'email': admin_credentials['email'],
         'download_limit': 10000,
         'is_admin': True,
-        'saml': False
-
+        'saml': False,
+        'enable_search_history': True,
     }
+
+def test_user_updates(auth_client):
+    route = '/users/user/'
+    details = lambda: auth_client.get(route)
+    search_history_enabled = lambda: details().data.get('enable_search_history')
+
+    assert search_history_enabled()
+
+    response = auth_client.patch(route, {'enable_search_history': False}, content_type='application/json')
+    assert response.status_code == 200
+
+    assert not search_history_enabled()
diff --git a/backend/users/views.py b/backend/users/views.py
index 778d774ca..0e5268c53 100644
--- a/backend/users/views.py
+++ b/backend/users/views.py
@@ -6,8 +6,10 @@
 from rest_framework.response import Response
 from rest_framework.status import HTTP_404_NOT_FOUND
 from rest_framework.views import APIView
-
+from rest_framework.viewsets import ModelViewSet
+from rest_framework.permissions import IsAuthenticated
 from djangosaml2.views import LogoutView
+from .serializers import CustomUserDetailsSerializer
 
 
 def redirect_confirm(request, key):
@@ -44,3 +46,8 @@ class SamlLogoutView(LogoutView):
     @csrf_exempt
     def post(self, request, *args, **kwargs):
         return super().post(request, *args, **kwargs)
+
+class UserViewSet(ModelViewSet):
+    permission_classes = [IsAuthenticated]
+    serializer_class = CustomUserDetailsSerializer
+

From 6901468bfa1747e88f10c0f94daf6cf4ba959bb2 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 16:28:44 +0200
Subject: [PATCH 03/50] add settings update to api service

---
 frontend/src/app/services/api.service.ts  | 10 +++-
 frontend/src/app/services/auth.service.ts | 69 +++++++++++------------
 2 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts
index a7f27ace3..ad5aa95d3 100644
--- a/frontend/src/app/services/api.service.ts
+++ b/frontend/src/app/services/api.service.ts
@@ -13,7 +13,7 @@ import {
 } from '@ngx-resource/core';
 
 import { HttpClient } from '@angular/common/http';
-import { timer } from 'rxjs';
+import { Observable, timer } from 'rxjs';
 import { filter, switchMap, take } from 'rxjs/operators';
 import { environment } from '../../environments/environment';
 import { ImageInfo } from '../image-view/image-view.component';
@@ -344,4 +344,12 @@ export class ApiService extends Resource {
             }
         );
     }
+
+    /** send PATCH request to update settings for the user */
+    public updateUserSettings(details: Partial<UserResponse>): Observable<UserResponse> {
+        return this.http.patch<UserResponse>(
+            this.authApiRoute('user'),
+            details
+        );
+    }
 }
diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts
index 0d60c73e2..42692db02 100644
--- a/frontend/src/app/services/auth.service.ts
+++ b/frontend/src/app/services/auth.service.ts
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/member-ordering */
 import { Injectable, OnDestroy } from '@angular/core';
 import { Router } from '@angular/router';
 import {
@@ -77,7 +78,7 @@ export class AuthService implements OnDestroy {
             .getUser()
             .pipe(takeUntil(this.destroy$))
             .subscribe(
-                (result) => this.setAuth(this.transformUserResponse(result)),
+                (result) => this.setAuth(this.parseUserResponse(result)),
                 () => this.purgeAuth()
             );
     }
@@ -91,41 +92,6 @@ export class AuthService implements OnDestroy {
         return Promise.resolve(currentUser);
     }
 
-    /**
-     * Transforms backend user response to User object
-     *
-     * @param result User response data
-     * @returns User object
-     */
-    private transformUserResponse(
-        result: UserResponse
-    ): User {
-        return new User(
-            result.id,
-            result.username,
-            result.is_admin,
-            result.download_limit == null ? 0 : result.download_limit,
-            result.saml
-        );
-    }
-
-    /**
-     * Deserializes localStorage user
-     *
-     * @param serializedUser serialized currentUser
-     * @returns User object
-     */
-    private deserializeUser(serializedUser: string): User {
-        const parsed = JSON.parse(serializedUser);
-        return new User(
-            parsed['id'],
-            parsed['username'],
-            parsed['is_admin'],
-            parsed['download_limit'],
-            parsed['isSamlLogin']
-        );
-    }
-
     checkUser(): Observable<UserResponse> {
         return this.apiService.getUser();
     }
@@ -137,7 +103,7 @@ export class AuthService implements OnDestroy {
         const loginRequest$ = this.apiService.login(username, password);
         return loginRequest$.pipe(
             mergeMap(() => this.checkUser()),
-            tap((res) => this.setAuth(this.transformUserResponse(res))),
+            tap((res) => this.setAuth(this.parseUserResponse(res))),
             catchError((error) => {
                 console.error(error);
                 return throwError(error);
@@ -200,4 +166,33 @@ export class AuthService implements OnDestroy {
             newPassword2
         );
     }
+
+    public updateSettings(update: Partial<User>) {
+        return this.apiService.updateUserSettings(update).pipe(
+            tap((res) => this.setAuth(this.parseUserResponse(res))),
+            catchError((error) => {
+                console.error(error);
+                return throwError(error);
+            })
+        );
+    }
+
+    /**
+     * Transforms backend user response to User object
+     *
+     * @param result User response data
+     * @returns User object
+     */
+    private parseUserResponse(
+        result: UserResponse
+    ): User {
+        return new User(
+            result.id,
+            result.username,
+            result.is_admin,
+            result.download_limit == null ? 0 : result.download_limit,
+            result.saml
+        );
+    }
+
 }

From 0b4521a7502746380e95ac4451db2c778f6d2a00 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 16:33:30 +0200
Subject: [PATCH 04/50] add settings component

---
 frontend/src/app/app.module.ts                |  7 ++++++
 .../src/app/settings/settings.component.html  |  5 ++++
 .../src/app/settings/settings.component.scss  |  0
 .../app/settings/settings.component.spec.ts   | 24 +++++++++++++++++++
 .../src/app/settings/settings.component.ts    | 15 ++++++++++++
 5 files changed, 51 insertions(+)
 create mode 100644 frontend/src/app/settings/settings.component.html
 create mode 100644 frontend/src/app/settings/settings.component.scss
 create mode 100644 frontend/src/app/settings/settings.component.spec.ts
 create mode 100644 frontend/src/app/settings/settings.component.ts

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 0fd77585b..261617462 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -93,6 +93,7 @@ import { CorpusFilterComponent } from './corpus-selection/corpus-filter/corpus-f
 import { DatePickerComponent } from './corpus-selection/corpus-filter/date-picker/date-picker.component';
 import { CorpusInfoComponent } from './corpus-info/corpus-info.component';
 import { FieldInfoComponent } from './corpus-info/field-info/field-info.component';
+import { SettingsComponent } from './settings/settings.component';
 
 
 export const appRoutes: Routes = [
@@ -165,6 +166,11 @@ export const appRoutes: Routes = [
         path: 'confirm-email/:key',
         component: VerifyEmailComponent,
     },
+    {
+        path: 'settings',
+        component: SettingsComponent,
+        canActivate: [LoggedOnGuard],
+    },
     {
         path: '',
         redirectTo: 'home',
@@ -234,6 +240,7 @@ export const declarations: any[] = [
     SearchResultsComponent,
     SearchSortingComponent,
     SelectFieldComponent,
+    SettingsComponent,
     SimilarityChartComponent,
     TermComparisonEditorComponent,
     TimeIntervalSliderComponent,
diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
new file mode 100644
index 000000000..f76b10953
--- /dev/null
+++ b/frontend/src/app/settings/settings.component.html
@@ -0,0 +1,5 @@
+<section class="section">
+    <div class="container is-readable">
+        <h1 class="title">Settings</h1>
+    </div>
+</section>
diff --git a/frontend/src/app/settings/settings.component.scss b/frontend/src/app/settings/settings.component.scss
new file mode 100644
index 000000000..e69de29bb
diff --git a/frontend/src/app/settings/settings.component.spec.ts b/frontend/src/app/settings/settings.component.spec.ts
new file mode 100644
index 000000000..365de8e9b
--- /dev/null
+++ b/frontend/src/app/settings/settings.component.spec.ts
@@ -0,0 +1,24 @@
+import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
+
+import { SettingsComponent } from './settings.component';
+import { commonTestBed } from '../common-test-bed';
+
+describe('SettingsComponent', () => {
+    let component: SettingsComponent;
+    let fixture: ComponentFixture<SettingsComponent>;
+
+
+    beforeEach(waitForAsync(() => {
+        commonTestBed().testingModule.compileComponents();
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(SettingsComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+});
diff --git a/frontend/src/app/settings/settings.component.ts b/frontend/src/app/settings/settings.component.ts
new file mode 100644
index 000000000..8f4b6f22f
--- /dev/null
+++ b/frontend/src/app/settings/settings.component.ts
@@ -0,0 +1,15 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+    selector: 'ia-settings',
+    templateUrl: './settings.component.html',
+    styleUrls: ['./settings.component.scss']
+})
+export class SettingsComponent implements OnInit {
+
+    constructor() { }
+
+    ngOnInit(): void {
+    }
+
+}

From 984a447a4c7e2869ad652e7d14b5cb92476cfc61 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 16:38:07 +0200
Subject: [PATCH 05/50] add search history setting component

---
 frontend/src/app/app.module.ts                |  2 ++
 .../search-history-setting.component.html     |  1 +
 .../search-history-setting.component.scss     |  0
 .../search-history-setting.component.spec.ts  | 23 +++++++++++++++++++
 .../search-history-setting.component.ts       | 15 ++++++++++++
 .../src/app/settings/settings.component.html  |  2 ++
 6 files changed, 43 insertions(+)
 create mode 100644 frontend/src/app/settings/search-history-setting/search-history-setting.component.html
 create mode 100644 frontend/src/app/settings/search-history-setting/search-history-setting.component.scss
 create mode 100644 frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts
 create mode 100644 frontend/src/app/settings/search-history-setting/search-history-setting.component.ts

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 261617462..0aba79e70 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -94,6 +94,7 @@ import { DatePickerComponent } from './corpus-selection/corpus-filter/date-picke
 import { CorpusInfoComponent } from './corpus-info/corpus-info.component';
 import { FieldInfoComponent } from './corpus-info/field-info/field-info.component';
 import { SettingsComponent } from './settings/settings.component';
+import { SearchHistorySettingComponent } from './settings/search-history-setting/search-history-setting.component';
 
 
 export const appRoutes: Routes = [
@@ -236,6 +237,7 @@ export const declarations: any[] = [
     ScrollToDirective,
     SearchComponent,
     SearchHistoryComponent,
+    SearchHistorySettingComponent,
     SearchRelevanceComponent,
     SearchResultsComponent,
     SearchSortingComponent,
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.html b/frontend/src/app/settings/search-history-setting/search-history-setting.component.html
new file mode 100644
index 000000000..7083aa8a7
--- /dev/null
+++ b/frontend/src/app/settings/search-history-setting/search-history-setting.component.html
@@ -0,0 +1 @@
+<p>search-history-setting works!</p>
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.scss b/frontend/src/app/settings/search-history-setting/search-history-setting.component.scss
new file mode 100644
index 000000000..e69de29bb
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts b/frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts
new file mode 100644
index 000000000..c99f701e4
--- /dev/null
+++ b/frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts
@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
+
+import { SearchHistorySettingComponent } from './search-history-setting.component';
+import { commonTestBed } from '../../common-test-bed';
+
+describe('SearchHistorySettingComponent', () => {
+    let component: SearchHistorySettingComponent;
+    let fixture: ComponentFixture<SearchHistorySettingComponent>;
+
+    beforeEach(waitForAsync(() => {
+        commonTestBed().testingModule.compileComponents();
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(SearchHistorySettingComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+});
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts b/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts
new file mode 100644
index 000000000..4104fc158
--- /dev/null
+++ b/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts
@@ -0,0 +1,15 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+    selector: 'ia-search-history-setting',
+    templateUrl: './search-history-setting.component.html',
+    styleUrls: ['./search-history-setting.component.scss']
+})
+export class SearchHistorySettingComponent implements OnInit {
+
+    constructor() { }
+
+    ngOnInit(): void {
+    }
+
+}
diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
index f76b10953..a52af6e24 100644
--- a/frontend/src/app/settings/settings.component.html
+++ b/frontend/src/app/settings/settings.component.html
@@ -1,5 +1,7 @@
 <section class="section">
     <div class="container is-readable">
         <h1 class="title">Settings</h1>
+
+        <ia-search-history-setting></ia-search-history-setting>
     </div>
 </section>

From b1581823b1441e4106d2df8de18e9ea0b1511de1 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 11:08:02 +0200
Subject: [PATCH 06/50] rename toggle search history component

---
 frontend/src/app/app.module.ts                    |  4 ++--
 .../search-history-setting.component.ts           | 15 ---------------
 frontend/src/app/settings/settings.component.html |  3 ++-
 .../toggle-search-history.component.html}         |  0
 .../toggle-search-history.component.scss}         |  0
 .../toggle-search-history.component.spec.ts}      | 10 +++++-----
 .../toggle-search-history.component.ts            | 15 +++++++++++++++
 7 files changed, 24 insertions(+), 23 deletions(-)
 delete mode 100644 frontend/src/app/settings/search-history-setting/search-history-setting.component.ts
 rename frontend/src/app/settings/{search-history-setting/search-history-setting.component.html => toggle-search-history/toggle-search-history.component.html} (100%)
 rename frontend/src/app/settings/{search-history-setting/search-history-setting.component.scss => toggle-search-history/toggle-search-history.component.scss} (100%)
 rename frontend/src/app/settings/{search-history-setting/search-history-setting.component.spec.ts => toggle-search-history/toggle-search-history.component.spec.ts} (57%)
 create mode 100644 frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 0aba79e70..92bfbfa92 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -94,7 +94,7 @@ import { DatePickerComponent } from './corpus-selection/corpus-filter/date-picke
 import { CorpusInfoComponent } from './corpus-info/corpus-info.component';
 import { FieldInfoComponent } from './corpus-info/field-info/field-info.component';
 import { SettingsComponent } from './settings/settings.component';
-import { SearchHistorySettingComponent } from './settings/search-history-setting/search-history-setting.component';
+import { ToggleSearchHistoryComponent } from './settings/toggle-search-history/toggle-search-history.component';
 
 
 export const appRoutes: Routes = [
@@ -237,7 +237,6 @@ export const declarations: any[] = [
     ScrollToDirective,
     SearchComponent,
     SearchHistoryComponent,
-    SearchHistorySettingComponent,
     SearchRelevanceComponent,
     SearchResultsComponent,
     SearchSortingComponent,
@@ -247,6 +246,7 @@ export const declarations: any[] = [
     TermComparisonEditorComponent,
     TimeIntervalSliderComponent,
     TimelineComponent,
+    ToggleSearchHistoryComponent,
     VerifyEmailComponent,
     VisualizationComponent,
     VisualizationFooterComponent,
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts b/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts
deleted file mode 100644
index 4104fc158..000000000
--- a/frontend/src/app/settings/search-history-setting/search-history-setting.component.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-
-@Component({
-    selector: 'ia-search-history-setting',
-    templateUrl: './search-history-setting.component.html',
-    styleUrls: ['./search-history-setting.component.scss']
-})
-export class SearchHistorySettingComponent implements OnInit {
-
-    constructor() { }
-
-    ngOnInit(): void {
-    }
-
-}
diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
index a52af6e24..0e3b19418 100644
--- a/frontend/src/app/settings/settings.component.html
+++ b/frontend/src/app/settings/settings.component.html
@@ -2,6 +2,7 @@
     <div class="container is-readable">
         <h1 class="title">Settings</h1>
 
-        <ia-search-history-setting></ia-search-history-setting>
+        <ia-toggle-search-history></ia-toggle-search-history>
+
     </div>
 </section>
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.html b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
similarity index 100%
rename from frontend/src/app/settings/search-history-setting/search-history-setting.component.html
rename to frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.scss b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.scss
similarity index 100%
rename from frontend/src/app/settings/search-history-setting/search-history-setting.component.scss
rename to frontend/src/app/settings/toggle-search-history/toggle-search-history.component.scss
diff --git a/frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts
similarity index 57%
rename from frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts
rename to frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts
index c99f701e4..13346a6b0 100644
--- a/frontend/src/app/settings/search-history-setting/search-history-setting.component.spec.ts
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts
@@ -1,18 +1,18 @@
 import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
 
-import { SearchHistorySettingComponent } from './search-history-setting.component';
+import { ToggleSearchHistoryComponent } from './toggle-search-history.component';
 import { commonTestBed } from '../../common-test-bed';
 
-describe('SearchHistorySettingComponent', () => {
-    let component: SearchHistorySettingComponent;
-    let fixture: ComponentFixture<SearchHistorySettingComponent>;
+describe('ToggleSearchHistoryComponent', () => {
+    let component: ToggleSearchHistoryComponent;
+    let fixture: ComponentFixture<ToggleSearchHistoryComponent>;
 
     beforeEach(waitForAsync(() => {
         commonTestBed().testingModule.compileComponents();
     }));
 
     beforeEach(() => {
-        fixture = TestBed.createComponent(SearchHistorySettingComponent);
+        fixture = TestBed.createComponent(ToggleSearchHistoryComponent);
         component = fixture.componentInstance;
         fixture.detectChanges();
     });
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
new file mode 100644
index 000000000..24f51414a
--- /dev/null
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
@@ -0,0 +1,15 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+    selector: 'ia-toggle-search-history',
+    templateUrl: './toggle-search-history.component.html',
+    styleUrls: ['./toggle-search-history.component.scss']
+})
+export class ToggleSearchHistoryComponent implements OnInit {
+
+    constructor() { }
+
+    ngOnInit(): void {
+    }
+
+}

From 092e50950ab5522538f197d39e42242a4ed8b0f8 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 16:43:10 +0200
Subject: [PATCH 07/50] add search history setting to frontend user model

---
 frontend/src/app/models/user.ts           | 4 +++-
 frontend/src/app/services/auth.service.ts | 3 ++-
 frontend/src/app/services/user.service.ts | 6 ++++--
 frontend/src/mock-data/user.ts            | 3 ++-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/frontend/src/app/models/user.ts b/frontend/src/app/models/user.ts
index 9d266096c..c8855c33b 100644
--- a/frontend/src/app/models/user.ts
+++ b/frontend/src/app/models/user.ts
@@ -7,6 +7,7 @@ export interface UserResponse {
     download_limit: number;
     is_admin: boolean;
     saml: boolean;
+    enable_search_history: boolean;
 }
 
 export class User {
@@ -15,6 +16,7 @@ export class User {
         public name,
         public isAdmin: boolean,
         public downloadLimit: number = 0, // The download limit for this user, will be 0 if there is no limit.
-        public isSamlLogin: boolean
+        public isSamlLogin: boolean,
+        public enableSearchHistory: boolean,
     ) {}
 }
diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts
index 42692db02..b2a43df57 100644
--- a/frontend/src/app/services/auth.service.ts
+++ b/frontend/src/app/services/auth.service.ts
@@ -191,7 +191,8 @@ export class AuthService implements OnDestroy {
             result.username,
             result.is_admin,
             result.download_limit == null ? 0 : result.download_limit,
-            result.saml
+            result.saml,
+            result.enable_search_history,
         );
     }
 
diff --git a/frontend/src/app/services/user.service.ts b/frontend/src/app/services/user.service.ts
index 530ec3bc4..a990ffbe7 100644
--- a/frontend/src/app/services/user.service.ts
+++ b/frontend/src/app/services/user.service.ts
@@ -42,7 +42,8 @@ export class UserService implements OnDestroy {
                 parsed['username'],
                 parsed['isAdmin'],
                 parsed['downloadLimit'],
-                parsed['isSamlLogin']
+                parsed['isSamlLogin'],
+                parsed['enable_search_history'],
             );
         } else {
             return false;
@@ -149,7 +150,8 @@ export class UserService implements OnDestroy {
             result.username,
             result.is_admin,
             result.download_limit == null ? 0 : result.download_limit,
-            result.saml != null
+            result.saml != null,
+            result.enable_search_history,
         );
 
         return this.currentUser;
diff --git a/frontend/src/mock-data/user.ts b/frontend/src/mock-data/user.ts
index 769e7dd48..b6595c2d3 100644
--- a/frontend/src/mock-data/user.ts
+++ b/frontend/src/mock-data/user.ts
@@ -8,7 +8,7 @@ export class UserServiceMock {
     }
 }
 
-export const mockUser: User = new User(42, 'mouse', false, 10000, false);
+export const mockUser: User = new User(42, 'mouse', false, 10000, false, true);
 
 export const mockUserResponse: UserResponse = {
     id: 42,
@@ -17,4 +17,5 @@ export const mockUserResponse: UserResponse = {
     email: 'mighty@mouse.com',
     download_limit: 10000,
     saml: false,
+    enable_search_history: true,
 };

From 72f4400a7131ca93f7512e4dd0d45bede550fa38 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 17:13:24 +0200
Subject: [PATCH 08/50] enable settings change

---
 frontend/src/app/services/auth.service.spec.ts |  1 +
 frontend/src/app/services/auth.service.ts      | 11 ++++++++++-
 .../toggle-search-history.component.html       | 10 +++++++++-
 .../toggle-search-history.component.ts         | 18 ++++++++++++++----
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/frontend/src/app/services/auth.service.spec.ts b/frontend/src/app/services/auth.service.spec.ts
index c81025273..63ca3a5d8 100644
--- a/frontend/src/app/services/auth.service.spec.ts
+++ b/frontend/src/app/services/auth.service.spec.ts
@@ -24,4 +24,5 @@ describe('AuthService', () => {
     it('should be created', () => {
         expect(service).toBeTruthy();
     });
+
 });
diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts
index b2a43df57..3a6604ed2 100644
--- a/frontend/src/app/services/auth.service.ts
+++ b/frontend/src/app/services/auth.service.ts
@@ -17,6 +17,7 @@ import { environment } from '../../environments/environment';
 import { User, UserResponse } from '../models';
 import { ApiService } from './api.service';
 import { SessionService } from './session.service';
+import * as _ from 'lodash';
 
 @Injectable({
     providedIn: 'root',
@@ -168,7 +169,7 @@ export class AuthService implements OnDestroy {
     }
 
     public updateSettings(update: Partial<User>) {
-        return this.apiService.updateUserSettings(update).pipe(
+        return this.apiService.updateUserSettings(this.encodeUserUpdate(update)).pipe(
             tap((res) => this.setAuth(this.parseUserResponse(res))),
             catchError((error) => {
                 console.error(error);
@@ -196,4 +197,12 @@ export class AuthService implements OnDestroy {
         );
     }
 
+    private encodeUserUpdate(update: Partial<User>): Partial<UserResponse> {
+        const changeKeys = {
+            enableSearchHistory: 'enable_search_history'
+        };
+        const transformKey = (value, key, obj) => changeKeys[key] || key;
+        return _.mapKeys(update, transformKey);
+    }
+
 }
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
index 7083aa8a7..78d1f9d16 100644
--- a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
@@ -1 +1,9 @@
-<p>search-history-setting works!</p>
+<div class="field">
+    <div class="control">
+        <label class="checkbox">
+            <input type="checkbox" [checked]="searchHistoryEnabled$ | async"
+                (change)="emitChange($event.target.checked)">
+            Save my search history
+        </label>
+    </div>
+</div>
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
index 24f51414a..66b4bd2c1 100644
--- a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
@@ -1,15 +1,25 @@
-import { Component, OnInit } from '@angular/core';
+import { Component } from '@angular/core';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+import { AuthService } from '../../services';
 
 @Component({
     selector: 'ia-toggle-search-history',
     templateUrl: './toggle-search-history.component.html',
     styleUrls: ['./toggle-search-history.component.scss']
 })
-export class ToggleSearchHistoryComponent implements OnInit {
+export class ToggleSearchHistoryComponent {
+    searchHistoryEnabled$: Observable<boolean>;
 
-    constructor() { }
+    constructor(private authService: AuthService) {
+        this.searchHistoryEnabled$ = this.authService.currentUser$.pipe(
+            map(user => user.enableSearchHistory)
+        );
+    }
 
-    ngOnInit(): void {
+    emitChange(setting: boolean) {
+        const data = { enableSearchHistory: setting };
+        this.authService.updateSettings(data).subscribe();
     }
 
 }

From 36c02115850f07cddaefb4403998f5ae74ec6585 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 17:37:44 +0200
Subject: [PATCH 09/50] styling & help text

---
 .../src/app/settings/settings.component.html  |  5 +++-
 .../toggle-search-history.component.html      | 24 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
index 0e3b19418..55ad7a1da 100644
--- a/frontend/src/app/settings/settings.component.html
+++ b/frontend/src/app/settings/settings.component.html
@@ -2,7 +2,10 @@
     <div class="container is-readable">
         <h1 class="title">Settings</h1>
 
-        <ia-toggle-search-history></ia-toggle-search-history>
+        <div class="box">
+            <h2 class="title is-5">Search history</h2>
 
+            <ia-toggle-search-history></ia-toggle-search-history>
+        </div>
     </div>
 </section>
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
index 78d1f9d16..50883ef42 100644
--- a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
@@ -5,5 +5,29 @@
                 (change)="emitChange($event.target.checked)">
             Save my search history
         </label>
+
+        <p class="help">
+
+        </p>
     </div>
+
+    <details class="content help">
+        <summary>More information</summary>
+
+        <p>
+            Saving your search history allows you to look up earlier queries.
+            You can view your saved history on the <a href="/search-history">search history page</a>.
+            It can be used to quickly get back to earlier queries, or to log your research process.
+        </p>
+        <p>
+            Search histories are stored on the I-analyzer server.
+            They are not shared with others, but developers may use search history
+            statistics to assess the level of interest in different corpora.
+            See our <a href="/privacy">privacy statement</a> for more information.
+        </p>
+        <p>
+            You can change this setting at any time. Doing so will not delete
+            your existing search history.
+        </p>
+    </details>
 </div>

From 9ef6ec52aa516461ca9e6123e5e28f5b072b3131 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 18 Jul 2023 17:58:17 +0200
Subject: [PATCH 10/50] add settings page to nav menu

---
 frontend/src/app/menu/menu.component.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/frontend/src/app/menu/menu.component.ts b/frontend/src/app/menu/menu.component.ts
index e7645c1e3..c031b3ed8 100644
--- a/frontend/src/app/menu/menu.component.ts
+++ b/frontend/src/app/menu/menu.component.ts
@@ -104,6 +104,13 @@ export class MenuComponent implements OnDestroy, OnInit {
                     this.router.navigate(['download-history']);
                 },
             },
+            {
+                label: 'Settings',
+                icon: 'fa fa-cog',
+                command: (click) => {
+                    this.router.navigate(['settings']);
+                }
+            },
             ...(this.isAdmin
                 ? [
                       {

From 04988f9f0166e6a59fc69e29c54c879541c4503b Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 10:26:58 +0200
Subject: [PATCH 11/50] add view to delete search history

---
 backend/api/tests/test_api_views.py | 41 ++++++++++++++++++++---------
 backend/api/views.py                |  7 +++++
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/backend/api/tests/test_api_views.py b/backend/api/tests/test_api_views.py
index 2a9c33874..9c4d1d53a 100644
--- a/backend/api/tests/test_api_views.py
+++ b/backend/api/tests/test_api_views.py
@@ -4,19 +4,11 @@
 from addcorpus.models import Corpus
 from rest_framework.status import is_success
 
-def test_search_history_view(admin_user, admin_client):
-    corpus = Corpus.objects.create(name = 'mock-corpus', description = '')
-
-    # get search history
-    response = admin_client.get('/api/search_history/')
-    assert is_success(response.status_code)
-    assert len(response.data) == 0
-
-    # add a query to search history
-    data = {
+def mock_query_data(user, corpus_name):
+    return {
         'aborted': False,
-        'corpus': 'mock-corpus',
-        'user': admin_user.id,
+        'corpus': corpus_name,
+        'user': user.id,
         'started': datetime.now().isoformat(),
         'completed': datetime.now().isoformat(),
         'query_json': {
@@ -27,6 +19,17 @@ def test_search_history_view(admin_user, admin_client):
         'total_results': 10,
         'transferred': 0,
     }
+
+def test_search_history_view(admin_user, admin_client):
+    corpus = Corpus.objects.create(name = 'mock-corpus', description = '')
+
+    # get search history
+    response = admin_client.get('/api/search_history/')
+    assert is_success(response.status_code)
+    assert len(response.data) == 0
+
+    # add a query to search history
+    data = mock_query_data(admin_user, 'mock-corpus')
     response = admin_client.post('/api/search_history/', data, content_type='application/json')
     assert is_success(response.status_code)
 
@@ -36,6 +39,20 @@ def test_search_history_view(admin_user, admin_client):
     assert len(response.data) == 1
 
 
+def test_delete_search_history(auth_client, auth_user, db):
+    mock_corpus = 'mock-corpus'
+    corpus = Corpus.objects.create(name = mock_corpus, description = '')
+    query = mock_query_data(auth_user, mock_corpus)
+    auth_client.post('/api/search_history/', query, content_type='application/json')
+
+    assert len(auth_user.queries.all()) == 1
+
+    response = auth_client.post('/api/search_history/delete_all/')
+    assert is_success(response.status_code)
+
+    assert len(auth_user.queries.all()) == 0
+
+
 def test_task_status_view(transactional_db, admin_client, celery_worker):
     bad_request = {
         'bad_key': 'data'
diff --git a/backend/api/views.py b/backend/api/views.py
index 6e6f6be54..2dbff15fb 100644
--- a/backend/api/views.py
+++ b/backend/api/views.py
@@ -4,6 +4,7 @@
 from api.serializers import QuerySerializer
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.exceptions import APIException
+from rest_framework.decorators import action
 import logging
 from api.utils import check_json_keys
 from celery import current_app as celery_app
@@ -22,6 +23,12 @@ class QueryViewset(viewsets.ModelViewSet):
     def get_queryset(self):
         return self.request.user.queries.all()
 
+    @action(detail=False, methods=['post'])
+    def delete_all(self, request):
+        queries = self.get_queryset()
+        queries.delete()
+        return Response('success')
+
 class TaskStatusView(APIView):
     '''
     Get the status of an array of backend tasks (working/done/failed),

From 4adfb373430d7e5b851dc757b8bc9446d6391c6d Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 10:32:08 +0200
Subject: [PATCH 12/50] add delete request to api service

---
 frontend/src/app/services/api.service.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts
index ad5aa95d3..a9461b7f3 100644
--- a/frontend/src/app/services/api.service.ts
+++ b/frontend/src/app/services/api.service.ts
@@ -144,10 +144,14 @@ export class ApiService extends Resource {
         TaskResult
     >;
 
-    public saveQuery(options: QueryDb) {
+    public saveQuery(options: QueryDb): Promise<any> {
         return this.http.post('/api/search_history/', options).toPromise();
     }
 
+    public deleteSearchHistory(): Observable<any> {
+        return this.http.post('/api/search_history/delete_all/', {});
+    }
+
     @ResourceAction({
         method: ResourceRequestMethod.Post,
         path: '/download/search_results',

From 6f55e83dbf2a55a0cc075a57a02629568be46f9d Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 10:39:54 +0200
Subject: [PATCH 13/50] add delete search history component

---
 frontend/src/app/app.module.ts                |  2 ++
 .../delete-search-history.component.html      | 25 +++++++++++++++++++
 .../delete-search-history.component.scss      |  0
 .../delete-search-history.component.spec.ts   | 23 +++++++++++++++++
 .../delete-search-history.component.ts        | 19 ++++++++++++++
 .../src/app/settings/settings.component.html  |  8 +++++-
 .../toggle-search-history.component.ts        |  2 +-
 7 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100644 frontend/src/app/settings/delete-search-history/delete-search-history.component.html
 create mode 100644 frontend/src/app/settings/delete-search-history/delete-search-history.component.scss
 create mode 100644 frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts
 create mode 100644 frontend/src/app/settings/delete-search-history/delete-search-history.component.ts

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 92bfbfa92..7cf8ea2c5 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -95,6 +95,7 @@ import { CorpusInfoComponent } from './corpus-info/corpus-info.component';
 import { FieldInfoComponent } from './corpus-info/field-info/field-info.component';
 import { SettingsComponent } from './settings/settings.component';
 import { ToggleSearchHistoryComponent } from './settings/toggle-search-history/toggle-search-history.component';
+import { DeleteSearchHistoryComponent } from './settings/delete-search-history/delete-search-history.component';
 
 
 export const appRoutes: Routes = [
@@ -193,6 +194,7 @@ export const declarations: any[] = [
     CorpusSelectorComponent,
     DatePickerComponent,
     DateFilterComponent,
+    DeleteSearchHistoryComponent,
     DialogComponent,
     DocumentPageComponent,
     DocumentViewComponent,
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.html b/frontend/src/app/settings/delete-search-history/delete-search-history.component.html
new file mode 100644
index 000000000..45f55cec3
--- /dev/null
+++ b/frontend/src/app/settings/delete-search-history/delete-search-history.component.html
@@ -0,0 +1,25 @@
+<div class="field">
+    <div class="control">
+        <button class="button" (click)="showConfirm = !showConfirm">
+            <span class="icon"><fa-icon [icon]="faTrash"></fa-icon></span>
+            <span>Delete my search history</span>
+        </button>
+    </div>
+</div>
+
+<div class="modal" [class.is-active]="showConfirm">
+    <div class="modal-background"></div>
+    <div class="modal-card">
+        <header class="modal-card-head">
+            <p class="modal-card-title">Delete search history?</p>
+            <button class="delete" aria-label="close" (click)="showConfirm = false"></button>
+        </header>
+        <section class="modal-card-body">
+            <p>This action will delete all of your search history. This cannot be reversed.</p>
+        </section>
+        <footer class="modal-card-foot">
+            <button class="button is-danger" (click)="deleteHistory()">Delete search history</button>
+            <button class="button" (click)="showConfirm = false">Cancel</button>
+        </footer>
+    </div>
+</div>
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.scss b/frontend/src/app/settings/delete-search-history/delete-search-history.component.scss
new file mode 100644
index 000000000..e69de29bb
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts b/frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts
new file mode 100644
index 000000000..823fe9127
--- /dev/null
+++ b/frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts
@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
+
+import { DeleteSearchHistoryComponent } from './delete-search-history.component';
+import { commonTestBed } from '../../common-test-bed';
+
+describe('DeleteSearchHistoryComponent', () => {
+    let component: DeleteSearchHistoryComponent;
+    let fixture: ComponentFixture<DeleteSearchHistoryComponent>;
+
+    beforeEach(waitForAsync(() => {
+        commonTestBed().testingModule.compileComponents();
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(DeleteSearchHistoryComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+});
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.ts b/frontend/src/app/settings/delete-search-history/delete-search-history.component.ts
new file mode 100644
index 000000000..c2f430732
--- /dev/null
+++ b/frontend/src/app/settings/delete-search-history/delete-search-history.component.ts
@@ -0,0 +1,19 @@
+import { Component, OnInit } from '@angular/core';
+import { faTrash } from '@fortawesome/free-solid-svg-icons';
+import { ConfirmationService } from 'primeng/api';
+
+@Component({
+    selector: 'ia-delete-search-history',
+    templateUrl: './delete-search-history.component.html',
+    styleUrls: ['./delete-search-history.component.scss']
+})
+export class DeleteSearchHistoryComponent {
+    faTrash = faTrash;
+
+    showConfirm = false;
+
+    constructor() { }
+
+    deleteHistory() { }
+
+}
diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
index 55ad7a1da..bfe33ae05 100644
--- a/frontend/src/app/settings/settings.component.html
+++ b/frontend/src/app/settings/settings.component.html
@@ -5,7 +5,13 @@ <h1 class="title">Settings</h1>
         <div class="box">
             <h2 class="title is-5">Search history</h2>
 
-            <ia-toggle-search-history></ia-toggle-search-history>
+            <div class="block">
+                <ia-toggle-search-history></ia-toggle-search-history>
+            </div>
+
+            <div class="block">
+                <ia-delete-search-history></ia-delete-search-history>
+            </div>
         </div>
     </div>
 </section>
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
index 66b4bd2c1..245951d07 100644
--- a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
+++ b/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
@@ -13,7 +13,7 @@ export class ToggleSearchHistoryComponent {
 
     constructor(private authService: AuthService) {
         this.searchHistoryEnabled$ = this.authService.currentUser$.pipe(
-            map(user => user.enableSearchHistory)
+            map(user => user?.enableSearchHistory)
         );
     }
 

From a1f774650e350e2ff8a552f624ed164ed3feffe9 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 11:11:45 +0200
Subject: [PATCH 14/50] add search history settings component

---
 frontend/src/app/app.module.ts                |  6 +++--
 .../delete-search-history.component.html      |  0
 .../delete-search-history.component.scss      |  0
 .../delete-search-history.component.spec.ts   |  2 +-
 .../delete-search-history.component.ts        |  1 -
 .../search-history-settings.component.html    |  9 ++++++++
 .../search-history-settings.component.scss}   |  0
 .../search-history-settings.component.spec.ts | 23 +++++++++++++++++++
 .../search-history-settings.component.ts      | 15 ++++++++++++
 .../toggle-search-history.component.html      |  0
 .../toggle-search-history.component.scss      |  0
 .../toggle-search-history.component.spec.ts   |  2 +-
 .../toggle-search-history.component.ts        |  0
 .../src/app/settings/settings.component.html  | 10 +-------
 14 files changed, 54 insertions(+), 14 deletions(-)
 rename frontend/src/app/settings/{ => search-history-settings}/delete-search-history/delete-search-history.component.html (100%)
 rename frontend/src/app/settings/{ => search-history-settings}/delete-search-history/delete-search-history.component.scss (100%)
 rename frontend/src/app/settings/{ => search-history-settings}/delete-search-history/delete-search-history.component.spec.ts (92%)
 rename frontend/src/app/settings/{ => search-history-settings}/delete-search-history/delete-search-history.component.ts (89%)
 create mode 100644 frontend/src/app/settings/search-history-settings/search-history-settings.component.html
 rename frontend/src/app/settings/{toggle-search-history/toggle-search-history.component.scss => search-history-settings/search-history-settings.component.scss} (100%)
 create mode 100644 frontend/src/app/settings/search-history-settings/search-history-settings.component.spec.ts
 create mode 100644 frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
 rename frontend/src/app/settings/{ => search-history-settings}/toggle-search-history/toggle-search-history.component.html (100%)
 create mode 100644 frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.scss
 rename frontend/src/app/settings/{ => search-history-settings}/toggle-search-history/toggle-search-history.component.spec.ts (92%)
 rename frontend/src/app/settings/{ => search-history-settings}/toggle-search-history/toggle-search-history.component.ts (100%)

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 7cf8ea2c5..5176887eb 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -94,8 +94,9 @@ import { DatePickerComponent } from './corpus-selection/corpus-filter/date-picke
 import { CorpusInfoComponent } from './corpus-info/corpus-info.component';
 import { FieldInfoComponent } from './corpus-info/field-info/field-info.component';
 import { SettingsComponent } from './settings/settings.component';
-import { ToggleSearchHistoryComponent } from './settings/toggle-search-history/toggle-search-history.component';
-import { DeleteSearchHistoryComponent } from './settings/delete-search-history/delete-search-history.component';
+import { ToggleSearchHistoryComponent } from './settings/search-history-settings/toggle-search-history/toggle-search-history.component';
+import { DeleteSearchHistoryComponent } from './settings/search-history-settings/delete-search-history/delete-search-history.component';
+import { SearchHistorySettingsComponent } from './settings/search-history-settings/search-history-settings.component';
 
 
 export const appRoutes: Routes = [
@@ -239,6 +240,7 @@ export const declarations: any[] = [
     ScrollToDirective,
     SearchComponent,
     SearchHistoryComponent,
+    SearchHistorySettingsComponent,
     SearchRelevanceComponent,
     SearchResultsComponent,
     SearchSortingComponent,
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.html b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.html
similarity index 100%
rename from frontend/src/app/settings/delete-search-history/delete-search-history.component.html
rename to frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.html
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.scss b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.scss
similarity index 100%
rename from frontend/src/app/settings/delete-search-history/delete-search-history.component.scss
rename to frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.scss
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.spec.ts
similarity index 92%
rename from frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts
rename to frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.spec.ts
index 823fe9127..e3094c04e 100644
--- a/frontend/src/app/settings/delete-search-history/delete-search-history.component.spec.ts
+++ b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.spec.ts
@@ -1,7 +1,7 @@
 import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
 
 import { DeleteSearchHistoryComponent } from './delete-search-history.component';
-import { commonTestBed } from '../../common-test-bed';
+import { commonTestBed } from '../../../common-test-bed';
 
 describe('DeleteSearchHistoryComponent', () => {
     let component: DeleteSearchHistoryComponent;
diff --git a/frontend/src/app/settings/delete-search-history/delete-search-history.component.ts b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
similarity index 89%
rename from frontend/src/app/settings/delete-search-history/delete-search-history.component.ts
rename to frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
index c2f430732..fffb800d4 100644
--- a/frontend/src/app/settings/delete-search-history/delete-search-history.component.ts
+++ b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
@@ -1,6 +1,5 @@
 import { Component, OnInit } from '@angular/core';
 import { faTrash } from '@fortawesome/free-solid-svg-icons';
-import { ConfirmationService } from 'primeng/api';
 
 @Component({
     selector: 'ia-delete-search-history',
diff --git a/frontend/src/app/settings/search-history-settings/search-history-settings.component.html b/frontend/src/app/settings/search-history-settings/search-history-settings.component.html
new file mode 100644
index 000000000..1f9c7f02b
--- /dev/null
+++ b/frontend/src/app/settings/search-history-settings/search-history-settings.component.html
@@ -0,0 +1,9 @@
+<h2 class="title is-5">Search history</h2>
+
+<div class="block">
+    <ia-toggle-search-history></ia-toggle-search-history>
+</div>
+
+<div class="block">
+    <ia-delete-search-history></ia-delete-search-history>
+</div>
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.scss b/frontend/src/app/settings/search-history-settings/search-history-settings.component.scss
similarity index 100%
rename from frontend/src/app/settings/toggle-search-history/toggle-search-history.component.scss
rename to frontend/src/app/settings/search-history-settings/search-history-settings.component.scss
diff --git a/frontend/src/app/settings/search-history-settings/search-history-settings.component.spec.ts b/frontend/src/app/settings/search-history-settings/search-history-settings.component.spec.ts
new file mode 100644
index 000000000..b2de05b2b
--- /dev/null
+++ b/frontend/src/app/settings/search-history-settings/search-history-settings.component.spec.ts
@@ -0,0 +1,23 @@
+import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
+
+import { SearchHistorySettingsComponent } from './search-history-settings.component';
+import { commonTestBed } from '../../common-test-bed';
+
+describe('SearchHistorySettingsComponent', () => {
+    let component: SearchHistorySettingsComponent;
+    let fixture: ComponentFixture<SearchHistorySettingsComponent>;
+
+    beforeEach(waitForAsync(() => {
+        commonTestBed().testingModule.compileComponents();
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(SearchHistorySettingsComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+});
diff --git a/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts b/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
new file mode 100644
index 000000000..beee1975f
--- /dev/null
+++ b/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
@@ -0,0 +1,15 @@
+import { Component, OnInit } from '@angular/core';
+
+@Component({
+    selector: 'ia-search-history-settings',
+    templateUrl: './search-history-settings.component.html',
+    styleUrls: ['./search-history-settings.component.scss']
+})
+export class SearchHistorySettingsComponent implements OnInit {
+
+    constructor() { }
+
+    ngOnInit(): void {
+    }
+
+}
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html
similarity index 100%
rename from frontend/src/app/settings/toggle-search-history/toggle-search-history.component.html
rename to frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html
diff --git a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.scss b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.scss
new file mode 100644
index 000000000..e69de29bb
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.spec.ts
similarity index 92%
rename from frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts
rename to frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.spec.ts
index 13346a6b0..109ca90b6 100644
--- a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.spec.ts
+++ b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.spec.ts
@@ -1,7 +1,7 @@
 import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing';
 
 import { ToggleSearchHistoryComponent } from './toggle-search-history.component';
-import { commonTestBed } from '../../common-test-bed';
+import { commonTestBed } from '../../../common-test-bed';
 
 describe('ToggleSearchHistoryComponent', () => {
     let component: ToggleSearchHistoryComponent;
diff --git a/frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts
similarity index 100%
rename from frontend/src/app/settings/toggle-search-history/toggle-search-history.component.ts
rename to frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts
diff --git a/frontend/src/app/settings/settings.component.html b/frontend/src/app/settings/settings.component.html
index bfe33ae05..13910a889 100644
--- a/frontend/src/app/settings/settings.component.html
+++ b/frontend/src/app/settings/settings.component.html
@@ -3,15 +3,7 @@
         <h1 class="title">Settings</h1>
 
         <div class="box">
-            <h2 class="title is-5">Search history</h2>
-
-            <div class="block">
-                <ia-toggle-search-history></ia-toggle-search-history>
-            </div>
-
-            <div class="block">
-                <ia-delete-search-history></ia-delete-search-history>
-            </div>
+            <ia-search-history-settings></ia-search-history-settings>
         </div>
     </div>
 </section>

From f404c5f5509dfc6e32202cee26e22cc781152294 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 11:17:14 +0200
Subject: [PATCH 15/50] reorder search history explanation

---
 .../search-history-settings.component.html    | 16 ++++++++++++++
 .../toggle-search-history.component.html      | 21 +------------------
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/frontend/src/app/settings/search-history-settings/search-history-settings.component.html b/frontend/src/app/settings/search-history-settings/search-history-settings.component.html
index 1f9c7f02b..16177e1db 100644
--- a/frontend/src/app/settings/search-history-settings/search-history-settings.component.html
+++ b/frontend/src/app/settings/search-history-settings/search-history-settings.component.html
@@ -1,5 +1,21 @@
 <h2 class="title is-5">Search history</h2>
 
+<details class="content help">
+    <summary>More information</summary>
+
+    <p>
+        Saving your search history allows you to look up earlier queries.
+        You can view your saved history on the <a href="/search-history">search history page</a>.
+        It can be used to quickly get back to earlier queries, or to log your research process.
+    </p>
+    <p>
+        Search histories are stored on the I-analyzer server.
+        They are not shared with others, but developers may use search history
+        statistics to assess the level of interest in different corpora.
+        See our <a href="/privacy">privacy statement</a> for more information.
+    </p>
+</details>
+
 <div class="block">
     <ia-toggle-search-history></ia-toggle-search-history>
 </div>
diff --git a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html
index 50883ef42..9c7ad1e92 100644
--- a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html
+++ b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.html
@@ -7,27 +7,8 @@
         </label>
 
         <p class="help">
-
-        </p>
-    </div>
-
-    <details class="content help">
-        <summary>More information</summary>
-
-        <p>
-            Saving your search history allows you to look up earlier queries.
-            You can view your saved history on the <a href="/search-history">search history page</a>.
-            It can be used to quickly get back to earlier queries, or to log your research process.
-        </p>
-        <p>
-            Search histories are stored on the I-analyzer server.
-            They are not shared with others, but developers may use search history
-            statistics to assess the level of interest in different corpora.
-            See our <a href="/privacy">privacy statement</a> for more information.
-        </p>
-        <p>
             You can change this setting at any time. Doing so will not delete
             your existing search history.
         </p>
-    </details>
+    </div>
 </div>

From 86904f999fecfc237c50b0be516bccd0d8be9567 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 11:49:10 +0200
Subject: [PATCH 16/50] make api call to delete search history

---
 .../delete-search-history.component.ts             | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
index fffb800d4..c44008584 100644
--- a/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
+++ b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts
@@ -1,5 +1,7 @@
 import { Component, OnInit } from '@angular/core';
 import { faTrash } from '@fortawesome/free-solid-svg-icons';
+import { ApiService, NotificationService } from '../../../services';
+import { tap } from 'rxjs/operators';
 
 @Component({
     selector: 'ia-delete-search-history',
@@ -11,8 +13,14 @@ export class DeleteSearchHistoryComponent {
 
     showConfirm = false;
 
-    constructor() { }
-
-    deleteHistory() { }
+    constructor(private apiService: ApiService, private notificationService: NotificationService) { }
 
+    deleteHistory() {
+        this.apiService.deleteSearchHistory().pipe(
+            tap(() => this.showConfirm = false)
+        ).subscribe(
+            res => this.notificationService.showMessage('Search history deleted', 'success'),
+            err => this.notificationService.showMessage('Deleting search history failed', 'danger'),
+        );
+    }
 }

From ca823c5212770a00dcc276281c2e473336e8f83e Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 11:49:30 +0200
Subject: [PATCH 17/50] add notifications to search history toggle

---
 .../toggle-search-history.component.ts            | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts
index 245951d07..c6e6d1f7e 100644
--- a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts
+++ b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts
@@ -1,7 +1,7 @@
 import { Component } from '@angular/core';
-import { Observable } from 'rxjs';
+import { AuthService, NotificationService } from '../../../services';
 import { map } from 'rxjs/operators';
-import { AuthService } from '../../services';
+import { Observable } from 'rxjs';
 
 @Component({
     selector: 'ia-toggle-search-history',
@@ -11,7 +11,7 @@ import { AuthService } from '../../services';
 export class ToggleSearchHistoryComponent {
     searchHistoryEnabled$: Observable<boolean>;
 
-    constructor(private authService: AuthService) {
+    constructor(private authService: AuthService, private notificationService: NotificationService) {
         this.searchHistoryEnabled$ = this.authService.currentUser$.pipe(
             map(user => user?.enableSearchHistory)
         );
@@ -19,7 +19,14 @@ export class ToggleSearchHistoryComponent {
 
     emitChange(setting: boolean) {
         const data = { enableSearchHistory: setting };
-        this.authService.updateSettings(data).subscribe();
+        const succesMessage = `Search history will ${setting ? '' : 'not '} be saved from now on`;
+        this.authService.updateSettings(data).subscribe(
+            res => this.notificationService.showMessage(succesMessage, 'success'),
+            err => this.notificationService.showMessage(
+                'An error occured while trying to save your search history setting',
+                'danger'
+            ),
+        );
     }
 
 }

From b494dab2304d7bf5c8a5c55ba7ffb462d626d821 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 14:16:55 +0200
Subject: [PATCH 18/50] only send search history request when enabled

---
 frontend/src/app/services/search.service.ts | 35 +++++++++++++++------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/frontend/src/app/services/search.service.ts b/frontend/src/app/services/search.service.ts
index e5b2f5720..1689c1441 100644
--- a/frontend/src/app/services/search.service.ts
+++ b/frontend/src/app/services/search.service.ts
@@ -5,7 +5,7 @@ import { ElasticSearchService } from './elastic-search.service';
 import { QueryService } from './query.service';
 import {
     Corpus, QueryModel, SearchResults,
-    AggregateQueryFeedback, QueryDb
+    AggregateQueryFeedback, QueryDb, User
 } from '../models/index';
 import { AuthService } from './auth.service';
 
@@ -41,15 +41,15 @@ export class SearchService {
     public async search(queryModel: QueryModel
     ): Promise<SearchResults> {
         const user = await this.authService.getCurrentUserPromise();
-        const esQuery = queryModel.toEsQuery();
-        const query = new QueryDb(esQuery, queryModel.corpus.name, user.id);
-        query.started = new Date(Date.now());
-        const results = await this.elasticSearchService.search(
-            queryModel
-        );
-        query.total_results = results.total.value;
-        query.completed = new Date(Date.now());
-        this.queryService.save(query);
+        const request = () => this.elasticSearchService.search(queryModel);
+
+        let results: SearchResults;
+
+        if (user.enableSearchHistory) {
+            results = await this.saveQuery(queryModel, user, request);
+        } else {
+            results = await request();
+        }
 
         return {
             fields: queryModel.corpus.fields.filter((field) => field.resultsOverview),
@@ -84,4 +84,19 @@ export class SearchService {
         );
     }
 
+    /** execute a search request and save the action to the search history log */
+    private saveQuery(queryModel: QueryModel, user: User, searchRequest: () => Promise<SearchResults>): Promise<SearchResults> {
+        const esQuery = queryModel.toEsQuery();
+        const query = new QueryDb(esQuery, queryModel.corpus.name, user.id);
+        query.started = new Date(Date.now());
+
+        return searchRequest().then(results => {
+            query.total_results = results.total.value;
+            query.completed = new Date(Date.now());
+            this.queryService.save(query);
+            return results;
+        });
+    }
+
+
 }

From bb20b453daa0b085086c547bef07966db0616782 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 14:56:06 +0200
Subject: [PATCH 19/50] code quality

---
 frontend/src/app/services/search.service.ts | 53 ++++++++++++++-------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/frontend/src/app/services/search.service.ts b/frontend/src/app/services/search.service.ts
index 1689c1441..b185126c6 100644
--- a/frontend/src/app/services/search.service.ts
+++ b/frontend/src/app/services/search.service.ts
@@ -43,19 +43,17 @@ export class SearchService {
         const user = await this.authService.getCurrentUserPromise();
         const request = () => this.elasticSearchService.search(queryModel);
 
-        let results: SearchResults;
+        let resultsPromise: Promise<SearchResults>;
 
         if (user.enableSearchHistory) {
-            results = await this.saveQuery(queryModel, user, request);
+            resultsPromise = this.searchAndSave(queryModel, user, request);
         } else {
-            results = await request();
+            resultsPromise = request();
         }
 
-        return {
-            fields: queryModel.corpus.fields.filter((field) => field.resultsOverview),
-            total: results.total,
-            documents: results.documents,
-        } as SearchResults;
+        return resultsPromise.then(results =>
+            this.filterResultsFields(results, queryModel)
+        );
     }
 
     public async aggregateSearch<TKey>(
@@ -85,18 +83,39 @@ export class SearchService {
     }
 
     /** execute a search request and save the action to the search history log */
-    private saveQuery(queryModel: QueryModel, user: User, searchRequest: () => Promise<SearchResults>): Promise<SearchResults> {
-        const esQuery = queryModel.toEsQuery();
-        const query = new QueryDb(esQuery, queryModel.corpus.name, user.id);
-        query.started = new Date(Date.now());
-
-        return searchRequest().then(results => {
-            query.total_results = results.total.value;
-            query.completed = new Date(Date.now());
-            this.queryService.save(query);
+    private searchAndSave(queryModel: QueryModel, user: User, searchRequest: () => Promise<SearchResults>): Promise<SearchResults> {
+        return this.recordTime(searchRequest).then(([results, started, completed]) => {
+            this.saveQuery(queryModel, user, results, started, completed);
             return results;
         });
     }
 
+    /** execute a promise while noting the start and end time */
+    private recordTime<T>(makePromise: () => Promise<T>): Promise<[result: T, started: Date, completed: Date]> {
+        const started = new Date(Date.now());
+
+        return makePromise().then(result => {
+            const completed = new Date(Date.now());
+            return [result, started, completed];
+        });
+    }
+
+    /** save query data to search history */
+    private saveQuery(queryModel: QueryModel, user: User, results: SearchResults, started: Date, completed: Date) {
+        const esQuery = queryModel.toEsQuery();
+        const query = new QueryDb(esQuery, queryModel.corpus.name, user.id);
+        query.started = started;
+        query.total_results = results.total.value;
+        query.completed = completed;
+        this.queryService.save(query);
+    }
 
+    /** filter search results for fields included in resultsOverview of the corpus */
+    private filterResultsFields(results: SearchResults, queryModel: QueryModel): SearchResults {
+        return {
+            fields: queryModel.corpus.fields.filter((field) => field.resultsOverview),
+            total: results.total,
+            documents: results.documents,
+        } as SearchResults;
+    }
 }

From 1665409d81b83ffaf949fac67bb4eeaceced7ba2 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Wed, 19 Jul 2023 15:08:57 +0200
Subject: [PATCH 20/50] add unit test for searchService.search()

---
 .../src/app/services/search.service.spec.ts   | 18 +++++++++----
 frontend/src/mock-data/api.ts                 |  4 +++
 frontend/src/mock-data/auth.ts                |  5 ++++
 frontend/src/mock-data/elastic-search.ts      | 25 +++++++++++++------
 4 files changed, 40 insertions(+), 12 deletions(-)
 create mode 100644 frontend/src/mock-data/auth.ts

diff --git a/frontend/src/app/services/search.service.spec.ts b/frontend/src/app/services/search.service.spec.ts
index 6abf2ee76..50d5a258e 100644
--- a/frontend/src/app/services/search.service.spec.ts
+++ b/frontend/src/app/services/search.service.spec.ts
@@ -13,6 +13,10 @@ import { UserService } from './user.service';
 import { WordmodelsService } from './wordmodels.service';
 import { WordmodelsServiceMock } from '../../mock-data/wordmodels';
 import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { QueryModel } from '../models';
+import { mockCorpus } from '../../mock-data/corpus';
+import { AuthService } from './auth.service';
+import { AuthServiceMock } from '../../mock-data/auth';
 
 describe('SearchService', () => {
     beforeEach(() => {
@@ -24,18 +28,14 @@ describe('SearchService', () => {
             providers: [
                 SearchService,
                 ApiRetryService,
+                { provide: AuthService, useValue: new AuthServiceMock() },
                 { provide: ApiService, useValue: new ApiServiceMock() },
                 {
                     provide: ElasticSearchService,
                     useValue: new ElasticSearchServiceMock(),
                 },
                 QueryService,
-                UserService,
                 SessionService,
-                {
-                    provide: WordmodelsService,
-                    useValue: new WordmodelsServiceMock(),
-                },
             ],
         });
     });
@@ -43,4 +43,12 @@ describe('SearchService', () => {
     it('should be created', inject([SearchService], (service: SearchService) => {
         expect(service).toBeTruthy();
     }));
+
+    it('should search', inject([SearchService], async (service: SearchService) => {
+        const queryModel = new QueryModel(mockCorpus);
+        const results = await service.search(queryModel);
+        expect(results).toBeTruthy();
+        expect(results.total.value).toBeGreaterThan(0);
+    }));
+
 });
diff --git a/frontend/src/mock-data/api.ts b/frontend/src/mock-data/api.ts
index 30e8b4d09..b60eb5a95 100644
--- a/frontend/src/mock-data/api.ts
+++ b/frontend/src/mock-data/api.ts
@@ -62,4 +62,8 @@ export class ApiServiceMock {
     public fieldCoverage() {
         return Promise.resolve({});
     }
+
+    saveQuery() {
+        return Promise.resolve();
+    }
 }
diff --git a/frontend/src/mock-data/auth.ts b/frontend/src/mock-data/auth.ts
new file mode 100644
index 000000000..b4385c88f
--- /dev/null
+++ b/frontend/src/mock-data/auth.ts
@@ -0,0 +1,5 @@
+import { mockUser } from './user';
+
+export class AuthServiceMock {
+    getCurrentUserPromise = () => Promise.resolve(mockUser);
+}
diff --git a/frontend/src/mock-data/elastic-search.ts b/frontend/src/mock-data/elastic-search.ts
index 942f2fa3e..a1a49f946 100644
--- a/frontend/src/mock-data/elastic-search.ts
+++ b/frontend/src/mock-data/elastic-search.ts
@@ -1,5 +1,12 @@
-import { Corpus, FoundDocument, QueryModel } from '../app/models';
-import { EsQuery } from '../app/services';
+import { FoundDocument, SearchResults } from '../app/models';
+
+const mockDocumentResult = {
+    id: '0',
+    relevance: null,
+    fieldValues: {
+        content: 'Hello world!'
+    }
+};
 
 export class ElasticSearchServiceMock {
     /**
@@ -9,12 +16,16 @@ export class ElasticSearchServiceMock {
     }
 
     getDocumentById(): Promise<FoundDocument> {
+        return Promise.resolve(mockDocumentResult);
+    }
+
+    search(): Promise<SearchResults> {
         return Promise.resolve({
-            id: '0',
-            relevance: null,
-            fieldValues: {
-                content: 'Hello world!'
-            }
+            total: {
+                relation: 'eq',
+                value: 1
+            },
+            documents: [mockDocumentResult]
         });
     }
 }

From 5017146e634d33e223e5949fb688a39b77313346 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Thu, 27 Jul 2023 15:12:56 +0200
Subject: [PATCH 21/50] add settings module

---
 frontend/src/app/app.module.ts               |  2 ++
 frontend/src/app/settings/settings.module.ts | 23 ++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 frontend/src/app/settings/settings.module.ts

diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts
index 3ba07b933..1dd0b6d67 100644
--- a/frontend/src/app/app.module.ts
+++ b/frontend/src/app/app.module.ts
@@ -44,6 +44,7 @@ import { LoginModule } from './login/login.module';
 import { SearchModule } from './search/search.module';
 import { ManualModule } from './manual/manual.module';
 import { SettingsComponent } from './settings/settings.component';
+import { SettingsModule } from './settings/settings.module';
 
 
 export const appRoutes: Routes = [
@@ -148,6 +149,7 @@ export const imports: any[] = [
     ManualModule,
     MenuModule,
     SearchModule,
+    SettingsModule,
     SharedModule,
     WordModelsModule,
     RouterModule.forRoot(appRoutes, { relativeLinkResolution: 'legacy' }),
diff --git a/frontend/src/app/settings/settings.module.ts b/frontend/src/app/settings/settings.module.ts
new file mode 100644
index 000000000..6f1f833e3
--- /dev/null
+++ b/frontend/src/app/settings/settings.module.ts
@@ -0,0 +1,23 @@
+import { NgModule } from '@angular/core';
+import { SharedModule } from '../shared/shared.module';
+import { SettingsComponent } from './settings.component';
+import { SearchHistorySettingsComponent } from './search-history-settings/search-history-settings.component';
+import { DeleteSearchHistoryComponent } from './search-history-settings/delete-search-history/delete-search-history.component';
+import { ToggleSearchHistoryComponent } from './search-history-settings/toggle-search-history/toggle-search-history.component';
+
+
+
+@NgModule({
+    declarations: [
+        DeleteSearchHistoryComponent,
+        SearchHistorySettingsComponent,
+        SettingsComponent,
+        ToggleSearchHistoryComponent,
+    ],
+    imports: [
+        SharedModule
+    ], exports: [
+        SettingsComponent,
+    ]
+})
+export class SettingsModule { }

From 78b34645c3f7fd00465c3893591ad609bb061032 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 9 Aug 2023 13:43:13 +0200
Subject: [PATCH 22/50] use userprofile model

---
 .../0004_customuser_enable_search_history.py  | 18 ---------------
 backend/users/migrations/0004_userprofile.py  | 23 +++++++++++++++++++
 backend/users/models.py                       | 13 ++++++++++-
 3 files changed, 35 insertions(+), 19 deletions(-)
 delete mode 100644 backend/users/migrations/0004_customuser_enable_search_history.py
 create mode 100644 backend/users/migrations/0004_userprofile.py

diff --git a/backend/users/migrations/0004_customuser_enable_search_history.py b/backend/users/migrations/0004_customuser_enable_search_history.py
deleted file mode 100644
index c6964fb27..000000000
--- a/backend/users/migrations/0004_customuser_enable_search_history.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 4.1.9 on 2023-07-18 12:51
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('users', '0003_sitedomain'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='customuser',
-            name='enable_search_history',
-            field=models.BooleanField(default=True),
-        ),
-    ]
diff --git a/backend/users/migrations/0004_userprofile.py b/backend/users/migrations/0004_userprofile.py
new file mode 100644
index 000000000..40a0d2ebc
--- /dev/null
+++ b/backend/users/migrations/0004_userprofile.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.1.9 on 2023-08-09 11:41
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0003_sitedomain'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UserProfile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('enable_search_history', models.BooleanField(default=True, help_text='Whether to save the search history of this user')),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='profile', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]
diff --git a/backend/users/models.py b/backend/users/models.py
index 8c1c6878b..767dd941d 100644
--- a/backend/users/models.py
+++ b/backend/users/models.py
@@ -19,4 +19,15 @@ def has_access(self, corpus_name):
         return any(corpus for group in self.groups.all()
                    for corpus in group.corpora.filter(name=corpus_name))
 
-    enable_search_history = models.BooleanField(default=True)
+
+class UserProfile(models.Model):
+    user = models.OneToOneField(
+        to=CustomUser,
+        on_delete=models.CASCADE,
+        related_name='profile',
+    )
+
+    enable_search_history = models.BooleanField(
+        help_text='Whether to save the search history of this user',
+        default=True,
+    )

From c153996cb019e12ca22f72a5490e27f81530dab3 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 9 Aug 2023 13:46:31 +0200
Subject: [PATCH 23/50] add user profile to admin

---
 backend/users/admin.py  | 10 ++++++++--
 backend/users/models.py |  3 +++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/backend/users/admin.py b/backend/users/admin.py
index cf2ee2107..ec9ee1c56 100644
--- a/backend/users/admin.py
+++ b/backend/users/admin.py
@@ -1,5 +1,11 @@
 from django.contrib import admin
-from .models import CustomUser
+from .models import CustomUser, UserProfile
 from django.contrib.auth.admin import UserAdmin
 
-admin.site.register(CustomUser, UserAdmin)
+class InlineUserProfileAdmin(admin.StackedInline):
+    model = UserProfile
+
+class CustomUserAdmin(UserAdmin):
+    inlines = [InlineUserProfileAdmin]
+
+admin.site.register(CustomUser, CustomUserAdmin)
diff --git a/backend/users/models.py b/backend/users/models.py
index 767dd941d..aed69d11a 100644
--- a/backend/users/models.py
+++ b/backend/users/models.py
@@ -31,3 +31,6 @@ class UserProfile(models.Model):
         help_text='Whether to save the search history of this user',
         default=True,
     )
+
+    def __str__(self):
+        return f'Profile of {self.user.username}'

From 724bf87a48222a515ddd52351e100329fe57ae4c Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Tue, 15 Aug 2023 11:26:29 +0200
Subject: [PATCH 24/50] add profile assertin in user model test

---
 backend/users/tests/test_user_models.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/users/tests/test_user_models.py b/backend/users/tests/test_user_models.py
index f6ac7c7be..3d9153b89 100644
--- a/backend/users/tests/test_user_models.py
+++ b/backend/users/tests/test_user_models.py
@@ -10,11 +10,13 @@ def test_user_crud(db, user_credentials, admin_credentials):
     assert len(User.objects.all()) == 2
     assert admin.username == 'admin'
     assert user.email == 'basicuser@ianalyzer.com'
+    assert admin.profile
 
     admin.is_superuser = True
     admin.is_staff = True
     admin.save()
 
+
     admin.delete()
     user.delete()
 

From 2995aae9df29252037b9570c8121aeab2b524dce Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 9 Aug 2023 14:01:58 +0200
Subject: [PATCH 25/50] erialise user profile

add update customisation to user serialiser
---
 backend/users/serializers.py                | 19 ++++++++++++++++++-
 backend/users/tests/test_user_serializer.py |  8 ++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/backend/users/serializers.py b/backend/users/serializers.py
index 9622ad060..28b04e199 100644
--- a/backend/users/serializers.py
+++ b/backend/users/serializers.py
@@ -2,15 +2,32 @@
 from dj_rest_auth.registration.serializers import RegisterSerializer
 from rest_framework import serializers
 from django.db import transaction
+from users.models import UserProfile
+
+
+class UserProfileSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = UserProfile
+        fields = ['enable_search_history']
 
 
 class CustomUserDetailsSerializer(UserDetailsSerializer):
     is_admin = serializers.BooleanField(source='is_staff')
+    profile = UserProfileSerializer()
 
     class Meta(UserDetailsSerializer.Meta):
         fields = ('id', 'username', 'email', 'saml',
-                  'download_limit', 'is_admin', 'enable_search_history')
+                  'download_limit', 'is_admin', 'profile')
+
+
+    def update(self, instance, validated_data):
+        profile_data = validated_data.pop('profile', None)
+
+        if profile_data:
+            profile_serializer = UserProfileSerializer()
+            profile_serializer.update(instance.profile, profile_data)
 
+        return super().update(instance, validated_data)
 
 class CustomRegistrationSerializer(RegisterSerializer):
     saml = serializers.BooleanField(default=False)
diff --git a/backend/users/tests/test_user_serializer.py b/backend/users/tests/test_user_serializer.py
index f46a1a43c..61c9562e0 100644
--- a/backend/users/tests/test_user_serializer.py
+++ b/backend/users/tests/test_user_serializer.py
@@ -12,7 +12,9 @@ def test_user_serializer(auth_client,
         'download_limit': 10000,
         'is_admin': False,
         'saml': False,
-        'enable_search_history': True
+        'profile': {
+            'enable_search_history': True,
+        },
     }
 
 
@@ -26,7 +28,9 @@ def test_admin_serializer(admin_client, admin_credentials):
         'download_limit': 10000,
         'is_admin': True,
         'saml': False,
-        'enable_search_history': True,
+        'profile': {
+            'enable_search_history': True,
+        },
     }
 
 def test_user_updates(auth_client):

From 36c04633e49190608bf6e41ac54c2ff17d771359 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Tue, 15 Aug 2023 11:44:54 +0200
Subject: [PATCH 26/50] automatically create profile for new user

---
 backend/users/apps.py                       | 1 -
 backend/users/models.py                     | 1 +
 backend/users/signals.py                    | 9 ++++++++-
 backend/users/tests/test_user_serializer.py | 4 ++--
 4 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/backend/users/apps.py b/backend/users/apps.py
index 746cf1ff6..232707d8d 100644
--- a/backend/users/apps.py
+++ b/backend/users/apps.py
@@ -1,6 +1,5 @@
 from django.apps import AppConfig
 
-
 class UsersConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'users'
diff --git a/backend/users/models.py b/backend/users/models.py
index aed69d11a..154da9982 100644
--- a/backend/users/models.py
+++ b/backend/users/models.py
@@ -34,3 +34,4 @@ class UserProfile(models.Model):
 
     def __str__(self):
         return f'Profile of {self.user.username}'
+
diff --git a/backend/users/signals.py b/backend/users/signals.py
index 6a4119254..67198d742 100644
--- a/backend/users/signals.py
+++ b/backend/users/signals.py
@@ -3,7 +3,7 @@
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
-from .models import CustomUser
+from .models import CustomUser, UserProfile
 
 
 @receiver(post_save, sender=CustomUser)
@@ -28,3 +28,10 @@ def ensure_admin_email(sender, instance, created, **kwargs):
             print(f'Automatically verified email {instance.email} for {instance}')
         except Exception as e:
             print('Failed to automatically verify admin email', e, sep='\n')
+
+@receiver(post_save, sender=CustomUser)
+def create_user_profile(sender, instance, created, **kwargs):
+    if created:
+        UserProfile.objects.get_or_create(
+            user=instance
+        )
diff --git a/backend/users/tests/test_user_serializer.py b/backend/users/tests/test_user_serializer.py
index 61c9562e0..bd3858e82 100644
--- a/backend/users/tests/test_user_serializer.py
+++ b/backend/users/tests/test_user_serializer.py
@@ -36,11 +36,11 @@ def test_admin_serializer(admin_client, admin_credentials):
 def test_user_updates(auth_client):
     route = '/users/user/'
     details = lambda: auth_client.get(route)
-    search_history_enabled = lambda: details().data.get('enable_search_history')
+    search_history_enabled = lambda: details().data.get('profile').get('enable_search_history')
 
     assert search_history_enabled()
 
-    response = auth_client.patch(route, {'enable_search_history': False}, content_type='application/json')
+    response = auth_client.patch(route, {'profile': {'enable_search_history': False}}, content_type='application/json')
     assert response.status_code == 200
 
     assert not search_history_enabled()

From 04368e65a02ee2ce3fcaabcf053094b9917aef6b Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Tue, 15 Aug 2023 12:01:48 +0200
Subject: [PATCH 27/50] add user profiles in migration

---
 backend/users/migrations/0004_userprofile.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/backend/users/migrations/0004_userprofile.py b/backend/users/migrations/0004_userprofile.py
index 40a0d2ebc..b1237d8d4 100644
--- a/backend/users/migrations/0004_userprofile.py
+++ b/backend/users/migrations/0004_userprofile.py
@@ -4,6 +4,13 @@
 from django.db import migrations, models
 import django.db.models.deletion
 
+def add_user_profile(apps, schema_editor):
+    CustomUser = apps.get_model('users', 'CustomUser')
+    UserProfile = apps.get_model('users', 'UserProfile')
+    db_alias = schema_editor.connection.alias
+
+    for user in CustomUser.objects.all():
+        UserProfile.objects.get_or_create(user=user)
 
 class Migration(migrations.Migration):
 
@@ -20,4 +27,8 @@ class Migration(migrations.Migration):
                 ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='profile', to=settings.AUTH_USER_MODEL)),
             ],
         ),
+        migrations.RunPython(
+            add_user_profile,
+            reverse_code=migrations.RunPython.noop,
+        )
     ]

From c6f42362165c7372631e61f634be4185f646af6e Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Tue, 15 Aug 2023 13:15:47 +0200
Subject: [PATCH 28/50] adjust frontend; move api conversion to own module

---
 frontend/src/app/models/user.ts           |  6 +-
 frontend/src/app/services/auth.service.ts | 36 ++---------
 frontend/src/app/utils/user.spec.ts       | 79 +++++++++++++++++++++++
 frontend/src/app/utils/user.ts            | 44 +++++++++++++
 frontend/src/mock-data/user.ts            |  4 +-
 5 files changed, 136 insertions(+), 33 deletions(-)
 create mode 100644 frontend/src/app/utils/user.spec.ts
 create mode 100644 frontend/src/app/utils/user.ts

diff --git a/frontend/src/app/models/user.ts b/frontend/src/app/models/user.ts
index c8855c33b..481558979 100644
--- a/frontend/src/app/models/user.ts
+++ b/frontend/src/app/models/user.ts
@@ -1,5 +1,9 @@
 import * as _ from 'lodash';
 
+interface UserProfileResponse {
+    enable_search_history: boolean;
+}
+
 export interface UserResponse {
     id: number;
     username: string;
@@ -7,7 +11,7 @@ export interface UserResponse {
     download_limit: number;
     is_admin: boolean;
     saml: boolean;
-    enable_search_history: boolean;
+    profile: UserProfileResponse;
 }
 
 export class User {
diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts
index 3a6604ed2..4852e35a3 100644
--- a/frontend/src/app/services/auth.service.ts
+++ b/frontend/src/app/services/auth.service.ts
@@ -18,6 +18,7 @@ import { User, UserResponse } from '../models';
 import { ApiService } from './api.service';
 import { SessionService } from './session.service';
 import * as _ from 'lodash';
+import { encodeUserData, parseUserData } from '../utils/user';
 
 @Injectable({
     providedIn: 'root',
@@ -79,7 +80,7 @@ export class AuthService implements OnDestroy {
             .getUser()
             .pipe(takeUntil(this.destroy$))
             .subscribe(
-                (result) => this.setAuth(this.parseUserResponse(result)),
+                (result) => this.setAuth(parseUserData(result)),
                 () => this.purgeAuth()
             );
     }
@@ -104,7 +105,7 @@ export class AuthService implements OnDestroy {
         const loginRequest$ = this.apiService.login(username, password);
         return loginRequest$.pipe(
             mergeMap(() => this.checkUser()),
-            tap((res) => this.setAuth(this.parseUserResponse(res))),
+            tap((res) => this.setAuth(parseUserData(res))),
             catchError((error) => {
                 console.error(error);
                 return throwError(error);
@@ -169,8 +170,8 @@ export class AuthService implements OnDestroy {
     }
 
     public updateSettings(update: Partial<User>) {
-        return this.apiService.updateUserSettings(this.encodeUserUpdate(update)).pipe(
-            tap((res) => this.setAuth(this.parseUserResponse(res))),
+        return this.apiService.updateUserSettings(encodeUserData(update)).pipe(
+            tap((res) => this.setAuth(parseUserData(res))),
             catchError((error) => {
                 console.error(error);
                 return throwError(error);
@@ -178,31 +179,4 @@ export class AuthService implements OnDestroy {
         );
     }
 
-    /**
-     * Transforms backend user response to User object
-     *
-     * @param result User response data
-     * @returns User object
-     */
-    private parseUserResponse(
-        result: UserResponse
-    ): User {
-        return new User(
-            result.id,
-            result.username,
-            result.is_admin,
-            result.download_limit == null ? 0 : result.download_limit,
-            result.saml,
-            result.enable_search_history,
-        );
-    }
-
-    private encodeUserUpdate(update: Partial<User>): Partial<UserResponse> {
-        const changeKeys = {
-            enableSearchHistory: 'enable_search_history'
-        };
-        const transformKey = (value, key, obj) => changeKeys[key] || key;
-        return _.mapKeys(update, transformKey);
-    }
-
 }
diff --git a/frontend/src/app/utils/user.spec.ts b/frontend/src/app/utils/user.spec.ts
new file mode 100644
index 000000000..a43600813
--- /dev/null
+++ b/frontend/src/app/utils/user.spec.ts
@@ -0,0 +1,79 @@
+import * as _ from 'lodash';
+import { User, UserResponse } from '../models';
+import { parseUserData, encodeUserData } from './user';
+
+/**
+ * check if an object is a partial version of another object
+ *
+ * Verify that each key in `part` has the same value in `whole`,
+ * but ignore any properties of `whole` that are ommitted in `part`.
+ */
+const isPartialOf = <T>(part: Partial<T>, whole: T): boolean => {
+    const picked = _.pick(whole, _.keys(part));
+    return _.isEqual(part, picked);
+};
+
+const customMatchers = {
+    /** expect an object to be a partial version of another object */
+    toBePartialOf: (matchersUtil) => ({
+        compare: <T>(actual: Partial<T>, expected: T) => {
+            const pass = isPartialOf(actual, expected);
+            return { pass };
+        }
+    })
+};
+
+describe('user API conversion', () => {
+    let user: User;
+    let userResponse: UserResponse;
+
+    beforeEach(() => {
+        jasmine.addMatchers(customMatchers);
+    });
+
+    beforeEach(() => {
+        user = new User(
+            1,
+            'Hamlet',
+            false,
+            10000,
+            false,
+            true,
+        );
+    });
+
+    beforeEach(() => {
+        userResponse = {
+            id: 1,
+            username: 'Hamlet',
+            email: 'hamlet@elsinore.dk',
+            download_limit: 10000,
+            is_admin: false,
+            saml: false,
+            profile: {
+                enable_search_history: true,
+            }
+        };
+    });
+
+    it('should convert a user response to a user object', () => {
+        expect(parseUserData(userResponse)).toEqual(user);
+    });
+
+    it('should convert a user to a user response object', () => {
+        const encoded = encodeUserData(user);
+        (expect(encoded) as any).toBePartialOf(userResponse);
+    });
+
+    it('should define inverse functions', () => {
+        const encoded = encodeUserData(user);
+        const decoded = parseUserData(encoded as UserResponse);
+        expect(decoded).toEqual(user);
+
+        const parsed = parseUserData(userResponse);
+        const unparsed = encodeUserData(parsed);
+        // this has to be a partial match because User contains a subset of the information
+        // in the API
+        (expect(unparsed) as any).toBePartialOf(userResponse);
+    });
+});
diff --git a/frontend/src/app/utils/user.ts b/frontend/src/app/utils/user.ts
new file mode 100644
index 000000000..c0d44e644
--- /dev/null
+++ b/frontend/src/app/utils/user.ts
@@ -0,0 +1,44 @@
+import * as _ from 'lodash';
+import { User, UserResponse } from '../models';
+
+/* Transforms backend user response to User object
+*
+* @param result User response data
+* @returns User object
+*/
+export const parseUserData = (result: UserResponse): User => new User(
+    result.id,
+    result.username,
+    result.is_admin,
+    result.download_limit == null ? 0 : result.download_limit,
+    result.saml,
+    result.profile.enable_search_history,
+);
+
+/**
+ * Transfroms User data to backend UserResponse object
+ *
+ * Because this is used for patching, the data can be partial
+ *
+ * @param data (partial) User object
+ * @returns UserResponse object
+ */
+export const encodeUserData = (data: Partial<User>): Partial<UserResponse> => {
+    const changeKeys = {
+        name: 'username',
+        isAdmin: 'is_admin',
+        downloadLimit: 'download_limit',
+        isSamlLogin: 'saml',
+        enableSearchHistory: 'profile.enable_search_history'
+    };
+
+    const encoded = {};
+
+    _.keys(data).forEach(key => {
+        const value = data[key];
+        const path = changeKeys[key] ? _.toPath(changeKeys[key]) : key;
+        _.set(encoded, path, value);
+    });
+
+    return encoded;
+};
diff --git a/frontend/src/mock-data/user.ts b/frontend/src/mock-data/user.ts
index d243ba059..564cb28da 100644
--- a/frontend/src/mock-data/user.ts
+++ b/frontend/src/mock-data/user.ts
@@ -10,5 +10,7 @@ export const mockUserResponse: UserResponse = {
     email: 'mighty@mouse.com',
     download_limit: 10000,
     saml: false,
-    enable_search_history: true,
+    profile: {
+        enable_search_history: true,
+    },
 };

From 2fc1d72ae80beaa6004e7d6566a5f4d27bb5ae38 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 11 Sep 2023 15:01:03 +0200
Subject: [PATCH 29/50] code quality

---
 backend/users/models.py                                |  4 ++++
 frontend/src/app/services/search.service.ts            | 10 +++-------
 .../search-history-settings.component.ts               |  7 ++-----
 frontend/src/app/settings/settings.component.ts        |  7 ++-----
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/backend/users/models.py b/backend/users/models.py
index 154da9982..a0ed7b049 100644
--- a/backend/users/models.py
+++ b/backend/users/models.py
@@ -21,6 +21,10 @@ def has_access(self, corpus_name):
 
 
 class UserProfile(models.Model):
+    ''' User information that is not relevant to authentication.
+    E.g. settings, preferences, optional personal information.
+    '''
+
     user = models.OneToOneField(
         to=CustomUser,
         on_delete=models.CASCADE,
diff --git a/frontend/src/app/services/search.service.ts b/frontend/src/app/services/search.service.ts
index ff2249371..4f446a0b9 100644
--- a/frontend/src/app/services/search.service.ts
+++ b/frontend/src/app/services/search.service.ts
@@ -43,13 +43,9 @@ export class SearchService {
         const user = await this.authService.getCurrentUserPromise();
         const request = () => this.elasticSearchService.search(queryModel);
 
-        let resultsPromise: Promise<SearchResults>;
-
-        if (user.enableSearchHistory) {
-            resultsPromise = this.searchAndSave(queryModel, user, request);
-        } else {
-            resultsPromise = request();
-        }
+        const resultsPromise = user.enableSearchHistory ?
+            this.searchAndSave(queryModel, user, request) :
+            request();
 
         return resultsPromise.then(results =>
             this.filterResultsFields(results, queryModel)
diff --git a/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts b/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
index beee1975f..4096cfee3 100644
--- a/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
+++ b/frontend/src/app/settings/search-history-settings/search-history-settings.component.ts
@@ -1,15 +1,12 @@
-import { Component, OnInit } from '@angular/core';
+import { Component } from '@angular/core';
 
 @Component({
     selector: 'ia-search-history-settings',
     templateUrl: './search-history-settings.component.html',
     styleUrls: ['./search-history-settings.component.scss']
 })
-export class SearchHistorySettingsComponent implements OnInit {
+export class SearchHistorySettingsComponent {
 
     constructor() { }
 
-    ngOnInit(): void {
-    }
-
 }
diff --git a/frontend/src/app/settings/settings.component.ts b/frontend/src/app/settings/settings.component.ts
index 8f4b6f22f..855fc747c 100644
--- a/frontend/src/app/settings/settings.component.ts
+++ b/frontend/src/app/settings/settings.component.ts
@@ -1,15 +1,12 @@
-import { Component, OnInit } from '@angular/core';
+import { Component } from '@angular/core';
 
 @Component({
     selector: 'ia-settings',
     templateUrl: './settings.component.html',
     styleUrls: ['./settings.component.scss']
 })
-export class SettingsComponent implements OnInit {
+export class SettingsComponent {
 
     constructor() { }
 
-    ngOnInit(): void {
-    }
-
 }

From e4e71647d1bed9288640847b5baf96522911af03 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 11 Sep 2023 15:01:32 +0200
Subject: [PATCH 30/50] use create instead of get_or_create

since the object should not exist yet
---
 backend/users/signals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/users/signals.py b/backend/users/signals.py
index 67198d742..2939086e6 100644
--- a/backend/users/signals.py
+++ b/backend/users/signals.py
@@ -32,6 +32,6 @@ def ensure_admin_email(sender, instance, created, **kwargs):
 @receiver(post_save, sender=CustomUser)
 def create_user_profile(sender, instance, created, **kwargs):
     if created:
-        UserProfile.objects.get_or_create(
+        UserProfile.objects.create(
             user=instance
         )

From 0641efa26590c44e2ebd08ddea88b7ddc7b0fa52 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 4 Jul 2023 16:11:10 +0200
Subject: [PATCH 31/50] use ids in filenames

---
 backend/download/conftest.py               |  5 ++++-
 backend/download/create_csv.py             | 21 ++++++---------------
 backend/download/tasks.py                  | 12 ++++++------
 backend/download/tests/test_csv_results.py |  4 ++--
 backend/download/tests/test_full_data.py   |  2 +-
 backend/download/views.py                  |  6 +++---
 6 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/backend/download/conftest.py b/backend/download/conftest.py
index 1eb5e9b54..17cdd9109 100644
--- a/backend/download/conftest.py
+++ b/backend/download/conftest.py
@@ -50,6 +50,8 @@ def index_ml_mock_corpus(es_client, ml_mock_corpus):
 def index_mock_corpus(es_client, mock_corpus, index_small_mock_corpus, index_large_mock_corpus, index_ml_mock_corpus):
     yield mock_corpus
 
+
+
 def save_all_results_csv(mock_corpus, mock_corpus_specs):
     fields = mock_corpus_specs['fields']
     query = mock_corpus_specs['example_query']
@@ -61,7 +63,8 @@ def save_all_results_csv(mock_corpus, mock_corpus_specs):
         'route': '/search/{};query={}'.format(mock_corpus, query)
     }
     results = tasks.download_scroll(request_json)
-    filename = tasks.make_csv(results, request_json)
+    fake_id = mock_corpus + '_all_results'
+    filename = tasks.make_csv(results, request_json, fake_id)
 
     return filename
 
diff --git a/backend/download/create_csv.py b/backend/download/create_csv.py
index 01e01092b..f0f93b8e9 100644
--- a/backend/download/create_csv.py
+++ b/backend/download/create_csv.py
@@ -21,12 +21,10 @@ def write_file(filename, fieldnames, rows, dialect = 'excel'):
 
     return filepath
 
-def create_filename(descriptive_part, essential_suffix = '.csv'):
-    max_length = 255 - (len(essential_suffix) + len(settings.CSV_FILES_PATH))
-    truncated = descriptive_part[:min(max_length, len(descriptive_part))]
-    return truncated + essential_suffix
+def create_filename(download_id):
+    return f'{download_id}.csv'
 
-def search_results_csv(results, fields, query):
+def search_results_csv(results, fields, query, download_id):
     entries = []
     field_set = set(fields)
     field_set.update(['query'])
@@ -50,14 +48,14 @@ def search_results_csv(results, fields, query):
                     entry.update({highlight_field_name: soup.get_text()})
         entries.append(entry)
 
-    filename = create_filename(query)
+    filename = create_filename(download_id)
     field_set.discard('context')
     fieldnames = sorted(field_set)
     filepath = write_file(filename, fieldnames, entries, dialect = 'resultsDialect')
     return filepath
 
 
-def term_frequency_csv(queries, results, field_name, unit = None):
+def term_frequency_csv(queries, results, field_name, download_id, unit = None):
     has_token_counts = results[0].get('token_count', None) != None
     query_column = ['Query'] if len(queries) > 1 else []
     freq_columns = ['Term frequency', 'Relative term frequency (by # documents)', 'Total documents']
@@ -66,17 +64,10 @@ def term_frequency_csv(queries, results, field_name, unit = None):
 
     rows = term_frequency_csv_rows(queries, results, field_name, unit)
 
-    filename = term_frequency_filename(queries, field_name)
+    filename = create_filename(download_id)
     filepath = write_file(filename, fieldnames, rows)
     return filepath
 
-def term_frequency_filename(queries, field_name):
-    querystring = '_'.join(queries)
-    timestamp = datetime.now().isoformat(sep='_', timespec='minutes') # ensure csv filenames are unique with timestamp
-    suffix = '_' + timestamp + '.csv'
-    description = 'term_frequency_{}_{}'.format(field_name, querystring)
-    return create_filename(description, suffix)
-
 def term_frequency_csv_rows(queries, results, field_name, unit):
     for result in results:
         field_value = format_field_value(result['key'], unit)
diff --git a/backend/download/tasks.py b/backend/download/tasks.py
index 91c88f358..a47340ba6 100644
--- a/backend/download/tasks.py
+++ b/backend/download/tasks.py
@@ -37,9 +37,9 @@ def download_scroll(request_json, download_size=10000):
     return results
 
 @shared_task()
-def make_csv(results, request_json):
+def make_csv(results, request_json, log_id):
     query = create_query(request_json)
-    filepath = create_csv.search_results_csv(results, request_json['fields'], query)
+    filepath = create_csv.search_results_csv(results, request_json['fields'], query, log_id)
     return filepath
 
 
@@ -82,7 +82,7 @@ def download_search_results(request_json, user):
 
     make_chain = lambda: chain(
         download_scroll.s(request_json, download_limit),
-        make_csv.s(request_json),
+        make_csv.s(request_json, download.id),
         complete_download.s(download.id),
         csv_data_email.s(user.email, user.username),
     ).on_error(complete_failed_download.s(download.id))
@@ -90,12 +90,12 @@ def download_search_results(request_json, user):
     return try_download(make_chain, download)
 
 @shared_task()
-def make_term_frequency_csv(results_per_series, parameters_per_series):
+def make_term_frequency_csv(results_per_series, parameters_per_series, log_id):
     '''
     Export term frequency results to a csv.
     '''
     query_per_series, field_name, unit = extract_term_frequency_download_metadata(parameters_per_series)
-    return create_csv.term_frequency_csv(query_per_series, results_per_series, field_name, unit = unit)
+    return create_csv.term_frequency_csv(query_per_series, results_per_series, field_name, log_id, unit = unit)
 
 
 def term_frequency_full_data_tasks(parameters_per_series, visualization_type):
@@ -166,7 +166,7 @@ def download_full_data(request_json, user):
 
     make_chain = lambda : chain(
         task,
-        make_term_frequency_csv.s(parameters),
+        make_term_frequency_csv.s(parameters, download.id),
         complete_download.s(download.id),
         csv_data_email.s(user.email, user.username),
     ).on_error(complete_failed_download.s(download.id))
diff --git a/backend/download/tests/test_csv_results.py b/backend/download/tests/test_csv_results.py
index b6ff8b0da..f33a896ef 100644
--- a/backend/download/tests/test_csv_results.py
+++ b/backend/download/tests/test_csv_results.py
@@ -45,7 +45,7 @@
 def result_csv_with_highlights(csv_directory):
     route = 'parliament-netherlands_query=test'
     fields = ['speech']
-    file = create_csv.search_results_csv(hits(mock_es_result), fields, route)
+    file = create_csv.search_results_csv(hits(mock_es_result), fields, route, 0)
     return file
 
 def test_create_csv(result_csv_with_highlights):
@@ -190,7 +190,7 @@ def test_csv_encoding(ml_mock_corpus_results_csv):
 
 @pytest.fixture()
 def term_frequency_file(index_small_mock_corpus, csv_directory):
-    filename = create_csv.term_frequency_csv(mock_queries, mock_timeline_result, 'date', unit = 'year')
+    filename = create_csv.term_frequency_csv(mock_queries, mock_timeline_result, 'date', 0, unit = 'year')
     return filename
 
 
diff --git a/backend/download/tests/test_full_data.py b/backend/download/tests/test_full_data.py
index 385fb701b..fccca3e6d 100644
--- a/backend/download/tests/test_full_data.py
+++ b/backend/download/tests/test_full_data.py
@@ -24,7 +24,7 @@ def test_timeline_full_data(small_mock_corpus, index_small_mock_corpus, small_mo
 
     group = tasks.term_frequency_full_data_tasks(full_data_parameters, 'date_term_frequency')
     results = group.apply().get()
-    filename = tasks.make_term_frequency_csv(results, full_data_parameters)
+    filename = tasks.make_term_frequency_csv(results, full_data_parameters, 0)
 
     with open(filename) as f:
         reader = csv.DictReader(f)
diff --git a/backend/download/views.py b/backend/download/views.py
index 273bc672f..12d7de6ee 100644
--- a/backend/download/views.py
+++ b/backend/download/views.py
@@ -51,11 +51,11 @@ def post(self, request, *args, **kwargs):
             handle_tags_in_request(request)
             search_results = es_download.normal_search(
                 corpus_name, request.data['es_query'], request.data['size'])
-            csv_path = tasks.make_csv(search_results, request.data)
-            directory, filename = os.path.split(csv_path)
-            # Create download for download history
             download = Download.objects.create(
                 download_type='search_results', corpus=corpus, parameters=request.data, user=request.user)
+            csv_path = tasks.make_csv(search_results, request.data, download.id)
+            directory, filename = os.path.split(csv_path)
+            # Create download for download history
             download.complete(filename=filename)
             return send_csv_file(directory, filename, 'search_results', request.data['encoding'])
         except Exception as e:

From 08c00cb5dd3bdc0a75d10e97b6b6edd5b2ccb359 Mon Sep 17 00:00:00 2001
From: lukavdplas <lukavdplas@gmail.com>
Date: Tue, 4 Jul 2023 16:33:10 +0200
Subject: [PATCH 32/50] adjust download filename test

---
 backend/download/conftest.py                |  9 +++++----
 backend/download/tests/test_file_storage.py | 16 +++++++++++-----
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/backend/download/conftest.py b/backend/download/conftest.py
index 17cdd9109..5ef03873d 100644
--- a/backend/download/conftest.py
+++ b/backend/download/conftest.py
@@ -50,18 +50,19 @@ def index_ml_mock_corpus(es_client, ml_mock_corpus):
 def index_mock_corpus(es_client, mock_corpus, index_small_mock_corpus, index_large_mock_corpus, index_ml_mock_corpus):
     yield mock_corpus
 
-
-
-def save_all_results_csv(mock_corpus, mock_corpus_specs):
+def all_results_request_json(mock_corpus, mock_corpus_specs):
     fields = mock_corpus_specs['fields']
     query = mock_corpus_specs['example_query']
 
-    request_json = {
+    return {
         'corpus': mock_corpus,
         'es_query': MATCH_ALL,
         'fields': fields,
         'route': '/search/{};query={}'.format(mock_corpus, query)
     }
+
+def save_all_results_csv(mock_corpus, mock_corpus_specs):
+    request_json = all_results_request_json(mock_corpus, mock_corpus_specs)
     results = tasks.download_scroll(request_json)
     fake_id = mock_corpus + '_all_results'
     filename = tasks.make_csv(results, request_json, fake_id)
diff --git a/backend/download/tests/test_file_storage.py b/backend/download/tests/test_file_storage.py
index 767b23f46..0f96c30b7 100644
--- a/backend/download/tests/test_file_storage.py
+++ b/backend/download/tests/test_file_storage.py
@@ -1,7 +1,13 @@
+import os
 from download import tasks
+from download.conftest import all_results_request_json
+from download.models import Download
 
-def test_format_route_to_filename():
-    route = '/search/mock-corpus;query=test'
-    request_json = { 'route': route }
-    output = tasks.create_query(request_json)
-    assert output == 'mock-corpus_query=test'
+def test_download_filename(auth_user, small_mock_corpus, index_small_mock_corpus, small_mock_corpus_specs):
+    request = all_results_request_json(small_mock_corpus, small_mock_corpus_specs)
+    tasks.download_search_results(request, auth_user).apply()
+    download = Download.objects.latest('completed')
+    _, filename = os.path.split(download.filename)
+    name, ext = os.path.splitext(filename)
+    assert name == str(download.id)
+    assert ext == '.csv'

From d2af30f6a7d682536ae0871323f188c7b02959c8 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Fri, 6 Oct 2023 13:05:46 +0200
Subject: [PATCH 33/50] send proper filename from backend

---
 backend/download/mail.py   |  4 ++--
 backend/download/models.py |  7 +++++++
 backend/download/views.py  | 15 ++++++++-------
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/backend/download/mail.py b/backend/download/mail.py
index 5dc840ec1..3bd29ccaa 100644
--- a/backend/download/mail.py
+++ b/backend/download/mail.py
@@ -20,8 +20,8 @@ def send_csv_email(user_email, username, download_id):
 
     subject = 'I-Analyzer CSV download'
     from_email = settings.DEFAULT_FROM_EMAIL
-    path = Download.objects.get(id=download_id).filename
-    _, filename = os.path.split(path)
+    download = Download.objects.get(id=download_id)
+    filename = download.nice_filename()
 
     context = {
         'email_title': 'Download CSV',
diff --git a/backend/download/models.py b/backend/download/models.py
index ca49e8db6..83b3f6e4c 100644
--- a/backend/download/models.py
+++ b/backend/download/models.py
@@ -51,3 +51,10 @@ def complete(self, filename = None):
         self.filename = filename
         self.completed = datetime.now()
         self.save()
+
+    def nice_filename(self):
+        corpus_name = self.corpus.name
+        type_name = self.download_type
+        timestamp = self.completed.strftime('%Y-%m-%d %H:%M')
+
+        return f'{type_name} {corpus_name} {timestamp}.csv'
diff --git a/backend/download/views.py b/backend/download/views.py
index 12d7de6ee..4ee51d2ca 100644
--- a/backend/download/views.py
+++ b/backend/download/views.py
@@ -21,14 +21,15 @@
 
 logger = logging.getLogger()
 
-def send_csv_file(directory, filename, download_type, encoding, format=None):
+def send_csv_file(download, directory, filename, download_type, encoding, format=None):
     '''
     Perform final formatting and send a CSV file as a FileResponse
     '''
     converted_filename = convert_csv.convert_csv(
         directory, filename, download_type, encoding, format)
     path = os.path.join(directory, converted_filename)
-    return FileResponse(open(path, 'rb'), filename=filename, as_attachment=True)
+
+    return FileResponse(open(path, 'rb'), filename=download.nice_filename(), as_attachment=True)
 
 class ResultsDownloadView(APIView):
     '''
@@ -57,7 +58,7 @@ def post(self, request, *args, **kwargs):
             directory, filename = os.path.split(csv_path)
             # Create download for download history
             download.complete(filename=filename)
-            return send_csv_file(directory, filename, 'search_results', request.data['encoding'])
+            return send_csv_file(download, directory, filename, 'search_results', request.data['encoding'])
         except Exception as e:
             logger.error(e)
             raise APIException(detail = 'Download failed: could not generate csv file')
@@ -138,13 +139,13 @@ def get(self, request, *args, **kwargs):
         encoding = request.query_params.get('encoding', 'utf-8')
         format = request.query_params.get('table_format', None)
 
-        record = Download.objects.get(id=id)
-        if not record.user == request.user:
+        download = Download.objects.get(id=id)
+        if not download.user == request.user:
             raise PermissionDenied(detail='User has no access to this download')
 
         directory = settings.CSV_FILES_PATH
 
-        if not os.path.isfile(os.path.join(directory, record.filename)):
+        if not os.path.isfile(os.path.join(directory, download.filename)):
             raise NotFound(detail='File does not exist')
 
-        return send_csv_file(directory, record.filename, record.download_type, encoding, format)
+        return send_csv_file(download, directory, download.filename, download.download_type, encoding, format)

From 13d734e7c4d5323666155010e7abc2d87733f3d6 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 9 Oct 2023 16:20:25 +0200
Subject: [PATCH 34/50] remove ngram visualistion from dbnl

---
 backend/corpora/dbnl/dbnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/corpora/dbnl/dbnl.py b/backend/corpora/dbnl/dbnl.py
index b48a28963..99281c6aa 100644
--- a/backend/corpora/dbnl/dbnl.py
+++ b/backend/corpora/dbnl/dbnl.py
@@ -362,7 +362,7 @@ def _xml_files(self):
             transform_soup_func=utils.pad_content,
         ),
         es_mapping=main_content_mapping(token_counts=True),
-        visualizations=['wordcloud', 'ngram'],
+        visualizations=['wordcloud'],
     )
 
     has_content = FieldDefinition(

From 1ab16035e871cda699c2d3612294aafd12b44dc8 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 9 Oct 2023 16:21:42 +0200
Subject: [PATCH 35/50] add documentation

---
 documentation/Defining-corpus-fields.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/Defining-corpus-fields.md b/documentation/Defining-corpus-fields.md
index b53d8dab2..53dc30d89 100644
--- a/documentation/Defining-corpus-fields.md
+++ b/documentation/Defining-corpus-fields.md
@@ -47,7 +47,7 @@ The following properties determine how a field appears in the interface.
 
 `search_filter` can be set if the interface should include a search filter widget for the field. I-analyzer includes date filters, multiplechoice filters (used for keyword data), range filters, and boolean filters. See [filters.py](../backend/addcorpus/filters.py).
 
-`visualizations` optionally specifies a list of visualisations that apply for the field. Generally speaking, this is based on the type of data. For date fields and categorical/ordinal fields (usually keyword type), you can use `['resultscount', 'termfrequency']`. For text fields, you can use `['wordcloud', 'ngram']`.
+`visualizations` optionally specifies a list of visualisations that apply for the field. Generally speaking, this is based on the type of data. For date fields and categorical/ordinal fields (usually keyword type), you can use `['resultscount', 'termfrequency']`. For text fields, you can use `['wordcloud', 'ngram']`. However, the ngram visualisation also requires that your corpus has a date field.
 
 If a field includes the `'resultscount'` and/or `'termfrequency'` visualisations and it is not a date field, you can also specify `visualisation_sort`, which determines how to sort the x-axis of the graph. Default is `'value'`, where categories are sorted based on the y-axis value (i.e., frequency). You may specify that they should be sorted on `'key'`, so that categories are sorted alphabetically (for keywords) or small-to-large (for numbers).
 

From a8e85aa4cca792177d69d25fe81a0be657ed56b0 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 9 Oct 2023 16:35:16 +0200
Subject: [PATCH 36/50] add validation: ngram visualisation needs date field

---
 backend/addcorpus/models.py                | 10 ++++++-
 backend/addcorpus/tests/test_validators.py | 32 +++++++++++++++++++++-
 backend/addcorpus/validators.py            |  8 ++++++
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/backend/addcorpus/models.py b/backend/addcorpus/models.py
index 0c91431ee..4ec410382 100644
--- a/backend/addcorpus/models.py
+++ b/backend/addcorpus/models.py
@@ -8,7 +8,7 @@
 from addcorpus.validators import validate_language_code, validate_image_filename_extension, \
     validate_markdown_filename_extension, validate_es_mapping, validate_mimetype, validate_search_filter, \
     validate_name_is_not_a_route_parameter, validate_search_filter_with_mapping, validate_searchable_field_has_full_text_search, \
-    validate_visualizations_with_mapping, validate_implication
+    validate_visualizations_with_mapping, validate_implication, any_fields_with_ngram_visualisation, any_date_fields
 
 MAX_LENGTH_NAME = 126
 MAX_LENGTH_DESCRIPTION = 254
@@ -126,6 +126,14 @@ class CorpusConfiguration(models.Model):
     def __str__(self):
         return f'Configuration of <{self.corpus.name}>'
 
+    def clean(self):
+        # validate that ngram visualisations are only included if there is also a date field
+        validate_implication(
+            self.fields, self.fields,
+            'the ngram visualisation can only be used if the corpus has a field with date mapping',
+            any_fields_with_ngram_visualisation, any_date_fields
+        )
+
 FIELD_DISPLAY_TYPES = [
     ('text_content', 'text content'),
     (MappingType.TEXT.value, 'text'),
diff --git a/backend/addcorpus/tests/test_validators.py b/backend/addcorpus/tests/test_validators.py
index f5d95e6a3..7143183f5 100644
--- a/backend/addcorpus/tests/test_validators.py
+++ b/backend/addcorpus/tests/test_validators.py
@@ -1,5 +1,6 @@
 import pytest
-from addcorpus.es_mappings import int_mapping, text_mapping, keyword_mapping
+from addcorpus.models import Field
+from addcorpus.es_mappings import int_mapping, text_mapping, keyword_mapping, main_content_mapping, date_mapping
 from addcorpus.validators import *
 
 def test_validate_mimetype():
@@ -71,3 +72,32 @@ def test_filename_validation():
     with pytest.raises(ValidationError):
         validate_image_filename_extension('image.txt')
 
+def test_validate_ngram_has_date_field():
+    text_field = Field(
+        name='content',
+        es_mapping=main_content_mapping(),
+        visualizations=['wordcloud', 'ngram']
+    )
+
+    date_field = Field(
+        name='date',
+        es_mapping=date_mapping()
+    )
+
+    with_date_field = [text_field, date_field]
+    without_date_field = [text_field]
+
+    validate_implication(
+        with_date_field, with_date_field,
+        '',
+        any_fields_with_ngram_visualisation,
+        any_date_fields
+    )
+
+    with pytest.raises(ValidationError):
+        validate_implication(
+            without_date_field, without_date_field,
+            '',
+            any_fields_with_ngram_visualisation,
+            any_date_fields
+        )
diff --git a/backend/addcorpus/validators.py b/backend/addcorpus/validators.py
index fe11fb33d..a4d60ee8c 100644
--- a/backend/addcorpus/validators.py
+++ b/backend/addcorpus/validators.py
@@ -152,3 +152,11 @@ def validate_markdown_filename_extension(filename):
 def validate_image_filename_extension(filename):
     allowed = ['.jpeg', '.jpg', '.png', '.JPG']
     validate_filename_extension(filename, allowed)
+
+def any_date_fields(fields):
+    is_date = lambda field: primary_mapping_type(field.es_mapping) == 'date'
+    return any(map(is_date, fields))
+
+def any_fields_with_ngram_visualisation(fields):
+    has_ngram = lambda field: field.visualizations and 'ngram' in field.visualizations
+    return any(map(has_ngram, fields))

From 69ddc46e3138435e2774e7f646c2fb37feef496a Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Mon, 9 Oct 2023 16:47:08 +0200
Subject: [PATCH 37/50] fix validation logic

---
 backend/addcorpus/models.py                | 16 +++++++---------
 backend/addcorpus/tests/test_validators.py |  8 ++++----
 backend/addcorpus/validators.py            |  5 ++---
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/backend/addcorpus/models.py b/backend/addcorpus/models.py
index 4ec410382..880d853bf 100644
--- a/backend/addcorpus/models.py
+++ b/backend/addcorpus/models.py
@@ -8,7 +8,7 @@
 from addcorpus.validators import validate_language_code, validate_image_filename_extension, \
     validate_markdown_filename_extension, validate_es_mapping, validate_mimetype, validate_search_filter, \
     validate_name_is_not_a_route_parameter, validate_search_filter_with_mapping, validate_searchable_field_has_full_text_search, \
-    validate_visualizations_with_mapping, validate_implication, any_fields_with_ngram_visualisation, any_date_fields
+    validate_visualizations_with_mapping, validate_implication, any_date_fields, visualisations_require_date_field
 
 MAX_LENGTH_NAME = 126
 MAX_LENGTH_DESCRIPTION = 254
@@ -126,14 +126,6 @@ class CorpusConfiguration(models.Model):
     def __str__(self):
         return f'Configuration of <{self.corpus.name}>'
 
-    def clean(self):
-        # validate that ngram visualisations are only included if there is also a date field
-        validate_implication(
-            self.fields, self.fields,
-            'the ngram visualisation can only be used if the corpus has a field with date mapping',
-            any_fields_with_ngram_visualisation, any_date_fields
-        )
-
 FIELD_DISPLAY_TYPES = [
     ('text_content', 'text content'),
     (MappingType.TEXT.value, 'text'),
@@ -277,3 +269,9 @@ def clean(self):
             validate_implication(self.search_field_core, self.searchable, "Core search fields must be searchable")
         except ValidationError as e:
             warnings.warn(e.message)
+
+        validate_implication(
+            self.visualizations, self.corpus_configuration.fields.all(),
+            'The ngram visualisation requires a date field on the corpus',
+            visualisations_require_date_field, any_date_fields,
+        )
diff --git a/backend/addcorpus/tests/test_validators.py b/backend/addcorpus/tests/test_validators.py
index 7143183f5..fbab9c400 100644
--- a/backend/addcorpus/tests/test_validators.py
+++ b/backend/addcorpus/tests/test_validators.py
@@ -88,16 +88,16 @@ def test_validate_ngram_has_date_field():
     without_date_field = [text_field]
 
     validate_implication(
-        with_date_field, with_date_field,
+        text_field.visualizations, with_date_field,
         '',
-        any_fields_with_ngram_visualisation,
+        visualisations_require_date_field,
         any_date_fields
     )
 
     with pytest.raises(ValidationError):
         validate_implication(
-            without_date_field, without_date_field,
+            text_field.visualizations, without_date_field,
             '',
-            any_fields_with_ngram_visualisation,
+            visualisations_require_date_field,
             any_date_fields
         )
diff --git a/backend/addcorpus/validators.py b/backend/addcorpus/validators.py
index a4d60ee8c..a894fc074 100644
--- a/backend/addcorpus/validators.py
+++ b/backend/addcorpus/validators.py
@@ -157,6 +157,5 @@ def any_date_fields(fields):
     is_date = lambda field: primary_mapping_type(field.es_mapping) == 'date'
     return any(map(is_date, fields))
 
-def any_fields_with_ngram_visualisation(fields):
-    has_ngram = lambda field: field.visualizations and 'ngram' in field.visualizations
-    return any(map(has_ngram, fields))
+def visualisations_require_date_field(visualisations):
+    return visualisations and 'ngram' in visualisations

From edd8f76350ab4538e69c97ad5c958a2ff5558d5d Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 11 Oct 2023 17:54:39 +0200
Subject: [PATCH 38/50] rename function

---
 backend/download/mail.py   | 2 +-
 backend/download/models.py | 2 +-
 backend/download/views.py  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/download/mail.py b/backend/download/mail.py
index 3bd29ccaa..0220edd58 100644
--- a/backend/download/mail.py
+++ b/backend/download/mail.py
@@ -21,7 +21,7 @@ def send_csv_email(user_email, username, download_id):
     subject = 'I-Analyzer CSV download'
     from_email = settings.DEFAULT_FROM_EMAIL
     download = Download.objects.get(id=download_id)
-    filename = download.nice_filename()
+    filename = download.descriptive_filename()
 
     context = {
         'email_title': 'Download CSV',
diff --git a/backend/download/models.py b/backend/download/models.py
index 83b3f6e4c..8e9936a7b 100644
--- a/backend/download/models.py
+++ b/backend/download/models.py
@@ -52,7 +52,7 @@ def complete(self, filename = None):
         self.completed = datetime.now()
         self.save()
 
-    def nice_filename(self):
+    def descriptive_filename(self):
         corpus_name = self.corpus.name
         type_name = self.download_type
         timestamp = self.completed.strftime('%Y-%m-%d %H:%M')
diff --git a/backend/download/views.py b/backend/download/views.py
index 4ee51d2ca..5510183a9 100644
--- a/backend/download/views.py
+++ b/backend/download/views.py
@@ -29,7 +29,7 @@ def send_csv_file(download, directory, filename, download_type, encoding, format
         directory, filename, download_type, encoding, format)
     path = os.path.join(directory, converted_filename)
 
-    return FileResponse(open(path, 'rb'), filename=download.nice_filename(), as_attachment=True)
+    return FileResponse(open(path, 'rb'), filename=download.descriptive_filename(), as_attachment=True)
 
 class ResultsDownloadView(APIView):
     '''

From ee9bc90e1c4ba3d2087b4fead181e3cfe9d32122 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 11 Oct 2023 17:58:14 +0200
Subject: [PATCH 39/50] fewer parameters in send_csv_file

---
 backend/download/views.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/download/views.py b/backend/download/views.py
index 5510183a9..0e40bfa67 100644
--- a/backend/download/views.py
+++ b/backend/download/views.py
@@ -21,12 +21,12 @@
 
 logger = logging.getLogger()
 
-def send_csv_file(download, directory, filename, download_type, encoding, format=None):
+def send_csv_file(download, directory, encoding, format=None):
     '''
     Perform final formatting and send a CSV file as a FileResponse
     '''
     converted_filename = convert_csv.convert_csv(
-        directory, filename, download_type, encoding, format)
+        directory, download.filename, download.download_type, encoding, format)
     path = os.path.join(directory, converted_filename)
 
     return FileResponse(open(path, 'rb'), filename=download.descriptive_filename(), as_attachment=True)
@@ -58,7 +58,7 @@ def post(self, request, *args, **kwargs):
             directory, filename = os.path.split(csv_path)
             # Create download for download history
             download.complete(filename=filename)
-            return send_csv_file(download, directory, filename, 'search_results', request.data['encoding'])
+            return send_csv_file(download, directory, request.data['encoding'])
         except Exception as e:
             logger.error(e)
             raise APIException(detail = 'Download failed: could not generate csv file')
@@ -148,4 +148,4 @@ def get(self, request, *args, **kwargs):
         if not os.path.isfile(os.path.join(directory, download.filename)):
             raise NotFound(detail='File does not exist')
 
-        return send_csv_file(download, directory, download.filename, download.download_type, encoding, format)
+        return send_csv_file(download, directory, encoding, format)

From 7a8877882f48fc2884974fbe48759d278f0d24a9 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 11 Oct 2023 18:00:16 +0200
Subject: [PATCH 40/50] local variable for log_id

---
 backend/download/tests/test_full_data.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/download/tests/test_full_data.py b/backend/download/tests/test_full_data.py
index fccca3e6d..47e553310 100644
--- a/backend/download/tests/test_full_data.py
+++ b/backend/download/tests/test_full_data.py
@@ -24,7 +24,8 @@ def test_timeline_full_data(small_mock_corpus, index_small_mock_corpus, small_mo
 
     group = tasks.term_frequency_full_data_tasks(full_data_parameters, 'date_term_frequency')
     results = group.apply().get()
-    filename = tasks.make_term_frequency_csv(results, full_data_parameters, 0)
+    log_id = 0 # fake ID
+    filename = tasks.make_term_frequency_csv(results, full_data_parameters, log_id)
 
     with open(filename) as f:
         reader = csv.DictReader(f)

From e603cf99fb80f1baa9cd113c95096c5400fe573a Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 11 Oct 2023 18:00:59 +0200
Subject: [PATCH 41/50] use underscores in filename

---
 backend/download/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/download/models.py b/backend/download/models.py
index 8e9936a7b..2e5e9bcd1 100644
--- a/backend/download/models.py
+++ b/backend/download/models.py
@@ -57,4 +57,4 @@ def descriptive_filename(self):
         type_name = self.download_type
         timestamp = self.completed.strftime('%Y-%m-%d %H:%M')
 
-        return f'{type_name} {corpus_name} {timestamp}.csv'
+        return f'{type_name}__{corpus_name}__{timestamp}.csv'

From e52b09edfaffac37ac1ef79b6edae956c1f94e15 Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Thu, 12 Oct 2023 10:45:39 +0200
Subject: [PATCH 42/50] increment version number

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 3791d2b5e..bca2e7ad6 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "i-analyzer",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "license": "MIT",
   "scripts": {
     "postinstall": "yarn install-back && yarn install-front",

From fdbc99be052c9938ccb2445bed08e9bf7b91de31 Mon Sep 17 00:00:00 2001
From: Luka van der Plas <43678097+lukavdplas@users.noreply.github.com>
Date: Tue, 17 Oct 2023 16:01:54 +0200
Subject: [PATCH 43/50] Add issue templates

---
 .github/ISSUE_TEMPLATE/bug_report.md      | 36 +++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..5a8565682
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Let us know about something that isn't working right
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+### What went wrong?
+
+Describe what happened.
+
+### Expected behavior
+
+What did you expect to happen?
+
+### Screenshots
+
+If applicable, please add a screenshot of the problem! 
+
+### Which version?
+
+Please specify where you encountered the issue:
+
+- [ ] https://ianalyzer.hum.uu.nl
+- [ ] https://peopleandparliament.hum.uu.nl
+- [ ] https://peace.sites.uu.nl/
+- [ ] a server hosted elsewhere (i.e. not by the research software lab)
+- [ ] a local server
+
+If this happened on local or third-party server, it helps if you can be more specific about the version. Please include the version number (e.g. "3.2.4") or a commit hash if you know it!
+
+### To reproduce
+
+How can a developer replicate the issue? Please provide any information you can. For example: "I went to https://ianalyzer.hum.uu.nl/search/troonredes?date=1814-01-01:1972-01-01 and then clicked on *Download CSV*. I pressed *cancel* and then I clicked *Download CSV* again."
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..042278843
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for something new
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

From 45b5ba5ce8865e3603e9f0e4a74a59deee11d8f7 Mon Sep 17 00:00:00 2001
From: Jelte van Boheemen <j.vanboheemen@uu.nl>
Date: Mon, 23 Oct 2023 11:08:42 +0200
Subject: [PATCH 44/50] Add SamlSessionMiddleware to common_settings

---
 backend/ianalyzer/common_settings.py | 2 +-
 package.json                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/ianalyzer/common_settings.py b/backend/ianalyzer/common_settings.py
index 986a11b91..06b8fcaf0 100644
--- a/backend/ianalyzer/common_settings.py
+++ b/backend/ianalyzer/common_settings.py
@@ -49,7 +49,7 @@
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-
+    'djangosaml2.middleware.SamlSessionMiddleware',
 ]
 
 ROOT_URLCONF = 'ianalyzer.urls'
diff --git a/package.json b/package.json
index bca2e7ad6..fbae7a7a5 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "i-analyzer",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "license": "MIT",
   "scripts": {
     "postinstall": "yarn install-back && yarn install-front",

From 8ffbd2e05463bd3533bffb6e040214e5bff608df Mon Sep 17 00:00:00 2001
From: Luka van der Plas <lukavdplas@gmail.com>
Date: Wed, 25 Oct 2023 17:07:19 +0200
Subject: [PATCH 45/50] fix image tab missing text

---
 .../src/app/document-view/document-view.component.html     | 2 +-
 .../src/app/document-view/document-view.component.spec.ts  | 7 +++++++
 frontend/src/app/services/corpus.service.spec.ts           | 3 +++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/frontend/src/app/document-view/document-view.component.html b/frontend/src/app/document-view/document-view.component.html
index dde15325d..c7505664b 100644
--- a/frontend/src/app/document-view/document-view.component.html
+++ b/frontend/src/app/document-view/document-view.component.html
@@ -29,7 +29,7 @@
                 <ng-template iaTabPanel *ngFor="let field of contentFields" [id]="field.name" [title]="field.displayName" [icon]="tabIcons.text">
                     <div class="content" [innerHtml]="highlightedInnerHtml(field)"></div>
                 </ng-template>
-                <ng-template iaTabPanel id="scan" *ngIf="showScanTab">
+                <ng-template iaTabPanel id="scan" *ngIf="showScanTab" title="Image" [icon]="tabIcons.scan">
                     <ia-image-view [corpus]="corpus" [document]="document"></ia-image-view>
                 </ng-template>
             </ia-tabs>
diff --git a/frontend/src/app/document-view/document-view.component.spec.ts b/frontend/src/app/document-view/document-view.component.spec.ts
index 863dd2631..6fa470a3a 100644
--- a/frontend/src/app/document-view/document-view.component.spec.ts
+++ b/frontend/src/app/document-view/document-view.component.spec.ts
@@ -41,4 +41,11 @@ describe('DocumentViewComponent', () => {
         const element = debug[0].nativeElement;
         expect(element.textContent).toBe('Hello world!');
     });
+
+    it('should create tabs', () => {
+        const debug = fixture.debugElement.queryAll(By.css('a[role=tab]'));
+        expect(debug.length).toBe(2);
+        expect(debug[0].attributes['id']).toBe('tab-speech');
+        expect(debug[1].attributes['id']).toBe('tab-scan');
+    });
 });
diff --git a/frontend/src/app/services/corpus.service.spec.ts b/frontend/src/app/services/corpus.service.spec.ts
index 26364246a..3d4d30dff 100644
--- a/frontend/src/app/services/corpus.service.spec.ts
+++ b/frontend/src/app/services/corpus.service.spec.ts
@@ -199,6 +199,8 @@ describe('CorpusService', () => {
             expect(items.length).toBe(1);
             const corpus = _.first(items);
 
+            expect(corpus.scan_image_type).toBe('png');
+
             const fieldData = [
                 {
                     description: 'Banking concern to which the report belongs.',
@@ -275,6 +277,7 @@ describe('CorpusService', () => {
                     expect(result[key]).toEqual(expected[key]);
                 });
             });
+
         });
     });
 });

From 84818b5cf4feef016ebe6b1770910d362345f82e Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Thu, 5 Oct 2023 15:54:05 +0200
Subject: [PATCH 46/50] increase parliament-nl max_date

---
 backend/corpora/parliament/netherlands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/corpora/parliament/netherlands.py b/backend/corpora/parliament/netherlands.py
index 49f4ba5a1..7e3f51e6b 100644
--- a/backend/corpora/parliament/netherlands.py
+++ b/backend/corpora/parliament/netherlands.py
@@ -124,7 +124,7 @@ class ParliamentNetherlands(Parliament, XMLCorpusDefinition):
     title = "People & Parliament (Netherlands)"
     description = "Speeches from the Eerste Kamer and Tweede Kamer"
     min_date = datetime(year=1815, month=1, day=1)
-    max_date = datetime(year=2020, month=12, day=31)
+    max_date = datetime(year=2022, month=12, day=31)
     data_directory = settings.PP_NL_DATA
     data_directory_recent = settings.PP_NL_RECENT_DATA
     word_model_path = getattr(settings, 'PP_NL_WM', None)

From 344470a01924a73da803fc2ae94802ff8c046603 Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Fri, 6 Oct 2023 11:22:23 +0200
Subject: [PATCH 47/50] add term vectors on top position by default

---
 backend/addcorpus/es_mappings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/addcorpus/es_mappings.py b/backend/addcorpus/es_mappings.py
index a2f58418f..54bddbaa0 100644
--- a/backend/addcorpus/es_mappings.py
+++ b/backend/addcorpus/es_mappings.py
@@ -1,4 +1,4 @@
-def main_content_mapping(token_counts=True, stopword_analysis=False, stemming_analysis=False, updated_highlighting=False):
+def main_content_mapping(token_counts=True, stopword_analysis=False, stemming_analysis=False, updated_highlighting=True):
     '''
     Mapping for the main content field. Options:
 

From 5e88d508310110f170c11ab1c7a8f33d02bd144c Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Fri, 6 Oct 2023 13:13:39 +0200
Subject: [PATCH 48/50] add settings dutchannualreports

---
 backend/corpora/dutchannualreports/dutchannualreports.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/backend/corpora/dutchannualreports/dutchannualreports.py b/backend/corpora/dutchannualreports/dutchannualreports.py
index d4f4c7038..6a7c89168 100644
--- a/backend/corpora/dutchannualreports/dutchannualreports.py
+++ b/backend/corpora/dutchannualreports/dutchannualreports.py
@@ -12,8 +12,8 @@
 from addcorpus.corpus import XMLCorpusDefinition, FieldDefinition
 from media.image_processing import get_pdf_info, retrieve_pdf, pdf_pages, build_partial_pdf
 from addcorpus.load_corpus import corpus_dir
-
 from addcorpus.es_mappings import keyword_mapping, main_content_mapping
+from addcorpus.es_settings import es_settings
 
 from media.media_url import media_url
 
@@ -48,6 +48,10 @@ class DutchAnnualReports(XMLCorpusDefinition):
 
     dutchannualreports_map = {}
 
+    @property
+    def es_settings(self):
+        return es_settings(self.languages[0], stopword_analyzer=True, stemming_analyzer=True)
+
     with open(op.join(corpus_dir('dutchannualreports'), 'dutchannualreports_mapping.csv')) as f:
         reader = csv.DictReader(f)
         for line in reader:

From c27598c6ae15f85e6b322a22a7ee2061c71d73f9 Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Wed, 11 Oct 2023 12:30:16 +0200
Subject: [PATCH 49/50] change order of languages and es_settings in ecco

---
 backend/corpora/ecco/ecco.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/backend/corpora/ecco/ecco.py b/backend/corpora/ecco/ecco.py
index d23ef196b..a6b517dac 100644
--- a/backend/corpora/ecco/ecco.py
+++ b/backend/corpora/ecco/ecco.py
@@ -29,11 +29,6 @@ class Ecco(XMLCorpusDefinition):
     description_page = 'ecco.md'
     min_date = datetime(year=1700, month=1, day=1)
     max_date = datetime(year=1800, month=12, day=31)
-
-    @property
-    def es_settings(self):
-        return es_settings(self.languages[0], stopword_analyzer=True, stemming_analyzer=True)
-
     data_directory = settings.ECCO_DATA
     es_index = getattr(settings, 'ECCO_ES_INDEX', 'ecco')
     image = 'ecco.jpg'
@@ -47,6 +42,10 @@ def es_settings(self):
 
     meta_pattern = re.compile('^\d+\_DocMetadata\.xml$')
 
+    @property
+    def es_settings(self):
+        return es_settings(self.languages[0], stopword_analyzer=True, stemming_analyzer=True)
+
     def sources(self, start=min_date, end=max_date):
         logging.basicConfig(filename='ecco.log', level=logging.INFO)
 

From 86c524dfbf4ec0c0d3804a9b4020300016931a8f Mon Sep 17 00:00:00 2001
From: BeritJanssen <berit.janssen@gmail.com>
Date: Thu, 12 Oct 2023 15:09:48 +0200
Subject: [PATCH 50/50] update Finland-old max date

---
 backend/corpora/parliament/finland-old.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/corpora/parliament/finland-old.py b/backend/corpora/parliament/finland-old.py
index 59bf354c6..7e654b941 100644
--- a/backend/corpora/parliament/finland-old.py
+++ b/backend/corpora/parliament/finland-old.py
@@ -14,7 +14,7 @@
 class ParliamentFinlandOld(Parliament, CSVCorpusDefinition):
     title = 'People and Parliament (Finland, 1863-1905)'
     description = 'Speeches from the early Finnish estates'
-    max_date = datetime(year=1905, month=12, day=31)
+    max_date = datetime(year=1906, month=12, day=31)
     min_date = datetime(year=1863, month=1, day=1)
     data_directory = settings.PP_FINLAND_OLD_DATA
     es_index = getattr(settings, 'PP_FINLAND_OLD_INDEX', 'parliament-finland-old')