From bf9258b6ffb4742d517ba0d0b0432938ac160473 Mon Sep 17 00:00:00 2001 From: Daniel Gray Date: Thu, 11 Jul 2024 08:01:51 +0200 Subject: [PATCH 1/2] added Django filter plugin and updated testing pipeline - added flag to validate_templates tests, to ignore django_filters templates --- .github/workflows/testing.yml | 2 +- app/app/settings.py | 1 + requirements.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cee182ac..48fd9ed2 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -41,7 +41,7 @@ jobs: cp .env.testing app/.env cd app/ mkdir -p static_files - python manage.py validate_templates + python manage.py validate_templates --ignore-app django_filters - name: Run Tests run: | cp .env.testing app/.env diff --git a/app/app/settings.py b/app/app/settings.py index e03a5c95..db46e5d6 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -48,6 +48,7 @@ "general", "simple_history", "accounts", + "django_filters", ] # Add django-extensions to the installed apps if DEBUG is True diff --git a/requirements.txt b/requirements.txt index f6d8f0c2..4d793de9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ whitenoise pillow python-magic pypdf +django-filter From 5f700a7368ea1beb19af4ab36393b1339300e49e Mon Sep 17 00:00:00 2001 From: Daniel Gray Date: Thu, 11 Jul 2024 09:45:09 +0200 Subject: [PATCH 2/2] added filters updated search - Updated search templates - Added search filters - Updated search View - Added tests --- app/app/views.py | 60 ++++---- app/general/filters.py | 59 ++++++++ app/general/tests/test_filter.py | 91 ++++++++++++ app/general/tests/test_view_search.py | 75 ++++++++++ app/static/css/styles.css | 9 ++ app/templates/app/search.html | 201 +++++++++++++++++--------- 6 files changed, 399 insertions(+), 96 deletions(-) create mode 100644 app/general/filters.py create mode 100644 app/general/tests/test_filter.py create mode 100644 app/general/tests/test_view_search.py diff --git a/app/app/views.py b/app/app/views.py index 8285f7ce..9e0eb798 100644 --- a/app/app/views.py +++ b/app/app/views.py @@ -1,12 +1,11 @@ -import os - -from django.contrib.postgres.search import SearchHeadline, SearchQuery, SearchRank -from django.core.paginator import Paginator +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Count from django.http import HttpResponse from django.shortcuts import get_object_or_404, render +from django.utils.http import urlencode from django.utils.translation import gettext as _ +from general.filters import DocumentFileFilter from general.models import DocumentFile, Institution, Language, Project, Subject @@ -230,39 +229,40 @@ def institutions(request): def search(request): - q = request.GET.get("q") - - if q: - queue = SearchQuery(q) - search_headline = SearchHeadline("document_data", queue) - - documents = ( - DocumentFile.objects.annotate(rank=SearchRank("search_vector", queue)) - .annotate(search_headline=search_headline) - .filter(search_vector=queue) - .order_by("-rank") - ) - - else: - documents = None + page_number = request.GET.get("page", "1") + if not page_number.isdigit(): + page_number = "1" - # Create a Paginator instance with the documents and the number of items per page - paginator = Paginator(documents, 10) if documents else None # Show 10 documents per page + f = DocumentFileFilter(request.GET, queryset=DocumentFile.objects.all()) - # Get the page number from the request's GET parameters - page_number = request.GET.get("page") + template = "app/search.html" - # Use the get_page method to get the Page object for that page number - page_obj = paginator.get_page(page_number) if paginator else None + paginator = Paginator(f.qs, 5) # 5 documents per page - feature_flag = os.getenv("FEATURE_FLAG", False) + try: + page_obj = paginator.page(page_number) + except PageNotAnInteger: + page_obj = paginator.page(1) + except EmptyPage: + page_obj = paginator.page(paginator.num_pages) - template = "app/search.html" context = { + "search_results": paginator.page(page_obj.number), + "filter": f, "documents": page_obj, - "current_page": "search", - "document_count": len(documents) if documents else 0, - "feature_flag": feature_flag, + "search_params": pagination_url(request), } return render(request, template_name=template, context=context) + + +def pagination_url(request): + url_params = { + "search": request.GET.get("search", ""), + "document_type": request.GET.getlist("document_type", []), + "institution": request.GET.getlist("institution", []), + "subjects": request.GET.getlist("subjects", []), + "languages": request.GET.getlist("languages", []), + } + + return "?" + urlencode(url_params, doseq=True) diff --git a/app/general/filters.py b/app/general/filters.py new file mode 100644 index 00000000..5f3ecbc4 --- /dev/null +++ b/app/general/filters.py @@ -0,0 +1,59 @@ +import django_filters +from django import forms +from django.contrib.postgres.search import SearchHeadline, SearchQuery, SearchRank +from django.db.models import F +from django_filters import ModelMultipleChoiceFilter, MultipleChoiceFilter + +from general.models import DocumentFile, Institution, Language, Subject + + +class DocumentFileFilter(django_filters.FilterSet): + search = django_filters.CharFilter(method="filter_search", label="Search") + + institution = ModelMultipleChoiceFilter( + queryset=Institution.objects.all(), widget=forms.CheckboxSelectMultiple + ) + document_type = MultipleChoiceFilter( + choices=DocumentFile.document_type_choices, widget=forms.CheckboxSelectMultiple + ) + subjects = ModelMultipleChoiceFilter( + queryset=Subject.objects.all(), widget=forms.CheckboxSelectMultiple + ) + languages = ModelMultipleChoiceFilter( + queryset=Language.objects.all(), widget=forms.CheckboxSelectMultiple + ) + + class Meta: + model = DocumentFile + fields = [ + "document_type", + "institution", + "subjects", + "languages", + ] + + def filter_queryset(self, queryset): + queryset = super().filter_queryset(queryset) + + search = self.form.cleaned_data.get("search", "") + queue = SearchQuery(search.strip()) + search_rank = SearchRank(F("search_vector"), queue) + search_headline = SearchHeadline("document_data", queue) + queryset = ( + queryset.annotate( + rank=search_rank, + search_headline=search_headline, + ) + .defer("document_data") + .select_related("institution") + ).order_by("-rank") + + return queryset + + def filter_search(self, queryset, name, value): + if value: + queue = SearchQuery(value.strip()) + return queryset.filter(search_vector=queue) + + else: + return queryset diff --git a/app/general/tests/test_filter.py b/app/general/tests/test_filter.py new file mode 100644 index 00000000..1cc5650c --- /dev/null +++ b/app/general/tests/test_filter.py @@ -0,0 +1,91 @@ +import unittest + +from django.test import TestCase + +from general.filters import DocumentFileFilter +from general.models import DocumentFile, Institution, Language, Subject + + +class TestSearchFilter(TestCase): + def setUp(self): + self.institution1 = Institution.objects.create(name="Institution 1") + self.institution2 = Institution.objects.create(name="Institution 2") + + # Create languages + self.language1 = Language.objects.create(name="English", iso_code="EN") + self.language2 = Language.objects.create(name="Afrikaans", iso_code="AF") + + # Create subjects + self.subject1 = Subject.objects.create(name="Science") + self.subject2 = Subject.objects.create(name="Math") + + # Create DocumentFiles + self.doc1 = DocumentFile.objects.create( + title="Document 1", + document_data="Document 1 content", + institution=self.institution1, + document_type="glossary", + ) + self.doc1.subjects.add(self.subject1) + self.doc1.languages.add(self.language1) + + self.doc2 = DocumentFile.objects.create( + title="Document 2", + document_data="Document 2 content", + institution=self.institution2, + document_type="glossary", + ) + self.doc2.subjects.add(self.subject2) + self.doc2.languages.add(self.language2) + + def test_institution_filter(self): + data = {"institution": [self.institution1.id]} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 1) + self.assertIn(self.doc1, filter.qs) + + def test_document_type_filter(self): + data = {"document_type": ["glossary"]} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 2) + self.assertIn(self.doc1, filter.qs) + self.assertIn(self.doc2, filter.qs) + + def test_subjects_filter(self): + data = {"subjects": [self.subject1.id]} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 1) + self.assertIn(self.doc1, filter.qs) + + def test_languages_filter(self): + data = {"languages": [self.language1.id]} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 1) + self.assertIn(self.doc1, filter.qs) + + def test_combined_filters(self): + data = { + "institution": [self.institution1.id], + "document_type": ["glossary"], + "subjects": [self.subject1.id], + "languages": [self.language1.id], + } + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 1) + self.assertIn(self.doc1, filter.qs) + + def test_search_filter(self): + data = {"search": "Document"} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 2) + self.assertIn(self.doc1, filter.qs) + self.assertIn(self.doc2, filter.qs) + + data = {"search": "Document 1"} + filter = DocumentFileFilter(data=data) + self.assertEqual(len(filter.qs), 1) + self.assertIn(self.doc1, filter.qs) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/general/tests/test_view_search.py b/app/general/tests/test_view_search.py new file mode 100644 index 00000000..5d0386ab --- /dev/null +++ b/app/general/tests/test_view_search.py @@ -0,0 +1,75 @@ +import unittest + +from django.test import Client, TestCase +from django.urls import reverse + +from general.models import DocumentFile, Institution, Language, Subject + + +class SearchViewTest(TestCase): + def setUp(self): + # Create institutions + self.institution1 = Institution.objects.create(name="Institution 1") + self.institution2 = Institution.objects.create(name="Institution 2") + + # Create languages + self.language1 = Language.objects.create(name="English", iso_code="EN") + self.language2 = Language.objects.create(name="Afrikaans", iso_code="AF") + + # Create subjects + self.subject1 = Subject.objects.create(name="Science") + self.subject2 = Subject.objects.create(name="Math") + + # Create DocumentFiles + for i in range(10): + doc = DocumentFile.objects.create( + title=f"Document {i + 1}", + institution=self.institution1 if i % 2 == 0 else self.institution2, + document_type="report" if i % 2 == 0 else "article", + document_data="Document {i + 1} content", + ) + doc.subjects.add(self.subject1 if i % 2 == 0 else self.subject2) + doc.languages.add(self.language1 if i % 2 == 0 else self.language2) + + def test_search_pagination(self): + client = Client() + response = client.get(reverse("search"), {"page": "1"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context["documents"]), 5) + + response = client.get(reverse("search"), {"page": "2"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context["documents"]), 5) + + response = client.get(reverse("search"), {"page": "3"}) + self.assertEqual(response.status_code, 200) + + def test_search_filtering(self): + client = Client() + response = client.get(reverse("search"), {"search": "Document 1"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context["documents"][0].title, "Document 1") + + def test_invalid_page_number(self): + client = Client() + response = client.get(reverse("search"), {"page": "invalid"}) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context["documents"]), 5) + + def test_combined_filters(self): + client = Client() + response = client.get( + reverse("search"), + { + "institution": self.institution1.id, + "document_type": "report", + "subjects": self.subject1.id, + "languages": self.language1.id, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context["documents"]), 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/static/css/styles.css b/app/static/css/styles.css index 438c5e2e..d6e97f89 100755 --- a/app/static/css/styles.css +++ b/app/static/css/styles.css @@ -611,3 +611,12 @@ html { display: flex; } } + +/*Search Css*/ +.checkbox-container { + cursor: pointer; + max-height: 130px; /* Adjust based on your line-height and padding to show only 4 rows */ + overflow-y: auto; /* Enable vertical scrollbar when content overflows */ + border: 1px solid #ced4da; /* Bootstrap's form control border color */ + border-radius: 0.25rem; /* Bootstrap's form control border radius */ +} diff --git a/app/templates/app/search.html b/app/templates/app/search.html index 55264498..4709731b 100644 --- a/app/templates/app/search.html +++ b/app/templates/app/search.html @@ -3,84 +3,153 @@ {% load i18n %} {% block content %} -
-
+
{% trans "Search a term" %}
- {% if feature_flag == "search_feature" %} - {# test section start#} -
-
- -
- + + +
+ + +
+
+
+ {% for document in search_results %} +
+
    +
  • + {% trans "Title:" %} + {{ document.title }} +
    • +
  • + {% trans "Institution:" %} + {{ document.institution }} +
    • +
  • + {% trans "Headline:" %} + {{ document.search_headline|safe }} +
    • +
  • + {% trans "File:" %} + {{ document.uploaded_file }} +
    • +
  • + {% trans "License:" %} + {{ document.license }} +
    • +
  • + {% trans "License:" %} + {{ document.document_type }} +
    • +
  • + {% trans "Mime Type:" %} + {{ document.mime_type }} +
    • +
  • + {% trans "Rank:" %} + {{ document.rank }} +
    • +
+
+ {% endfor %} +
+
+
+
+ +
+
+
+ +
+
+
+
+ +
+ {% for checkbox in filter.form.institution %} +
+ {{ checkbox.tag }} + +
+ {% endfor %}
- -
- {% trans "Results:" %} - {{ documents|length }}
-
- {% for document in documents %} -
-
    -
  • - {% trans "Title:" %} - {{ document.title }} -
    • -
  • - {% trans "Institution:" %} - {{ document.institution }} -
    • -
  • - {% trans "Headline:" %} - {{ document.search_headline|safe }} -
    • -
  • - {% trans "File:" %} - {{ document.uploaded_file }} -
    • -
  • - {% trans "License:" %} - {{ document.license }} -
    • -
  • - {% trans "Mime Type:" %} - {{ document.mime_type }} -
    • -
  • - {% trans "Rank:" %} - {{ document.rank }} -
    • -
+
+
+ +
+ {% for checkbox in filter.form.document_type %} +
+ {{ checkbox.tag }} + +
+ {% endfor %}
- {% endfor %} +
- +
+ +
+
+