diff --git a/app/app/views.py b/app/app/views.py index 3d6190e9..22d6aa71 100644 --- a/app/app/views.py +++ b/app/app/views.py @@ -5,8 +5,8 @@ from django.utils.http import urlencode from django.utils.translation import gettext as _ -from general.filters import DocumentFileFilter -from general.models import DocumentFile, Institution, Language, Project, Subject +from general.filters import DocumentFilter +from general.models import Document, Institution, Language, Project, Subject def health(request): @@ -154,7 +154,7 @@ def institution_detail(request, institution_id): institution = get_object_or_404(Institution, id=institution_id) projects = Project.objects.filter(institution=institution).order_by("name") - documents = DocumentFile.objects.filter(institution=institution).order_by("title") + documents = Document.objects.filter(institution=institution).order_by("title") context = { "current_page": "institution_detail", @@ -171,7 +171,7 @@ def documents(request): template = "app/documents.html" documents = ( - DocumentFile.objects.select_related("institution") + Document.objects.select_related("institution") .prefetch_related("subjects", "languages") .order_by("title") ) @@ -227,7 +227,7 @@ def document_detail(request, document_id): template = "app/document_detail.html" document = get_object_or_404( - DocumentFile.objects.select_related("institution").prefetch_related( + Document.objects.select_related("institution").prefetch_related( Prefetch("subjects", queryset=Subject.objects.order_by("name")), Prefetch("languages", queryset=Language.objects.order_by("name")), ), @@ -248,8 +248,8 @@ def languages(request): Language.objects.all() .prefetch_related( Prefetch( - "documentfile_set", - queryset=DocumentFile.objects.only("id", "title", "languages").order_by("title"), + "document_set", + queryset=Document.objects.only("id", "title", "languages").order_by("title"), ), Prefetch( "project_set", @@ -261,7 +261,7 @@ def languages(request): language_data = [] for language in languages: - documents = language.documentfile_set.all() + documents = language.document_set.all() projects = language.project_set.all() language_data.append( { @@ -285,8 +285,8 @@ def subjects(request): Subject.objects.all() .prefetch_related( Prefetch( - "documentfile_set", - queryset=DocumentFile.objects.only("id", "title", "subjects").order_by("title"), + "document_set", + queryset=Document.objects.only("id", "title", "subjects").order_by("title"), ), Prefetch( "project_set", @@ -299,7 +299,7 @@ def subjects(request): subject_data = [] for subject in subjects: - documents = subject.documentfile_set.all() + documents = subject.document_set.all() projects = subject.project_set.all() if documents or projects: subject_data.append( @@ -311,7 +311,6 @@ def subjects(request): ) paginator = Paginator(subject_data, 10) - page_number = request.GET.get("page") page_obj = paginator.get_page(page_number) @@ -329,7 +328,7 @@ def institutions(request): # https://docs.djangoproject.com/en/stable/topics/db/aggregation/#combining-multiple-aggregations subquery = ( - DocumentFile.objects.filter(institution=OuterRef("id")) + Document.objects.filter(institution=OuterRef("id")) .order_by() .annotate(count=Func(F("id"), function="Count")) ) @@ -371,7 +370,7 @@ def institutions(request): def search(request): - f = DocumentFileFilter(request.GET, queryset=DocumentFile.objects.all()) + f = DocumentFilter(request.GET, queryset=Document.objects.all()) template = "app/search.html" diff --git a/app/general/filters.py b/app/general/filters.py index 26e44d9c..0eaa4882 100644 --- a/app/general/filters.py +++ b/app/general/filters.py @@ -12,10 +12,10 @@ from django.utils.translation import gettext_lazy as _ from django_filters import ModelMultipleChoiceFilter, MultipleChoiceFilter -from general.models import DocumentFile, Institution, Language, Project, Subject +from general.models import Document, Institution, Language, Project, Subject -class DocumentFileFilter(django_filters.FilterSet): +class DocumentFilter(django_filters.FilterSet): search = django_filters.CharFilter(method="ignore", label=_("Search")) institution = ModelMultipleChoiceFilter( @@ -35,7 +35,7 @@ class DocumentFileFilter(django_filters.FilterSet): ) class Meta: - model = DocumentFile + model = Document fields = [ "institution", "subjects", diff --git a/app/general/management/commands/dev_update_vector_search.py b/app/general/management/commands/dev_update_vector_search.py index 56298789..9b1628c8 100644 --- a/app/general/management/commands/dev_update_vector_search.py +++ b/app/general/management/commands/dev_update_vector_search.py @@ -1,12 +1,12 @@ from django.core.management.base import BaseCommand -from general.models import DocumentFile +from general.models import Document class Command(BaseCommand): help = "Updating the Vector Search index on document_file." def handle(self, *args, **options): - for document_file in DocumentFile.objects.all(): + for document_file in Document.objects.all(): document_file.save() # This line updates the vector search for the document file print(f"Updated {document_file.title}.") diff --git a/app/general/management/commands/import_documents.py b/app/general/management/commands/import_documents.py index 9f001543..f116167d 100644 --- a/app/general/management/commands/import_documents.py +++ b/app/general/management/commands/import_documents.py @@ -11,7 +11,7 @@ from django.core.files.base import ContentFile from django.core.management.base import BaseCommand -from general.models import DocumentFile +from general.models import Document from general.service.extract_text import GetTextError, pdf_to_text @@ -42,7 +42,7 @@ def save_data(self, file_path, file_name): content_file = ContentFile(f.read(), name=file_name) try: - instance = DocumentFile( + instance = Document( title=file_name, document_data=pdf_to_text(file_path), uploaded_file=content_file, diff --git a/app/general/migrations/0013_rename_documentfile_document_and_more.py b/app/general/migrations/0013_rename_documentfile_document_and_more.py new file mode 100644 index 00000000..f432997b --- /dev/null +++ b/app/general/migrations/0013_rename_documentfile_document_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.0.8 on 2024-08-17 14:47 + +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('general', '0012_alter_documentfile_search_vector'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.RenameModel( + old_name='DocumentFile', + new_name='Document', + ), + migrations.RenameModel( + old_name='HistoricalDocumentFile', + new_name='HistoricalDocument', + ), + migrations.RenameIndex( + model_name='document', + new_name='general_doc_search__12340c_gin', + old_name='general_doc_search__752b22_gin', + ), + ] diff --git a/app/general/models.py b/app/general/models.py index 064d7b8a..0e56d660 100644 --- a/app/general/models.py +++ b/app/general/models.py @@ -82,7 +82,7 @@ def __str__(self): return self.name -class DocumentFile(models.Model): +class Document(models.Model): file_validators = [FileExtensionValidator(["pdf"])] # names and abbreviations based on diff --git a/app/general/tests/test_document_file.py b/app/general/tests/test_document.py similarity index 89% rename from app/general/tests/test_document_file.py rename to app/general/tests/test_document.py index 6eb1a651..a7248c2a 100644 --- a/app/general/tests/test_document_file.py +++ b/app/general/tests/test_document.py @@ -3,10 +3,10 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase -from general.models import DocumentFile, Institution, Language, Subject +from general.models import Document, Institution, Language, Subject -class DocumentFileTest(TestCase): +class DocumentTest(TestCase): def setUp(self): self.subject = Subject.objects.create(name="Test Subject") self.language = Language.objects.create(name="Test Language", iso_code="TL") @@ -22,7 +22,7 @@ def setUp(self): self.document_type = "Glossary" self.institution = self.institution - self.document = DocumentFile.objects.create( + self.document = Document.objects.create( title=self.title, url=self.url, uploaded_file=self.uploaded_file, @@ -35,8 +35,8 @@ def setUp(self): self.document.languages.add(self.language) def test_document_creation(self): - self.assertEqual(DocumentFile.objects.count(), 1) - self.assertEqual(DocumentFile.objects.get().title, self.title) + self.assertEqual(Document.objects.count(), 1) + self.assertEqual(Document.objects.get().title, self.title) def test_document_str_representation(self): # Test __str__ method self.assertEqual(str(self.document), self.title) diff --git a/app/general/tests/test_document_admin_file.py b/app/general/tests/test_document_admin.py similarity index 89% rename from app/general/tests/test_document_admin_file.py rename to app/general/tests/test_document_admin.py index 76c8a5c8..11d14b0a 100644 --- a/app/general/tests/test_document_admin_file.py +++ b/app/general/tests/test_document_admin.py @@ -3,11 +3,11 @@ from django.core.files.uploadedfile import SimpleUploadedFile -from general.admin import DocumentFileForm +from general.admin import DocumentForm from general.models import Institution -class TestDocumentFileForm(unittest.TestCase): +class TestDocumentForm(unittest.TestCase): def __init__(self, methodName: str = "runTest"): super().__init__(methodName) self.form = None @@ -33,7 +33,7 @@ def test_clean_without_url_and_file(self): "description": "Test description", } - form = DocumentFileForm(tests_form) + form = DocumentForm(tests_form) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["url"], ["Either URL or uploaded file must be provided."]) self.assertEqual( @@ -53,7 +53,7 @@ def test_clean_without_file(self): "description": "", } - form = DocumentFileForm(tests_form) + form = DocumentForm(tests_form) self.assertTrue(form.is_valid()) # @@ -70,7 +70,7 @@ def test_clean_without_url(self): "description": "Test description", } - form = DocumentFileForm(tests_form, files={"uploaded_file": self.file_mock}) + form = DocumentForm(tests_form, files={"uploaded_file": self.file_mock}) self.assertTrue(form.is_valid()) def test_clean_with_large_file(self): @@ -87,7 +87,7 @@ def test_clean_with_large_file(self): "description": "Test description", } - form = DocumentFileForm(tests_form, files={"uploaded_file": self.file_mock}) + form = DocumentForm(tests_form, files={"uploaded_file": self.file_mock}) self.assertFalse(form.is_valid()) self.assertIn("uploaded_file", form.errors) self.assertEqual(form.errors["uploaded_file"], ["File size must not exceed 10MB."]) diff --git a/app/general/tests/test_document_detail.py b/app/general/tests/test_document_detail.py index b6051b21..d6ea0c1b 100644 --- a/app/general/tests/test_document_detail.py +++ b/app/general/tests/test_document_detail.py @@ -1,7 +1,7 @@ from django.test import TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Language, Project, Subject +from general.models import Document, Institution, Language, Project, Subject class DocumentDetailViewTest(TestCase): @@ -14,7 +14,7 @@ def setUp(self): self.language1 = Language.objects.create(name="Afrikaans", iso_code="af") self.language2 = Language.objects.create(name="English", iso_code="en") - self.document = DocumentFile.objects.create( + self.document = Document.objects.create( title="Afrikaans_HL_P1_Feb-March_2011", description="This is a description of the document.", url="https://externaldocumentrepository.com/document1", diff --git a/app/general/tests/test_documents.py b/app/general/tests/test_documents.py index 6d6f1d46..9f39a890 100644 --- a/app/general/tests/test_documents.py +++ b/app/general/tests/test_documents.py @@ -1,7 +1,7 @@ from django.test import TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Language, Subject +from general.models import Document, Institution, Language, Subject class DocumentViewTests(TestCase): @@ -16,14 +16,14 @@ def setUp(self): name="Institution", logo="institution_logo.png" ) - self.document5 = DocumentFile.objects.create( + self.document5 = Document.objects.create( title="Document 5", institution=self.institution, ) self.document5.subjects.add(self.subject1) self.document5.languages.add(self.language1) - self.document6 = DocumentFile.objects.create( + self.document6 = Document.objects.create( title="Document 6", institution=self.institution, ) diff --git a/app/general/tests/test_filter.py b/app/general/tests/test_filter.py index 579f8348..36f36190 100644 --- a/app/general/tests/test_filter.py +++ b/app/general/tests/test_filter.py @@ -2,8 +2,8 @@ from django.test import TestCase -from general.filters import DocumentFileFilter -from general.models import DocumentFile, Institution, Language, Project, Subject +from general.filters import DocumentFilter +from general.models import Document, Institution, Language, Project, Subject class TestSearchFilter(TestCase): @@ -19,8 +19,8 @@ def setUp(self): self.subject1 = Subject.objects.create(name="Science") self.subject2 = Subject.objects.create(name="Math") - # Create DocumentFiles - self.doc1 = DocumentFile.objects.create( + # Create Documents + self.doc1 = Document.objects.create( title="Document 1", document_data="Document 1 content", institution=self.institution1, @@ -29,7 +29,7 @@ def setUp(self): self.doc1.subjects.add(self.subject1) self.doc1.languages.add(self.language1) - self.doc2 = DocumentFile.objects.create( + self.doc2 = Document.objects.create( title="Document 2", document_data="Document 2 content", institution=self.institution2, @@ -48,7 +48,7 @@ def setUp(self): def test_institution_filter(self): data = {"institution": [self.institution1.id]} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 2) # TODO: ordering between documents and projects are not yet defined @@ -56,14 +56,14 @@ def test_institution_filter(self): def test_subjects_filter(self): data = {"subjects": [self.subject1.id]} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 1) self.assertEqual(qs[0]["id"], self.doc1.id) def test_languages_filter(self): data = {"languages": [self.language1.id]} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 1) self.assertEqual(qs[0]["id"], self.doc1.id) @@ -74,34 +74,34 @@ def test_combined_filters(self): "subjects": [self.subject1.id], "languages": [self.language1.id], } - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 1) self.assertEqual(qs[0]["id"], self.doc1.id) def test_search_filter_documents(self): data = {"search": "Document"} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 2) self.assertCountEqual([qs[0]["id"], qs[1]["id"]], [self.doc1.id, self.doc2.id]) data = {"search": "Document 1"} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 1) self.assertEqual(qs[0]["id"], self.doc1.id) def test_search_filter_projects(self): data = {"search": "Project 1"} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs = filter.qs self.assertEqual(len(qs), 1) self.assertEqual(qs[0]["id"], self.project1.id) def test_search_filter_combined(self): data = {"search": "1"} - filter = DocumentFileFilter(data=data) + filter = DocumentFilter(data=data) qs_ids = [x["id"] for x in filter.qs] expected_ids = [self.doc1.id, self.project1.id, self.institution1.id] self.assertCountEqual(qs_ids, expected_ids) diff --git a/app/general/tests/test_import_documents.py b/app/general/tests/test_import_documents.py index 8725d085..a18de80b 100644 --- a/app/general/tests/test_import_documents.py +++ b/app/general/tests/test_import_documents.py @@ -6,7 +6,7 @@ from faker import Faker from general.management.commands.import_documents import Command -from general.models import DocumentFile, Institution +from general.models import Document, Institution class TestHandleFile(unittest.TestCase): @@ -20,11 +20,11 @@ def setUp(self): def tearDown(self): try: - document_file = DocumentFile.objects.get(title=self.name) + document_file = Document.objects.get(title=self.name) path = document_file.uploaded_file.path if os.path.isfile(path): os.remove(path) - except DocumentFile.DoesNotExist: + except Document.DoesNotExist: pass def test_handle_file_pdf(self): @@ -52,6 +52,6 @@ def test_save_data(self): ) command.save_data(self.test_file, self.name) - document_file = DocumentFile.objects.get(title=self.name) + document_file = Document.objects.get(title=self.name) self.assertEqual(document_file.title, self.name) self.assertIn("Lorem ipsum dolor", document_file.document_data) diff --git a/app/general/tests/test_institution_detail.py b/app/general/tests/test_institution_detail.py index ae9a0015..2904e598 100644 --- a/app/general/tests/test_institution_detail.py +++ b/app/general/tests/test_institution_detail.py @@ -1,8 +1,7 @@ -from django.http import Http404 from django.test import TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Project +from general.models import Institution class InstitutionDetailViewTests(TestCase): diff --git a/app/general/tests/test_languages.py b/app/general/tests/test_languages.py index 0701c1d9..ffbb5bbf 100644 --- a/app/general/tests/test_languages.py +++ b/app/general/tests/test_languages.py @@ -1,7 +1,7 @@ from django.test import TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Language, Project +from general.models import Document, Institution, Language, Project class LanguagesViewTest(TestCase): @@ -16,12 +16,12 @@ def setUp(self): self.language1 = Language.objects.create(name="English", iso_code="lang1") self.language2 = Language.objects.create(name="Spanish", iso_code="lang2") - self.document1 = DocumentFile.objects.create( + self.document1 = Document.objects.create( title="Document 1", institution=self.institution, document_type="report" ) self.document1.languages.add(self.language1) - self.document2 = DocumentFile.objects.create( + self.document2 = Document.objects.create( title="Document 2", institution=self.institution, document_type="report" ) self.document2.languages.add(self.language2) diff --git a/app/general/tests/test_view_search.py b/app/general/tests/test_view_search.py index 3db28d7c..2a3b7af8 100644 --- a/app/general/tests/test_view_search.py +++ b/app/general/tests/test_view_search.py @@ -3,7 +3,7 @@ from django.test import Client, TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Language, Subject +from general.models import Document, Institution, Language, Subject class SearchViewTest(TestCase): @@ -20,9 +20,9 @@ def setUp(self): self.subject1 = Subject.objects.create(name="Science") self.subject2 = Subject.objects.create(name="Math") - # Create DocumentFiles + # Create documents for i in range(10): - doc = DocumentFile.objects.create( + doc = Document.objects.create( title=f"Document {i + 1}", institution=self.institution1 if i % 2 == 0 else self.institution2, document_type="report" if i % 2 == 0 else "article", diff --git a/app/general/tests/test_views_institution.py b/app/general/tests/test_views_institution.py index 3b623f18..9690c56f 100644 --- a/app/general/tests/test_views_institution.py +++ b/app/general/tests/test_views_institution.py @@ -1,7 +1,7 @@ from django.test import Client, TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Project +from general.models import Document, Institution, Project class InstitutionsViewTestCase(TestCase): @@ -19,8 +19,8 @@ def setUp(self): Project.objects.create(name="Test Project 1", institution=inst1) Project.objects.create(name="Test Project 2", institution=inst1) - DocumentFile.objects.create(title="Test document 1", institution=inst1) - DocumentFile.objects.create(title="Test document 2", institution=inst1) + Document.objects.create(title="Test document 1", institution=inst1) + Document.objects.create(title="Test document 2", institution=inst1) self.url = reverse("institutions") diff --git a/app/general/tests/tests_subject.py b/app/general/tests/tests_subject.py index 9f61bd97..2b02ae19 100644 --- a/app/general/tests/tests_subject.py +++ b/app/general/tests/tests_subject.py @@ -1,7 +1,7 @@ from django.test import TestCase from django.urls import reverse -from general.models import DocumentFile, Institution, Project, Subject +from general.models import Document, Institution, Project, Subject class TestSubjects(TestCase): @@ -16,12 +16,12 @@ def setUp(self): self.subject1 = Subject.objects.create(name="Mathematics") self.subject2 = Subject.objects.create(name="Science") - self.document1 = DocumentFile.objects.create( + self.document1 = Document.objects.create( title="Document 1", institution=self.institution, document_type="report" ) self.document1.subjects.add(self.subject1) - self.document2 = DocumentFile.objects.create( + self.document2 = Document.objects.create( title="Document 2", institution=self.institution, document_type="report" ) self.document2.subjects.add(self.subject2)