From 6ffa7bfc74603c491b1b7a05858568967af59c3c Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Fri, 10 Nov 2023 07:48:36 -0500 Subject: [PATCH 01/26] fix: Replaced whitelist_externals with allowlist_externals in tox and removed tox-battery (#138) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5725d943..fa71d60a 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ commands = python -m coverage xml [testenv:quality] -whitelist_externals = +allowlist_externals = make deps = -r{toxinidir}/requirements/quality.txt From f505d30f23afb6bd41084083eb0e93098cdf7278 Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Mon, 16 Oct 2023 13:58:46 -0400 Subject: [PATCH 02/26] feat: add install-local target for local devstack convenience --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0be8b17d..6b16034d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean quality requirements validate test test-python quality-python +.PHONY: clean quality requirements validate test test-python quality-python install-local clean: find . -name '__pycache__' -exec rm -rf {} + @@ -55,3 +55,7 @@ test-python: clean ## run tests using pytest and generate coverage report pytest test: test-python ## run tests and generate coverage report + +install-local: ## installs your local edx-search into the LMS and CMS python virtualenvs + docker exec -t edx.devstack.lms bash -c '. /edx/app/edxapp/venvs/edxapp/bin/activate && cd /edx/app/edxapp/edx-platform && pip uninstall -y edx-search && pip install -e /edx/src/edx-search && pip freeze | grep edx-search' + docker exec -t edx.devstack.cms bash -c '. /edx/app/edxapp/venvs/edxapp/bin/activate && cd /edx/app/edxapp/edx-platform && pip uninstall -y edx-search && pip install -e /edx/src/edx-search && pip freeze | grep edx-search' From decfa2fec2056b3b6210868f018cf11d70974bb1 Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Wed, 15 Nov 2023 09:04:36 -0500 Subject: [PATCH 03/26] fix: be clear that perform search is not generic --- search/api.py | 2 +- search/tests/test_engines.py | 4 ++-- search/views.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/search/api.py b/search/api.py index 81c26090..f016a54a 100644 --- a/search/api.py +++ b/search/api.py @@ -41,7 +41,7 @@ class NoSearchEngineError(Exception): """ -def perform_search( +def perform_course_search( search_term, user=None, size=10, diff --git a/search/tests/test_engines.py b/search/tests/test_engines.py index 659d4cc9..3340340d 100644 --- a/search/tests/test_engines.py +++ b/search/tests/test_engines.py @@ -13,7 +13,7 @@ from django.test.utils import override_settings from elasticsearch import exceptions from elasticsearch.helpers import BulkIndexError -from search.api import NoSearchEngineError, perform_search +from search.api import NoSearchEngineError, perform_course_search from search.elastic import RESERVED_CHARACTERS from search.tests.mock_search_engine import (MockSearchEngine, json_date_to_datetime) @@ -238,7 +238,7 @@ class TestNone(TestCase): def test_perform_search(self): """ search opertaion should yeild an exception with no search engine """ with self.assertRaises(NoSearchEngineError): - perform_search("abc test") + perform_course_search("abc test") @override_settings(SEARCH_ENGINE="search.elastic.ElasticSearchEngine") diff --git a/search/views.py b/search/views.py index c59f7463..0c30d426 100644 --- a/search/views.py +++ b/search/views.py @@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST from eventtracking import tracker as track -from .api import perform_search, course_discovery_search, course_discovery_filter_fields +from .api import perform_course_search, course_discovery_search, course_discovery_filter_fields from .initializer import SearchInitializer # log appears to be standard name used for logger @@ -96,7 +96,7 @@ def do_search(request, course_id=None): } ) - results = perform_search( + results = perform_course_search( search_term, user=request.user, size=size, From de830900a44216f54a8357b702ffb041876ba0cf Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Wed, 15 Nov 2023 10:39:14 -0500 Subject: [PATCH 04/26] fix: use the search method rather than helper method which hides args the search_string helper method just obscures the importance of the args --- search/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search/api.py b/search/api.py index f016a54a..48cb4d42 100644 --- a/search/api.py +++ b/search/api.py @@ -63,8 +63,8 @@ def perform_course_search( if not searcher: raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE") - results = searcher.search_string( - search_term, + results = searcher.search( + query_string=search_term, field_dictionary=field_dictionary, filter_dictionary=filter_dictionary, exclude_dictionary=exclude_dictionary, From 7a200a04a77f33a87179d525148e1f19bf3ef0e8 Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Wed, 15 Nov 2023 10:08:03 -0500 Subject: [PATCH 05/26] fix: plain pytest does not work, get it out of the default target ci uses tox, which uses the manage.py test that requires a running elastic --- Makefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6b16034d..8e1cfd0a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -.PHONY: clean quality requirements validate test test-python quality-python install-local +.PHONY: clean quality requirements validate test test-with-es quality-python install-local clean: find . -name '__pycache__' -exec rm -rf {} + find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + coverage erase rm -rf coverage htmlcov rm -fr build/ @@ -51,10 +51,7 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy sed '/^[dD]jango==/d' requirements/testing.txt > requirements/testing.tmp mv requirements/testing.tmp requirements/testing.txt -test-python: clean ## run tests using pytest and generate coverage report - pytest - -test: test-python ## run tests and generate coverage report +test: test_with_es ## run tests and generate coverage report install-local: ## installs your local edx-search into the LMS and CMS python virtualenvs docker exec -t edx.devstack.lms bash -c '. /edx/app/edxapp/venvs/edxapp/bin/activate && cd /edx/app/edxapp/edx-platform && pip uninstall -y edx-search && pip install -e /edx/src/edx-search && pip freeze | grep edx-search' From c6aabb6cf168696aa08b6715b9c31891827eb9da Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Wed, 15 Nov 2023 09:43:10 -0500 Subject: [PATCH 06/26] feat: add arg and setting to trigger more complete search log We don't want to trigger logging on every single ES search or force logging into other courseware search uses, so arg + setting just for courseware search. Fairly awkward test brought to you via the repository's 100% coverage policy, but it does ensure the log line is reached and does not error. --- search/api.py | 2 ++ search/elastic.py | 4 ++++ search/search_engine_base.py | 1 + search/tests/mock_search_engine.py | 1 + search/tests/tests.py | 9 +++++++++ 5 files changed, 17 insertions(+) diff --git a/search/api.py b/search/api.py index 48cb4d42..177e909a 100644 --- a/search/api.py +++ b/search/api.py @@ -62,6 +62,7 @@ def perform_course_search( ) if not searcher: raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE") + log_search_params = getattr(settings, "SEARCH_COURSEWARE_CONTENT_LOG_PARAMS", False) results = searcher.search( query_string=search_term, @@ -70,6 +71,7 @@ def perform_course_search( exclude_dictionary=exclude_dictionary, size=size, from_=from_, + log_search_params=log_search_params, ) # post-process the result diff --git a/search/elastic.py b/search/elastic.py index f895564a..4e38093a 100644 --- a/search/elastic.py +++ b/search/elastic.py @@ -477,6 +477,7 @@ def search(self, aggregation_terms=None, exclude_ids=None, use_field_match=False, + log_search_params=False, **kwargs): # pylint: disable=arguments-differ, unused-argument """ Implements call to search the index for the desired content. @@ -653,6 +654,9 @@ def search(self, if aggregation_terms: body["aggs"] = _process_aggregation_terms(aggregation_terms) + if log_search_params: + log.info(f"full elastic search body {body}") + try: es_response = self._es.search(index=self._prefixed_index_name, body=body, **kwargs) except exceptions.ElasticsearchException as ex: diff --git a/search/search_engine_base.py b/search/search_engine_base.py index e2b9ef1f..49b2e367 100644 --- a/search/search_engine_base.py +++ b/search/search_engine_base.py @@ -48,6 +48,7 @@ def search(self, filter_dictionary=None, exclude_dictionary=None, aggregation_terms=None, + log_search_params=False, **kwargs): # pylint: disable=too-many-arguments """ Search for matching documents within the search index. diff --git a/search/tests/mock_search_engine.py b/search/tests/mock_search_engine.py index 10f243e4..52894800 100644 --- a/search/tests/mock_search_engine.py +++ b/search/tests/mock_search_engine.py @@ -340,6 +340,7 @@ def search(self, filter_dictionary=None, exclude_dictionary=None, aggregation_terms=None, + log_search_params=False, **kwargs): # pylint: disable=too-many-arguments """ Perform search upon documents within index. diff --git a/search/tests/tests.py b/search/tests/tests.py index 62ea00cd..3fe5047b 100644 --- a/search/tests/tests.py +++ b/search/tests/tests.py @@ -130,6 +130,15 @@ def test_find_string(self): response = self.searcher.search_string(test_string) self.assertEqual(response["total"], 3) + def test_log_params(self): + """ Test that if you turn on detailed logging, search doesn't explode. """ + test_string = "A test string" + self.searcher.index([{"content": {"name": test_string}}]) + + # search string + response = self.searcher.search(query_string=test_string, log_search_params=True) + self.assertEqual(response["total"], 1) + def test_field(self): """ test matching on a field """ test_string = "A test string" From 7b4df58743bea4bf5bd7432b7be696fdc25a6024 Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Wed, 15 Nov 2023 11:31:15 -0500 Subject: [PATCH 07/26] chore: bump version --- edxsearch/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 0506050a..3af5eb5f 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.6.0' +__version__ = '3.7.0' From 97ddab0da8ebf00d79c227024bf200e08f3a9789 Mon Sep 17 00:00:00 2001 From: Andy Shultz Date: Thu, 16 Nov 2023 09:16:59 -0500 Subject: [PATCH 08/26] fix: cannot rename perform-search to clearer name, breaks tests platform does not use this, but platform tests do, so the clarity is not worth the trouble and upgrade risk --- edxsearch/__init__.py | 2 +- search/api.py | 2 +- search/tests/test_engines.py | 4 ++-- search/views.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 3af5eb5f..0f6ce6a7 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.7.0' +__version__ = '3.7.1' diff --git a/search/api.py b/search/api.py index 177e909a..893dacce 100644 --- a/search/api.py +++ b/search/api.py @@ -41,7 +41,7 @@ class NoSearchEngineError(Exception): """ -def perform_course_search( +def perform_search( search_term, user=None, size=10, diff --git a/search/tests/test_engines.py b/search/tests/test_engines.py index 3340340d..659d4cc9 100644 --- a/search/tests/test_engines.py +++ b/search/tests/test_engines.py @@ -13,7 +13,7 @@ from django.test.utils import override_settings from elasticsearch import exceptions from elasticsearch.helpers import BulkIndexError -from search.api import NoSearchEngineError, perform_course_search +from search.api import NoSearchEngineError, perform_search from search.elastic import RESERVED_CHARACTERS from search.tests.mock_search_engine import (MockSearchEngine, json_date_to_datetime) @@ -238,7 +238,7 @@ class TestNone(TestCase): def test_perform_search(self): """ search opertaion should yeild an exception with no search engine """ with self.assertRaises(NoSearchEngineError): - perform_course_search("abc test") + perform_search("abc test") @override_settings(SEARCH_ENGINE="search.elastic.ElasticSearchEngine") diff --git a/search/views.py b/search/views.py index 0c30d426..c59f7463 100644 --- a/search/views.py +++ b/search/views.py @@ -10,7 +10,7 @@ from django.views.decorators.http import require_POST from eventtracking import tracker as track -from .api import perform_course_search, course_discovery_search, course_discovery_filter_fields +from .api import perform_search, course_discovery_search, course_discovery_filter_fields from .initializer import SearchInitializer # log appears to be standard name used for logger @@ -96,7 +96,7 @@ def do_search(request, course_id=None): } ) - results = perform_course_search( + results = perform_search( search_term, user=request.user, size=size, From 091a4790954ca9bef368edf0c655032861b8091e Mon Sep 17 00:00:00 2001 From: David Nuon Date: Thu, 4 Jan 2024 13:19:10 -0800 Subject: [PATCH 09/26] feat: add timing for results filtering (#142) * feat: add timing for results filtering --- search/api.py | 42 +++++++++- search/tests/test_api_timing_events.py | 107 +++++++++++++++++++++++++ search/utils.py | 42 ++++++++++ 3 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 search/tests/test_api_timing_events.py diff --git a/search/api.py b/search/api.py index 893dacce..fde7010d 100644 --- a/search/api.py +++ b/search/api.py @@ -1,13 +1,13 @@ """ search business logic implementations """ from datetime import datetime - from django.conf import settings +from eventtracking import tracker as track from .filter_generator import SearchFilterGenerator from .search_engine_base import SearchEngine from .result_processor import SearchResultProcessor -from .utils import DateRange +from .utils import DateRange, Timer # Default filters that we support, override using COURSE_DISCOVERY_FILTERS setting if desired DEFAULT_FILTER_FIELDS = ["org", "modes", "language"] @@ -52,10 +52,13 @@ def perform_search( """ # field_, filter_ and exclude_dictionary(s) can be overridden by calling application # field_dictionary includes course if course_id provided + filter_generation_timer = Timer() + filter_generation_timer.start() (field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters( user=user, course_id=course_id ) + filter_generation_timer.stop() searcher = SearchEngine.get_search_engine( getattr(settings, "COURSEWARE_CONTENT_INDEX_NAME", "courseware_content") @@ -64,6 +67,9 @@ def perform_search( raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE") log_search_params = getattr(settings, "SEARCH_COURSEWARE_CONTENT_LOG_PARAMS", False) + search_timer = Timer() + search_timer.start() + results = searcher.search( query_string=search_term, field_dictionary=field_dictionary, @@ -74,6 +80,9 @@ def perform_search( log_search_params=log_search_params, ) + processing_timer = Timer() + processing_timer.start() + # post-process the result for result in results["results"]: result["data"] = SearchResultProcessor.process_result(result["data"], search_term, user) @@ -81,9 +90,38 @@ def perform_search( results["access_denied_count"] = len([r for r in results["results"] if r["data"] is None]) results["results"] = [r for r in results["results"] if r["data"] is not None] + processing_timer.stop() + search_timer.stop() + + emit_api_timing_event(search_term, course_id, filter_generation_timer, processing_timer, search_timer) return results +def emit_api_timing_event(search_term, course_id, filter_generation_timer, processing_timer, search_timer): + """ + Emit the timing events for the search API + """ + track.emit("edx.course.search.executed", { + "search_term": search_term, + "course_id": course_id, + "filter_generation_time": { + "start": filter_generation_timer.start_time, + "end": filter_generation_timer.end_time, + "elapsed": filter_generation_timer.elapsed_time, + }, + "processing_time": { + "start": processing_timer.start_time_string, + "end": processing_timer.start_time_string, + "elapsed": processing_timer.elapsed_time, + }, + "search_time": { + "start": search_timer.start_time_string, + "end": search_timer.start_time_string, + "elapsed": search_timer.elapsed_time, + }, + }) + + def course_discovery_search(search_term=None, size=20, from_=0, field_dictionary=None): """ Course Discovery activities against the search engine index of course details diff --git a/search/tests/test_api_timing_events.py b/search/tests/test_api_timing_events.py new file mode 100644 index 00000000..1c11347e --- /dev/null +++ b/search/tests/test_api_timing_events.py @@ -0,0 +1,107 @@ +""" Tests for timing functionality """ + +import datetime +from unittest.mock import patch, call + +from django.test import TestCase +from django.test.utils import override_settings +from search.tests.mock_search_engine import MockSearchEngine +from search.utils import Timer +from search.api import emit_api_timing_event + + +@override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") +class TimingEventsTest(TestCase): + """ Tests to see if timing events are emitted""" + + def setUp(self): + super().setUp() + MockSearchEngine.destroy() + patcher = patch('search.api.track') + self.mock_track = patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + MockSearchEngine.destroy() + super().tearDown() + + def test_perform_search(self): + search_term = "testing search" + course_id = "mock.course.id" + + filter_generation_timer = Timer() + filter_generation_timer.start() + filter_generation_timer.stop() + + search_timer = Timer() + search_timer.start() + search_timer.stop() + + processing_timer = Timer() + processing_timer.start() + processing_timer.stop() + + emit_api_timing_event(search_term, course_id, filter_generation_timer, processing_timer, search_timer) + timing_event_call = self.mock_track.emit.mock_calls[0] + expected_call = call("edx.course.search.executed", { + "search_term": search_term, + "course_id": course_id, + "filter_generation_time": { + "start": filter_generation_timer.start_time, + "end": filter_generation_timer.end_time, + "elapsed": filter_generation_timer.elapsed_time, + }, + "processing_time": { + "start": processing_timer.start_time_string, + "end": processing_timer.start_time_string, + "elapsed": processing_timer.elapsed_time, + }, + "search_time": { + "start": search_timer.start_time_string, + "end": search_timer.start_time_string, + "elapsed": search_timer.elapsed_time, + }, + }) + self.assertEqual(timing_event_call, expected_call) + + +class TimerTest(TestCase): + """ + Timer Test Case + """ + + def test_start_timer(self): + timer = Timer() + timer.start() + timer.stop() + self.assertIsNotNone(timer.start_time) + self.assertIsNotNone(timer.end_time) + + def test_elapsed_time(self): + # pylint: disable=protected-access + + start = datetime.datetime(2024, 1, 1, 0, 0, 0, 0) + end = start + datetime.timedelta(seconds=5) + + timer = Timer() + timer._start_time = start + timer._end_time = end + + self.assertEqual(timer.elapsed_time, 5) + self.assertEqual(timer.start_time, start) + self.assertEqual(timer.end_time, end) + + def test_elapsed_time_string(self): + # pylint: disable=protected-access + + start = datetime.datetime(2024, 1, 1, 0, 0, 0, 0) + end = start + datetime.timedelta(seconds=5) + + timer = Timer() + timer._start_time = start + timer._end_time = end + + self.assertEqual(timer.elapsed_time, 5) + self.assertEqual(timer.start_time_string, "2024-01-01T00:00:00") + self.assertEqual(timer.end_time_string, "2024-01-01T00:00:05") + self.assertGreaterEqual(timer.end_time, timer.start_time) diff --git a/search/utils.py b/search/utils.py index a5b4a424..ded4a759 100644 --- a/search/utils.py +++ b/search/utils.py @@ -1,6 +1,7 @@ """ Utility classes to support others """ import importlib +import datetime from collections.abc import Iterable @@ -65,3 +66,44 @@ def upper_string(self): def lower_string(self): """ use isoformat for _lower date's string format """ return self._lower.isoformat() + + +class Timer: + + """ Simple timer class to measure elapsed time """ + def __init__(self): + self._start_time = None + self._end_time = None + + def start(self): + """ Start the timer """ + self._start_time = datetime.datetime.now() + + def stop(self): + """ Stop the timer """ + self._end_time = datetime.datetime.now() + + @property + def start_time(self): + """ Return the start time """ + return self._start_time + + @property + def end_time(self): + """ Return the end time """ + return self._end_time + + @property + def start_time_string(self): + """ use isoformat for the start time """ + return self._start_time.isoformat() + + @property + def end_time_string(self): + """ use isoformat for the end time """ + return self._end_time.isoformat() + + @property + def elapsed_time(self): + """ Return the elapsed time """ + return (self._end_time - self._start_time).seconds From 585c74c6bfdc39400cbdb26144098ce0220596c3 Mon Sep 17 00:00:00 2001 From: David Nuon Date: Mon, 8 Jan 2024 10:56:25 -0800 Subject: [PATCH 10/26] fix: update version number (#143) --- edxsearch/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 0f6ce6a7..9f65a02f 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.7.1' +__version__ = '3.8.1' From 983065a49040eb7f9816198c27466986a4fdd132 Mon Sep 17 00:00:00 2001 From: David Nuon Date: Tue, 9 Jan 2024 12:53:16 -0800 Subject: [PATCH 11/26] fix: Fix event logging to log proper start and end time (#144) --- edxsearch/__init__.py | 2 +- search/api.py | 8 ++++---- search/tests/test_api_timing_events.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 9f65a02f..4807b08e 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.8.1' +__version__ = '3.8.2' diff --git a/search/api.py b/search/api.py index fde7010d..43d2ed18 100644 --- a/search/api.py +++ b/search/api.py @@ -110,13 +110,13 @@ def emit_api_timing_event(search_term, course_id, filter_generation_timer, proce "elapsed": filter_generation_timer.elapsed_time, }, "processing_time": { - "start": processing_timer.start_time_string, - "end": processing_timer.start_time_string, + "start": processing_timer.start_time, + "end": processing_timer.end_time, "elapsed": processing_timer.elapsed_time, }, "search_time": { - "start": search_timer.start_time_string, - "end": search_timer.start_time_string, + "start": search_timer.start_time, + "end": search_timer.end_time, "elapsed": search_timer.elapsed_time, }, }) diff --git a/search/tests/test_api_timing_events.py b/search/tests/test_api_timing_events.py index 1c11347e..dcf83c3e 100644 --- a/search/tests/test_api_timing_events.py +++ b/search/tests/test_api_timing_events.py @@ -52,13 +52,13 @@ def test_perform_search(self): "elapsed": filter_generation_timer.elapsed_time, }, "processing_time": { - "start": processing_timer.start_time_string, - "end": processing_timer.start_time_string, + "start": processing_timer.start_time, + "end": processing_timer.end_time, "elapsed": processing_timer.elapsed_time, }, "search_time": { - "start": search_timer.start_time_string, - "end": search_timer.start_time_string, + "start": search_timer.start_time, + "end": search_timer.end_time, "elapsed": search_timer.elapsed_time, }, }) From abad3bde1fdedf92dcb1680628a7a847589498b8 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Fri, 29 Mar 2024 19:47:04 +0500 Subject: [PATCH 12/26] build: adding python 3.11 and 3.12 support (#145) * feat: Add back 3.12 testing and bump the version. --------- Co-authored-by: Feanil Patel --- .github/workflows/ci.yml | 8 +- edxsearch/__init__.py | 2 +- requirements/base.txt | 78 +++++--- requirements/ci.txt | 38 ++-- requirements/constraints.txt | 4 +- requirements/dev.txt | 185 ++++++++++++------- requirements/pip-tools.txt | 27 ++- requirements/pip.txt | 8 +- requirements/quality.txt | 128 +++++++------ requirements/testing.txt | 93 ++++++---- search/elastic.py | 3 +- search/filter_generator.py | 2 - search/initializer.py | 2 - search/result_processor.py | 7 +- search/tests/mock_search_engine.py | 2 +- search/tests/test_course_discovery.py | 4 +- search/tests/test_course_discovery_views.py | 5 +- search/tests/test_engines.py | 1 - search/tests/test_mock_search_engine.py | 4 - search/tests/test_search_result_processor.py | 3 - search/tests/test_views.py | 9 +- search/tests/tests.py | 4 - search/tests/utils.py | 3 +- search/urls.py | 1 - search/views.py | 9 +- tox.ini | 20 +- 26 files changed, 377 insertions(+), 273 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e39fa0bb..37c133cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8'] - toxenv: [django32, django40, django42, quality] + python-version: ['3.8', '3.11', '3.12'] + toxenv: [django42, quality] steps: - uses: actions/checkout@v2 @@ -31,7 +31,9 @@ jobs: run: pip install -r requirements/pip.txt - name: Install Dependencies - run: pip install -r requirements/ci.txt + run: + pip install setuptools + pip install -r requirements/ci.txt - name: Run Tests env: diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 4807b08e..c673376f 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.8.2' +__version__ = '3.9.0' diff --git a/requirements/base.txt b/requirements/base.txt index 71cfd11e..78cb9cd8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,23 +4,26 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via kombu asgiref==3.7.2 # via django -backports-zoneinfo[tzdata]==0.2.1 +attrs==23.2.0 + # via openedx-events +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via + # -c requirements/constraints.txt # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via celery -celery==5.3.1 +celery==5.3.6 # via event-tracking -certifi==2023.5.7 +certifi==2024.2.2 # via elasticsearch -cffi==1.15.1 +cffi==1.16.0 # via pynacl -click==8.1.3 +click==8.1.7 # via # celery # click-didyoumean @@ -34,65 +37,78 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -code-annotations==1.3.0 +code-annotations==1.6.0 # via edx-toggles -django==3.2.19 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in # django-crum + # django-waffle # edx-django-utils # edx-toggles # event-tracking + # openedx-events django-crum==0.7.9 # via # edx-django-utils # edx-toggles -django-waffle==3.0.0 +django-waffle==4.1.0 # via # edx-django-utils # edx-toggles -edx-django-utils==5.5.0 +edx-django-utils==5.10.1 # via # edx-toggles # event-tracking -edx-toggles==5.0.0 - # via -r requirements/base.in + # openedx-events +edx-opaque-keys[django]==2.5.1 + # via openedx-events +edx-toggles==5.1.1 + # via + # -r requirements/base.in + # event-tracking elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in -event-tracking==2.1.0 +event-tracking==2.3.0 # via -r requirements/base.in -jinja2==3.1.2 +fastavro==1.9.4 + # via openedx-events +jinja2==3.1.3 # via code-annotations -kombu==5.3.1 +kombu==5.3.5 # via celery -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -newrelic==8.8.0 +newrelic==9.6.0 # via edx-django-utils -pbr==5.11.1 +openedx-events==9.5.2 + # via event-tracking +pbr==6.0.0 # via stevedore -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.43 # via click-repl -psutil==5.9.5 +psutil==5.9.8 # via edx-django-utils pycparser==2.21 # via cffi pymongo==3.13.0 - # via event-tracking + # via + # edx-opaque-keys + # event-tracking pynacl==1.5.0 # via edx-django-utils python-dateutil==2.8.2 # via celery -python-slugify==8.0.1 +python-slugify==8.0.4 # via code-annotations -pytz==2023.3 +pytz==2024.1 # via # django # event-tracking -pyyaml==6.0 +pyyaml==6.0.1 # via code-annotations six==1.16.0 # via @@ -104,22 +120,24 @@ stevedore==5.1.0 # via # code-annotations # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via python-slugify -typing-extensions==4.6.3 +typing-extensions==4.9.0 # via # asgiref + # edx-opaque-keys # kombu -tzdata==2023.3 +tzdata==2024.1 # via # backports-zoneinfo # celery -urllib3==1.26.16 +urllib3==1.26.18 # via elasticsearch -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index 0ed5aa69..bd57292b 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,27 +4,35 @@ # # make upgrade # -distlib==0.3.6 +cachetools==5.3.2 + # via tox +chardet==5.2.0 + # via tox +colorama==0.4.6 + # via tox +distlib==0.3.8 # via virtualenv -filelock==3.12.2 +filelock==3.13.1 # via # tox # virtualenv -packaging==23.1 - # via tox -platformdirs==3.8.0 - # via virtualenv -pluggy==1.2.0 - # via tox -py==1.11.0 +packaging==23.2 + # via + # pyproject-api + # tox +platformdirs==4.2.0 + # via + # tox + # virtualenv +pluggy==1.4.0 # via tox -six==1.16.0 +pyproject-api==1.6.1 # via tox tomli==2.0.1 - # via tox -tox==3.28.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # -r requirements/ci.in -virtualenv==20.23.1 + # pyproject-api + # tox +tox==4.13.0 + # via -r requirements/ci.in +virtualenv==20.25.0 # via tox diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 7c8a02cb..aa2ccdf1 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -15,6 +15,4 @@ # ddt >= 1.4.0 causing test failures ddt < 1.4.0 - -# greater version breaking quality build. fix in separate PR. -pylint==2.10.2 \ No newline at end of file +backports.zoneinfo;python_version<"3.9" \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 18a58447..4b4de066 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -14,42 +14,56 @@ asgiref==3.7.2 # -r requirements/quality.txt # -r requirements/testing.txt # django -astroid==2.7.3 +astroid==3.0.3 # via # -r requirements/quality.txt # pylint # pylint-celery -backports-zoneinfo[tzdata]==0.2.1 +attrs==23.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt + # openedx-events +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # -r requirements/quality.txt + # -r requirements/testing.txt # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # celery -build==0.10.0 +build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools -celery==5.3.1 +cachetools==5.3.2 + # via + # -r requirements/ci.txt + # tox +celery==5.3.6 # via # -r requirements/quality.txt # -r requirements/testing.txt # event-tracking -certifi==2023.5.7 +certifi==2024.2.2 # via # -r requirements/quality.txt # -r requirements/testing.txt # elasticsearch -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # pynacl -click==8.1.3 +chardet==5.2.0 + # via + # -r requirements/ci.txt + # tox +click==8.1.7 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt @@ -82,13 +96,17 @@ click-repl==0.3.0 # -r requirements/quality.txt # -r requirements/testing.txt # celery -code-annotations==1.3.0 +code-annotations==1.6.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-lint # edx-toggles -coverage[toml]==7.2.7 +colorama==0.4.6 + # via + # -r requirements/ci.txt + # tox +coverage[toml]==7.4.2 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -98,156 +116,176 @@ ddt==1.3.1 # -c requirements/constraints.txt # -r requirements/quality.txt # -r requirements/testing.txt -distlib==0.3.6 +dill==0.3.8 + # via + # -r requirements/quality.txt + # pylint +distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.19 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # -r requirements/testing.txt # django-crum + # django-waffle # edx-django-utils # edx-toggles # event-tracking + # openedx-events django-crum==0.7.9 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils # edx-toggles -django-waffle==3.0.0 +django-waffle==4.1.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils # edx-toggles -edx-django-utils==5.5.0 +edx-django-utils==5.10.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-toggles # event-tracking -edx-lint==5.3.4 + # openedx-events +edx-lint==5.3.6 # via -r requirements/quality.txt -edx-toggles==5.0.0 +edx-opaque-keys[django]==2.5.1 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # openedx-events +edx-toggles==5.1.1 # via # -r requirements/quality.txt # -r requirements/testing.txt + # event-tracking elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # -r requirements/testing.txt -event-tracking==2.1.0 +event-tracking==2.3.0 # via # -r requirements/quality.txt # -r requirements/testing.txt -exceptiongroup==1.1.1 +exceptiongroup==1.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # pytest -filelock==3.12.2 +fastavro==1.9.4 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # openedx-events +filelock==3.13.1 # via # -r requirements/ci.txt # tox # virtualenv +importlib-metadata==7.0.1 + # via + # -r requirements/pip-tools.txt + # build iniconfig==2.0.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # pytest -isort==5.12.0 +isort==5.13.2 # via # -r requirements/quality.txt # pylint -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/quality.txt # -r requirements/testing.txt # code-annotations -kombu==5.3.1 +kombu==5.3.5 # via # -r requirements/quality.txt # -r requirements/testing.txt # celery -lazy-object-proxy==1.9.0 - # via - # -r requirements/quality.txt - # astroid -markupsafe==2.1.3 +markupsafe==2.1.5 # via # -r requirements/quality.txt # -r requirements/testing.txt # jinja2 -mccabe==0.6.1 +mccabe==0.7.0 # via # -r requirements/quality.txt # pylint -mock==5.0.2 +mock==5.1.0 # via # -r requirements/quality.txt # -r requirements/testing.txt -newrelic==8.8.0 +newrelic==9.6.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils -packaging==23.1 +openedx-events==9.5.2 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # event-tracking +packaging==23.2 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # -r requirements/testing.txt # build + # pyproject-api # pytest # tox -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # stevedore -pip-tools==6.13.0 +pip-tools==7.4.0 # via -r requirements/pip-tools.txt -platformdirs==3.8.0 +platformdirs==4.2.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint + # tox # virtualenv -pluggy==1.2.0 +pluggy==1.4.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # -r requirements/testing.txt # pytest # tox -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.43 # via # -r requirements/quality.txt # -r requirements/testing.txt # click-repl -psutil==5.9.5 +psutil==5.9.8 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils -py==1.11.0 - # via - # -r requirements/ci.txt - # tox -pycodestyle==2.10.0 +pycodestyle==2.11.1 # via -r requirements/quality.txt pycparser==2.21 # via # -r requirements/quality.txt # -r requirements/testing.txt # cffi -pylint==2.10.2 +pylint==3.0.3 # via - # -c requirements/constraints.txt # -r requirements/quality.txt # edx-lint # pylint-celery @@ -257,7 +295,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.3 +pylint-django==2.5.5 # via # -r requirements/quality.txt # edx-lint @@ -270,17 +308,23 @@ pymongo==3.13.0 # via # -r requirements/quality.txt # -r requirements/testing.txt + # edx-opaque-keys # event-tracking pynacl==1.5.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils +pyproject-api==1.6.1 + # via + # -r requirements/ci.txt + # tox pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.0 + # pip-tools +pytest==8.0.1 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -294,31 +338,29 @@ python-dateutil==2.8.2 # -r requirements/quality.txt # -r requirements/testing.txt # celery -python-slugify==8.0.1 +python-slugify==8.0.4 # via # -r requirements/quality.txt # -r requirements/testing.txt # code-annotations -pytz==2023.3 +pytz==2024.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # django # event-tracking -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # code-annotations six==1.16.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # -r requirements/testing.txt # edx-lint # event-tracking # python-dateutil - # tox sqlparse==0.4.4 # via # -r requirements/quality.txt @@ -330,15 +372,12 @@ stevedore==5.1.0 # -r requirements/testing.txt # code-annotations # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/quality.txt # -r requirements/testing.txt # python-slugify -toml==0.10.2 - # via - # -r requirements/quality.txt - # pylint tomli==2.0.1 # via # -r requirements/ci.txt @@ -347,54 +386,62 @@ tomli==2.0.1 # -r requirements/testing.txt # build # coverage + # pip-tools + # pylint + # pyproject-api # pyproject-hooks # pytest # tox -tox==3.28.0 +tomlkit==0.12.3 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # -r requirements/ci.txt -typing-extensions==4.6.3 + # -r requirements/quality.txt + # pylint +tox==4.13.0 + # via -r requirements/ci.txt +typing-extensions==4.9.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # asgiref + # astroid + # edx-opaque-keys # kombu -tzdata==2023.3 + # pylint +tzdata==2024.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # backports-zoneinfo # celery -urllib3==1.26.16 +urllib3==1.26.18 # via # -r requirements/quality.txt # -r requirements/testing.txt # elasticsearch -vine==5.0.0 +vine==5.1.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # amqp # celery # kombu -virtualenv==20.23.1 +virtualenv==20.25.0 # via # -r requirements/ci.txt # tox -wcwidth==0.2.6 +wcwidth==0.2.13 # via # -r requirements/quality.txt # -r requirements/testing.txt # prompt-toolkit -wheel==0.40.0 +wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools -wrapt==1.12.1 +zipp==3.17.0 # via - # -r requirements/quality.txt - # astroid + # -r requirements/pip-tools.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 8620fabd..44c48d99 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,23 +1,32 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -build==0.10.0 +build==1.0.3 # via pip-tools -click==8.1.3 +click==8.1.7 # via pip-tools -packaging==23.1 +importlib-metadata==7.0.1 # via build -pip-tools==6.13.0 +packaging==23.2 + # via build +pip-tools==7.4.0 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools tomli==2.0.1 - # via build -wheel==0.40.0 + # via + # build + # pip-tools + # pyproject-hooks +wheel==0.42.0 # via pip-tools +zipp==3.17.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip.txt b/requirements/pip.txt index 8a5a6e3b..677bf038 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # @@ -8,7 +8,7 @@ wheel==0.37.1 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==22.0.3 +pip==24.0 # via -r requirements/pip.in -setuptools==59.8.0 +setuptools==69.1.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 832f3232..4a8c3857 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/testing.txt # kombu @@ -12,32 +12,37 @@ asgiref==3.7.2 # via # -r requirements/testing.txt # django -astroid==2.7.3 +astroid==3.0.3 # via # pylint # pylint-celery -backports-zoneinfo[tzdata]==0.2.1 +attrs==23.2.0 # via # -r requirements/testing.txt + # openedx-events +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # -r requirements/testing.txt # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/testing.txt # celery -celery==5.3.1 +celery==5.3.6 # via # -r requirements/testing.txt # event-tracking -certifi==2023.5.7 +certifi==2024.2.2 # via # -r requirements/testing.txt # elasticsearch -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/testing.txt # pynacl -click==8.1.3 +click==8.1.7 # via # -r requirements/testing.txt # celery @@ -62,12 +67,12 @@ click-repl==0.3.0 # via # -r requirements/testing.txt # celery -code-annotations==1.3.0 +code-annotations==1.6.0 # via # -r requirements/testing.txt # edx-lint # edx-toggles -coverage[toml]==7.2.7 +coverage[toml]==7.4.2 # via # -r requirements/quality.in # -r requirements/testing.txt @@ -76,109 +81,125 @@ ddt==1.3.1 # via # -c requirements/constraints.txt # -r requirements/testing.txt -django==3.2.19 +dill==0.3.8 + # via pylint +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/testing.txt # django-crum + # django-waffle # edx-django-utils # edx-toggles # event-tracking + # openedx-events django-crum==0.7.9 # via # -r requirements/testing.txt # edx-django-utils # edx-toggles -django-waffle==3.0.0 +django-waffle==4.1.0 # via # -r requirements/testing.txt # edx-django-utils # edx-toggles -edx-django-utils==5.5.0 +edx-django-utils==5.10.1 # via # -r requirements/testing.txt # edx-toggles # event-tracking -edx-lint==5.3.4 + # openedx-events +edx-lint==5.3.6 # via -r requirements/quality.in -edx-toggles==5.0.0 - # via -r requirements/testing.txt +edx-opaque-keys[django]==2.5.1 + # via + # -r requirements/testing.txt + # openedx-events +edx-toggles==5.1.1 + # via + # -r requirements/testing.txt + # event-tracking elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/testing.txt -event-tracking==2.1.0 +event-tracking==2.3.0 # via -r requirements/testing.txt -exceptiongroup==1.1.1 +exceptiongroup==1.2.0 # via # -r requirements/testing.txt # pytest +fastavro==1.9.4 + # via + # -r requirements/testing.txt + # openedx-events iniconfig==2.0.0 # via # -r requirements/testing.txt # pytest -isort==5.12.0 +isort==5.13.2 # via pylint -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/testing.txt # code-annotations -kombu==5.3.1 +kombu==5.3.5 # via # -r requirements/testing.txt # celery -lazy-object-proxy==1.9.0 - # via astroid -markupsafe==2.1.3 +markupsafe==2.1.5 # via # -r requirements/testing.txt # jinja2 -mccabe==0.6.1 +mccabe==0.7.0 # via pylint -mock==5.0.2 +mock==5.1.0 # via -r requirements/testing.txt -newrelic==8.8.0 +newrelic==9.6.0 # via # -r requirements/testing.txt # edx-django-utils -packaging==23.1 +openedx-events==9.5.2 + # via + # -r requirements/testing.txt + # event-tracking +packaging==23.2 # via # -r requirements/testing.txt # pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/testing.txt # stevedore -platformdirs==3.8.0 +platformdirs==4.2.0 # via pylint -pluggy==1.2.0 +pluggy==1.4.0 # via # -r requirements/testing.txt # pytest -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.43 # via # -r requirements/testing.txt # click-repl -psutil==5.9.5 +psutil==5.9.8 # via # -r requirements/testing.txt # edx-django-utils -pycodestyle==2.10.0 +pycodestyle==2.11.1 # via -r requirements/quality.in pycparser==2.21 # via # -r requirements/testing.txt # cffi -pylint==2.10.2 +pylint==3.0.3 # via - # -c requirements/constraints.txt # edx-lint # pylint-celery # pylint-django # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.3 +pylint-django==2.5.5 # via edx-lint pylint-plugin-utils==0.8.2 # via @@ -187,12 +208,13 @@ pylint-plugin-utils==0.8.2 pymongo==3.13.0 # via # -r requirements/testing.txt + # edx-opaque-keys # event-tracking pynacl==1.5.0 # via # -r requirements/testing.txt # edx-django-utils -pytest==7.4.0 +pytest==8.0.1 # via # -r requirements/testing.txt # pytest-cov @@ -202,16 +224,16 @@ python-dateutil==2.8.2 # via # -r requirements/testing.txt # celery -python-slugify==8.0.1 +python-slugify==8.0.4 # via # -r requirements/testing.txt # code-annotations -pytz==2023.3 +pytz==2024.1 # via # -r requirements/testing.txt # django # event-tracking -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/testing.txt # code-annotations @@ -230,43 +252,43 @@ stevedore==5.1.0 # -r requirements/testing.txt # code-annotations # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/testing.txt # python-slugify -toml==0.10.2 - # via pylint tomli==2.0.1 # via # -r requirements/testing.txt # coverage + # pylint # pytest -typing-extensions==4.6.3 +tomlkit==0.12.3 + # via pylint +typing-extensions==4.9.0 # via # -r requirements/testing.txt # asgiref + # astroid + # edx-opaque-keys # kombu -tzdata==2023.3 + # pylint +tzdata==2024.1 # via # -r requirements/testing.txt # backports-zoneinfo # celery -urllib3==1.26.16 +urllib3==1.26.18 # via # -r requirements/testing.txt # elasticsearch -vine==5.0.0 +vine==5.1.0 # via # -r requirements/testing.txt # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.13 # via # -r requirements/testing.txt # prompt-toolkit -wrapt==1.12.1 - # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/testing.txt b/requirements/testing.txt index ba18e4d5..9a1d1263 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/base.txt # kombu @@ -12,28 +12,33 @@ asgiref==3.7.2 # via # -r requirements/base.txt # django -backports-zoneinfo[tzdata]==0.2.1 +attrs==23.2.0 # via # -r requirements/base.txt + # openedx-events +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # -r requirements/base.txt # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/base.txt # celery -celery==5.3.1 +celery==5.3.6 # via # -r requirements/base.txt # event-tracking -certifi==2023.5.7 +certifi==2024.2.2 # via # -r requirements/base.txt # elasticsearch -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/base.txt # pynacl -click==8.1.3 +click==8.1.7 # via # -r requirements/base.txt # celery @@ -54,11 +59,11 @@ click-repl==0.3.0 # via # -r requirements/base.txt # celery -code-annotations==1.3.0 +code-annotations==1.6.0 # via # -r requirements/base.txt # edx-toggles -coverage[toml]==7.2.7 +coverage[toml]==7.4.2 # via # -r requirements/testing.in # pytest-cov @@ -70,67 +75,84 @@ ddt==1.3.1 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt # django-crum + # django-waffle # edx-django-utils # edx-toggles # event-tracking + # openedx-events django-crum==0.7.9 # via # -r requirements/base.txt # edx-django-utils # edx-toggles -django-waffle==3.0.0 +django-waffle==4.1.0 # via # -r requirements/base.txt # edx-django-utils # edx-toggles -edx-django-utils==5.5.0 +edx-django-utils==5.10.1 # via # -r requirements/base.txt # edx-toggles # event-tracking -edx-toggles==5.0.0 - # via -r requirements/base.txt + # openedx-events +edx-opaque-keys[django]==2.5.1 + # via + # -r requirements/base.txt + # openedx-events +edx-toggles==5.1.1 + # via + # -r requirements/base.txt + # event-tracking elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt -event-tracking==2.1.0 +event-tracking==2.3.0 # via -r requirements/base.txt -exceptiongroup==1.1.1 +exceptiongroup==1.2.0 # via pytest +fastavro==1.9.4 + # via + # -r requirements/base.txt + # openedx-events iniconfig==2.0.0 # via pytest -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/base.txt # code-annotations -kombu==5.3.1 +kombu==5.3.5 # via # -r requirements/base.txt # celery -markupsafe==2.1.3 +markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -mock==5.0.2 +mock==5.1.0 # via -r requirements/testing.in -newrelic==8.8.0 +newrelic==9.6.0 # via # -r requirements/base.txt # edx-django-utils -packaging==23.1 +openedx-events==9.5.2 + # via + # -r requirements/base.txt + # event-tracking +packaging==23.2 # via pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/base.txt # stevedore -pluggy==1.2.0 +pluggy==1.4.0 # via pytest -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.43 # via # -r requirements/base.txt # click-repl -psutil==5.9.5 +psutil==5.9.8 # via # -r requirements/base.txt # edx-django-utils @@ -141,12 +163,13 @@ pycparser==2.21 pymongo==3.13.0 # via # -r requirements/base.txt + # edx-opaque-keys # event-tracking pynacl==1.5.0 # via # -r requirements/base.txt # edx-django-utils -pytest==7.4.0 +pytest==8.0.1 # via pytest-cov pytest-cov==4.1.0 # via -r requirements/testing.in @@ -154,16 +177,16 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # celery -python-slugify==8.0.1 +python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations -pytz==2023.3 +pytz==2024.1 # via # -r requirements/base.txt # django # event-tracking -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/base.txt # code-annotations @@ -181,6 +204,7 @@ stevedore==5.1.0 # -r requirements/base.txt # code-annotations # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/base.txt @@ -189,27 +213,28 @@ tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.6.3 +typing-extensions==4.9.0 # via # -r requirements/base.txt # asgiref + # edx-opaque-keys # kombu -tzdata==2023.3 +tzdata==2024.1 # via # -r requirements/base.txt # backports-zoneinfo # celery -urllib3==1.26.16 +urllib3==1.26.18 # via # -r requirements/base.txt # elasticsearch -vine==5.0.0 +vine==5.1.0 # via # -r requirements/base.txt # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.13 # via # -r requirements/base.txt # prompt-toolkit diff --git a/search/elastic.py b/search/elastic.py index 4e38093a..6a942ce5 100644 --- a/search/elastic.py +++ b/search/elastic.py @@ -469,6 +469,7 @@ def remove(self, doc_ids, **kwargs): log.exception("An error occurred while removing documents from the index: %r", valid_errors) raise + # pylint: disable=arguments-renamed, unused-argument def search(self, query_string=None, field_dictionary=None, @@ -478,7 +479,7 @@ def search(self, exclude_ids=None, use_field_match=False, log_search_params=False, - **kwargs): # pylint: disable=arguments-differ, unused-argument + **kwargs): """ Implements call to search the index for the desired content. diff --git a/search/filter_generator.py b/search/filter_generator.py index 8fbdabb4..5adacb86 100644 --- a/search/filter_generator.py +++ b/search/filter_generator.py @@ -14,8 +14,6 @@ class SearchFilterGenerator: Users of this search app will override this class and update setting for SEARCH_FILTER_GENERATOR """ - # disabling pylint violations because overriders will want to use these - # pylint: disable=unused-argument, no-self-use def filter_dictionary(self, **kwargs): """ base implementation which filters via start_date """ return {"start_date": DateRange(None, datetime.utcnow())} diff --git a/search/initializer.py b/search/initializer.py index 307fa533..04c6edb7 100644 --- a/search/initializer.py +++ b/search/initializer.py @@ -12,8 +12,6 @@ class SearchInitializer: Users of this search app will override this class and update setting for SEARCH_INITIALIZER """ - # disabling pylint violations because overriders will want to use these - # pylint: disable=unused-argument, no-self-use def initialize(self, **kwargs): """ empty base implementation """ diff --git a/search/result_processor.py b/search/result_processor.py index 6674cca1..24ba1b57 100644 --- a/search/result_processor.py +++ b/search/result_processor.py @@ -17,7 +17,7 @@ ELLIPSIS = '' # log appears to be standard name used for logger -log = logging.getLogger(__name__) # pylint: disable=invalid-name +log = logging.getLogger(__name__) class SearchResultProcessor: @@ -87,8 +87,7 @@ def decorate_matches(match_in, match_word): ) return match_in - # disabling pylint violations because overriders will want to use these - def should_remove(self, user): # pylint: disable=unused-argument, no-self-use + def should_remove(self, user): # pylint: disable=unused-argument """ Override this in a class in order to add in last-chance access checks to the search process Your application will want to make this decision @@ -116,7 +115,7 @@ def process_result(cls, dictionary, match_phrase, user): try: srp.add_properties() # protect around any problems introduced by subclasses within their properties - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: log.exception("error processing properties for %s - %s: will remove from results", json.dumps(dictionary, cls=DjangoJSONEncoder), str(ex)) return None diff --git a/search/tests/mock_search_engine.py b/search/tests/mock_search_engine.py index 52894800..4974c80e 100644 --- a/search/tests/mock_search_engine.py +++ b/search/tests/mock_search_engine.py @@ -190,7 +190,7 @@ def add_agg_value(agg_value): for document in aggregated_documents: add_agg_value(document[aggregate]) - total = sum([terms[term] for term in terms]) + total = sum([terms[term] for term in terms]) # pylint: disable=consider-using-generator return total, terms diff --git a/search/tests/test_course_discovery.py b/search/tests/test_course_discovery.py index cad33b4a..0c823f50 100644 --- a/search/tests/test_course_discovery.py +++ b/search/tests/test_course_discovery.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # Some of the subclasses that get used as settings-overrides will yield this pylint # error, but they do get used when included as part of the override_settings -# pylint: disable=too-few-public-methods -# pylint: disable=too-many-ancestors """ Tests for search functionalty """ import copy @@ -84,7 +82,7 @@ def get_and_index(cls, searcher, update_dict=None, remove_fields=None): @override_settings(COURSEWARE_CONTENT_INDEX_NAME=TEST_INDEX_NAME) @override_settings(COURSEWARE_INFO_INDEX_NAME=TEST_INDEX_NAME) # Any class that inherits from TestCase will cause too-many-public-methods pylint error -class TestMockCourseDiscoverySearch(TestCase, SearcherMixin): # pylint: disable=too-many-public-methods +class TestMockCourseDiscoverySearch(TestCase, SearcherMixin): """ Tests course discovery activities """ diff --git a/search/tests/test_course_discovery_views.py b/search/tests/test_course_discovery_views.py index 918b6ccf..f01a442a 100644 --- a/search/tests/test_course_discovery_views.py +++ b/search/tests/test_course_discovery_views.py @@ -7,11 +7,8 @@ from .test_views import MockSearchUrlTest from .test_course_discovery import DemoCourse -# Any class that inherits from TestCase will cause too-many-public-methods pylint error -# pylint: disable=too-many-public-methods - -@override_settings(ELASTIC_FIELD_MAPPINGS={ # pylint: disable=too-many-ancestors +@override_settings(ELASTIC_FIELD_MAPPINGS={ "start_date": {"type": "date"}, "enrollment_start": {"type": "date"}, "enrollment_end": {"type": "date"} diff --git a/search/tests/test_engines.py b/search/tests/test_engines.py index 659d4cc9..7888041c 100644 --- a/search/tests/test_engines.py +++ b/search/tests/test_engines.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # Some of the subclasses that get used as settings-overrides will yield this pylint # error, but they do get used when included as part of the override_settings -# pylint: disable=too-few-public-methods """ Tests for search functionality """ import json diff --git a/search/tests/test_mock_search_engine.py b/search/tests/test_mock_search_engine.py index 896a9558..13cc7867 100644 --- a/search/tests/test_mock_search_engine.py +++ b/search/tests/test_mock_search_engine.py @@ -11,10 +11,6 @@ from search.utils import DateRange -# Any class that inherits from TestCase will cause too-many-public-methods pylint error -# pylint: disable=too-many-public-methods - - @override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") @override_settings(ELASTIC_FIELD_MAPPINGS={"start_date": {"type": "date"}}) class MockSpecificSearchTests(TestCase, SearcherMixin): diff --git a/search/tests/test_search_result_processor.py b/search/tests/test_search_result_processor.py index a144a3dc..2a2efbd3 100644 --- a/search/tests/test_search_result_processor.py +++ b/search/tests/test_search_result_processor.py @@ -7,8 +7,6 @@ from search.result_processor import SearchResultProcessor, ELLIPSIS -# Any class that inherits from TestCase will cause too-many-public-methods pylint error -# pylint: disable=too-many-public-methods @ddt.ddt class SearchResultProcessorTests(TestCase): """ Tests to check SearchResultProcessor is working as desired """ @@ -297,7 +295,6 @@ class TestSearchResultProcessor(SearchResultProcessor): Override the SearchResultProcessor so that we get the additional (inferred) properties and can identify results that should be removed due to access restriction """ - # pylint: disable=no-self-use @property def additional_property(self): """ additional property that should appear within processed results """ diff --git a/search/tests/test_views.py b/search/tests/test_views.py index 92958f1e..7c3aff46 100644 --- a/search/tests/test_views.py +++ b/search/tests/test_views.py @@ -17,7 +17,6 @@ # Any class that inherits from TestCase will cause too-many-public-methods pylint error -# pylint: disable=too-many-public-methods @override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") @override_settings(ELASTIC_FIELD_MAPPINGS={"start_date": {"type": "date"}}) @override_settings(COURSEWARE_CONTENT_INDEX_NAME=TEST_INDEX_NAME) @@ -46,11 +45,11 @@ def tearDown(self): def assert_no_events_were_emitted(self): """Ensures no events were emitted since the last event related assertion""" - self.assertFalse(self.mock_tracker.emit.called) # pylint: disable=maybe-no-member + self.assertFalse(self.mock_tracker.emit.called) def assert_search_initiated_event(self, search_term, size, page): """Ensures an search initiated event was emitted""" - initiated_search_call = self.mock_tracker.emit.mock_calls[0] # pylint: disable=maybe-no-member + initiated_search_call = self.mock_tracker.emit.mock_calls[0] expected_result = call('edx.course.search.initiated', { "search_term": str(search_term), "page_size": size, @@ -60,7 +59,7 @@ def assert_search_initiated_event(self, search_term, size, page): def assert_results_returned_event(self, search_term, size, page, total): """Ensures an results returned event was emitted""" - returned_results_call = self.mock_tracker.emit.mock_calls[1] # pylint: disable=maybe-no-member + returned_results_call = self.mock_tracker.emit.mock_calls[1] expected_result = call('edx.course.search.results_displayed', { "search_term": str(search_term), "page_size": size, @@ -71,7 +70,7 @@ def assert_results_returned_event(self, search_term, size, page, total): def assert_initiated_return_events(self, search_term, size, page, total): """Asserts search initiated and results returned events were emitted""" - self.assertEqual(self.mock_tracker.emit.call_count, 2) # pylint: disable=maybe-no-member + self.assertEqual(self.mock_tracker.emit.call_count, 2) self.assert_search_initiated_event(search_term, size, page) self.assert_results_returned_event(search_term, size, page, total) diff --git a/search/tests/tests.py b/search/tests/tests.py index 3fe5047b..35383315 100644 --- a/search/tests/tests.py +++ b/search/tests/tests.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # Some of the subclasses that get used as settings-overrides will yield this pylint # error, but they do get used when included as part of the override_settings -# pylint: disable=too-few-public-methods -# pylint: disable=too-many-ancestors """ Tests for search functionalty """ from datetime import datetime @@ -20,8 +18,6 @@ from .mock_search_engine import MockSearchEngine -# Any class that inherits from TestCase will cause too-many-public-methods pylint error -# pylint: disable=too-many-public-methods @override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") @override_settings(ELASTIC_FIELD_MAPPINGS={"start_date": {"type": "date"}}) @override_settings(MOCK_SEARCH_BACKING_FILE=None) diff --git a/search/tests/utils.py b/search/tests/utils.py index d8c5e074..5d38d616 100644 --- a/search/tests/utils.py +++ b/search/tests/utils.py @@ -73,14 +73,13 @@ def search(self, class ErroringIndexEngine(MockSearchEngine): """ Override to generate search engine error to test """ - def index(self, sources, **kwargs): # pylint: disable=unused-argument, arguments-differ + def index(self, sources, **kwargs): raise Exception("There is a problem here") class ErroringElasticImpl(Elasticsearch): """ Elasticsearch implementation that throws exceptions""" - # pylint: disable=unused-argument def search(self, **kwargs): # pylint: disable=arguments-differ """ this will definitely fail """ raise exceptions.ElasticsearchException("This search operation failed") diff --git a/search/urls.py b/search/urls.py index ddf59c42..10439d54 100644 --- a/search/urls.py +++ b/search/urls.py @@ -8,7 +8,6 @@ COURSE_ID_PATTERN = getattr(settings, "COURSE_ID_PATTERN", r'(?P[^/+]+(/|\+)[^/+]+(/|\+)[^/]+)') # urlpatterns is the standard name to use here -# pylint: disable=invalid-name urlpatterns = [ path('', views.do_search, name='do_search'), re_path(r'^{}$'.format(COURSE_ID_PATTERN), views.do_search, name='do_search'), diff --git a/search/views.py b/search/views.py index c59f7463..cda536b2 100644 --- a/search/views.py +++ b/search/views.py @@ -1,6 +1,5 @@ """ handle requests for courseware search http requests """ # This contains just the url entry points to use if desired, which currently has only one -# pylint: disable=too-few-public-methods import logging @@ -14,7 +13,7 @@ from .initializer import SearchInitializer # log appears to be standard name used for logger -log = logging.getLogger(__name__) # pylint: disable=invalid-name +log = logging.getLogger(__name__) def _process_pagination_values(request): @@ -123,8 +122,7 @@ def do_search(request, course_id=None): } log.debug(str(invalid_err)) - # Allow for broad exceptions here - this is an entry point from external reference - except Exception as err: # pylint: disable=broad-except + except Exception as err: results = { "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term) } @@ -209,8 +207,7 @@ def course_discovery(request): } log.debug(str(invalid_err)) - # Allow for broad exceptions here - this is an entry point from external reference - except Exception as err: # pylint: disable=broad-except + except Exception as err: results = { "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term) } diff --git a/tox.ini b/tox.ini index fa71d60a..19cc118f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,24 +1,26 @@ [tox] -envlist = py38-django{32,40,42},quality +envlist = py{38,311,312}-django{42},quality [testenv] -setenv = +setenv = DJANGO_SETTINGS_MODULE = edxsearch.settings PYTHONPATH = {toxinidir} -deps = - django32: Django>=3.2,<4.0 - django40: Django>=4.0,<4.1 +deps = + setuptools + wheel django42: Django>=4.2,<4.3 -r {toxinidir}/requirements/testing.txt -commands = +commands = python -Wd -m coverage run manage.py test --settings=settings {posargs} python -m coverage xml [testenv:quality] -allowlist_externals = +allowlist_externals = make -deps = +deps = + setuptools + wheel -r{toxinidir}/requirements/quality.txt -commands = +commands = make quality From 72c04ee21cca9d5ae047ba0895ca4d434710c588 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 1 Apr 2024 14:18:50 -0400 Subject: [PATCH 13/26] docs: Add a long_description. The default empty long_description is failing when we try to push it to PyPI so make the README the long_description and be clear that it's markdown and not RST. We bump the version since we've already tagged the old version that failed to release. --- edxsearch/__init__.py | 2 +- setup.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index c673376f..835caa16 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.9.0' +__version__ = '3.9.1' diff --git a/setup.py b/setup.py index b12433d0..9685ebea 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ """ Setup to allow pip installs of edx-search module """ import os import re +from pathlib import Path from setuptools import setup @@ -47,11 +48,16 @@ def get_version(*file_paths): VERSION = get_version('edxsearch', '__init__.py') +# read the contents of your README file +this_directory = Path(__file__).parent +long_description = (this_directory / "README.md").read_text() setup( name='edx-search', version=VERSION, description='Search and index routines for index access', + long_description=long_description, + long_description_content_type='text/markdown', author='edX', author_email='oscm@edx.org', url='https://github.com/openedx/edx-search', From a9b4f969d79ed6aed10917a2310e654660dd4ff2 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Fri, 14 Jun 2024 17:00:40 -0300 Subject: [PATCH 14/26] build: Update codecov and use token Update codecov to the latest version and start using the org-wide token for uploads. See https://github.com/openedx/wg-frontend/issues/179 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37c133cb..9b543e78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: - name: Run Coverage if: matrix.python-version == '3.8' && matrix.toxenv=='django42' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true From 46db3ab7b5ca8913d0a3bdc4477e3cb8b4814e4e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Jul 2024 12:00:35 -0400 Subject: [PATCH 15/26] chore: Run `make upgrade` with Python 3.11 --- requirements/base.txt | 77 ++++++++++----------- requirements/ci.txt | 22 +++--- requirements/dev.txt | 136 ++++++++++++++----------------------- requirements/pip-tools.txt | 21 ++---- requirements/pip.txt | 2 +- requirements/quality.txt | 98 ++++++++++++-------------- requirements/testing.txt | 82 ++++++++++------------ 7 files changed, 183 insertions(+), 255 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 78cb9cd8..df520b14 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,25 +1,20 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # amqp==5.2.0 # via kombu -asgiref==3.7.2 +asgiref==3.8.1 # via django attrs==23.2.0 # via openedx-events -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # celery - # kombu billiard==4.2.0 # via celery -celery==5.3.6 +celery==5.4.0 # via event-tracking -certifi==2024.2.2 +certifi==2024.7.4 # via elasticsearch cffi==1.16.0 # via pynacl @@ -31,15 +26,15 @@ click==8.1.7 # click-repl # code-annotations # edx-django-utils -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -code-annotations==1.6.0 +code-annotations==1.8.0 # via edx-toggles -django==3.2.24 +django==4.2.14 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -57,14 +52,18 @@ django-waffle==4.1.0 # via # edx-django-utils # edx-toggles -edx-django-utils==5.10.1 +edx-ccx-keys==1.3.0 + # via openedx-events +edx-django-utils==5.14.2 # via # edx-toggles # event-tracking # openedx-events -edx-opaque-keys[django]==2.5.1 - # via openedx-events -edx-toggles==5.1.1 +edx-opaque-keys[django]==2.10.0 + # via + # edx-ccx-keys + # openedx-events +edx-toggles==5.2.0 # via # -r requirements/base.in # event-tracking @@ -72,27 +71,29 @@ elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in -event-tracking==2.3.0 - # via -r requirements/base.in -fastavro==1.9.4 +event-tracking==2.4.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/base.in +fastavro==1.9.5 # via openedx-events -jinja2==3.1.3 +jinja2==3.1.4 # via code-annotations -kombu==5.3.5 +kombu==5.3.7 # via celery markupsafe==2.1.5 # via jinja2 -newrelic==9.6.0 +newrelic==9.12.0 # via edx-django-utils -openedx-events==9.5.2 +openedx-events==9.11.0 # via event-tracking pbr==6.0.0 # via stevedore -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via click-repl -psutil==5.9.8 +psutil==6.0.0 # via edx-django-utils -pycparser==2.21 +pycparser==2.22 # via cffi pymongo==3.13.0 # via @@ -100,39 +101,33 @@ pymongo==3.13.0 # event-tracking pynacl==1.5.0 # via edx-django-utils -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via celery python-slugify==8.0.4 # via code-annotations pytz==2024.1 - # via - # django - # event-tracking + # via event-tracking pyyaml==6.0.1 # via code-annotations six==1.16.0 # via + # edx-ccx-keys # event-tracking # python-dateutil -sqlparse==0.4.4 +sqlparse==0.5.1 # via django -stevedore==5.1.0 +stevedore==5.2.0 # via # code-annotations # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via python-slugify -typing-extensions==4.9.0 - # via - # asgiref - # edx-opaque-keys - # kombu +typing-extensions==4.12.2 + # via edx-opaque-keys tzdata==2024.1 - # via - # backports-zoneinfo - # celery -urllib3==1.26.18 + # via celery +urllib3==1.26.19 # via elasticsearch vine==5.1.0 # via diff --git a/requirements/ci.txt b/requirements/ci.txt index bd57292b..564c0f58 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -cachetools==5.3.2 +cachetools==5.4.0 # via tox chardet==5.2.0 # via tox @@ -12,27 +12,23 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.15.4 # via # tox # virtualenv -packaging==23.2 +packaging==24.1 # via # pyproject-api # tox -platformdirs==4.2.0 +platformdirs==4.2.2 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via tox -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox -tomli==2.0.1 - # via - # pyproject-api - # tox -tox==4.13.0 +tox==4.16.0 # via -r requirements/ci.in -virtualenv==20.25.0 +virtualenv==20.26.3 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 4b4de066..f8f96994 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -9,12 +9,12 @@ amqp==5.2.0 # -r requirements/quality.txt # -r requirements/testing.txt # kombu -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # django -astroid==3.0.3 +astroid==3.2.4 # via # -r requirements/quality.txt # pylint @@ -24,32 +24,25 @@ attrs==23.2.0 # -r requirements/quality.txt # -r requirements/testing.txt # openedx-events -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # -r requirements/quality.txt - # -r requirements/testing.txt - # celery - # kombu billiard==4.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # celery -build==1.0.3 +build==1.2.1 # via # -r requirements/pip-tools.txt # pip-tools -cachetools==5.3.2 +cachetools==5.4.0 # via # -r requirements/ci.txt # tox -celery==5.3.6 +celery==5.4.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # event-tracking -certifi==2024.2.2 +certifi==2024.7.4 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -77,7 +70,7 @@ click==8.1.7 # edx-django-utils # edx-lint # pip-tools -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -96,7 +89,7 @@ click-repl==0.3.0 # -r requirements/quality.txt # -r requirements/testing.txt # celery -code-annotations==1.6.0 +code-annotations==1.8.0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -106,7 +99,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.2 +coverage[toml]==7.6.0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -124,7 +117,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.24 +django==4.2.14 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -147,21 +140,27 @@ django-waffle==4.1.0 # -r requirements/testing.txt # edx-django-utils # edx-toggles -edx-django-utils==5.10.1 +edx-ccx-keys==1.3.0 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # openedx-events +edx-django-utils==5.14.2 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-toggles # event-tracking # openedx-events -edx-lint==5.3.6 +edx-lint==5.3.7 # via -r requirements/quality.txt -edx-opaque-keys[django]==2.5.1 +edx-opaque-keys[django]==2.10.0 # via # -r requirements/quality.txt # -r requirements/testing.txt + # edx-ccx-keys # openedx-events -edx-toggles==5.1.1 +edx-toggles==5.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -171,29 +170,21 @@ elasticsearch==7.13.4 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # -r requirements/testing.txt -event-tracking==2.3.0 - # via - # -r requirements/quality.txt - # -r requirements/testing.txt -exceptiongroup==1.2.0 +event-tracking==2.4.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # -r requirements/testing.txt - # pytest -fastavro==1.9.4 +fastavro==1.9.5 # via # -r requirements/quality.txt # -r requirements/testing.txt # openedx-events -filelock==3.13.1 +filelock==3.15.4 # via # -r requirements/ci.txt # tox # virtualenv -importlib-metadata==7.0.1 - # via - # -r requirements/pip-tools.txt - # build iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -203,12 +194,12 @@ isort==5.13.2 # via # -r requirements/quality.txt # pylint -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/quality.txt # -r requirements/testing.txt # code-annotations -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -226,17 +217,17 @@ mock==5.1.0 # via # -r requirements/quality.txt # -r requirements/testing.txt -newrelic==9.6.0 +newrelic==9.12.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils -openedx-events==9.5.2 +openedx-events==9.11.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # event-tracking -packaging==23.2 +packaging==24.1 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt @@ -251,40 +242,40 @@ pbr==6.0.0 # -r requirements/quality.txt # -r requirements/testing.txt # stevedore -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.txt -platformdirs==4.2.0 +platformdirs==4.2.2 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # -r requirements/testing.txt # pytest # tox -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via # -r requirements/quality.txt # -r requirements/testing.txt # click-repl -psutil==5.9.8 +psutil==6.0.0 # via # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils -pycodestyle==2.11.1 +pycodestyle==2.12.0 # via -r requirements/quality.txt -pycparser==2.21 +pycparser==2.22 # via # -r requirements/quality.txt # -r requirements/testing.txt # cffi -pylint==3.0.3 +pylint==3.2.6 # via # -r requirements/quality.txt # edx-lint @@ -315,25 +306,25 @@ pynacl==1.5.0 # -r requirements/quality.txt # -r requirements/testing.txt # edx-django-utils -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via # -r requirements/ci.txt # tox -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.0.1 +pytest==8.3.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # pytest-cov -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via # -r requirements/quality.txt # -r requirements/testing.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -347,7 +338,6 @@ pytz==2024.1 # via # -r requirements/quality.txt # -r requirements/testing.txt - # django # event-tracking pyyaml==6.0.1 # via @@ -358,15 +348,16 @@ six==1.16.0 # via # -r requirements/quality.txt # -r requirements/testing.txt + # edx-ccx-keys # edx-lint # event-tracking # python-dateutil -sqlparse==0.4.4 +sqlparse==0.5.1 # via # -r requirements/quality.txt # -r requirements/testing.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -378,42 +369,23 @@ text-unidecode==1.3 # -r requirements/quality.txt # -r requirements/testing.txt # python-slugify -tomli==2.0.1 +tomlkit==0.13.0 # via - # -r requirements/ci.txt - # -r requirements/pip-tools.txt # -r requirements/quality.txt - # -r requirements/testing.txt - # build - # coverage - # pip-tools # pylint - # pyproject-api - # pyproject-hooks - # pytest - # tox -tomlkit==0.12.3 - # via - # -r requirements/quality.txt - # pylint -tox==4.13.0 +tox==4.16.0 # via -r requirements/ci.txt -typing-extensions==4.9.0 +typing-extensions==4.12.2 # via # -r requirements/quality.txt # -r requirements/testing.txt - # asgiref - # astroid # edx-opaque-keys - # kombu - # pylint tzdata==2024.1 # via # -r requirements/quality.txt # -r requirements/testing.txt - # backports-zoneinfo # celery -urllib3==1.26.18 +urllib3==1.26.19 # via # -r requirements/quality.txt # -r requirements/testing.txt @@ -425,7 +397,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.25.0 +virtualenv==20.26.3 # via # -r requirements/ci.txt # tox @@ -434,14 +406,10 @@ wcwidth==0.2.13 # -r requirements/quality.txt # -r requirements/testing.txt # prompt-toolkit -wheel==0.42.0 +wheel==0.43.0 # via # -r requirements/pip-tools.txt # pip-tools -zipp==3.17.0 - # via - # -r requirements/pip-tools.txt - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 44c48d99..b544e9f5 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,32 +1,23 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -build==1.0.3 +build==1.2.1 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==7.0.1 +packaging==24.1 # via build -packaging==23.2 - # via build -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.in -pyproject-hooks==1.0.0 - # via - # build - # pip-tools -tomli==2.0.1 +pyproject-hooks==1.1.0 # via # build # pip-tools - # pyproject-hooks -wheel==0.42.0 +wheel==0.43.0 # via pip-tools -zipp==3.17.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip.txt b/requirements/pip.txt index 677bf038..2b919459 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade diff --git a/requirements/quality.txt b/requirements/quality.txt index 4a8c3857..dcc955c6 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -8,11 +8,11 @@ amqp==5.2.0 # via # -r requirements/testing.txt # kombu -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/testing.txt # django -astroid==3.0.3 +astroid==3.2.4 # via # pylint # pylint-celery @@ -20,21 +20,15 @@ attrs==23.2.0 # via # -r requirements/testing.txt # openedx-events -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # -r requirements/testing.txt - # celery - # kombu billiard==4.2.0 # via # -r requirements/testing.txt # celery -celery==5.3.6 +celery==5.4.0 # via # -r requirements/testing.txt # event-tracking -certifi==2024.2.2 +certifi==2024.7.4 # via # -r requirements/testing.txt # elasticsearch @@ -53,7 +47,7 @@ click==8.1.7 # code-annotations # edx-django-utils # edx-lint -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/testing.txt # celery @@ -67,12 +61,12 @@ click-repl==0.3.0 # via # -r requirements/testing.txt # celery -code-annotations==1.6.0 +code-annotations==1.8.0 # via # -r requirements/testing.txt # edx-lint # edx-toggles -coverage[toml]==7.4.2 +coverage[toml]==7.6.0 # via # -r requirements/quality.in # -r requirements/testing.txt @@ -83,7 +77,7 @@ ddt==1.3.1 # -r requirements/testing.txt dill==0.3.8 # via pylint -django==3.2.24 +django==4.2.14 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/testing.txt @@ -103,19 +97,24 @@ django-waffle==4.1.0 # -r requirements/testing.txt # edx-django-utils # edx-toggles -edx-django-utils==5.10.1 +edx-ccx-keys==1.3.0 + # via + # -r requirements/testing.txt + # openedx-events +edx-django-utils==5.14.2 # via # -r requirements/testing.txt # edx-toggles # event-tracking # openedx-events -edx-lint==5.3.6 +edx-lint==5.3.7 # via -r requirements/quality.in -edx-opaque-keys[django]==2.5.1 +edx-opaque-keys[django]==2.10.0 # via # -r requirements/testing.txt + # edx-ccx-keys # openedx-events -edx-toggles==5.1.1 +edx-toggles==5.2.0 # via # -r requirements/testing.txt # event-tracking @@ -123,13 +122,11 @@ elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/testing.txt -event-tracking==2.3.0 - # via -r requirements/testing.txt -exceptiongroup==1.2.0 +event-tracking==2.4.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/testing.txt - # pytest -fastavro==1.9.4 +fastavro==1.9.5 # via # -r requirements/testing.txt # openedx-events @@ -139,11 +136,11 @@ iniconfig==2.0.0 # pytest isort==5.13.2 # via pylint -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/testing.txt # code-annotations -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/testing.txt # celery @@ -155,15 +152,15 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/testing.txt -newrelic==9.6.0 +newrelic==9.12.0 # via # -r requirements/testing.txt # edx-django-utils -openedx-events==9.5.2 +openedx-events==9.11.0 # via # -r requirements/testing.txt # event-tracking -packaging==23.2 +packaging==24.1 # via # -r requirements/testing.txt # pytest @@ -171,27 +168,27 @@ pbr==6.0.0 # via # -r requirements/testing.txt # stevedore -platformdirs==4.2.0 +platformdirs==4.2.2 # via pylint -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/testing.txt # pytest -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via # -r requirements/testing.txt # click-repl -psutil==5.9.8 +psutil==6.0.0 # via # -r requirements/testing.txt # edx-django-utils -pycodestyle==2.11.1 +pycodestyle==2.12.0 # via -r requirements/quality.in -pycparser==2.21 +pycparser==2.22 # via # -r requirements/testing.txt # cffi -pylint==3.0.3 +pylint==3.2.6 # via # edx-lint # pylint-celery @@ -214,13 +211,13 @@ pynacl==1.5.0 # via # -r requirements/testing.txt # edx-django-utils -pytest==8.0.1 +pytest==8.3.1 # via # -r requirements/testing.txt # pytest-cov -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/testing.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/testing.txt # celery @@ -231,7 +228,6 @@ python-slugify==8.0.4 pytz==2024.1 # via # -r requirements/testing.txt - # django # event-tracking pyyaml==6.0.1 # via @@ -240,14 +236,15 @@ pyyaml==6.0.1 six==1.16.0 # via # -r requirements/testing.txt + # edx-ccx-keys # edx-lint # event-tracking # python-dateutil -sqlparse==0.4.4 +sqlparse==0.5.1 # via # -r requirements/testing.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/testing.txt # code-annotations @@ -257,28 +254,17 @@ text-unidecode==1.3 # via # -r requirements/testing.txt # python-slugify -tomli==2.0.1 - # via - # -r requirements/testing.txt - # coverage - # pylint - # pytest -tomlkit==0.12.3 +tomlkit==0.13.0 # via pylint -typing-extensions==4.9.0 +typing-extensions==4.12.2 # via # -r requirements/testing.txt - # asgiref - # astroid # edx-opaque-keys - # kombu - # pylint tzdata==2024.1 # via # -r requirements/testing.txt - # backports-zoneinfo # celery -urllib3==1.26.18 +urllib3==1.26.19 # via # -r requirements/testing.txt # elasticsearch diff --git a/requirements/testing.txt b/requirements/testing.txt index 9a1d1263..cf3cb6eb 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade @@ -8,7 +8,7 @@ amqp==5.2.0 # via # -r requirements/base.txt # kombu -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/base.txt # django @@ -16,21 +16,15 @@ attrs==23.2.0 # via # -r requirements/base.txt # openedx-events -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # -r requirements/base.txt - # celery - # kombu billiard==4.2.0 # via # -r requirements/base.txt # celery -celery==5.3.6 +celery==5.4.0 # via # -r requirements/base.txt # event-tracking -certifi==2024.2.2 +certifi==2024.7.4 # via # -r requirements/base.txt # elasticsearch @@ -47,7 +41,7 @@ click==8.1.7 # click-repl # code-annotations # edx-django-utils -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/base.txt # celery @@ -59,11 +53,11 @@ click-repl==0.3.0 # via # -r requirements/base.txt # celery -code-annotations==1.6.0 +code-annotations==1.8.0 # via # -r requirements/base.txt # edx-toggles -coverage[toml]==7.4.2 +coverage[toml]==7.6.0 # via # -r requirements/testing.in # pytest-cov @@ -90,17 +84,22 @@ django-waffle==4.1.0 # -r requirements/base.txt # edx-django-utils # edx-toggles -edx-django-utils==5.10.1 +edx-ccx-keys==1.3.0 + # via + # -r requirements/base.txt + # openedx-events +edx-django-utils==5.14.2 # via # -r requirements/base.txt # edx-toggles # event-tracking # openedx-events -edx-opaque-keys[django]==2.5.1 +edx-opaque-keys[django]==2.10.0 # via # -r requirements/base.txt + # edx-ccx-keys # openedx-events -edx-toggles==5.1.1 +edx-toggles==5.2.0 # via # -r requirements/base.txt # event-tracking @@ -108,21 +107,21 @@ elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt -event-tracking==2.3.0 - # via -r requirements/base.txt -exceptiongroup==1.2.0 - # via pytest -fastavro==1.9.4 +event-tracking==2.4.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/base.txt +fastavro==1.9.5 # via # -r requirements/base.txt # openedx-events iniconfig==2.0.0 # via pytest -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/base.txt # code-annotations -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/base.txt # celery @@ -132,31 +131,31 @@ markupsafe==2.1.5 # jinja2 mock==5.1.0 # via -r requirements/testing.in -newrelic==9.6.0 +newrelic==9.12.0 # via # -r requirements/base.txt # edx-django-utils -openedx-events==9.5.2 +openedx-events==9.11.0 # via # -r requirements/base.txt # event-tracking -packaging==23.2 +packaging==24.1 # via pytest pbr==6.0.0 # via # -r requirements/base.txt # stevedore -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via # -r requirements/base.txt # click-repl -psutil==5.9.8 +psutil==6.0.0 # via # -r requirements/base.txt # edx-django-utils -pycparser==2.21 +pycparser==2.22 # via # -r requirements/base.txt # cffi @@ -169,11 +168,11 @@ pynacl==1.5.0 # via # -r requirements/base.txt # edx-django-utils -pytest==8.0.1 +pytest==8.3.1 # via pytest-cov -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/testing.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/base.txt # celery @@ -184,7 +183,6 @@ python-slugify==8.0.4 pytz==2024.1 # via # -r requirements/base.txt - # django # event-tracking pyyaml==6.0.1 # via @@ -193,13 +191,14 @@ pyyaml==6.0.1 six==1.16.0 # via # -r requirements/base.txt + # edx-ccx-keys # event-tracking # python-dateutil -sqlparse==0.4.4 +sqlparse==0.5.1 # via # -r requirements/base.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/base.txt # code-annotations @@ -209,22 +208,15 @@ text-unidecode==1.3 # via # -r requirements/base.txt # python-slugify -tomli==2.0.1 - # via - # coverage - # pytest -typing-extensions==4.9.0 +typing-extensions==4.12.2 # via # -r requirements/base.txt - # asgiref # edx-opaque-keys - # kombu tzdata==2024.1 # via # -r requirements/base.txt - # backports-zoneinfo # celery -urllib3==1.26.18 +urllib3==1.26.19 # via # -r requirements/base.txt # elasticsearch From f996125794f0d31a7a94671baac486f413c4cd0a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Jul 2024 13:27:18 -0400 Subject: [PATCH 16/26] style: Update pylint config and fix pylint issues. --- pylintrc | 7 ++++--- search/result_processor.py | 2 +- search/tests/mock_search_engine.py | 3 +-- search/views.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pylintrc b/pylintrc index 8038c460..81bd0e53 100644 --- a/pylintrc +++ b/pylintrc @@ -64,7 +64,7 @@ # SERIOUSLY. # # ------------------------------ -# Generated by edx-lint version: 5.3.0 +# Generated by edx-lint version: 5.3.7 # ------------------------------ [MASTER] ignore = @@ -259,6 +259,7 @@ enable = useless-suppression, disable = bad-indentation, + broad-exception-raised, consider-using-f-string, duplicate-code, file-ignored, @@ -380,6 +381,6 @@ ext-import-graph = int-import-graph = [EXCEPTIONS] -overgeneral-exceptions = Exception +overgeneral-exceptions = builtins.Exception -# e79d284912469e90087c79e80233e873551b6ca9 +# 1a9b016a2ea1ded8c5b79b0e520a5566eba73b62 diff --git a/search/result_processor.py b/search/result_processor.py index 24ba1b57..8f54f2a7 100644 --- a/search/result_processor.py +++ b/search/result_processor.py @@ -115,7 +115,7 @@ def process_result(cls, dictionary, match_phrase, user): try: srp.add_properties() # protect around any problems introduced by subclasses within their properties - except Exception as ex: + except Exception as ex: # pylint: disable=broad-exception-caught log.exception("error processing properties for %s - %s: will remove from results", json.dumps(dictionary, cls=DjangoJSONEncoder), str(ex)) return None diff --git a/search/tests/mock_search_engine.py b/search/tests/mock_search_engine.py index 4974c80e..0b498f0a 100644 --- a/search/tests/mock_search_engine.py +++ b/search/tests/mock_search_engine.py @@ -385,8 +385,7 @@ def score_documents(documents_to_search): while documents_to_search: current_doc = documents_to_search[0] score = len([d for d in documents_to_search if d == current_doc]) - if score > max_score: - max_score = score + max_score = max(max_score, score) documents_to_search = [d for d in documents_to_search if d != current_doc] data = copy.copy(current_doc) diff --git a/search/views.py b/search/views.py index cda536b2..e9e0bcc4 100644 --- a/search/views.py +++ b/search/views.py @@ -122,7 +122,7 @@ def do_search(request, course_id=None): } log.debug(str(invalid_err)) - except Exception as err: + except Exception as err: # pylint: disable=broad-exception-caught results = { "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term) } @@ -207,7 +207,7 @@ def course_discovery(request): } log.debug(str(invalid_err)) - except Exception as err: + except Exception as err: # pylint: disable=broad-exception-caught results = { "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term) } From b07a1f827619f0add5ae071efbc2d2d216876f6a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Jul 2024 13:57:41 -0400 Subject: [PATCH 17/26] build: Drop support for Python 3.8 --- .github/workflows/ci.yml | 6 ++++-- edxsearch/__init__.py | 2 +- setup.py | 4 ++-- tox.ini | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b543e78..6b547442 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8', '3.11', '3.12'] + python-version: ['3.11', '3.12'] toxenv: [django42, quality] steps: @@ -38,10 +38,12 @@ jobs: - name: Run Tests env: TOXENV: ${{ matrix.toxenv }} + # Sleep is needed to give elasticsearch enough time to startup. + # Longer term we should switch to the upstream ES github action to start up the server run: sleep 10 && tox - name: Run Coverage - if: matrix.python-version == '3.8' && matrix.toxenv=='django42' + if: matrix.python-version == '3.12' && matrix.toxenv=='django42' uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 835caa16..6020c5ce 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.9.1' +__version__ = '4.0.0' diff --git a/setup.py b/setup.py index 9685ebea..eec63771 100755 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ def get_version(*file_paths): 'License :: OSI Approved :: GNU Affero General Public License v3', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Framework :: Django', 'Framework :: Django :: 2.2', 'Framework :: Django :: 3.0', diff --git a/tox.ini b/tox.ini index 19cc118f..20d4e9cb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,311,312}-django{42},quality +envlist = py{311,312}-django{42},quality [testenv] setenv = From e392d8e5bc5e1bd9848b70fb6082cd600837cfe9 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Jul 2024 13:58:17 -0400 Subject: [PATCH 18/26] test: Use modern commands for docker in testing. --- Makefile | 4 ++-- docker-compose.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8e1cfd0a..bea3f88d 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,10 @@ validate: clean tox test.start_elasticsearch: - docker-compose up -d + docker compose up -d test.stop_elasticsearch: - docker-compose stop + docker compose stop test_with_es: clean test.start_elasticsearch coverage run --source='.' manage.py test diff --git a/docker-compose.yml b/docker-compose.yml index 5ad8701c..2be4003b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.2' services: test_elasticsearch: From 797a86262e1bb337f498fbedc29706e45465e39d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Jul 2024 14:09:59 -0400 Subject: [PATCH 19/26] build: Correct how the dependencies get installed. Without this change we were getting a weird error that the 'install' package could not be found because github action was combining the two lines into a single `pip install setuptools pip install -r requirements/ci.txt` call. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b547442..431a1b06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: run: pip install -r requirements/pip.txt - name: Install Dependencies - run: + run: | pip install setuptools pip install -r requirements/ci.txt From b48d1fb19dc1599631b5f8fe78e31cbb0181706a Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Mon, 22 Jul 2024 01:39:54 +0330 Subject: [PATCH 20/26] fix: master branch sunset As it mentioned in the gh-action-pypi-publish repo, the master branch version has been sunset and we should use release/v1. --- .github/workflows/pypi-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 50d307a9..49cb69d1 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -27,7 +27,7 @@ jobs: run: python setup.py sdist bdist_wheel - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_UPLOAD_TOKEN }} From 5f87ce1a2f1db72e0a16d197a6c6ab9f62ce5644 Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Thu, 30 May 2024 09:36:28 +0300 Subject: [PATCH 21/26] chore: add python upgrade requirements file --- .../workflows/upgrade-python-requirements.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/upgrade-python-requirements.yml diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml new file mode 100644 index 00000000..bf567d65 --- /dev/null +++ b/.github/workflows/upgrade-python-requirements.yml @@ -0,0 +1,27 @@ +name: Upgrade Python Requirements + +on: + schedule: + - cron: "0 0 * * 1" + workflow_dispatch: + inputs: + branch: + description: "Target branch against which to create requirements PR" + required: true + default: 'master' + +jobs: + call-upgrade-python-requirements-workflow: + uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master + with: + branch: ${{ github.event.inputs.branch || 'master' }} + # optional parameters below; fill in if you'd like github or email notifications + # user_reviewers: "" + # team_reviewers: "" + # email_address: "" + # send_success_notification: false + secrets: + requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} + requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} + edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} + edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} From beaa83173c773e790d19b7bc090f7dff1577da33 Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Thu, 30 May 2024 09:36:44 +0300 Subject: [PATCH 22/26] chore: add catalog-info.yml --- catalog-info.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 catalog-info.yaml diff --git a/catalog-info.yaml b/catalog-info.yaml new file mode 100644 index 00000000..2e9a40e4 --- /dev/null +++ b/catalog-info.yaml @@ -0,0 +1,19 @@ +# This file records information about this repo. Its use is described in OEP-55: +# https://open-edx-proposals.readthedocs.io/en/latest/processes/oep-0055-proc-project-maintainers.html + +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: 'edx-search' + description: "A django application to provide access to search services" + links: + - url: "https://github.com/openedx/edx-search" + title: "openedx/edx-search" + icon: "GitHub" + annotations: + openedx.org/arch-interest-groups: "" + openedx.org/release: "main" +spec: + owner: group:openedx-unmaintained + type: 'service' + lifecycle: 'production' From 1a534e4233fcaf6eadefb907305671b4bb8655cd Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Thu, 29 Aug 2024 09:28:47 +0300 Subject: [PATCH 23/26] fix: remove annotations and links portion --- catalog-info.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 2e9a40e4..8209c1c0 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -6,13 +6,7 @@ kind: Component metadata: name: 'edx-search' description: "A django application to provide access to search services" - links: - - url: "https://github.com/openedx/edx-search" - title: "openedx/edx-search" - icon: "GitHub" - annotations: - openedx.org/arch-interest-groups: "" - openedx.org/release: "main" + links: [] spec: owner: group:openedx-unmaintained type: 'service' From 2d5b6c99411b80fbc5c1d933d085c3d67c4ad36d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 9 Sep 2024 09:58:49 -0400 Subject: [PATCH 24/26] build: Switch to ubuntu-latest for builds This code does not have any dependencies that are specific to any specific version of ubuntu. So instead of testing on a specific version and then needing to do work to keep the versions up-to-date, we switch to the ubuntu-latest target which should be sufficient for testing purposes. This work is being done as a part of https://github.com/openedx/platform-roadmap/issues/377 closes https://github.com/openedx/edx-search/issues/154 --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 431a1b06..b553a7e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: ['3.11', '3.12'] toxenv: [django42, quality] diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 49cb69d1..94a11e2f 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -7,7 +7,7 @@ on: jobs: push: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout From 91686e96699cb8678bd12e2ef42ac51128c13f42 Mon Sep 17 00:00:00 2001 From: Jillian Date: Thu, 24 Oct 2024 00:02:58 +1030 Subject: [PATCH 25/26] feat: add Meilisearch-compatible search engine with tests (#164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The goal of this change is to introduce a search engine that is compatible with the edx-search API but that uses Meilisearch instead of Elasticsearch. That way, we can replace one by the other across edx-platform by simply changing a single SEARCH_ENGINE django setting. There are a couple of differences between Meilisearch and Elasticsearch: 1. Filterable attributes must be defined explicitly. 2. No support for datetime objects, which must be converted to timestamps (with an extra field to store the timezone). 3. No special characters allowed in the primary key values, such that we must hash course IDs before we can use them as primary key values. Note that this PR does not introduce any breaking change. This is an opt-in engine that anyone is free to use. There is some setup work for every search feature: see the engine module documentation for more information. See the corresponding conversation here: https://github.com/openedx/frontend-app-authoring/issues/1334#issuecomment-2401805382 * fix: `make test` command Unit test command was failing because manage.py was pointing to an incorrect module by default. * feat: add convenient "make compile-requirements" command This command makes it possible to compile requirements without upgrading them. Note that the `make upgrade` command still works with this change. --------- Co-authored-by: Régis Behmo --- Makefile | 19 +- edxsearch/__init__.py | 2 +- edxsearch/settings.py | 29 ++ requirements/base.in | 1 + requirements/base.txt | 29 +- requirements/dev.txt | 43 +++ requirements/quality.txt | 34 ++ requirements/testing.txt | 34 ++ search/meilisearch.py | 540 +++++++++++++++++++++++++++++++ search/tests/test_meilisearch.py | 359 ++++++++++++++++++++ settings.py | 132 -------- tox.ini | 2 +- 12 files changed, 1079 insertions(+), 145 deletions(-) create mode 100644 search/meilisearch.py create mode 100644 search/tests/test_meilisearch.py delete mode 100644 settings.py diff --git a/Makefile b/Makefile index bea3f88d..adff3746 100644 --- a/Makefile +++ b/Makefile @@ -34,23 +34,26 @@ test_with_es: clean test.start_elasticsearch coverage run --source='.' manage.py test make test.stop_elasticsearch -upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in +compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade +compile-requirements: ## Re-compile *.in requirements to *.txt (without upgrading) pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! pip-compile --rebuild --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in - pip-compile --rebuild --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in pip install -qr requirements/pip.txt pip install -qr requirements/pip-tools.txt - pip-compile --rebuild --upgrade -o requirements/base.txt requirements/base.in - pip-compile --rebuild --upgrade -o requirements/testing.txt requirements/testing.in - pip-compile --rebuild --upgrade -o requirements/quality.txt requirements/quality.in - pip-compile --rebuild --upgrade -o requirements/ci.txt requirements/ci.in - pip-compile --rebuild --upgrade -o requirements/dev.txt requirements/dev.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/base.txt requirements/base.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/testing.txt requirements/testing.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/quality.txt requirements/quality.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/ci.txt requirements/ci.in + pip-compile --rebuild ${COMPILE_OPTS} -o requirements/dev.txt requirements/dev.in # Let tox control the Django version for tests sed '/^[dD]jango==/d' requirements/testing.txt > requirements/testing.tmp mv requirements/testing.tmp requirements/testing.txt +upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in + $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" + test: test_with_es ## run tests and generate coverage report install-local: ## installs your local edx-search into the LMS and CMS python virtualenvs diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 6020c5ce..d00a3e59 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '4.0.0' +__version__ = '4.1.0' diff --git a/edxsearch/settings.py b/edxsearch/settings.py index c6aee223..e200eaff 100644 --- a/edxsearch/settings.py +++ b/edxsearch/settings.py @@ -53,6 +53,7 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'eventtracking.django', 'waffle', ) @@ -99,3 +100,31 @@ # https://docs.djangoproject.com/en/1.6/howto/static-files/ STATIC_URL = '/static/' + +# EVENT TRACKING ################################# + +TRACK_MAX_EVENT = 50000 + +TRACKING_BACKENDS = { + 'logger': { + 'ENGINE': 'track.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking' + } + } +} + +# We're already logging events, and we don't want to capture user +# names/passwords. Heartbeat events are likely not interesting. +TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] + +EVENT_TRACKING_ENABLED = True +EVENT_TRACKING_BACKENDS = { + 'logger': { + 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking', + 'max_event_size': TRACK_MAX_EVENT, + } + } +} diff --git a/requirements/base.in b/requirements/base.in index 034b08a6..a758596f 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -15,3 +15,4 @@ Django # Web application framework elasticsearch>=7.8.0,<8.0.0 edx-toggles event-tracking +meilisearch diff --git a/requirements/base.txt b/requirements/base.txt index df520b14..c5679100 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,18 +6,26 @@ # amqp==5.2.0 # via kombu +annotated-types==0.7.0 + # via pydantic asgiref==3.8.1 # via django attrs==23.2.0 # via openedx-events billiard==4.2.0 # via celery +camel-converter[pydantic]==4.0.1 + # via meilisearch celery==5.4.0 # via event-tracking certifi==2024.7.4 - # via elasticsearch + # via + # elasticsearch + # requests cffi==1.16.0 # via pynacl +charset-normalizer==3.4.0 + # via requests click==8.1.7 # via # celery @@ -77,12 +85,16 @@ event-tracking==2.4.0 # -r requirements/base.in fastavro==1.9.5 # via openedx-events +idna==3.10 + # via requests jinja2==3.1.4 # via code-annotations kombu==5.3.7 # via celery markupsafe==2.1.5 # via jinja2 +meilisearch==0.31.5 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils openedx-events==9.11.0 @@ -95,6 +107,10 @@ psutil==6.0.0 # via edx-django-utils pycparser==2.22 # via cffi +pydantic==2.9.2 + # via camel-converter +pydantic-core==2.23.4 + # via pydantic pymongo==3.13.0 # via # edx-opaque-keys @@ -109,6 +125,8 @@ pytz==2024.1 # via event-tracking pyyaml==6.0.1 # via code-annotations +requests==2.32.3 + # via meilisearch six==1.16.0 # via # edx-ccx-keys @@ -124,11 +142,16 @@ stevedore==5.2.0 text-unidecode==1.3 # via python-slugify typing-extensions==4.12.2 - # via edx-opaque-keys + # via + # edx-opaque-keys + # pydantic + # pydantic-core tzdata==2024.1 # via celery urllib3==1.26.19 - # via elasticsearch + # via + # elasticsearch + # requests vine==5.1.0 # via # amqp diff --git a/requirements/dev.txt b/requirements/dev.txt index f8f96994..3a858be5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,6 +9,11 @@ amqp==5.2.0 # -r requirements/quality.txt # -r requirements/testing.txt # kombu +annotated-types==0.7.0 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # pydantic asgiref==3.8.1 # via # -r requirements/quality.txt @@ -37,6 +42,11 @@ cachetools==5.4.0 # via # -r requirements/ci.txt # tox +camel-converter[pydantic]==4.0.1 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # meilisearch celery==5.4.0 # via # -r requirements/quality.txt @@ -47,6 +57,7 @@ certifi==2024.7.4 # -r requirements/quality.txt # -r requirements/testing.txt # elasticsearch + # requests cffi==1.16.0 # via # -r requirements/quality.txt @@ -56,6 +67,11 @@ chardet==5.2.0 # via # -r requirements/ci.txt # tox +charset-normalizer==3.4.0 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # requests click==8.1.7 # via # -r requirements/pip-tools.txt @@ -185,6 +201,11 @@ filelock==3.15.4 # -r requirements/ci.txt # tox # virtualenv +idna==3.10 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # requests iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -213,6 +234,10 @@ mccabe==0.7.0 # via # -r requirements/quality.txt # pylint +meilisearch==0.31.5 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt mock==5.1.0 # via # -r requirements/quality.txt @@ -275,6 +300,16 @@ pycparser==2.22 # -r requirements/quality.txt # -r requirements/testing.txt # cffi +pydantic==2.9.2 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # camel-converter +pydantic-core==2.23.4 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # pydantic pylint==3.2.6 # via # -r requirements/quality.txt @@ -344,6 +379,11 @@ pyyaml==6.0.1 # -r requirements/quality.txt # -r requirements/testing.txt # code-annotations +requests==2.32.3 + # via + # -r requirements/quality.txt + # -r requirements/testing.txt + # meilisearch six==1.16.0 # via # -r requirements/quality.txt @@ -380,6 +420,8 @@ typing-extensions==4.12.2 # -r requirements/quality.txt # -r requirements/testing.txt # edx-opaque-keys + # pydantic + # pydantic-core tzdata==2024.1 # via # -r requirements/quality.txt @@ -390,6 +432,7 @@ urllib3==1.26.19 # -r requirements/quality.txt # -r requirements/testing.txt # elasticsearch + # requests vine==5.1.0 # via # -r requirements/quality.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index dcc955c6..52990d46 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,6 +8,10 @@ amqp==5.2.0 # via # -r requirements/testing.txt # kombu +annotated-types==0.7.0 + # via + # -r requirements/testing.txt + # pydantic asgiref==3.8.1 # via # -r requirements/testing.txt @@ -24,6 +28,10 @@ billiard==4.2.0 # via # -r requirements/testing.txt # celery +camel-converter[pydantic]==4.0.1 + # via + # -r requirements/testing.txt + # meilisearch celery==5.4.0 # via # -r requirements/testing.txt @@ -32,10 +40,15 @@ certifi==2024.7.4 # via # -r requirements/testing.txt # elasticsearch + # requests cffi==1.16.0 # via # -r requirements/testing.txt # pynacl +charset-normalizer==3.4.0 + # via + # -r requirements/testing.txt + # requests click==8.1.7 # via # -r requirements/testing.txt @@ -130,6 +143,10 @@ fastavro==1.9.5 # via # -r requirements/testing.txt # openedx-events +idna==3.10 + # via + # -r requirements/testing.txt + # requests iniconfig==2.0.0 # via # -r requirements/testing.txt @@ -150,6 +167,8 @@ markupsafe==2.1.5 # jinja2 mccabe==0.7.0 # via pylint +meilisearch==0.31.5 + # via -r requirements/testing.txt mock==5.1.0 # via -r requirements/testing.txt newrelic==9.12.0 @@ -188,6 +207,14 @@ pycparser==2.22 # via # -r requirements/testing.txt # cffi +pydantic==2.9.2 + # via + # -r requirements/testing.txt + # camel-converter +pydantic-core==2.23.4 + # via + # -r requirements/testing.txt + # pydantic pylint==3.2.6 # via # edx-lint @@ -233,6 +260,10 @@ pyyaml==6.0.1 # via # -r requirements/testing.txt # code-annotations +requests==2.32.3 + # via + # -r requirements/testing.txt + # meilisearch six==1.16.0 # via # -r requirements/testing.txt @@ -260,6 +291,8 @@ typing-extensions==4.12.2 # via # -r requirements/testing.txt # edx-opaque-keys + # pydantic + # pydantic-core tzdata==2024.1 # via # -r requirements/testing.txt @@ -268,6 +301,7 @@ urllib3==1.26.19 # via # -r requirements/testing.txt # elasticsearch + # requests vine==5.1.0 # via # -r requirements/testing.txt diff --git a/requirements/testing.txt b/requirements/testing.txt index cf3cb6eb..d7b68223 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -8,6 +8,10 @@ amqp==5.2.0 # via # -r requirements/base.txt # kombu +annotated-types==0.7.0 + # via + # -r requirements/base.txt + # pydantic asgiref==3.8.1 # via # -r requirements/base.txt @@ -20,6 +24,10 @@ billiard==4.2.0 # via # -r requirements/base.txt # celery +camel-converter[pydantic]==4.0.1 + # via + # -r requirements/base.txt + # meilisearch celery==5.4.0 # via # -r requirements/base.txt @@ -28,10 +36,15 @@ certifi==2024.7.4 # via # -r requirements/base.txt # elasticsearch + # requests cffi==1.16.0 # via # -r requirements/base.txt # pynacl +charset-normalizer==3.4.0 + # via + # -r requirements/base.txt + # requests click==8.1.7 # via # -r requirements/base.txt @@ -115,6 +128,10 @@ fastavro==1.9.5 # via # -r requirements/base.txt # openedx-events +idna==3.10 + # via + # -r requirements/base.txt + # requests iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -129,6 +146,8 @@ markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 +meilisearch==0.31.5 + # via -r requirements/base.txt mock==5.1.0 # via -r requirements/testing.in newrelic==9.12.0 @@ -159,6 +178,14 @@ pycparser==2.22 # via # -r requirements/base.txt # cffi +pydantic==2.9.2 + # via + # -r requirements/base.txt + # camel-converter +pydantic-core==2.23.4 + # via + # -r requirements/base.txt + # pydantic pymongo==3.13.0 # via # -r requirements/base.txt @@ -188,6 +215,10 @@ pyyaml==6.0.1 # via # -r requirements/base.txt # code-annotations +requests==2.32.3 + # via + # -r requirements/base.txt + # meilisearch six==1.16.0 # via # -r requirements/base.txt @@ -212,6 +243,8 @@ typing-extensions==4.12.2 # via # -r requirements/base.txt # edx-opaque-keys + # pydantic + # pydantic-core tzdata==2024.1 # via # -r requirements/base.txt @@ -220,6 +253,7 @@ urllib3==1.26.19 # via # -r requirements/base.txt # elasticsearch + # requests vine==5.1.0 # via # -r requirements/base.txt diff --git a/search/meilisearch.py b/search/meilisearch.py new file mode 100644 index 00000000..75636526 --- /dev/null +++ b/search/meilisearch.py @@ -0,0 +1,540 @@ +""" +This is a search engine for Meilisearch. It implements the edx-search's SearchEngine +API, such that it can be setup as a drop-in replacement for the ElasticSearchEngine. To +switch to this engine, you should run a Meilisearch instance and define the following +setting: + + SEARCH_ENGINE = "search.meilisearch.MeilisearchEngine" + +You will then need to create the new indices by running: + + ./manage.py lms shell -c "import search.meilisearch; search.meilisearch.create_indexes()" + +For more information about the Meilisearch API in Python, check +https://github.com/meilisearch/meilisearch-python + +When implementing a new index, you might discover that you need to list explicit filterable +fields. Typically, you try to index new documents, and Meilisearch fails with the +following response: + + meilisearch.errors.MeilisearchApiError: MeilisearchApiError. Error code: invalid_search_filter. + Error message: Attribute `field3` is not filterable. Available filterable attributes are: + `field1 field2 _pk`. + +In such cases, the filterable field should be added to INDEX_FILTERABLES below. And you should +then run the `create_indexes()` function again, as indicated above. + +This search engine was tested for the following indexes: + +1. course_info ("course discovery"): + - Enable the course discovery feature: FEATURES["ENABLE_COURSE_DISCOVERY"] = True + - A search bar appears in the LMS landing page. + - Content is automatically indexed every time a course's "schedule & details" are + edited in the studio, course content is edited or the "reindex" button is clicked. + +2. courseware_content ("courseware search"): + - Enable the courseware search waffle flag: + + ./manage.py lms waffle_flag --create --everyone courseware.mfe_courseware_search + + - Enable the following feature flags: + + FEATURES["ENABLE_COURSEWARE_INDEX"] = True + FEATURES["ENABLE_COURSEWARE_SEARCH"] = True + + - Courseware content will be indexed by editing course sections and units. + - Alternatively, click the "Reindex" button in the Studio. + - Alternatively, index all courses by running: ./manage.py cms reindex_course --active + - In the learning MFE, a course search bar appears when opening a course. + +Note that the index names could be tuned with the COURSEWARE_INFO_INDEX_NAME and +COURSEWARE_CONTENT_INDEX_NAME settings. However, if you decide to change these settings, +beware that many other applications do not respect them... + +When facing issues with Meilisearch during indexing, you may want to look at the +Meilisearch logs. You might notice that some indexing tasks failed. In such cases, you +can troubleshoot these tasks by printing them with: + + ./manage.py lms shell -c "import search.meilisearch; search.meilisearch.print_failed_meilisearch_tasks()" +""" + +from copy import deepcopy +from datetime import datetime +import hashlib +import json +import logging +import typing as t + +import meilisearch + +from django.conf import settings +from django.utils import timezone + +from search.search_engine_base import SearchEngine +from search.utils import ValueRange + + +MEILISEARCH_API_KEY = getattr(settings, "MEILISEARCH_API_KEY", "") +MEILISEARCH_URL = getattr(settings, "MEILISEARCH_URL", "http://meilisearch") +MEILISEARCH_INDEX_PREFIX = getattr(settings, "MEILISEARCH_INDEX_PREFIX", "") + + +logger = logging.getLogger(__name__) + + +PRIMARY_KEY_FIELD_NAME = "_pk" +UTC_OFFSET_SUFFIX = "__utcoffset" + + +# In Meilisearch, we need to explicitly list fields for which we expect to define +# filters and aggregation functions. +# This is different than Elasticsearch where we can aggregate results over any field. +# Here, we list facet fields per index. +# Reference: https://www.meilisearch.com/docs/learn/filtering_and_sorting/search_with_facet_filters +# Note that index names are hard-coded here, because they are hardcoded anyway across all of edx-search. +INDEX_FILTERABLES: dict[str, list[str]] = { + getattr(settings, "COURSEWARE_INFO_INDEX_NAME", "course_info"): [ + "language", # aggregate by language, mode, org + "modes", + "org", + "catalog_visibility", # exclude visibility="none" + "enrollment_end", # include only enrollable courses + ], + getattr(settings, "COURSEWARE_CONTENT_INDEX_NAME", "courseware_content"): [ + PRIMARY_KEY_FIELD_NAME, # exclude some specific documents based on ID + "course", # search courseware content by course + "org", # used during indexing + "start_date", # limit search to started courses + ], +} + + +class MeilisearchEngine(SearchEngine): + """ + Meilisearch-compatible search engine. We work very hard to produce an output that is + compliant with edx-search's ElasticSearchEngine. + """ + + def __init__(self, index=None): + super().__init__(index=index) + self.meilisearch_index = get_meilisearch_index(self.index_name) + + @property + def meilisearch_index_name(self): + """ + The index UID is its name. + """ + return self.meilisearch_index.uid + + def index(self, sources: list[dict[str, t.Any]], **kwargs): + """ + Index a number of documents, which can have just any type. + """ + logger.info( + "Index request: index=%s sources=%s kwargs=%s", + self.meilisearch_index_name, + sources, + kwargs, + ) + processed_documents = [process_document(source) for source in sources] + self.meilisearch_index.add_documents( + processed_documents, serializer=DocumentEncoder + ) + + def search( + self, + query_string=None, + field_dictionary=None, + filter_dictionary=None, + exclude_dictionary=None, + aggregation_terms=None, + # exclude_ids=None, # deprecated + # use_field_match=False, # deprecated + log_search_params=False, + **kwargs, + ): # pylint: disable=too-many-arguments + """ + See meilisearch docs: https://www.meilisearch.com/docs/reference/api/search + """ + opt_params = get_search_params( + field_dictionary=field_dictionary, + filter_dictionary=filter_dictionary, + exclude_dictionary=exclude_dictionary, + aggregation_terms=aggregation_terms, + **kwargs, + ) + if log_search_params: + logger.info("Search query: opt_params=%s", opt_params) + meilisearch_results = self.meilisearch_index.search(query_string, opt_params) + processed_results = process_results(meilisearch_results, self.index_name) + return processed_results + + def remove(self, doc_ids, **kwargs): + """ + Removing documents from the index is as simple as deleting the the documents + with the corresponding primary key. + """ + logger.info( + "Remove request: index=%s, doc_ids=%s kwargs=%s", + self.meilisearch_index_name, + doc_ids, + kwargs, + ) + doc_pks = [id2pk(doc_id) for doc_id in doc_ids] + if doc_pks: + self.meilisearch_index.delete_documents(doc_pks) + + +class DocumentEncoder(json.JSONEncoder): + """ + Custom encoder, useful in particular to encode datetime fields. + Ref: https://github.com/meilisearch/meilisearch-python?tab=readme-ov-file#custom-serializer-for-documents- + """ + + def default(self, o): + if isinstance(o, datetime): + return str(o) + return super().default(o) + + +def print_failed_meilisearch_tasks(count: int = 10): + """ + Useful function for troubleshooting. + + Since indexing tasks are asynchronous, sometimes they fail and it's tricky to figure + out why. This will print failed tasks to stdout. + """ + client = get_meilisearch_client() + for result in client.task_handler.get_tasks( + {"statuses": "failed", "limit": count} + ).results: + print(result) + + +def create_indexes(index_filterables: dict[str, list[str]] = None): + """ + This is an initialization function that creates indexes and makes sure that they + support the right facetting. + + The `index_filterables` will default to `INDEX_FILTERABLES` if undefined. Developers + can use this function to configure their own indices. + """ + if index_filterables is None: + index_filterables = INDEX_FILTERABLES + + client = get_meilisearch_client() + for index_name, filterables in index_filterables.items(): + meilisearch_index_name = get_meilisearch_index_name(index_name) + try: + index = client.get_index(meilisearch_index_name) + except meilisearch.errors.MeilisearchApiError as e: + if e.code != "index_not_found": + raise + client.create_index( + meilisearch_index_name, {"primaryKey": PRIMARY_KEY_FIELD_NAME} + ) + # Get the index again + index = client.get_index(meilisearch_index_name) + + # Update filterables if there are some new elements + if filterables: + existing_filterables = set(index.get_filterable_attributes()) + if not set(filterables).issubset(existing_filterables): + all_filterables = list(existing_filterables.union(filterables)) + index.update_filterable_attributes(all_filterables) + + +def get_meilisearch_index(index_name: str): + """ + Return a meilisearch index. + + Note that the index may not exist, and it will be created on first insertion. + ideally, the initialisation function `create_indexes` should be run first. + """ + meilisearch_client = get_meilisearch_client() + meilisearch_index_name = get_meilisearch_index_name(index_name) + return meilisearch_client.index(meilisearch_index_name) + + +def get_meilisearch_client(): + return meilisearch.Client(MEILISEARCH_URL, api_key=MEILISEARCH_API_KEY) + + +def get_meilisearch_index_name(index_name: str) -> str: + """ + Return the index name in Meilisearch associated to a hard-coded index name. + + This is useful for multi-tenant Meilisearch: just define a different prefix for + every tenant. + + Usually, meilisearch API keys are allowed to access only certain index prefixes. + Make sure that your API key matches the prefix. + """ + return MEILISEARCH_INDEX_PREFIX + index_name + + +def process_document(doc: dict[str, t.Any]) -> dict[str, t.Any]: + """ + Process document before indexing. + + We make a copy to avoid modifying the source document. + """ + processed = process_nested_document(doc) + + # Add primary key field + processed[PRIMARY_KEY_FIELD_NAME] = id2pk(doc["id"]) + + return processed + + +def process_nested_document(doc: dict[str, t.Any]) -> dict[str, t.Any]: + """ + Process nested dict inside top-level Meilisearch document. + """ + processed = {} + for key, value in doc.items(): + if isinstance(value, timezone.datetime): + # Convert datetime objects to timestamp, and store the timezone in a + # separate field with a suffix given by UTC_OFFSET_SUFFIX. + utcoffset = None + if value.tzinfo: + utcoffset = value.utcoffset().seconds + processed[key] = value.timestamp() + processed[f"{key}{UTC_OFFSET_SUFFIX}"] = utcoffset + elif isinstance(value, dict): + processed[key] = process_nested_document(value) + else: + # Pray that there are not datetime objects inside lists. + # If there are, they will be converted to str by the DocumentEncoder. + processed[key] = value + return processed + + +def id2pk(value: str) -> str: + """ + Convert a document "id" field into a primary key that is compatible with Meilisearch. + + This step is necessary because the "id" is typically a course id, which includes + colon ":" characters, which are not supported by Meilisearch. Source: + https://www.meilisearch.com/docs/learn/getting_started/primary_key#formatting-the-document-id + """ + return hashlib.sha1(value.encode()).hexdigest() + + +def get_search_params( + field_dictionary=None, + filter_dictionary=None, + exclude_dictionary=None, + aggregation_terms=None, + **kwargs, +) -> dict[str, t.Any]: + """ + Return a dictionary of parameters that should be passed to the Meilisearch client + `.search()` method. + """ + params = {"showRankingScore": True} + + # Aggregation + if aggregation_terms: + params["facets"] = list(aggregation_terms.keys()) + + # Exclusion and inclusion filters + filters = [] + if field_dictionary: + filters += get_filter_rules(field_dictionary) + if filter_dictionary: + filters += get_filter_rules(filter_dictionary, optional=True) + if exclude_dictionary: + filters += get_filter_rules(exclude_dictionary, exclude=True) + if filters: + params["filter"] = filters + + # Offset/Size + if "from_" in kwargs: + params["offset"] = kwargs["from_"] + if "size" in kwargs: + params["limit"] = kwargs["size"] + + return params + + +def get_filter_rules( + rule_dict: dict[str, t.Any], exclude: bool = False, optional: bool = False +) -> list[str]: + """ + Convert inclusion/exclusion rules. + """ + rules = [] + for key, value in rule_dict.items(): + if isinstance(value, list): + for v in value: + rules.append( + get_filter_rule(key, v, exclude=exclude, optional=optional) + ) + else: + rules.append( + get_filter_rule(key, value, exclude=exclude, optional=optional) + ) + return rules + + +def get_filter_rule( + key: str, value: str, exclude: bool = False, optional: bool = False +) -> str: + """ + Meilisearch filter rule. + + See: https://www.meilisearch.com/docs/learn/filtering_and_sorting/filter_expression_reference + """ + prefix = "NOT " if exclude else "" + if key == "id": + key = PRIMARY_KEY_FIELD_NAME + value = id2pk(value) + if isinstance(value, str): + rule = f'{prefix}{key} = "{value}"' + elif isinstance(value, ValueRange): + constraints = [] + lower = value.lower + if isinstance(lower, timezone.datetime): + lower = lower.timestamp() + upper = value.upper + if isinstance(upper, timezone.datetime): + upper = upper.timestamp() + # I know that the following fails if value == 0, but we are being + # consistent with the behaviour in the elasticsearch engine. + if lower: + constraints.append(f"{key} >= {lower}") + if upper: + constraints.append(f"{key} <= {upper}") + rule = " AND ".join(constraints) + if len(constraints) > 1: + rule = f"({rule})" + else: + raise ValueError(f"Unknown value type: {value.__class__}") + if optional: + rule += f" OR {key} NOT EXISTS" + return rule + + +def process_results(results: dict[str, t.Any], index_name: str) -> dict[str, t.Any]: + """ + Convert results produced by Meilisearch into results that are compatible with the + edx-search engine API. + + Example input: + + { + 'hits': [ + { + 'pk': 'f381d4f1914235c9532576c0861d09b484ade634', + 'id': 'course-v1:OpenedX+DemoX+DemoCourse', + ... + "_rankingScore": 0.865, + }, + ... + ], + 'query': 'demo', + 'processingTimeMs': 0, + 'limit': 20, + 'offset': 0, + 'estimatedTotalHits': 1 + } + + Example output: + + { + 'took': 13, + 'total': 1, + 'max_score': 0.4001565, + 'results': [ + { + '_index': 'course_info', + '_type': '_doc', + '_id': 'course-v1:OpenedX+DemoX+DemoCourse', + '_ignored': ['content.overview.keyword'], # removed + 'data': { + 'id': 'course-v1:OpenedX+DemoX+DemoCourse', + 'course': 'course-v1:OpenedX+DemoX+DemoCourse', + 'content': { + 'display_name': 'Open edX Demo Course', + ... + }, + 'image_url': '/asset-v1:OpenedX+DemoX+DemoCourse+type@asset+block@thumbnail_demox.jpeg', + 'start': '2020-01-01T00:00:00+00:00', + ... + }, + 'score': 0.4001565 + } + ], + 'aggs': { + 'modes': { + 'terms': {'audit': 1}, + 'total': 1.0, + 'other': 0 + }, + 'org': { + 'terms': {'OpenedX': 1}, 'total': 1.0, 'other': 0 + }, + 'language': {'terms': {'en': 1}, 'total': 1.0, 'other': 0} + } + } + """ + # Base + processed = { + "took": results["processingTimeMs"], + "total": results["estimatedTotalHits"], + "results": [], + "aggs": {}, + } + + # Hits + max_score = 0 + for result in results["hits"]: + result = process_hit(result) + score = result.pop("_rankingScore") + max_score = max(max_score, score) + processed_result = { + "_id": result["id"], + "_index": index_name, + "_type": "_doc", + "data": result, + } + processed["results"].append(processed_result) + processed["max_score"] = max_score + + # Aggregates/Facets + for facet_name, facet_distribution in results.get("facetDistribution", {}).items(): + total = sum(facet_distribution.values()) + processed["aggs"][facet_name] = { + "terms": facet_distribution, + "total": total, + "other": 0, + } + return processed + + +def process_hit(hit: dict[str, t.Any]) -> dict[str, t.Any]: + """ + Convert a search result back to the ES format. + """ + processed = deepcopy(hit) + + # Remove primary key field + try: + processed.pop(PRIMARY_KEY_FIELD_NAME) + except KeyError: + pass + + # Convert datetime fields back to datetime + for key in list(processed.keys()): + if key.endswith(UTC_OFFSET_SUFFIX): + utcoffset = processed.pop(key) + key = key[: -len(UTC_OFFSET_SUFFIX)] + timestamp = hit[key] + tz = ( + timezone.get_fixed_timezone(timezone.timedelta(seconds=utcoffset)) + if utcoffset + else None + ) + processed[key] = timezone.datetime.fromtimestamp(timestamp, tz=tz) + return processed diff --git a/search/tests/test_meilisearch.py b/search/tests/test_meilisearch.py new file mode 100644 index 00000000..dc0e9f77 --- /dev/null +++ b/search/tests/test_meilisearch.py @@ -0,0 +1,359 @@ +""" +Test for the Meilisearch search engine. +""" + +from datetime import datetime +from unittest.mock import Mock + +import django.test +from django.utils import timezone +import pytest + +from search.utils import DateRange, ValueRange +import search.meilisearch + + +class DocumentEncoderTests(django.test.TestCase): + """ + JSON encoder unit tests. + """ + + def test_document_encode_without_timezone(self): + document = { + "date": timezone.datetime(2024, 12, 31, 5, 0, 0), + } + encoder = search.meilisearch.DocumentEncoder() + encoded = encoder.encode(document) + assert '{"date": "2024-12-31 05:00:00"}' == encoded + + def test_document_encode_with_timezone(self): + document = { + "date": timezone.datetime( + 2024, 12, 31, 5, 0, 0, tzinfo=timezone.get_fixed_timezone(0) + ), + } + encoder = search.meilisearch.DocumentEncoder() + encoded = encoder.encode(document) + assert '{"date": "2024-12-31 05:00:00+00:00"}' == encoded + + def test_document_encode_string(self): + document = { + "description": "I ♥ strings!", + } + encoder = search.meilisearch.DocumentEncoder() + encoded = encoder.encode(document) + assert '{"description": "I \\u2665 strings!"}' == encoded + + +class EngineTests(django.test.TestCase): + """ + MeilisearchEngine tests. + """ + + def test_index_empty_document(self): + assert not search.meilisearch.process_nested_document({}) + + def test_index_empty_document_raises_key_error(self): + with pytest.raises(KeyError): + search.meilisearch.process_document({}) + + def test_index(self): + document = { + "id": "abcd", + "name": "My name", + "title": "My title", + } + processed = search.meilisearch.process_document(document) + + # Check that the source document was not modified + self.assertNotIn(search.meilisearch.PRIMARY_KEY_FIELD_NAME, document) + + # "id" field is preserved + assert "abcd" == processed["id"] + + # Primary key field + # can be verified with: echo -n "abcd" | sha1sum + pk = "81fe8bfe87576c3ecb22426f8e57847382917acf" + assert pk == processed[search.meilisearch.PRIMARY_KEY_FIELD_NAME] + + # Additional fields + assert "My name" == processed["name"] + assert "My title" == processed["title"] + + def test_index_recursive(self): + document = {"field": {"value": timezone.datetime(2024, 1, 1)}} + processed = search.meilisearch.process_nested_document(document) + assert { + "field": { + "value": 1704067200.0, + "value__utcoffset": None, + } + } == processed + + def test_index_datetime_no_tz(self): + # No timezone + document = {"id": "1", "dt": timezone.datetime(2024, 1, 1)} + processed = search.meilisearch.process_document(document) + assert 1704067200.0 == processed["dt"] + assert processed["dt__utcoffset"] is None + # reverse serialisation + reverse = search.meilisearch.process_hit(processed) + assert document == reverse + + def test_index_datetime_with_tz(self): + # With timezone + document = { + "id": "1", + "dt": timezone.datetime( + 2024, + 1, + 1, + tzinfo=timezone.get_fixed_timezone(timezone.timedelta(seconds=3600)), + ), + } + processed = search.meilisearch.process_document(document) + assert 1704063600.0 == processed["dt"] + assert 3600 == processed["dt__utcoffset"] + # reverse serialisation + reverse = search.meilisearch.process_hit(processed) + assert document == reverse + + def test_search(self): + meilisearch_results = { + "hits": [ + { + "id": "id1", + search.meilisearch.PRIMARY_KEY_FIELD_NAME: search.meilisearch.id2pk( + "id1" + ), + "title": "title 1", + "_rankingScore": 0.8, + }, + { + "id": "id2", + search.meilisearch.PRIMARY_KEY_FIELD_NAME: search.meilisearch.id2pk( + "id2" + ), + "title": "title 2", + "_rankingScore": 0.2, + }, + ], + "query": "demo", + "processingTimeMs": 14, + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2, + } + processed_results = search.meilisearch.process_results( + meilisearch_results, "index_name" + ) + assert 14 == processed_results["took"] + assert 2 == processed_results["total"] + assert 0.8 == processed_results["max_score"] + + assert 2 == len(processed_results["results"]) + assert { + "_id": "id1", + "_index": "index_name", + "_type": "_doc", + "data": { + "id": "id1", + "title": "title 1", + }, + } == processed_results["results"][0] + + assert { + "_id": "id2", + "_index": "index_name", + "_type": "_doc", + "data": { + "id": "id2", + "title": "title 2", + }, + } == processed_results["results"][1] + + def test_search_with_facets(self): + meilisearch_results = { + "hits": [], + "query": "", + "processingTimeMs": 1, + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "facetDistribution": { + "modes": {"audit": 1, "honor": 3}, + "facet2": {"val1": 1, "val2": 2, "val3": 3}, + }, + } + processed_results = search.meilisearch.process_results( + meilisearch_results, "index_name" + ) + aggs = processed_results["aggs"] + assert { + "terms": {"audit": 1, "honor": 3}, + "total": 4.0, + "other": 0, + } == aggs["modes"] + + def test_search_params(self): + params = search.meilisearch.get_search_params() + self.assertTrue(params["showRankingScore"]) + + params = search.meilisearch.get_search_params(from_=0) + assert 0 == params["offset"] + + def test_search_params_exclude_dictionary(self): + # Simple value + params = search.meilisearch.get_search_params( + exclude_dictionary={"course_visibility": "none"} + ) + assert ['NOT course_visibility = "none"'] == params["filter"] + + # Multiple IDs + params = search.meilisearch.get_search_params( + exclude_dictionary={"id": ["1", "2"]} + ) + assert [ + f'NOT {search.meilisearch.PRIMARY_KEY_FIELD_NAME} = "{search.meilisearch.id2pk("1")}"', + f'NOT {search.meilisearch.PRIMARY_KEY_FIELD_NAME} = "{search.meilisearch.id2pk("2")}"', + ] == params["filter"] + + def test_search_params_field_dictionary(self): + params = search.meilisearch.get_search_params( + field_dictionary={ + "course": "course-v1:testorg+test1+alpha", + "org": "testorg", + } + ) + assert [ + 'course = "course-v1:testorg+test1+alpha"', + 'org = "testorg"', + ] == params["filter"] + + def test_search_params_filter_dictionary(self): + params = search.meilisearch.get_search_params( + filter_dictionary={"key": "value"} + ) + assert ['key = "value" OR key NOT EXISTS'] == params["filter"] + + def test_search_params_value_range(self): + params = search.meilisearch.get_search_params( + filter_dictionary={"value": ValueRange(lower=1, upper=2)} + ) + assert ["(value >= 1 AND value <= 2) OR value NOT EXISTS"] == params["filter"] + + params = search.meilisearch.get_search_params( + filter_dictionary={"value": ValueRange(lower=1)} + ) + assert ["value >= 1 OR value NOT EXISTS"] == params["filter"] + + def test_search_params_date_range(self): + params = search.meilisearch.get_search_params( + filter_dictionary={ + "enrollment_end": DateRange( + lower=datetime(2024, 1, 1), upper=datetime(2024, 1, 2) + ) + } + ) + assert [ + "(enrollment_end >= 1704067200.0 AND enrollment_end <= 1704153600.0) OR enrollment_end NOT EXISTS" + ] == params["filter"] + + params = search.meilisearch.get_search_params( + filter_dictionary={"enrollment_end": DateRange(lower=datetime(2024, 1, 1))} + ) + assert [ + "enrollment_end >= 1704067200.0 OR enrollment_end NOT EXISTS" + ] == params["filter"] + + def test_engine_init(self): + engine = search.meilisearch.MeilisearchEngine(index="my_index") + assert engine.meilisearch_index_name == "my_index" + + def test_engine_index(self): + engine = search.meilisearch.MeilisearchEngine(index="my_index") + engine.meilisearch_index.add_documents = Mock() + document = { + "id": "abcd", + "name": "My name", + "title": "My title", + } + processed_document = { + # Primary key field + # can be verified with: echo -n "abcd" | sha1sum + "_pk": "81fe8bfe87576c3ecb22426f8e57847382917acf", + "id": "abcd", + "name": "My name", + "title": "My title", + } + engine.index(sources=[document]) + engine.meilisearch_index.add_documents.assert_called_with( + [processed_document], + serializer=search.meilisearch.DocumentEncoder, + ) + + def test_engine_search(self): + engine = search.meilisearch.MeilisearchEngine(index="my_index") + engine.meilisearch_index.search = Mock(return_value={ + "hits": [ + { + "pk": "f381d4f1914235c9532576c0861d09b484ade634", + "id": "course-v1:OpenedX+DemoX+DemoCourse", + "_rankingScore": 0.865, + }, + ], + "query": "demo", + "processingTimeMs": 0, + "limit": 20, + "offset": 0, + "estimatedTotalHits": 1 + }) + + results = engine.search( + query_string="abc", + field_dictionary={ + "course": "course-v1:testorg+test1+alpha", + "org": "testorg", + }, + filter_dictionary={"key": "value"}, + exclude_dictionary={"id": ["abcd"]}, + aggregation_terms={"org": 1, "course": 2}, + log_search_params=True, + ) + + engine.meilisearch_index.search.assert_called_with("abc", { + "showRankingScore": True, + "facets": ["org", "course"], + "filter": [ + 'course = "course-v1:testorg+test1+alpha"', + 'org = "testorg"', 'key = "value" OR key NOT EXISTS', + 'NOT _pk = "81fe8bfe87576c3ecb22426f8e57847382917acf"', + ] + }) + assert results == { + "aggs": {}, + "max_score": 0.865, + "results": [ + { + "_id": "course-v1:OpenedX+DemoX+DemoCourse", + "_index": "my_index", + "_type": "_doc", + "data": { + "id": "course-v1:OpenedX+DemoX+DemoCourse", + "pk": "f381d4f1914235c9532576c0861d09b484ade634", + }, + }, + ], + "took": 0, + "total": 1, + } + + def test_engine_remove(self): + engine = search.meilisearch.MeilisearchEngine(index="my_index") + engine.meilisearch_index.delete_documents = Mock() + # Primary key field + # can be verified with: echo -n "abcd" | sha1sum + doc_id = "abcd" + doc_pk = "81fe8bfe87576c3ecb22426f8e57847382917acf" + engine.remove(doc_ids=[doc_id]) + engine.meilisearch_index.delete_documents.assert_called_with([doc_pk]) diff --git a/settings.py b/settings.py deleted file mode 100644 index 65362a40..00000000 --- a/settings.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Django settings for edxsearch test project. - -For more information on this file, see -https://docs.djangoproject.com/en/1.6/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.6/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) - -import os -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -# This is just a container for running tests, it's okay to allow it to be -# defaulted here if not present in environment settings -SECRET_KEY = os.environ.get('SECRET_KEY', '@krr4&!u8#g&2^(q53e3xu_kux$3rm=)7s3m1mjg2%$#u($-g4') - -# SECURITY WARNING: don't run with debug turned on in production! -# This is just a container for running tests -DEBUG = True - -ALLOWED_HOSTS = [] - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': ( - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ) - } - }, -] - - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'eventtracking.django', - 'waffle', -) - -MIDDLEWARE = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'waffle.middleware.WaffleMiddleware', -) - -ROOT_URLCONF = 'search.urls' - -WSGI_APPLICATION = 'edxsearch.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.6/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - -# Internationalization -# https://docs.djangoproject.com/en/1.6/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.6/howto/static-files/ - -STATIC_URL = '/static/' - - -############################## EVENT TRACKING ################################# - -TRACK_MAX_EVENT = 50000 - -TRACKING_BACKENDS = { - 'logger': { - 'ENGINE': 'track.backends.logger.LoggerBackend', - 'OPTIONS': { - 'name': 'tracking' - } - } -} - -# We're already logging events, and we don't want to capture user -# names/passwords. Heartbeat events are likely not interesting. -TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] - -EVENT_TRACKING_ENABLED = True -EVENT_TRACKING_BACKENDS = { - 'logger': { - 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', - 'OPTIONS': { - 'name': 'tracking', - 'max_event_size': TRACK_MAX_EVENT, - } - } -} - diff --git a/tox.ini b/tox.ini index 20d4e9cb..c9e94021 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = django42: Django>=4.2,<4.3 -r {toxinidir}/requirements/testing.txt commands = - python -Wd -m coverage run manage.py test --settings=settings {posargs} + python -Wd -m coverage run manage.py test {posargs} python -m coverage xml [testenv:quality] From 28fa16acd98356cbe3553b2ee7aefdf81875230c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 31 Oct 2024 22:11:56 +0100 Subject: [PATCH 26/26] fix: wait for meilisearch index creation to succeed (#166) In `search.meilisearch.create_indexes`, we were not waiting for the index creation tasks to complete. This was causing a potential race condition, where the `create_indexes` function would fail because it took a few seconds for the index creation to succeed. See the relevant conversation here: https://github.com/openedx/edx-platform/pull/35743#issuecomment-2450115310 --- edxsearch/__init__.py | 2 +- search/meilisearch.py | 95 +++++++++++++++++++++++--------- search/tests/test_meilisearch.py | 88 +++++++++++++++++++++-------- 3 files changed, 133 insertions(+), 52 deletions(-) diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index d00a3e59..71727cf1 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '4.1.0' +__version__ = '4.1.1' diff --git a/search/meilisearch.py b/search/meilisearch.py index 75636526..9e09e426 100644 --- a/search/meilisearch.py +++ b/search/meilisearch.py @@ -115,9 +115,20 @@ class MeilisearchEngine(SearchEngine): compliant with edx-search's ElasticSearchEngine. """ - def __init__(self, index=None): + def __init__(self, index=None) -> None: super().__init__(index=index) - self.meilisearch_index = get_meilisearch_index(self.index_name) + self._meilisearch_index: t.Optional[meilisearch.index.Index] = None + + @property + def meilisearch_index(self) -> meilisearch.index.Index: + """ + Lazy load meilisearch index. + """ + if self._meilisearch_index is None: + meilisearch_index_name = get_meilisearch_index_name(self.index_name) + meilisearch_client = get_meilisearch_client() + self._meilisearch_index = meilisearch_client.index(meilisearch_index_name) + return self._meilisearch_index @property def meilisearch_index_name(self): @@ -211,7 +222,7 @@ def print_failed_meilisearch_tasks(count: int = 10): print(result) -def create_indexes(index_filterables: dict[str, list[str]] = None): +def create_indexes(index_filterables: t.Optional[dict[str, list[str]]] = None): """ This is an initialization function that creates indexes and makes sure that they support the right facetting. @@ -225,38 +236,68 @@ def create_indexes(index_filterables: dict[str, list[str]] = None): client = get_meilisearch_client() for index_name, filterables in index_filterables.items(): meilisearch_index_name = get_meilisearch_index_name(index_name) - try: - index = client.get_index(meilisearch_index_name) - except meilisearch.errors.MeilisearchApiError as e: - if e.code != "index_not_found": - raise - client.create_index( - meilisearch_index_name, {"primaryKey": PRIMARY_KEY_FIELD_NAME} - ) - # Get the index again - index = client.get_index(meilisearch_index_name) + index = get_or_create_meilisearch_index(client, meilisearch_index_name) + update_index_filterables(client, index, filterables) - # Update filterables if there are some new elements - if filterables: - existing_filterables = set(index.get_filterable_attributes()) - if not set(filterables).issubset(existing_filterables): - all_filterables = list(existing_filterables.union(filterables)) - index.update_filterable_attributes(all_filterables) +def get_or_create_meilisearch_index( + client: meilisearch.Client, index_name: str +) -> meilisearch.index.Index: + """ + Get an index. If it does not exist, create it. -def get_meilisearch_index(index_name: str): + This will fail with a RuntimeError if we fail to create the index. It will fail with + a MeilisearchApiError in other failure cases. """ - Return a meilisearch index. + try: + return client.get_index(index_name) + except meilisearch.errors.MeilisearchApiError as e: + if e.code != "index_not_found": + raise + task_info = client.create_index( + index_name, {"primaryKey": PRIMARY_KEY_FIELD_NAME} + ) + wait_for_task_to_succeed(client, task_info) + # Get the index again + return client.get_index(index_name) - Note that the index may not exist, and it will be created on first insertion. - ideally, the initialisation function `create_indexes` should be run first. + +def update_index_filterables( + client: meilisearch.Client, index: meilisearch.index.Index, filterables: list[str] +) -> None: """ - meilisearch_client = get_meilisearch_client() - meilisearch_index_name = get_meilisearch_index_name(index_name) - return meilisearch_client.index(meilisearch_index_name) + Make sure that the filterable fields of an index include the given list of fields. + + If existing fields are present, they are preserved. + """ + if not filterables: + return + existing_filterables = set(index.get_filterable_attributes()) + if set(filterables).issubset(existing_filterables): + # all filterables fields are already present + return + all_filterables = list(existing_filterables.union(filterables)) + task_info = index.update_filterable_attributes(all_filterables) + wait_for_task_to_succeed(client, task_info) + + +def wait_for_task_to_succeed( + client: meilisearch.Client, + task_info: meilisearch.task.TaskInfo, + timeout_in_ms: int = 5000, +) -> None: + """ + Wait for a Meilisearch task to succeed. If it does not, raise RuntimeError. + """ + task = client.wait_for_task(task_info.task_uid, timeout_in_ms=timeout_in_ms) + if task.status != "succeeded": + raise RuntimeError(f"Failed meilisearch task: {task}") def get_meilisearch_client(): + """ + Return a Meilisearch client with the right settings. + """ return meilisearch.Client(MEILISEARCH_URL, api_key=MEILISEARCH_API_KEY) @@ -332,7 +373,7 @@ def get_search_params( Return a dictionary of parameters that should be passed to the Meilisearch client `.search()` method. """ - params = {"showRankingScore": True} + params: dict[str, t.Any] = {"showRankingScore": True} # Aggregation if aggregation_terms: diff --git a/search/tests/test_meilisearch.py b/search/tests/test_meilisearch.py index dc0e9f77..844854f5 100644 --- a/search/tests/test_meilisearch.py +++ b/search/tests/test_meilisearch.py @@ -3,11 +3,13 @@ """ from datetime import datetime -from unittest.mock import Mock +from unittest.mock import Mock, patch import django.test from django.utils import timezone +import meilisearch import pytest +from requests import Response from search.utils import DateRange, ValueRange import search.meilisearch @@ -294,20 +296,22 @@ def test_engine_index(self): def test_engine_search(self): engine = search.meilisearch.MeilisearchEngine(index="my_index") - engine.meilisearch_index.search = Mock(return_value={ - "hits": [ - { - "pk": "f381d4f1914235c9532576c0861d09b484ade634", - "id": "course-v1:OpenedX+DemoX+DemoCourse", - "_rankingScore": 0.865, - }, - ], - "query": "demo", - "processingTimeMs": 0, - "limit": 20, - "offset": 0, - "estimatedTotalHits": 1 - }) + engine.meilisearch_index.search = Mock( + return_value={ + "hits": [ + { + "pk": "f381d4f1914235c9532576c0861d09b484ade634", + "id": "course-v1:OpenedX+DemoX+DemoCourse", + "_rankingScore": 0.865, + }, + ], + "query": "demo", + "processingTimeMs": 0, + "limit": 20, + "offset": 0, + "estimatedTotalHits": 1, + } + ) results = engine.search( query_string="abc", @@ -321,15 +325,19 @@ def test_engine_search(self): log_search_params=True, ) - engine.meilisearch_index.search.assert_called_with("abc", { - "showRankingScore": True, - "facets": ["org", "course"], - "filter": [ - 'course = "course-v1:testorg+test1+alpha"', - 'org = "testorg"', 'key = "value" OR key NOT EXISTS', - 'NOT _pk = "81fe8bfe87576c3ecb22426f8e57847382917acf"', - ] - }) + engine.meilisearch_index.search.assert_called_with( + "abc", + { + "showRankingScore": True, + "facets": ["org", "course"], + "filter": [ + 'course = "course-v1:testorg+test1+alpha"', + 'org = "testorg"', + 'key = "value" OR key NOT EXISTS', + 'NOT _pk = "81fe8bfe87576c3ecb22426f8e57847382917acf"', + ], + }, + ) assert results == { "aggs": {}, "max_score": 0.865, @@ -357,3 +365,35 @@ def test_engine_remove(self): doc_pk = "81fe8bfe87576c3ecb22426f8e57847382917acf" engine.remove(doc_ids=[doc_id]) engine.meilisearch_index.delete_documents.assert_called_with([doc_pk]) + + +class UtilitiesTests(django.test.TestCase): + """ + Tests associated to the utility functions of the meilisearch engine. + """ + + @patch.object(search.meilisearch, "wait_for_task_to_succeed") + def test_create_index(self, mock_wait_for_task_to_succeed) -> None: + class ClientMock: + """ + Mocked client + """ + number_of_calls = 0 + + def get_index(self, index_name): + """Mocked client.get_index method""" + self.number_of_calls += 1 + if self.number_of_calls == 1: + error = meilisearch.errors.MeilisearchApiError("", Response()) + error.code = "index_not_found" + raise error + if self.number_of_calls == 2: + return f"index created: {index_name}" + # We shouldn't be there + assert False + + client = Mock() + client.get_index = Mock(side_effect=ClientMock().get_index) + result = search.meilisearch.get_or_create_meilisearch_index(client, "my_index") + assert result == "index created: my_index" + mock_wait_for_task_to_succeed.assert_called_once()