diff --git a/README.md b/README.md index 007e7a7b..f3abad1c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,15 @@ where 3. `search` - the operation to find matching documents within the index. `doc_type` is supported as an optional keyword parameter to return results only with a certain doc_type +## Configuring for multi-tenancy + +The modules exposes a setting `ELASTIC_SEARCH_INDEX_PREFIX` to enable so that the indices for multiple clients do not collide. + +```python +SearchEngine(index_name="test") +``` + +When invoked, this line will create an index named `test` on Elastic Search. Setting `ELASTIC_SEARCH_INDEX_PREFIX="client1_"` will instead create the index `client1_test`. ## Index documents Index documents are passed to the search application as python dictionaries, along with a `doc_type` document type, which is also optionally supported as a way to return only certain document types from a search. diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 2432599e..a33f2731 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.4.0' +__version__ = '3.5.0' diff --git a/search/elastic.py b/search/elastic.py index 701378cd..f895564a 100644 --- a/search/elastic.py +++ b/search/elastic.py @@ -304,16 +304,16 @@ def mappings(self): we'll load them again from Elasticsearch """ # Try loading the mapping from the cache. - mapping = ElasticSearchEngine.get_mappings(self.index_name) + mapping = ElasticSearchEngine.get_mappings(self._prefixed_index_name) # Fall back to Elasticsearch if not mapping: mapping = self._es.indices.get_mapping( - index=self.index_name - ).get(self.index_name, {}).get("mappings", {}) + index=self._prefixed_index_name + ).get(self._prefixed_index_name, {}).get("mappings", {}) # Cache the mapping, if one was retrieved if mapping: - ElasticSearchEngine.set_mappings(self.index_name, mapping) + ElasticSearchEngine.set_mappings(self._prefixed_index_name, mapping) return mapping @@ -323,14 +323,27 @@ def _clear_mapping(self): Next time ES mappings is are requested. """ - ElasticSearchEngine.set_mappings(self.index_name, {}) + ElasticSearchEngine.set_mappings(self._prefixed_index_name, {}) def __init__(self, index=None): super().__init__(index) es_config = getattr(settings, "ELASTIC_SEARCH_CONFIG", [{}]) self._es = getattr(settings, "ELASTIC_SEARCH_IMPL", Elasticsearch)(es_config) - if not self._es.indices.exists(index=self.index_name): - self._es.indices.create(index=self.index_name) + params = None + + if not self._es.indices.exists(index=self._prefixed_index_name): + self._es.indices.create(index=self._prefixed_index_name, params=params) + + @property + def _prefixed_index_name(self): + """ + Property that returns the defined index_name with the configured + prefix. + + To be used anywhere the index_name is required. + """ + prefix = getattr(settings, "ELASTIC_SEARCH_INDEX_PREFIX", "") + return prefix + self.index_name def _check_mappings(self, body): """ @@ -396,9 +409,10 @@ def field_property(field_name, field_value): } if new_properties: + self._es.indices.put_mapping( - index=self.index_name, - body={"properties": new_properties} + index=self._prefixed_index_name, + body={"properties": new_properties}, ) self._clear_mapping() @@ -417,7 +431,7 @@ def index(self, sources, **kwargs): id_ = source.get("id") log.debug("indexing object with id %s", id_) action = { - "_index": self.index_name, + "_index": self._prefixed_index_name, "_id": id_, "_source": source } @@ -437,14 +451,13 @@ def remove(self, doc_ids, **kwargs): """ Implements call to remove the documents from the index """ - try: actions = [] for doc_id in doc_ids: log.debug("Removing document with id %s", doc_id) action = { "_op_type": "delete", - "_index": self.index_name, + "_index": self._prefixed_index_name, "_id": doc_id } actions.append(action) @@ -568,7 +581,6 @@ def search(self, """ log.debug("searching index with %s", query_string) - elastic_queries = [] elastic_filters = [] @@ -642,7 +654,7 @@ def search(self, body["aggs"] = _process_aggregation_terms(aggregation_terms) try: - es_response = self._es.search(index=self.index_name, body=body, **kwargs) + es_response = self._es.search(index=self._prefixed_index_name, body=body, **kwargs) except exceptions.ElasticsearchException as ex: log.exception("error while searching index - %r", ex) raise diff --git a/search/tests/test_engines.py b/search/tests/test_engines.py index 2b4f32bd..659d4cc9 100644 --- a/search/tests/test_engines.py +++ b/search/tests/test_engines.py @@ -7,18 +7,35 @@ import json import os from datetime import datetime - from unittest.mock import patch + from django.test import TestCase from django.test.utils import override_settings from elasticsearch import exceptions from elasticsearch.helpers import BulkIndexError - -from search.api import perform_search, NoSearchEngineError +from search.api import NoSearchEngineError, perform_search from search.elastic import RESERVED_CHARACTERS -from search.tests.mock_search_engine import MockSearchEngine, json_date_to_datetime +from search.tests.mock_search_engine import (MockSearchEngine, + json_date_to_datetime) from search.tests.tests import MockSearchTests -from search.tests.utils import ErroringElasticImpl, SearcherMixin +from search.tests.utils import (TEST_INDEX_NAME, ErroringElasticImpl, + SearcherMixin) + + +@override_settings(ELASTIC_SEARCH_INDEX_PREFIX='prefixed_') +@override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine") +class ElasticSearchPrefixTests(MockSearchTests): + """ + Override that runs the same tests for ElasticSearchTests, + but with a prefixed index name. + """ + + @property + def index_name(self): + """ + The search index name to be used for this test. + """ + return f"prefixed_{TEST_INDEX_NAME}" @override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine") diff --git a/search/tests/tests.py b/search/tests/tests.py index bf7a7259..62ea00cd 100644 --- a/search/tests/tests.py +++ b/search/tests/tests.py @@ -27,6 +27,14 @@ @override_settings(MOCK_SEARCH_BACKING_FILE=None) class MockSearchTests(TestCase, SearcherMixin): """ Test operation of search activities """ + + @property + def index_name(self): + """ + The search index name to be used for this test. + """ + return TEST_INDEX_NAME + @property def _is_elastic(self): """ check search engine implementation, to manage cleanup differently """ @@ -39,11 +47,11 @@ def setUp(self): if self._is_elastic: _elasticsearch = Elasticsearch() # Make sure that we are fresh - _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404]) + _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404]) config_body = {} # ignore unexpected-keyword-arg; ES python client documents that it can be used - _elasticsearch.indices.create(index=TEST_INDEX_NAME, ignore=400, body=config_body) + _elasticsearch.indices.create(index=self.index_name, ignore=400, body=config_body) else: MockSearchEngine.destroy() self._searcher = None @@ -55,7 +63,7 @@ def tearDown(self): if self._is_elastic: _elasticsearch = Elasticsearch() # ignore unexpected-keyword-arg; ES python client documents that it can be used - _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404]) + _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404]) else: MockSearchEngine.destroy()