Merge pull request #130 from open-craft/keith/shared-elasticsearch

chore: add ELASTIC_SEARCH_INDEX_PREFIX setting to prefix indices

README.md

@@ -34,6 +34,15 @@ where
 
 3. `search` - the operation to find matching documents within the index. `doc_type` is supported as an optional keyword parameter to return results only with a certain doc_type
 
+## Configuring for multi-tenancy
+
+The modules exposes a setting `ELASTIC_SEARCH_INDEX_PREFIX` to enable so that the indices for multiple clients do not collide.
+
+```python
+SearchEngine(index_name="test")
+```
+
+When invoked, this line will create an index named `test` on Elastic Search. Setting `ELASTIC_SEARCH_INDEX_PREFIX="client1_"` will instead create the index `client1_test`.
 
 ## Index documents
 Index documents are passed to the search application as python dictionaries, along with a `doc_type` document type, which is also optionally supported as a way to return only certain document types from a search.

edxsearch/__init__.py

@@ -1,3 +1,3 @@
 """ Container module for testing / demoing search """
 
-__version__ = '3.4.0'
+__version__ = '3.5.0'

search/elastic.py

@@ -304,16 +304,16 @@ def mappings(self):
         we'll load them again from Elasticsearch
         """
         # Try loading the mapping from the cache.
-        mapping = ElasticSearchEngine.get_mappings(self.index_name)
+        mapping = ElasticSearchEngine.get_mappings(self._prefixed_index_name)
 
         # Fall back to Elasticsearch
         if not mapping:
             mapping = self._es.indices.get_mapping(
-                index=self.index_name
-            ).get(self.index_name, {}).get("mappings", {})
+                index=self._prefixed_index_name
+            ).get(self._prefixed_index_name, {}).get("mappings", {})
             # Cache the mapping, if one was retrieved
             if mapping:
-                ElasticSearchEngine.set_mappings(self.index_name, mapping)
+                ElasticSearchEngine.set_mappings(self._prefixed_index_name, mapping)
 
         return mapping
 
@@ -323,14 +323,27 @@ def _clear_mapping(self):
 
         Next time ES mappings is are requested.
         """
-        ElasticSearchEngine.set_mappings(self.index_name, {})
+        ElasticSearchEngine.set_mappings(self._prefixed_index_name, {})
 
     def __init__(self, index=None):
         super().__init__(index)
         es_config = getattr(settings, "ELASTIC_SEARCH_CONFIG", [{}])
         self._es = getattr(settings, "ELASTIC_SEARCH_IMPL", Elasticsearch)(es_config)
-        if not self._es.indices.exists(index=self.index_name):
-            self._es.indices.create(index=self.index_name)
+        params = None
+
+        if not self._es.indices.exists(index=self._prefixed_index_name):
+            self._es.indices.create(index=self._prefixed_index_name, params=params)
+
+    @property
+    def _prefixed_index_name(self):
+        """
+        Property that returns the defined index_name with the configured
+        prefix.
+
+        To be used anywhere the index_name is required.
+        """
+        prefix = getattr(settings, "ELASTIC_SEARCH_INDEX_PREFIX", "")
+        return prefix + self.index_name
 
     def _check_mappings(self, body):
         """
@@ -396,9 +409,10 @@ def field_property(field_name, field_value):
         }
 
         if new_properties:
+
             self._es.indices.put_mapping(
-                index=self.index_name,
-                body={"properties": new_properties}
+                index=self._prefixed_index_name,
+                body={"properties": new_properties},
             )
             self._clear_mapping()
 
@@ -417,7 +431,7 @@ def index(self, sources, **kwargs):
                 id_ = source.get("id")
                 log.debug("indexing object with id %s", id_)
                 action = {
-                    "_index": self.index_name,
+                    "_index": self._prefixed_index_name,
                     "_id": id_,
                     "_source": source
                 }
@@ -437,14 +451,13 @@ def remove(self, doc_ids, **kwargs):
         """
         Implements call to remove the documents from the index
         """
-
         try:
             actions = []
             for doc_id in doc_ids:
                 log.debug("Removing document with id %s", doc_id)
                 action = {
                     "_op_type": "delete",
-                    "_index": self.index_name,
+                    "_index": self._prefixed_index_name,
                     "_id": doc_id
                 }
                 actions.append(action)
@@ -568,7 +581,6 @@ def search(self,
         """
 
         log.debug("searching index with %s", query_string)
-
         elastic_queries = []
         elastic_filters = []
 
@@ -642,7 +654,7 @@ def search(self,
             body["aggs"] = _process_aggregation_terms(aggregation_terms)
 
         try:
-            es_response = self._es.search(index=self.index_name, body=body, **kwargs)
+            es_response = self._es.search(index=self._prefixed_index_name, body=body, **kwargs)
         except exceptions.ElasticsearchException as ex:
             log.exception("error while searching index - %r", ex)
             raise

search/tests/test_engines.py

@@ -7,18 +7,35 @@
 import json
 import os
 from datetime import datetime
-
 from unittest.mock import patch
+
 from django.test import TestCase
 from django.test.utils import override_settings
 from elasticsearch import exceptions
 from elasticsearch.helpers import BulkIndexError
-
-from search.api import perform_search, NoSearchEngineError
+from search.api import NoSearchEngineError, perform_search
 from search.elastic import RESERVED_CHARACTERS
-from search.tests.mock_search_engine import MockSearchEngine, json_date_to_datetime
+from search.tests.mock_search_engine import (MockSearchEngine,
+                                             json_date_to_datetime)
 from search.tests.tests import MockSearchTests
-from search.tests.utils import ErroringElasticImpl, SearcherMixin
+from search.tests.utils import (TEST_INDEX_NAME, ErroringElasticImpl,
+                                SearcherMixin)
+
+
+@override_settings(ELASTIC_SEARCH_INDEX_PREFIX='prefixed_')
+@override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine")
+class ElasticSearchPrefixTests(MockSearchTests):
+    """
+    Override that runs the same tests for ElasticSearchTests,
+    but with a prefixed index name.
+    """
+
+    @property
+    def index_name(self):
+        """
+        The search index name to be used for this test.
+        """
+        return f"prefixed_{TEST_INDEX_NAME}"
 
 
 @override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine")

search/tests/tests.py

@@ -27,6 +27,14 @@
 @override_settings(MOCK_SEARCH_BACKING_FILE=None)
 class MockSearchTests(TestCase, SearcherMixin):
     """ Test operation of search activities """
+
+    @property
+    def index_name(self):
+        """
+        The search index name to be used for this test.
+        """
+        return TEST_INDEX_NAME
+
     @property
     def _is_elastic(self):
         """ check search engine implementation, to manage cleanup differently """
@@ -39,11 +47,11 @@ def setUp(self):
         if self._is_elastic:
             _elasticsearch = Elasticsearch()
             # Make sure that we are fresh
-            _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404])
+            _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404])
 
             config_body = {}
             # ignore unexpected-keyword-arg; ES python client documents that it can be used
-            _elasticsearch.indices.create(index=TEST_INDEX_NAME, ignore=400, body=config_body)
+            _elasticsearch.indices.create(index=self.index_name, ignore=400, body=config_body)
         else:
             MockSearchEngine.destroy()
         self._searcher = None
@@ -55,7 +63,7 @@ def tearDown(self):
         if self._is_elastic:
             _elasticsearch = Elasticsearch()
             # ignore unexpected-keyword-arg; ES python client documents that it can be used
-            _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404])
+            _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404])
         else:
             MockSearchEngine.destroy()