bump elasticsearch version (#4303)

* bump elasticsearch version

* update images on docker hub

* remove uneccessary code

* fix typo

* fix indexing api tests

* use search instead of outdated percolate

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - "6379"
 
   elastic:
-    build: elastic
+    image: elasticsearch:6.7.1
     command: elasticsearch -E network.host=0.0.0.0 -E http.cors.enabled=true -E http.cors.allow-origin=* -E rest.action.multi.allow_explicit_index=false
     ports:
       - "9100:9200"

requirements.in

@@ -17,8 +17,8 @@ django-webpack-loader==0.5.0
 djangorestframework==3.7.7
 edx-api-client==0.6.1
 edx-opaque-keys==0.4
-elasticsearch-dsl==5.4.0
-elasticsearch==5.5.3
+elasticsearch-dsl==6.3.1
+elasticsearch==6.3.1
 factory_boy
 faker
 html5lib==0.999999999

requirements.txt

@@ -38,8 +38,8 @@ djangorestframework==3.7.7
 draftjs-exporter==2.0.0   # via wagtail
 edx-api-client==0.6.1
 edx-opaque-keys==0.4
-elasticsearch-dsl==5.4.0
-elasticsearch==5.5.3
+elasticsearch-dsl==6.3.1
+elasticsearch==6.3.1
 factory-boy==2.8.1
 faker==0.9.1
 html5lib==0.999999999

search/api.py

@@ -283,11 +283,22 @@ def _search_percolate_queries(program_enrollment):
     # We don't need this to search for percolator queries and
     # it causes a dynamic mapping failure so we need to remove it
     del doc['_id']
-    result = conn.percolate(percolate_index, GLOBAL_DOC_TYPE, body={"doc": doc})
+
+    body = {
+        "query": {
+            "percolate": {
+                "field": "query",
+                "document": doc
+            }
+        }
+    }
+
+    result = conn.search(percolate_index, GLOBAL_DOC_TYPE, body=body)
     failures = result.get('_shards', {}).get('failures', [])
     if len(failures) > 0:
         raise PercolateException("Failed to percolate: {}".format(failures))
-    return [int(row['_id']) for row in result['matches']]
+
+    return [int(row['_id']) for row in result['hits']['hits']]
 
 
 def adjust_search_for_percolator(search):
@@ -333,7 +344,7 @@ def document_needs_updating(enrollment):
 
     conn = get_conn()
     try:
-        document = conn.get(index=index, id=enrollment.id)
+        document = conn.get(index=index, doc_type=GLOBAL_DOC_TYPE, id=enrollment.id)
     except NotFoundError:
         return True
     serialized_enrollment = serialize_program_enrolled_user(enrollment)

search/api_test.py

@@ -398,7 +398,7 @@ def test_percolate_failure(self, mock_on_commit):
             profile = ProfileFactory.create(filled_out=True)
         program_enrollment = ProgramEnrollmentFactory.create(user=profile.user)
         with self.assertRaises(PercolateException) as ex, patch(
-            'search.api.get_conn', return_value=Mock(percolate=Mock(return_value=failure_payload))
+            'search.api.get_conn', return_value=Mock(search=Mock(return_value=failure_payload))
         ):
             search_percolate_queries(program_enrollment.id, "doesnt_matter")
         assert ex.exception.args[0] == "Failed to percolate: {}".format(failures)
@@ -498,7 +498,7 @@ def test_populate_query_inactive_memberships(self, is_active, has_profile, mock_
 
         with patch('search.api.get_conn') as es_mock:
             populate_query_memberships(query.id)
-            assert es_mock.return_value.percolate.call_count == (1 if has_profile and is_active else 0)
+            assert es_mock.return_value.search.call_count == (1 if has_profile and is_active else 0)
 
         assert PercolateQueryMembership.objects.filter(user=user, query=query).count() == (
             1 if is_active else 0

search/indexing_api.py

@@ -3,6 +3,7 @@
 """
 import logging
 
+from django.conf import settings
 from elasticsearch.helpers import bulk
 from elasticsearch.exceptions import NotFoundError
 
@@ -230,6 +231,8 @@
     }
 }
 
+INDEX_WILDCARD = '{index_name}_*'.format(index_name=settings.ELASTICSEARCH_INDEX)
+
 
 def _index_chunk(chunk, *, index):
     """
@@ -562,7 +565,7 @@ def delete_indices():
         aliases = get_aliases(index_type)
         for alias in aliases:
             if conn.indices.exists(alias):
-                conn.indices.delete(alias)
+                conn.indices.delete_alias(index=INDEX_WILDCARD, name=alias)
 
 
 # pylint: disable=too-many-locals
@@ -587,7 +590,7 @@ def recreate_index():
         temp_alias = make_alias_name(index_type, is_reindexing=True)
         if conn.indices.exists_alias(name=temp_alias):
             # Deletes both alias and backing indexes
-            conn.indices.delete(temp_alias)
+            conn.indices.delete_alias(index=INDEX_WILDCARD, name=temp_alias)
 
         # Point temp_alias toward new backing index
         conn.indices.put_alias(index=backing_index, name=temp_alias)

search/indexing_api_test.py

@@ -91,7 +91,7 @@ def search(self, index_type):
     def get_percolate_query(self, _id):
         """Get percolate query"""
         index = get_default_alias(PERCOLATE_INDEX_TYPE)
-        return self.conn.get(id=_id, index=index)
+        return self.conn.get(id=_id, doc_type=GLOBAL_DOC_TYPE, index=index)
 
     def get_mappings(self, index_type):
         """Gets mapping data"""
@@ -788,6 +788,8 @@ def test_index_percolate_query(self):
             '_id': str(percolate_query_id),
             '_index': es.get_default_backing_index(PERCOLATE_INDEX_TYPE),
             '_source': query,
+            '_seq_no': 0,
+            '_primary_term': 1,
             '_type': GLOBAL_DOC_TYPE,
             '_version': 1,
             'found': True,
@@ -802,6 +804,8 @@ def test_delete_percolate_queries(self):
                 '_id': str(percolate_query.id),
                 '_index': es.get_default_backing_index(PERCOLATE_INDEX_TYPE),
                 '_source': query,
+                '_seq_no': 0,
+                '_primary_term': 1,
                 '_type': GLOBAL_DOC_TYPE,
                 '_version': 1,
                 'found': True,

travis/Dockerfile-travis-watch

@@ -1,4 +1,4 @@
-FROM mitodl/micromasters_watch_travis_ba4c35
+FROM mitodl/micromasters_watch_travis_960cf1
 
 WORKDIR /src
 

travis/Dockerfile-travis-web

@@ -1,4 +1,4 @@
-FROM mitodl/micromasters_web_travis_ba4c35
+FROM mitodl/micromasters_web_travis_960cf1
 
 WORKDIR /tmp