From c7d579740f8bdabd4dbcdd67baf7858021b4228f Mon Sep 17 00:00:00 2001 From: tasawernawaz Date: Mon, 22 Apr 2019 16:47:40 +0500 Subject: [PATCH 1/2] bump elasticsearch version --- docker-compose.yml | 2 +- elastic/Dockerfile | 5 ----- requirements.in | 4 ++-- requirements.txt | 4 ++-- search/api.py | 17 ++++++++++++++--- search/api_test.py | 4 ++-- search/indexing_api.py | 7 +++++-- search/indexing_api_test.py | 6 +++++- travis/Dockerfile-travis-watch | 2 +- travis/Dockerfile-travis-web | 2 +- 10 files changed, 33 insertions(+), 20 deletions(-) delete mode 100644 elastic/Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 5355f2b4eb..85e9470de4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - "6379" elastic: - build: elastic + image: elasticsearch:6.7.1 command: elasticsearch -E network.host=0.0.0.0 -E http.cors.enabled=true -E http.cors.allow-origin=* -E rest.action.multi.allow_explicit_index=false ports: - "9100:9200" diff --git a/elastic/Dockerfile b/elastic/Dockerfile deleted file mode 100644 index 02e1aa3d8d..0000000000 --- a/elastic/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM elasticsearch:5.6.6 - -# Lower elasticsearch memory usage limit -RUN sed -i 's/-Xms2g/-Xms512m/' /etc/elasticsearch/jvm.options -RUN sed -i 's/-Xmx2g/-Xmx512m/' /etc/elasticsearch/jvm.options diff --git a/requirements.in b/requirements.in index c2c7fe4587..85a3752d18 100644 --- a/requirements.in +++ b/requirements.in @@ -17,8 +17,8 @@ django-webpack-loader==0.5.0 djangorestframework==3.7.7 edx-api-client==0.6.1 edx-opaque-keys==0.4 -elasticsearch-dsl==5.4.0 -elasticsearch==5.5.3 +elasticsearch-dsl==6.3.1 +elasticsearch==6.3.1 factory_boy faker html5lib==0.999999999 diff --git a/requirements.txt b/requirements.txt index f153fdb8a1..e6cd9545a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,8 +38,8 @@ djangorestframework==3.7.7 draftjs-exporter==2.0.0 # via wagtail edx-api-client==0.6.1 edx-opaque-keys==0.4 -elasticsearch-dsl==5.4.0 -elasticsearch==5.5.3 +elasticsearch-dsl==6.3.1 +elasticsearch==6.3.1 factory-boy==2.8.1 faker==0.9.1 html5lib==0.999999999 diff --git a/search/api.py b/search/api.py index bf8b978109..dafe495bb3 100644 --- a/search/api.py +++ b/search/api.py @@ -283,11 +283,22 @@ def _search_percolate_queries(program_enrollment): # We don't need this to search for percolator queries and # it causes a dynamic mapping failure so we need to remove it del doc['_id'] - result = conn.percolate(percolate_index, GLOBAL_DOC_TYPE, body={"doc": doc}) + + body = { + "query": { + "percolate": { + "field": "query", + "document": doc + } + } + } + + result = conn.search(percolate_index, GLOBAL_DOC_TYPE, body=body) failures = result.get('_shards', {}).get('failures', []) if len(failures) > 0: raise PercolateException("Failed to percolate: {}".format(failures)) - return [int(row['_id']) for row in result['matches']] + + return [int(row['_id']) for row in result['hits']['hits']] def adjust_search_for_percolator(search): @@ -333,7 +344,7 @@ def document_needs_updating(enrollment): conn = get_conn() try: - document = conn.get(index=index, id=enrollment.id) + document = conn.get(index=index, doc_type=GLOBAL_DOC_TYPE, id=enrollment.id) except NotFoundError: return True serialized_enrollment = serialize_program_enrolled_user(enrollment) diff --git a/search/api_test.py b/search/api_test.py index 1a11701e13..2b4392a505 100644 --- a/search/api_test.py +++ b/search/api_test.py @@ -398,7 +398,7 @@ def test_percolate_failure(self, mock_on_commit): profile = ProfileFactory.create(filled_out=True) program_enrollment = ProgramEnrollmentFactory.create(user=profile.user) with self.assertRaises(PercolateException) as ex, patch( - 'search.api.get_conn', return_value=Mock(percolate=Mock(return_value=failure_payload)) + 'search.api.get_conn', return_value=Mock(search=Mock(return_value=failure_payload)) ): search_percolate_queries(program_enrollment.id, "doesnt_matter") assert ex.exception.args[0] == "Failed to percolate: {}".format(failures) @@ -498,7 +498,7 @@ def test_populate_query_inactive_memberships(self, is_active, has_profile, mock_ with patch('search.api.get_conn') as es_mock: populate_query_memberships(query.id) - assert es_mock.return_value.percolate.call_count == (1 if has_profile and is_active else 0) + assert es_mock.return_value.search.call_count == (1 if has_profile and is_active else 0) assert PercolateQueryMembership.objects.filter(user=user, query=query).count() == ( 1 if is_active else 0 diff --git a/search/indexing_api.py b/search/indexing_api.py index f60767f45c..d4b093fe2d 100644 --- a/search/indexing_api.py +++ b/search/indexing_api.py @@ -3,6 +3,7 @@ """ import logging +from django.conf import settings from elasticsearch.helpers import bulk from elasticsearch.exceptions import NotFoundError @@ -230,6 +231,8 @@ } } +TEST_INDEX_WILDCARD = '{index_name}_*'.format(index_name=settings.ELASTICSEARCH_INDEX) + def _index_chunk(chunk, *, index): """ @@ -562,7 +565,7 @@ def delete_indices(): aliases = get_aliases(index_type) for alias in aliases: if conn.indices.exists(alias): - conn.indices.delete(alias) + conn.indices.delete_alias(index=TEST_INDEX_WILDCARD, name=alias) # pylint: disable=too-many-locals @@ -587,7 +590,7 @@ def recreate_index(): temp_alias = make_alias_name(index_type, is_reindexing=True) if conn.indices.exists_alias(name=temp_alias): # Deletes both alias and backing indexes - conn.indices.delete(temp_alias) + conn.indices.delete_alias(index=TEST_INDEX_WILDCARD, name=temp_alias) # Point temp_alias toward new backing index conn.indices.put_alias(index=backing_index, name=temp_alias) diff --git a/search/indexing_api_test.py b/search/indexing_api_test.py index 16450416ce..32146bb215 100644 --- a/search/indexing_api_test.py +++ b/search/indexing_api_test.py @@ -91,7 +91,7 @@ def search(self, index_type): def get_percolate_query(self, _id): """Get percolate query""" index = get_default_alias(PERCOLATE_INDEX_TYPE) - return self.conn.get(id=_id, index=index) + return self.conn.get(id=_id, doc_type=GLOBAL_DOC_TYPE, index=index) def get_mappings(self, index_type): """Gets mapping data""" @@ -788,6 +788,8 @@ def test_index_percolate_query(self): '_id': str(percolate_query_id), '_index': es.get_default_backing_index(PERCOLATE_INDEX_TYPE), '_source': query, + '_seq_no': 0, + '_primary_term': 1, '_type': GLOBAL_DOC_TYPE, '_version': 1, 'found': True, @@ -802,6 +804,8 @@ def test_delete_percolate_queries(self): '_id': str(percolate_query.id), '_index': es.get_default_backing_index(PERCOLATE_INDEX_TYPE), '_source': query, + '_seq_no': 0, + '_primary_term': 1, '_type': GLOBAL_DOC_TYPE, '_version': 1, 'found': True, diff --git a/travis/Dockerfile-travis-watch b/travis/Dockerfile-travis-watch index 162b8f1ba6..392429eed9 100644 --- a/travis/Dockerfile-travis-watch +++ b/travis/Dockerfile-travis-watch @@ -1,4 +1,4 @@ -FROM mitodl/micromasters_watch_travis_ba4c35 +FROM mitodl/micromasters_watch_travis_960cf1 WORKDIR /src diff --git a/travis/Dockerfile-travis-web b/travis/Dockerfile-travis-web index a420ea51f0..808c3358d3 100644 --- a/travis/Dockerfile-travis-web +++ b/travis/Dockerfile-travis-web @@ -1,4 +1,4 @@ -FROM mitodl/micromasters_web_travis_ba4c35 +FROM mitodl/micromasters_web_travis_960cf1 WORKDIR /tmp From 8886a8d84231feba7767b1c15fdabf69763c722a Mon Sep 17 00:00:00 2001 From: tasawernawaz Date: Tue, 30 Apr 2019 13:00:58 +0500 Subject: [PATCH 2/2] rename wiledcard --- search/indexing_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search/indexing_api.py b/search/indexing_api.py index d4b093fe2d..98c8408ba8 100644 --- a/search/indexing_api.py +++ b/search/indexing_api.py @@ -231,7 +231,7 @@ } } -TEST_INDEX_WILDCARD = '{index_name}_*'.format(index_name=settings.ELASTICSEARCH_INDEX) +INDEX_WILDCARD = '{index_name}_*'.format(index_name=settings.ELASTICSEARCH_INDEX) def _index_chunk(chunk, *, index): @@ -565,7 +565,7 @@ def delete_indices(): aliases = get_aliases(index_type) for alias in aliases: if conn.indices.exists(alias): - conn.indices.delete_alias(index=TEST_INDEX_WILDCARD, name=alias) + conn.indices.delete_alias(index=INDEX_WILDCARD, name=alias) # pylint: disable=too-many-locals @@ -590,7 +590,7 @@ def recreate_index(): temp_alias = make_alias_name(index_type, is_reindexing=True) if conn.indices.exists_alias(name=temp_alias): # Deletes both alias and backing indexes - conn.indices.delete_alias(index=TEST_INDEX_WILDCARD, name=temp_alias) + conn.indices.delete_alias(index=INDEX_WILDCARD, name=temp_alias) # Point temp_alias toward new backing index conn.indices.put_alias(index=backing_index, name=temp_alias)