Skip to content

Commit

Permalink
Upgrade from ES6 to ES8.
Browse files Browse the repository at this point in the history 
* ES URL needs to include http://
* No more doc_type (mapping types)
* response.hits is now an object rather than a number
  • Loading branch information
@nick8325
nick8325 committed Feb 13, 2024
1 parent a10c120 commit 3337a84
Show file tree
Hide file tree
Showing 7 changed files with 23 additions and 40 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ A Makefile is provided to simplify tasks.

```
export ES_ENABLED=true
export ELASTICSEARCH_HOST=localhost:9200
export ELASTICSEARCH_HOST=http://localhost:9200
```

## Create test resources
Expand Down
40 changes: 12 additions & 28 deletions karp-backend/src/karp/search_infrastructure/elasticsearch6/es_mapping_repo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
logger = logging.getLogger("karp")

KARP_CONFIGINDEX = "karp_config"
KARP_CONFIGINDEX_TYPE = "configs"


class Es6MappingRepository: # noqa: D101
Expand Down Expand Up @@ -39,12 +38,10 @@ def ensure_config_index_exist(self) -> None: # noqa: D102
"refresh_interval": -1,
},
"mappings": {
KARP_CONFIGINDEX_TYPE: {
"dynamic": False,
"properties": {
"index_name": {"type": "text"},
"alias_name": {"type": "text"},
},
"dynamic": False,
"properties": {
"index_name": {"type": "text"},
"alias_name": {"type": "text"},
}
},
},
Expand Down Expand Up @@ -72,7 +69,6 @@ def _update_config(self, resource_id: str) -> dict[str, str]:
self.es.index(
index=self._config_index,
id=resource_id,
doc_type=KARP_CONFIGINDEX_TYPE,
body=names,
)
return names
Expand All @@ -96,19 +92,15 @@ def delete_from_config(self, resource_id: str) -> None:

def get_index_name(self, resource_id: str) -> str: # noqa: D102
try:
res = self.es.get(
index=self._config_index, id=resource_id, doc_type=KARP_CONFIGINDEX_TYPE
)
res = self.es.get(index=self._config_index, id=resource_id)
except es_exceptions.NotFoundError as err:
logger.info("didn't find index_name for resource '%s' details: %s", resource_id, err)
return self._update_config(resource_id)["index_name"]
return res["_source"]["index_name"]

def get_alias_name(self, resource_id: str) -> str: # noqa: D102
try:
res = self.es.get(
index=self._config_index, id=resource_id, doc_type=KARP_CONFIGINDEX_TYPE
)
res = self.es.get(index=self._config_index, id=resource_id)
except es_exceptions.NotFoundError as err:
logger.info("didn't find alias_name for resource '%s' details: %s", resource_id, err)
return self._update_config(resource_id)["alias_name"]
Expand Down Expand Up @@ -144,16 +136,12 @@ def _init_field_mapping(
aliases = self._get_all_aliases()
mapping: Dict[str, Dict[str, Dict[str, Dict[str, Dict]]]] = self.es.indices.get_mapping()
for alias, index in aliases:
if (
"mappings" in mapping[index]
and "entry" in mapping[index]["mappings"]
and "properties" in mapping[index]["mappings"]["entry"]
):
if "mappings" in mapping[index] and "properties" in mapping[index]["mappings"]:
field_mapping[alias] = Es6MappingRepository.get_analyzed_fields_from_mapping(
mapping[index]["mappings"]["entry"]["properties"]
mapping[index]["mappings"]["properties"]
)
sortable_fields[alias] = Es6MappingRepository.create_sortable_map_from_mapping(
mapping[index]["mappings"]["entry"]["properties"]
mapping[index]["mappings"]["properties"]
)
return field_mapping, sortable_fields

Expand Down Expand Up @@ -226,20 +214,16 @@ def on_publish_resource( # noqa: ANN201, D102
self, alias_name: str, index_name: str
):
mapping = self._get_index_mappings(index=index_name)
if (
"mappings" in mapping[index_name]
and "entry" in mapping[index_name]["mappings"]
and "properties" in mapping[index_name]["mappings"]["entry"]
):
if "mappings" in mapping[index_name] and "properties" in mapping[index_name]["mappings"]:
self.analyzed_fields[
alias_name
] = Es6MappingRepository.get_analyzed_fields_from_mapping(
mapping[index_name]["mappings"]["entry"]["properties"]
mapping[index_name]["mappings"]["properties"]
)
self.sortable_fields[
alias_name
] = Es6MappingRepository.create_sortable_map_from_mapping(
mapping[index_name]["mappings"]["entry"]["properties"]
mapping[index_name]["mappings"]["properties"]
)

@staticmethod
Expand Down
9 changes: 5 additions & 4 deletions karp-backend/src/karp/search_infrastructure/queries/es6_search_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def format_entry(entry): # noqa: ANN202
}

result = {
"total": response.hits.total,
"total": response.hits.total.value,
"hits": [format_entry(entry) for entry in response],
}
return result
Expand Down Expand Up @@ -186,6 +186,7 @@ def search_with_query(self, query: EsQuery): # noqa: ANN201, D102, C901
alias_name = self.mapping_repo.get_alias_name(resource)
s = es_dsl.Search(index=alias_name)
s = self.add_runtime_mappings(s, field_names)
s = s.extra(track_total_hits=True) # get accurate hits numbers

if es_query is not None:
s = s.query(es_query)
Expand All @@ -204,11 +205,11 @@ def search_with_query(self, query: EsQuery): # noqa: ANN201, D102, C901
result["hits"][query.resources[i]] = self._format_result(
query.resources, response
).get("hits", [])
result["total"] += response.hits.total
result["total"] += response.hits.total.value
if query.lexicon_stats:
if "distribution" not in result:
result["distribution"] = {}
result["distribution"][query.resources[i]] = response.hits.total
result["distribution"][query.resources[i]] = response.hits.total.value
else:
result = self._extracted_from_search_with_query_47(query, es_query, field_names)

Expand All @@ -219,7 +220,7 @@ def _extracted_from_search_with_query_47(self, query, es_query, field_names): #
alias_names = [
self.mapping_repo.get_alias_name(resource) for resource in query.resources
]
s = es_dsl.Search(using=self.es, index=alias_names, doc_type="entry")
s = es_dsl.Search(using=self.es, index=alias_names)
s = self.add_runtime_mappings(s, field_names)
if es_query is not None:
s = s.query(es_query)
Expand Down
4 changes: 1 addition & 3 deletions karp-backend/src/karp/search_infrastructure/repositories/es6_indicies.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def create_index(self, resource_id: str, config): # noqa: ANN201, D102

body = {
"settings": settings,
"mappings": {"entry": mapping},
"mappings": mapping,
}

index_alias_name = self.mapping_repo.create_index_and_alias_name(resource_id)
Expand Down Expand Up @@ -94,7 +94,6 @@ def add_entries( # noqa: D102, ANN201
{
"_index": index_name,
"_id": entry.id,
"_type": "entry",
"_source": entry.entry,
}
)
Expand All @@ -117,7 +116,6 @@ def delete_entry( # noqa: ANN201, D102
try:
self.es.delete(
index=index_name,
doc_type="entry",
id=entry_id,
refresh=True,
)
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ aiomysql = { version = "^0.1.1", optional = true }
aiosqlite = { version = "^0.17.0", optional = true }
alembic = "^1.8.1"
asgi-correlation-id = "^3.0.1"
elasticsearch = "^6"
elasticsearch-dsl = "^6"
elasticsearch = "^8"
elasticsearch-dsl = "^8"
environs = "^9.3.4"
fastapi = "^0.89.0"
injector = "^0.20.1"
Expand Down
2 changes: 1 addition & 1 deletion tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from starlette.config import environ

environ["TESTING"] = "True"
environ["ELASTICSEARCH_HOST"] = "localhost:9202"
environ["ELASTICSEARCH_HOST"] = "http://localhost:9202"
environ["CONSOLE_LOG_LEVEL"] = "DEBUG"

from . import common_data, utils # nopep8 # noqa: E402, F401
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def sqlite_session_factory(in_memory_sqlite_db): # noqa: ANN201
def setup_environment() -> None:
os.environ["TESTING"] = "1"
os.environ["AUTH_JWT_PUBKEY_PATH"] = "assets/testing/pubkey.pem"
os.environ["ELASTICSEARCH_HOST"] = "localhost:9202"
os.environ["ELASTICSEARCH_HOST"] = "http://localhost:9202"


@pytest.fixture(scope="session")
Expand Down

0 comments on commit 3337a84

Please sign in to comment.