From e21a6992ac3b448d6be0648812e3ff4fa31f16db Mon Sep 17 00:00:00 2001 From: Michael Chouinard Date: Wed, 15 May 2024 12:25:14 -0400 Subject: [PATCH 1/3] [PROTOTYPE - DO NOT MERGE] Opensearch prototype --- api/Makefile | 11 +- api/docker-compose.yml | 46 ++++++++ api/poetry.lock | 54 +++++---- api/pyproject.toml | 1 + api/src/adapters/opensearch/__init__.py | 0 .../opensearch/populate_search_index.py | 82 +++++++++++++ .../api/schemas/extension/schema_fields.py | 6 +- .../search_opportunities.py | 108 ++++++++++++++++++ 8 files changed, 285 insertions(+), 23 deletions(-) create mode 100644 api/src/adapters/opensearch/__init__.py create mode 100644 api/src/adapters/opensearch/populate_search_index.py diff --git a/api/Makefile b/api/Makefile index f2774d3a7..119ac007d 100644 --- a/api/Makefile +++ b/api/Makefile @@ -100,7 +100,7 @@ start-debug: run-logs: start docker-compose logs --follow --no-color $(APP_NAME) -init: build init-db +init: build init-db init-opensearch clean-volumes: ## Remove project docker volumes (which includes the DB state) docker-compose down --volumes @@ -110,6 +110,15 @@ stop: check: format-check lint db-check-migrations test +################################################## +# Opensearch +################################################## + +init-opensearch: + docker-compose up --detach opensearch-node1 + docker-compose up --detach opensearch-dashboards + + ################################################## # DB & migrations ################################################## diff --git a/api/docker-compose.yml b/api/docker-compose.yml index a364c74c3..ba1df7c76 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -12,6 +12,46 @@ services: volumes: - grantsdbdata:/var/lib/postgresql/data + + opensearch-node1: + image: opensearchproject/opensearch:latest + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster # Name the cluster + - node.name=opensearch-node1 # Name the node that will run in this container + - discovery.type=single-node # Nodes to look for when discovering the cluster + - bootstrap.memory_lock=true # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM + - DISABLE_INSTALL_DEMO_CONFIG=true # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch + - DISABLE_SECURITY_PLUGIN=true # Disables Security plugin + ulimits: + memlock: + soft: -1 # Set memlock to unlimited (no soft or hard limit) + hard: -1 + nofile: + soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container + ports: + - 9200:9200 # REST API + - 9600:9600 # Performance Analyzer + networks: + - opensearch-net # All of the containers will join the same Docker bridge network + + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest + container_name: opensearch-dashboards + ports: + - 5601:5601 # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + environment: + - 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200"]' + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true # disables security dashboards plugin in OpenSearch Dashboards + networks: + - opensearch-net + grants-api: build: context: . @@ -28,6 +68,12 @@ services: - .:/api depends_on: - grants-db + - opensearch-node1 + volumes: grantsdbdata: + opensearch-data1: + +networks: + opensearch-net: \ No newline at end of file diff --git a/api/poetry.lock b/api/poetry.lock index 2b372c21e..1f6957750 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -978,7 +978,7 @@ files = [ [package.dependencies] marshmallow = [ - {version = ">=3.13.0,<4.0"}, + {version = ">=3.13.0,<4.0", optional = true, markers = "python_version < \"3.7\" or extra != \"enum\""}, {version = ">=3.18.0,<4.0", optional = true, markers = "python_version >= \"3.7\" and extra == \"enum\""}, ] typeguard = {version = ">=2.4.1,<4.0.0", optional = true, markers = "extra == \"union\""} @@ -1106,6 +1106,30 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "opensearch-py" +version = "2.5.0" +description = "Python client for OpenSearch" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<4,>=2.7" +files = [ + {file = "opensearch-py-2.5.0.tar.gz", hash = "sha256:0dde4ac7158a717d92a8cd81964cb99705a4b80bcf9258ba195b9a9f23f5226d"}, + {file = "opensearch_py-2.5.0-py2.py3-none-any.whl", hash = "sha256:cf093a40e272b60663f20417fc1264ac724dcf1e03c1a4542a6b44835b1e6c49"}, +] + +[package.dependencies] +certifi = ">=2022.12.07" +python-dateutil = "*" +requests = ">=2.4.0,<3.0.0" +six = "*" +urllib3 = ">=1.26.18,<2" + +[package.extras] +async = ["aiohttp (>=3,<4)"] +develop = ["black", "botocore", "coverage (<8.0.0)", "jinja2", "mock", "myst-parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +docs = ["aiohttp (>=3,<4)", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +kerberos = ["requests-kerberos"] + [[package]] name = "packaging" version = "24.0" @@ -1563,7 +1587,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1571,16 +1594,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1597,7 +1612,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1605,7 +1619,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1952,20 +1965,19 @@ files = [ [[package]] name = "urllib3" -version = "2.2.1" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "watchdog" @@ -2061,4 +2073,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "c53875955c1b910c3d4aa1748dce786e3cfa6f507895d7ca4111391333decb13" +content-hash = "9dbad8c12af5c08b839f7aeb13b42e14339a0e02f81e34d3d8eae31fd5541b1e" diff --git a/api/pyproject.toml b/api/pyproject.toml index f0a06b447..65253ff82 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -22,6 +22,7 @@ gunicorn = "^22.0.0" psycopg = { extras = ["binary"], version = "^3.1.10" } pydantic-settings = "^2.0.3" flask-cors = "^4.0.0" +opensearch-py = "^2.5.0" [tool.poetry.group.dev.dependencies] black = "^23.9.1" diff --git a/api/src/adapters/opensearch/__init__.py b/api/src/adapters/opensearch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/src/adapters/opensearch/populate_search_index.py b/api/src/adapters/opensearch/populate_search_index.py new file mode 100644 index 000000000..cae4c0e0c --- /dev/null +++ b/api/src/adapters/opensearch/populate_search_index.py @@ -0,0 +1,82 @@ +from opensearchpy import OpenSearch + +from src.adapters.db import PostgresDBClient +from src.api.opportunities_v0_1.opportunity_schemas import OpportunitySchema +from src.db.models.opportunity_models import Opportunity +import src.logging +import src.adapters.db as db + +INDEX = "test-opportunity-index" + +# https://opensearch.org/docs/latest/install-and-configure/configuring-opensearch/index-settings/ +# TODO - for local, we'll probably be fine with 1 shard, 1 replica +# but when we get this non-local, we'll probably want a way to configure +# them via an env var and probably set several other fields +# Until we're in AWS, probably won't need to touch this +index_settings = { + "settings": { + "index": { + # Note these are also the defaults + "number_of_shards": 1, + "number_of_replicas": 1 + } + } +} + +# +def create_index(index_name: str, opensearch_client: OpenSearch) -> None: + # TODO - more config (ie. alias) + + # hacky approach to delete and remake it + # We would not actually do it this way + exist_response = opensearch_client.indices.exists(index=index_name) + if exist_response: + opensearch_client.indices.delete(index=index_name) + + # https://opensearch.org/docs/latest/api-reference/index-apis/create-index/ + response = opensearch_client.indices.create(index=index_name, body=index_settings) + print(response) + +def insert(opensearch_client: OpenSearch, db_session: db.Session) -> None: + opportunities = db_session.query(Opportunity) + for opp in opportunities: + + # Don't index drafts or opportunities without a status + if opp.is_draft or opp.opportunity_status is None: + continue + + body = OpportunitySchema().dump(opp) + print(body) + + # TODO - use the bulk endpoint instead + # however that requires making raw JSON text - which I'll do later + opensearch_client.index(index=INDEX, body=body, id=opp.opportunity_id, refresh=True) + + +def get_client() -> OpenSearch: + # TODO - I'm certain we'll need to adjust these values (turning off auth non-locally is bad) + # TODO - make a config class for these + + # If you are running inside of docker, set the host to host.docker.internal + return OpenSearch( + hosts=[{"host": "localhost", "port": 9200}], + http_compress=True, + use_ssl=False, + verify_certs=False, + ssl_assert_hostname=False, + ssl_show_warn=False + ) + +def main(): + with src.logging.init("opensearch-load"): + opensearch_client = get_client() + + create_index(INDEX, opensearch_client) + + db_client = PostgresDBClient() + + with db_client.get_session() as db_session: + insert(opensearch_client, db_session) + + +main() \ No newline at end of file diff --git a/api/src/api/schemas/extension/schema_fields.py b/api/src/api/schemas/extension/schema_fields.py index 97b08636d..0c9587c8d 100644 --- a/api/src/api/schemas/extension/schema_fields.py +++ b/api/src/api/schemas/extension/schema_fields.py @@ -39,7 +39,11 @@ class MixinField(original_fields.Field): } def __init__(self, **kwargs: typing.Any) -> None: - super().__init__(**kwargs) + # TODO - this was needed to process the response from OpenSearch + # without configuring a bunch of fields - we might want to consider + # this in some way? I feel like allowing none by default should be the + # behavior, or at least if the field is not required. + super().__init__(allow_none=True, **kwargs) # The actual error mapping used for a specific instance self._error_mapping: dict[str, MarshmallowErrorContainer] = {} diff --git a/api/src/services/opportunities_v0_1/search_opportunities.py b/api/src/services/opportunities_v0_1/search_opportunities.py index 064534e8a..4bb517fb2 100644 --- a/api/src/services/opportunities_v0_1/search_opportunities.py +++ b/api/src/services/opportunities_v0_1/search_opportunities.py @@ -1,11 +1,14 @@ import logging +import math from typing import Any, Sequence, Tuple +from opensearchpy import OpenSearch from pydantic import BaseModel, Field from sqlalchemy import Select, asc, desc, nulls_last, or_, select from sqlalchemy.orm import InstrumentedAttribute, noload, selectinload import src.adapters.db as db +from src.api.opportunities_v0_1.opportunity_schemas import OpportunitySchema from src.db.models.opportunity_models import ( CurrentOpportunitySummary, LinkOpportunitySummaryApplicantType, @@ -187,11 +190,116 @@ def _add_order_by( return stmt.order_by(nulls_last(sort_fn(field))) +def opensearch_approach(search_params: SearchOpportunityParams) -> Tuple[Sequence[Opportunity], PaginationInfo]: + client = OpenSearch( + hosts=[{"host": "host.docker.internal", "port": 9200}], + http_compress=True, + use_ssl=False, + verify_certs=False, + ssl_assert_hostname=False, + ssl_show_warn=False + ) + + + body = { + "track_total_hits": True, # TODO - is this needed? + "size": search_params.pagination.page_size, + "from": (search_params.pagination.page_offset - 1) * search_params.pagination.page_size, + "query": {} + } + + must_filters = [] + non_scoring_filters = [] + + + if search_params.query: + must_filters.append({ + "multi_match": { + "query": search_params.query, + "fields": ["agency^16", "opportunity_title^2", "opportunity_number^12", "summary.summary_description", "opportunity_assistance_listings.assistance_listing_number^10", "opportunity_assistance_listings.program_title^4"], + "type": "best_fields", + "tie_breaker": 0.3 + } + } + ) + + + if search_params.filters: + if search_params.filters.agency: + non_scoring_filters.append({ + "terms": {"agency.keyword": search_params.filters.agency["one_of"]} + }) + + if search_params.filters.opportunity_status: + non_scoring_filters.append({ + "terms": {"opportunity_status": search_params.filters.opportunity_status["one_of"]} + }) + + if search_params.filters.applicant_type: + non_scoring_filters.append({ + "terms": {"summary.applicant_types": search_params.filters.applicant_type["one_of"]} + }) + if search_params.filters.funding_category: + non_scoring_filters.append({ + "terms": {"summary.funding_categories": search_params.filters.funding_category["one_of"]} + }) + if search_params.filters.funding_instrument: + non_scoring_filters.append({ + "terms": {"summary.funding_instruments": search_params.filters.funding_instrument["one_of"]} + }) + + body["query"]["bool"] = {} + if must_filters: + body["query"]["bool"]["must"] = must_filters + if non_scoring_filters: + body["query"]["bool"]["filter"] = non_scoring_filters + + result = client.search(body=body, index="test-opportunity-index") + print(result) + + raw_opps = [opp for opp in result["hits"]["hits"]] + + opportunities = [] + + for opp_raw in raw_opps: + + opp = opp_raw["_source"] + score = opp_raw["_score"] + # TODO - we have to attach the opportunity ID like this because the field + # isn't set by Marshmallow when loading - maybe work around that with inheritance? + opportunity = OpportunitySchema().load(opp) + opportunity["opportunity_id"] = opp["opportunity_id"] + + # TODO Hack to let me easily see scores in search + # Might be useful to just return that in the response + # and have some way to enable it to display in search results? + opportunity["opportunity_title"] = f"[{round(score, 2)}] " + opportunity["opportunity_title"] + + opportunities.append(opportunity) + + + total_records = result["hits"]["total"]["value"] + + pagination_info = PaginationInfo( + page_offset=search_params.pagination.page_offset, + page_size=search_params.pagination.page_size, + order_by=search_params.pagination.order_by, + sort_direction=search_params.pagination.sort_direction, + total_records=total_records, + # TODO - seems to be off by one + total_pages=int(math.ceil(total_records / search_params.pagination.page_size)) + ) + + return opportunities, pagination_info + def search_opportunities( db_session: db.Session, raw_search_params: dict ) -> Tuple[Sequence[Opportunity], PaginationInfo]: search_params = SearchOpportunityParams.model_validate(raw_search_params) + if True: + return opensearch_approach(search_params) + """ We create an inner query which handles all of the filtering and returns a set of opportunity IDs for the outer query to filter against. This query From f3f452617066d7bc5fe9cb22b3280ee95546f479 Mon Sep 17 00:00:00 2001 From: nava-platform-bot Date: Wed, 15 May 2024 17:09:27 +0000 Subject: [PATCH 2/3] Update OpenAPI spec --- api/openapi.generated.yml | 646 ++++++++++++++++++++++++++++---------- 1 file changed, 488 insertions(+), 158 deletions(-) diff --git a/api/openapi.generated.yml b/api/openapi.generated.yml index b6b756ae0..7ab4b9858 100644 --- a/api/openapi.generated.yml +++ b/api/openapi.generated.yml @@ -36,26 +36,36 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/Healthcheck' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: &id001 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: &id002 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Successful response '503': content: @@ -64,24 +74,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id001 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id002 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Service Unavailable tags: - Health @@ -93,7 +111,9 @@ paths: name: FF-Enable-Opportunity-Log-Msg description: Whether to log a message in the opportunity endpoint schema: - type: boolean + type: + - boolean + - 'null' required: false responses: '200': @@ -103,28 +123,38 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: type: array items: $ref: '#/components/schemas/OpportunityV0' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: &id003 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: &id004 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Successful response '422': content: @@ -133,24 +163,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id003 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id004 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Validation error '401': content: @@ -159,24 +197,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id003 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id004 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Authentication error tags: - Opportunity v0 @@ -215,28 +261,38 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: type: array items: $ref: '#/components/schemas/Opportunity' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: &id005 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: &id006 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Successful response '422': content: @@ -245,24 +301,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id005 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id006 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Validation error '401': content: @@ -271,24 +335,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id005 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id006 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Authentication error tags: - Opportunity v0.1 @@ -373,26 +445,36 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/OpportunityV0' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: &id007 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: &id008 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Successful response '401': content: @@ -401,24 +483,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id007 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id008 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Authentication error '404': content: @@ -427,24 +517,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id007 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id008 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Not found tags: - Opportunity v0 @@ -483,26 +581,36 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/Opportunity' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: &id009 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: &id010 - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Successful response '401': content: @@ -511,24 +619,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id009 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id010 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Authentication error '404': content: @@ -537,24 +653,32 @@ paths: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: $ref: '#/components/schemas/ErrorResponse' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints type: *id009 - allOf: + anyOf: - $ref: '#/components/schemas/PaginationInfo' + - type: 'null' warnings: - type: array + type: + - array + - 'null' items: type: *id010 - allOf: + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' description: Not found tags: - Opportunity v0.1 @@ -584,23 +708,33 @@ components: type: object properties: page_offset: - type: integer + type: + - integer + - 'null' description: The page number that was fetched example: 1 page_size: - type: integer + type: + - integer + - 'null' description: The size of the page fetched example: 25 total_records: - type: integer + type: + - integer + - 'null' description: The total number of records fetchable example: 42 total_pages: - type: integer + type: + - integer + - 'null' description: The total number of pages that can be fetched example: 2 order_by: - type: string + type: + - string + - 'null' description: The field that the records were sorted by example: id sort_direction: @@ -610,52 +744,75 @@ components: - descending type: - string + - 'null' + - 'null' ValidationIssue: type: object properties: type: - type: string + type: + - string + - 'null' description: The type of error message: - type: string + type: + - string + - 'null' description: The message to return field: - type: string + type: + - string + - 'null' description: The field that failed Healthcheck: type: object properties: message: - type: string + type: + - string + - 'null' ErrorResponse: type: object properties: message: - type: string + type: + - string + - 'null' description: The message to return data: description: The REST resource object + type: + - 'null' status_code: - type: integer + type: + - integer + - 'null' description: The HTTP status code errors: - type: array + type: + - array + - 'null' items: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ValidationIssue' + - type: 'null' OpportunitySorting: type: object properties: order_by: - type: string + type: + - string + - 'null' enum: - opportunity_id - agency - opportunity_number - created_at - updated_at + - null description: The field to sort the response by sort_direction: description: Whether to sort the response ascending or descending @@ -664,6 +821,8 @@ components: - descending type: - string + - 'null' + - 'null' required: - order_by - sort_direction @@ -671,12 +830,16 @@ components: type: object properties: page_size: - type: integer + type: + - integer + - 'null' minimum: 1 description: The size of the page to fetch example: 25 page_offset: - type: integer + type: + - integer + - 'null' minimum: 1 description: The page number to fetch, starts counting from 1 example: 1 @@ -687,7 +850,9 @@ components: type: object properties: opportunity_title: - type: string + type: + - string + - 'null' description: The title of the opportunity to search for example: research category: @@ -702,16 +867,22 @@ components: - O type: - string + - 'null' + - 'null' sorting: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunitySorting' + - type: 'null' paging: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/Pagination' + - type: 'null' required: - paging - sorting @@ -719,20 +890,28 @@ components: type: object properties: opportunity_id: - type: integer + type: + - integer + - 'null' readOnly: true description: The internal ID of the opportunity example: 12345 opportunity_number: - type: string + type: + - string + - 'null' description: The funding opportunity number example: ABC-123-XYZ-001 opportunity_title: - type: string + type: + - string + - 'null' description: The title of the opportunity example: Research into conservation techniques agency: - type: string + type: + - string + - 'null' description: The agency who created the opportunity example: US-ABC category: @@ -747,32 +926,46 @@ components: - O type: - string + - 'null' + - 'null' category_explanation: - type: string + type: + - string + - 'null' description: Explanation of the category when the category is 'O' (other) example: null revision_number: - type: integer + type: + - integer + - 'null' description: The current revision number of the opportunity, counting starts at 0 example: 0 modified_comments: - type: string + type: + - string + - 'null' description: Details regarding what modification was last made example: null created_at: - type: string + type: + - string + - 'null' format: date-time readOnly: true updated_at: - type: string + type: + - string + - 'null' format: date-time readOnly: true FundingInstrumentFilter: type: object properties: one_of: - type: array + type: + - array + - 'null' minItems: 1 items: enum: @@ -782,11 +975,15 @@ components: - other type: - string + - 'null' + - 'null' FundingCategoryFilter: type: object properties: one_of: - type: array + type: + - array + - 'null' minItems: 1 items: enum: @@ -818,11 +1015,15 @@ components: - other type: - string + - 'null' + - 'null' ApplicantTypeFilter: type: object properties: one_of: - type: array + type: + - array + - 'null' minItems: 1 items: enum: @@ -845,11 +1046,15 @@ components: - unrestricted type: - string + - 'null' + - 'null' OpportunityStatusFilter: type: object properties: one_of: - type: array + type: + - array + - 'null' minItems: 1 items: enum: @@ -859,14 +1064,20 @@ components: - archived type: - string + - 'null' + - 'null' AgencyFilter: type: object properties: one_of: - type: array + type: + - array + - 'null' minItems: 1 items: - type: string + type: + - string + - 'null' minLength: 2 example: US-ABC OpportunitySearchFilter: @@ -875,33 +1086,45 @@ components: funding_instrument: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/FundingInstrumentFilter' + - type: 'null' funding_category: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/FundingCategoryFilter' + - type: 'null' applicant_type: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/ApplicantTypeFilter' + - type: 'null' opportunity_status: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunityStatusFilter' + - type: 'null' agency: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/AgencyFilter' + - type: 'null' OpportunityPagination: type: object properties: order_by: - type: string + type: + - string + - 'null' enum: - opportunity_id - opportunity_number @@ -909,6 +1132,7 @@ components: - post_date - close_date - agency_code + - null description: The field to sort the response by sort_direction: description: Whether to sort the response ascending or descending @@ -917,13 +1141,19 @@ components: - descending type: - string + - 'null' + - 'null' page_size: - type: integer + type: + - integer + - 'null' minimum: 1 description: The size of the page to fetch example: 25 page_offset: - type: integer + type: + - integer + - 'null' minimum: 1 description: The page number to fetch, starts counting from 1 example: 1 @@ -936,7 +1166,9 @@ components: type: object properties: query: - type: string + type: + - string + - 'null' minLength: 1 maxLength: 100 description: Query string which searches against several text fields @@ -944,25 +1176,33 @@ components: filters: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunitySearchFilter' + - type: 'null' pagination: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunityPagination' + - type: 'null' required: - pagination OpportunityAssistanceListing: type: object properties: program_title: - type: string + type: + - string + - 'null' description: The name of the program, see https://sam.gov/content/assistance-listings for more detail example: Space Technology assistance_listing_number: - type: string + type: + - string + - 'null' description: The assistance listing number, see https://sam.gov/content/assistance-listings for more detail example: '43.012' @@ -970,123 +1210,179 @@ components: type: object properties: summary_description: - type: string + type: + - string + - 'null' description: The summary of the opportunity example: This opportunity aims to unravel the mysteries of the universe. is_cost_sharing: - type: boolean + type: + - boolean + - 'null' description: Whether or not the opportunity has a cost sharing/matching requirement is_forecast: - type: boolean + type: + - boolean + - 'null' description: Whether the opportunity is forecasted, that is, the information is only an estimate and not yet official example: false close_date: - type: string + type: + - string + - 'null' format: date description: The date that the opportunity will close - only set if is_forecast=False close_date_description: - type: string + type: + - string + - 'null' description: Optional details regarding the close date example: Proposals are due earlier than usual. post_date: - type: string + type: + - string + - 'null' format: date description: The date the opportunity was posted archive_date: - type: string + type: + - string + - 'null' format: date description: When the opportunity will be archived expected_number_of_awards: - type: integer + type: + - integer + - 'null' description: The number of awards the opportunity is expected to award example: 10 estimated_total_program_funding: - type: integer + type: + - integer + - 'null' description: The total program funding of the opportunity in US Dollars example: 10000000 award_floor: - type: integer + type: + - integer + - 'null' description: The minimum amount an opportunity would award example: 10000 award_ceiling: - type: integer + type: + - integer + - 'null' description: The maximum amount an opportunity would award example: 100000 additional_info_url: - type: string + type: + - string + - 'null' description: A URL to a website that can provide additional information about the opportunity example: grants.gov additional_info_url_description: - type: string + type: + - string + - 'null' description: The text to display for the additional_info_url link example: Click me for more info forecasted_post_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast forecasted_close_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be close once posted. forecasted_close_date_description: - type: string + type: + - string + - 'null' description: Forecasted opportunity only. Optional details regarding the forecasted closed date. example: Proposals will probably be due on this date forecasted_award_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor plans to award the opportunity. forecasted_project_start_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor expects the award recipient should start their project fiscal_year: - type: integer + type: + - integer + - 'null' description: Forecasted opportunity only. The fiscal year the project is expected to be funded and launched funding_category_description: - type: string + type: + - string + - 'null' description: Additional information about the funding category example: Economic Support applicant_eligibility_description: - type: string + type: + - string + - 'null' description: Additional information about the types of applicants that are eligible example: All types of domestic applicants are eligible to apply agency_code: - type: string + type: + - string + - 'null' description: The agency who owns the opportunity example: US-ABC agency_name: - type: string + type: + - string + - 'null' description: The name of the agency who owns the opportunity example: US Alphabetical Basic Corp agency_phone_number: - type: string + type: + - string + - 'null' description: The phone number of the agency who owns the opportunity example: 123-456-7890 agency_contact_description: - type: string + type: + - string + - 'null' description: Information regarding contacting the agency who owns the opportunity example: For more information, reach out to Jane Smith at agency US-ABC agency_email_address: - type: string + type: + - string + - 'null' description: The contact email of the agency who owns the opportunity example: fake_email@grants.gov agency_email_address_description: - type: string + type: + - string + - 'null' description: The text for the link to the agency email address example: Click me to email the agency funding_instruments: - type: array + type: + - array + - 'null' items: enum: - cooperative_agreement @@ -1095,8 +1391,12 @@ components: - other type: - string + - 'null' + - 'null' funding_categories: - type: array + type: + - array + - 'null' items: enum: - recovery_act @@ -1127,8 +1427,12 @@ components: - other type: - string + - 'null' + - 'null' applicant_types: - type: array + type: + - array + - 'null' items: enum: - state_governments @@ -1150,24 +1454,34 @@ components: - unrestricted type: - string + - 'null' + - 'null' Opportunity: type: object properties: opportunity_id: - type: integer + type: + - integer + - 'null' readOnly: true description: The internal ID of the opportunity example: 12345 opportunity_number: - type: string + type: + - string + - 'null' description: The funding opportunity number example: ABC-123-XYZ-001 opportunity_title: - type: string + type: + - string + - 'null' description: The title of the opportunity example: Research into conservation techniques agency: - type: string + type: + - string + - 'null' description: The agency who created the opportunity example: US-ABC category: @@ -1182,22 +1496,32 @@ components: - other type: - string + - 'null' + - 'null' category_explanation: - type: string + type: + - string + - 'null' description: Explanation of the category when the category is 'O' (other) example: null opportunity_assistance_listings: - type: array + type: + - array + - 'null' items: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunityAssistanceListing' + - type: 'null' summary: type: - object - allOf: + - 'null' + anyOf: - $ref: '#/components/schemas/OpportunitySummary' + - type: 'null' opportunity_status: description: The current status of the opportunity example: !!python/object/apply:src.constants.lookup_constants.OpportunityStatus @@ -1209,12 +1533,18 @@ components: - archived type: - string + - 'null' + - 'null' created_at: - type: string + type: + - string + - 'null' format: date-time readOnly: true updated_at: - type: string + type: + - string + - 'null' format: date-time readOnly: true securitySchemes: From db51054c645475de3f7c2b4e6eb283a8fed0231e Mon Sep 17 00:00:00 2001 From: Michael Chouinard Date: Thu, 16 May 2024 11:24:30 -0400 Subject: [PATCH 3/3] Just a few updates to the query box --- api/src/adapters/opensearch/populate_search_index.py | 1 - .../services/opportunities_v0_1/search_opportunities.py | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/api/src/adapters/opensearch/populate_search_index.py b/api/src/adapters/opensearch/populate_search_index.py index cae4c0e0c..6c9f58ed3 100644 --- a/api/src/adapters/opensearch/populate_search_index.py +++ b/api/src/adapters/opensearch/populate_search_index.py @@ -46,7 +46,6 @@ def insert(opensearch_client: OpenSearch, db_session: db.Session) -> None: continue body = OpportunitySchema().dump(opp) - print(body) # TODO - use the bulk endpoint instead # however that requires making raw JSON text - which I'll do later diff --git a/api/src/services/opportunities_v0_1/search_opportunities.py b/api/src/services/opportunities_v0_1/search_opportunities.py index 4bb517fb2..c6eef2fd2 100644 --- a/api/src/services/opportunities_v0_1/search_opportunities.py +++ b/api/src/services/opportunities_v0_1/search_opportunities.py @@ -214,11 +214,10 @@ def opensearch_approach(search_params: SearchOpportunityParams) -> Tuple[Sequenc if search_params.query: must_filters.append({ - "multi_match": { + "simple_query_string": { "query": search_params.query, - "fields": ["agency^16", "opportunity_title^2", "opportunity_number^12", "summary.summary_description", "opportunity_assistance_listings.assistance_listing_number^10", "opportunity_assistance_listings.program_title^4"], - "type": "best_fields", - "tie_breaker": 0.3 + "default_operator": "AND", + "fields": ["agency.keyword^16", "opportunity_title^2", "opportunity_number^12", "summary.summary_description", "opportunity_assistance_listings.assistance_listing_number^10", "opportunity_assistance_listings.program_title^4"], } } ) @@ -254,8 +253,8 @@ def opensearch_approach(search_params: SearchOpportunityParams) -> Tuple[Sequenc if non_scoring_filters: body["query"]["bool"]["filter"] = non_scoring_filters + print(body) result = client.search(body=body, index="test-opportunity-index") - print(result) raw_opps = [opp for opp in result["hits"]["hits"]]