From c0966aab019abd2c1a7148b89d664bc6246facbc Mon Sep 17 00:00:00 2001 From: Michael Chouinard <46358556+chouinar@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:07:36 -0400 Subject: [PATCH] [Issue #2084] Connect the API to use the search index (navapbc/simpler-grants-gov#63) Fixes #2084 Make the v1 search opportunity endpoint connect to the search index and return results. Adjust the structure of the response to be more flexible going forward. The actual building of the search request / parsing the response is pretty simple. Other than having to map some field names, that logic is mostly contained in the builder I made in the prior PR. However, there is a lot of configuration and other API components that had to be modified as part of this including: * Adjusting the API response schema (to better support facet counts) * Piping through the search client + index alias name configuration. * A monumental amount of test cases to verify everything is connected / behavior works in a way we expect - note that I did not test relevancy as that'll break anytime we adjust something. Note that the change in API schema means the API does not work with the frontend, but there are a few hacky changes you can make to connect them. In [BaseApi.ts](https://github.com/navapbc/simpler-grants-gov/blob/main/frontend/src/app/api/BaseApi.ts#L47) change the version to `v1`. In [SearchOpportunityAPI.ts](https://github.com/navapbc/simpler-grants-gov/blob/main/frontend/src/app/api/SearchOpportunityAPI.ts#L56) add `response.data = response.data.opportunities;` to the end of the `searchOpportunities` method. With that, the local frontend will work. To actually get everything running locally, you can run: ```sh make db-recreate make init make db-seed-local args="--iterations 10" poetry run flask load-search-data load-opportunity-data make run-logs npm run dev ``` Then go to http://localhost:3000/search --------- Co-authored-by: nava-platform-bot --- api/openapi.generated.yml | 248 +++++- api/src/adapters/search/__init__.py | 3 +- api/src/adapters/search/flask_opensearch.py | 47 ++ api/src/adapters/search/opensearch_client.py | 5 +- .../opportunities_v1/opportunity_routes.py | 75 +- .../opportunities_v1/opportunity_schemas.py | 155 +++- api/src/api/response.py | 1 + .../api/schemas/extension/schema_fields.py | 8 +- api/src/app.py | 8 + .../backend/load_opportunities_to_index.py | 4 +- api/src/search/search_config.py | 19 + .../opportunities_v1/search_opportunities.py | 134 ++- api/tests/conftest.py | 13 +- .../src/api/opportunities_v1/conftest.py | 4 +- .../test_opportunity_route_search.py | 771 +++++++++++++++++- 15 files changed, 1375 insertions(+), 120 deletions(-) create mode 100644 api/src/adapters/search/flask_opensearch.py create mode 100644 api/src/search/search_config.py diff --git a/api/openapi.generated.yml b/api/openapi.generated.yml index c05f960ef..b17a53122 100644 --- a/api/openapi.generated.yml +++ b/api/openapi.generated.yml @@ -145,6 +145,61 @@ paths: application/json: schema: $ref: '#/components/schemas/OpportunitySearchRequestV1' + examples: + example1: + summary: No filters + value: + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: ascending + example2: + summary: All filters + value: + query: research + filters: + agency: + one_of: + - USAID + - ARPAH + applicant_type: + one_of: + - state_governments + - county_governments + - individuals + funding_category: + one_of: + - recovery_act + - arts + - natural_resources + funding_instrument: + one_of: + - cooperative_agreement + - grant + opportunity_status: + one_of: + - forecasted + - posted + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: descending + example3: + summary: Query & opportunity_status filters + value: + query: research + filters: + opportunity_status: + one_of: + - forecasted + - posted + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: descending security: - ApiKeyAuth: [] /v0.1/opportunities/search: @@ -755,7 +810,7 @@ components: items: type: string minLength: 2 - example: US-ABC + example: USAID OpportunitySearchFilterV1: type: object properties: @@ -784,12 +839,13 @@ components: - object allOf: - $ref: '#/components/schemas/AgencyFilterV1' - OpportunityPagination: + OpportunityPaginationV1: type: object properties: order_by: type: string enum: + - relevancy - opportunity_id - opportunity_number - opportunity_title @@ -837,19 +893,23 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunityPagination' + - $ref: '#/components/schemas/OpportunityPaginationV1' required: - pagination OpportunityAssistanceListingV1: type: object properties: program_title: - type: string + type: + - string + - 'null' description: The name of the program, see https://sam.gov/content/assistance-listings for more detail example: Space Technology assistance_listing_number: - type: string + type: + - string + - 'null' description: The assistance listing number, see https://sam.gov/content/assistance-listings for more detail example: '43.012' @@ -857,11 +917,15 @@ components: type: object properties: summary_description: - type: string + type: + - string + - 'null' description: The summary of the opportunity example: This opportunity aims to unravel the mysteries of the universe. is_cost_sharing: - type: boolean + type: + - boolean + - 'null' description: Whether or not the opportunity has a cost sharing/matching requirement is_forecast: @@ -870,106 +934,154 @@ components: is only an estimate and not yet official example: false close_date: - type: string + type: + - string + - 'null' format: date description: The date that the opportunity will close - only set if is_forecast=False close_date_description: - type: string + type: + - string + - 'null' description: Optional details regarding the close date example: Proposals are due earlier than usual. post_date: - type: string + type: + - string + - 'null' format: date description: The date the opportunity was posted archive_date: - type: string + type: + - string + - 'null' format: date description: When the opportunity will be archived expected_number_of_awards: - type: integer + type: + - integer + - 'null' description: The number of awards the opportunity is expected to award example: 10 estimated_total_program_funding: - type: integer + type: + - integer + - 'null' description: The total program funding of the opportunity in US Dollars example: 10000000 award_floor: - type: integer + type: + - integer + - 'null' description: The minimum amount an opportunity would award example: 10000 award_ceiling: - type: integer + type: + - integer + - 'null' description: The maximum amount an opportunity would award example: 100000 additional_info_url: - type: string + type: + - string + - 'null' description: A URL to a website that can provide additional information about the opportunity example: grants.gov additional_info_url_description: - type: string + type: + - string + - 'null' description: The text to display for the additional_info_url link example: Click me for more info forecasted_post_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast forecasted_close_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be close once posted. forecasted_close_date_description: - type: string + type: + - string + - 'null' description: Forecasted opportunity only. Optional details regarding the forecasted closed date. example: Proposals will probably be due on this date forecasted_award_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor plans to award the opportunity. forecasted_project_start_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor expects the award recipient should start their project fiscal_year: - type: integer + type: + - integer + - 'null' description: Forecasted opportunity only. The fiscal year the project is expected to be funded and launched funding_category_description: - type: string + type: + - string + - 'null' description: Additional information about the funding category example: Economic Support applicant_eligibility_description: - type: string + type: + - string + - 'null' description: Additional information about the types of applicants that are eligible example: All types of domestic applicants are eligible to apply agency_code: - type: string + type: + - string + - 'null' description: The agency who owns the opportunity example: US-ABC agency_name: - type: string + type: + - string + - 'null' description: The name of the agency who owns the opportunity example: US Alphabetical Basic Corp agency_phone_number: - type: string + type: + - string + - 'null' description: The phone number of the agency who owns the opportunity example: 123-456-7890 agency_contact_description: - type: string + type: + - string + - 'null' description: Information regarding contacting the agency who owns the opportunity example: For more information, reach out to Jane Smith at agency US-ABC agency_email_address: - type: string + type: + - string + - 'null' description: The contact email of the agency who owns the opportunity example: fake_email@grants.gov agency_email_address_description: - type: string + type: + - string + - 'null' description: The text for the link to the agency email address example: Click me to email the agency version_number: @@ -1046,19 +1158,24 @@ components: properties: opportunity_id: type: integer - readOnly: true description: The internal ID of the opportunity example: 12345 opportunity_number: - type: string + type: + - string + - 'null' description: The funding opportunity number example: ABC-123-XYZ-001 opportunity_title: - type: string + type: + - string + - 'null' description: The title of the opportunity example: Research into conservation techniques agency: - type: string + type: + - string + - 'null' description: The agency who created the opportunity example: US-ABC category: @@ -1073,8 +1190,12 @@ components: - other type: - string + - 'null' + - 'null' category_explanation: - type: string + type: + - string + - 'null' description: Explanation of the category when the category is 'O' (other) example: null opportunity_assistance_listings: @@ -1108,6 +1229,51 @@ components: type: string format: date-time readOnly: true + OpportunityFacetV1: + type: object + properties: + opportunity_status: + type: object + description: The counts of opportunity_status values in the full response + example: + posted: 1 + forecasted: 2 + additionalProperties: + type: integer + applicant_type: + type: object + description: The counts of applicant_type values in the full response + example: + state_governments: 3 + county_governments: 2 + city_or_township_governments: 1 + additionalProperties: + type: integer + funding_instrument: + type: object + description: The counts of funding_instrument values in the full response + example: + cooperative_agreement: 4 + grant: 3 + additionalProperties: + type: integer + funding_category: + type: object + description: The counts of funding_category values in the full response + example: + recovery_act: 2 + arts: 3 + agriculture: 5 + additionalProperties: + type: integer + agency: + type: object + description: The counts of agency values in the full response + example: + USAID: 4 + ARPAH: 3 + additionalProperties: + type: integer OpportunitySearchResponseV1: type: object properties: @@ -1128,6 +1294,12 @@ components: type: integer description: The HTTP status code example: 200 + facet_counts: + description: Counts of filter/facet values in the full response + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunityFacetV1' FundingInstrumentFilterV01: type: object properties: @@ -1257,7 +1429,7 @@ components: - object allOf: - $ref: '#/components/schemas/AgencyFilterV01' - OpportunityPagination1: + OpportunityPagination: type: object properties: order_by: @@ -1310,7 +1482,7 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunityPagination1' + - $ref: '#/components/schemas/OpportunityPagination' required: - pagination OpportunityAssistanceListingV01: diff --git a/api/src/adapters/search/__init__.py b/api/src/adapters/search/__init__.py index 6b2607a04..c44446964 100644 --- a/api/src/adapters/search/__init__.py +++ b/api/src/adapters/search/__init__.py @@ -1,4 +1,5 @@ from src.adapters.search.opensearch_client import SearchClient from src.adapters.search.opensearch_config import get_opensearch_config +from src.adapters.search.opensearch_query_builder import SearchQueryBuilder -__all__ = ["SearchClient", "get_opensearch_config"] +__all__ = ["SearchClient", "get_opensearch_config", "SearchQueryBuilder"] diff --git a/api/src/adapters/search/flask_opensearch.py b/api/src/adapters/search/flask_opensearch.py new file mode 100644 index 000000000..0fa195456 --- /dev/null +++ b/api/src/adapters/search/flask_opensearch.py @@ -0,0 +1,47 @@ +from functools import wraps +from typing import Callable, Concatenate, ParamSpec, TypeVar + +from flask import Flask, current_app + +from src.adapters.search import SearchClient + +_SEARCH_CLIENT_KEY = "search-client" + + +def register_search_client(search_client: SearchClient, app: Flask) -> None: + app.extensions[_SEARCH_CLIENT_KEY] = search_client + + +def get_search_client(app: Flask) -> SearchClient: + return app.extensions[_SEARCH_CLIENT_KEY] + + +P = ParamSpec("P") +T = TypeVar("T") + + +def with_search_client() -> Callable[[Callable[Concatenate[SearchClient, P], T]], Callable[P, T]]: + """ + Decorator for functions that need a search client. + + This decorator will return the shared search client object which + has an internal connection pool that is shared. + + Usage: + @with_search_client() + def foo(search_client: search.SearchClient): + ... + + @with_search_client() + def bar(search_client: search.SearchClient, x: int, y: int): + ... + """ + + def decorator(f: Callable[Concatenate[SearchClient, P], T]) -> Callable[P, T]: + @wraps(f) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + return f(get_search_client(current_app), *args, **kwargs) + + return wrapper + + return decorator diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index b2e5b2bea..cb97a9c8c 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -15,10 +15,7 @@ "default": { "type": "custom", "filter": ["lowercase", "custom_stemmer"], - # Change tokenization to whitespace as the default is very clunky - # with a lot of our IDs that have dashes in them. - # see: https://opensearch.org/docs/latest/analyzers/tokenizers/index/ - "tokenizer": "whitespace", + "tokenizer": "standard", } }, # Change the default stemming to use snowball which handles plural diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index d2cdd2490..4bb4484d6 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -2,6 +2,8 @@ import src.adapters.db as db import src.adapters.db.flask_db as flask_db +import src.adapters.search as search +import src.adapters.search.flask_opensearch as flask_opensearch import src.api.opportunities_v1.opportunity_schemas as opportunity_schemas import src.api.response as response from src.api.opportunities_v1.opportunity_blueprint import opportunity_blueprint @@ -24,20 +26,76 @@ See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) for further details. """ +examples = { + "example1": { + "summary": "No filters", + "value": { + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "ascending", + }, + }, + }, + "example2": { + "summary": "All filters", + "value": { + "query": "research", + "filters": { + "agency": {"one_of": ["USAID", "ARPAH"]}, + "applicant_type": { + "one_of": ["state_governments", "county_governments", "individuals"] + }, + "funding_category": {"one_of": ["recovery_act", "arts", "natural_resources"]}, + "funding_instrument": {"one_of": ["cooperative_agreement", "grant"]}, + "opportunity_status": {"one_of": ["forecasted", "posted"]}, + }, + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "descending", + }, + }, + }, + "example3": { + "summary": "Query & opportunity_status filters", + "value": { + "query": "research", + "filters": { + "opportunity_status": {"one_of": ["forecasted", "posted"]}, + }, + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "descending", + }, + }, + }, +} + @opportunity_blueprint.post("/opportunities/search") @opportunity_blueprint.input( - opportunity_schemas.OpportunitySearchRequestV1Schema, arg_name="search_params" + opportunity_schemas.OpportunitySearchRequestV1Schema, + arg_name="search_params", + examples=examples, ) -# many=True allows us to return a list of opportunity objects -@opportunity_blueprint.output(opportunity_schemas.OpportunitySearchResponseV1Schema) +@opportunity_blueprint.output(opportunity_schemas.OpportunitySearchResponseV1Schema()) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) -def opportunity_search(search_params: dict) -> response.ApiResponse: +@flask_opensearch.with_search_client() +def opportunity_search( + search_client: search.SearchClient, search_params: dict +) -> response.ApiResponse: add_extra_data_to_current_request_logs(flatten_dict(search_params, prefix="request.body")) logger.info("POST /v1/opportunities/search") - opportunities, pagination_info = search_opportunities(search_params) + opportunities, aggregations, pagination_info = search_opportunities( + search_client, search_params + ) add_extra_data_to_current_request_logs( { @@ -48,12 +106,15 @@ def opportunity_search(search_params: dict) -> response.ApiResponse: logger.info("Successfully fetched opportunities") return response.ApiResponse( - message="Success", data=opportunities, pagination_info=pagination_info + message="Success", + data=opportunities, + facet_counts=aggregations, + pagination_info=pagination_info, ) @opportunity_blueprint.get("/opportunities/") -@opportunity_blueprint.output(opportunity_schemas.OpportunityGetResponseV1Schema) +@opportunity_blueprint.output(opportunity_schemas.OpportunityGetResponseV1Schema()) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) @flask_db.with_db_session() diff --git a/api/src/api/opportunities_v1/opportunity_schemas.py b/api/src/api/opportunities_v1/opportunity_schemas.py index 3c071f260..58a6dac36 100644 --- a/api/src/api/opportunities_v1/opportunity_schemas.py +++ b/api/src/api/opportunities_v1/opportunity_schemas.py @@ -13,15 +13,17 @@ class OpportunitySummaryV1Schema(Schema): summary_description = fields.String( + allow_none=True, metadata={ "description": "The summary of the opportunity", "example": "This opportunity aims to unravel the mysteries of the universe.", - } + }, ) is_cost_sharing = fields.Boolean( + allow_none=True, metadata={ "description": "Whether or not the opportunity has a cost sharing/matching requirement", - } + }, ) is_forecast = fields.Boolean( metadata={ @@ -31,147 +33,171 @@ class OpportunitySummaryV1Schema(Schema): ) close_date = fields.Date( + allow_none=True, metadata={ "description": "The date that the opportunity will close - only set if is_forecast=False", - } + }, ) close_date_description = fields.String( + allow_none=True, metadata={ "description": "Optional details regarding the close date", "example": "Proposals are due earlier than usual.", - } + }, ) post_date = fields.Date( + allow_none=True, metadata={ "description": "The date the opportunity was posted", - } + }, ) archive_date = fields.Date( + allow_none=True, metadata={ "description": "When the opportunity will be archived", - } + }, ) # not including unarchive date at the moment expected_number_of_awards = fields.Integer( + allow_none=True, metadata={ "description": "The number of awards the opportunity is expected to award", "example": 10, - } + }, ) estimated_total_program_funding = fields.Integer( + allow_none=True, metadata={ "description": "The total program funding of the opportunity in US Dollars", "example": 10_000_000, - } + }, ) award_floor = fields.Integer( + allow_none=True, metadata={ "description": "The minimum amount an opportunity would award", "example": 10_000, - } + }, ) award_ceiling = fields.Integer( + allow_none=True, metadata={ "description": "The maximum amount an opportunity would award", "example": 100_000, - } + }, ) additional_info_url = fields.String( + allow_none=True, metadata={ "description": "A URL to a website that can provide additional information about the opportunity", "example": "grants.gov", - } + }, ) additional_info_url_description = fields.String( + allow_none=True, metadata={ "description": "The text to display for the additional_info_url link", "example": "Click me for more info", - } + }, ) forecasted_post_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast" - } + }, ) forecasted_close_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the opportunity is expected to be close once posted." - } + }, ) forecasted_close_date_description = fields.String( + allow_none=True, metadata={ "description": "Forecasted opportunity only. Optional details regarding the forecasted closed date.", "example": "Proposals will probably be due on this date", - } + }, ) forecasted_award_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the grantor plans to award the opportunity." - } + }, ) forecasted_project_start_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the grantor expects the award recipient should start their project" - } + }, ) fiscal_year = fields.Integer( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The fiscal year the project is expected to be funded and launched" - } + }, ) funding_category_description = fields.String( + allow_none=True, metadata={ "description": "Additional information about the funding category", "example": "Economic Support", - } + }, ) applicant_eligibility_description = fields.String( + allow_none=True, metadata={ "description": "Additional information about the types of applicants that are eligible", "example": "All types of domestic applicants are eligible to apply", - } + }, ) agency_code = fields.String( + allow_none=True, metadata={ "description": "The agency who owns the opportunity", "example": "US-ABC", - } + }, ) agency_name = fields.String( + allow_none=True, metadata={ "description": "The name of the agency who owns the opportunity", "example": "US Alphabetical Basic Corp", - } + }, ) agency_phone_number = fields.String( + allow_none=True, metadata={ "description": "The phone number of the agency who owns the opportunity", "example": "123-456-7890", - } + }, ) agency_contact_description = fields.String( + allow_none=True, metadata={ "description": "Information regarding contacting the agency who owns the opportunity", "example": "For more information, reach out to Jane Smith at agency US-ABC", - } + }, ) agency_email_address = fields.String( + allow_none=True, metadata={ "description": "The contact email of the agency who owns the opportunity", "example": "fake_email@grants.gov", - } + }, ) agency_email_address_description = fields.String( + allow_none=True, metadata={ "description": "The text for the link to the agency email address", "example": "Click me to email the agency", - } + }, ) version_number = fields.Integer( @@ -185,50 +211,56 @@ class OpportunitySummaryV1Schema(Schema): class OpportunityAssistanceListingV1Schema(Schema): program_title = fields.String( + allow_none=True, metadata={ "description": "The name of the program, see https://sam.gov/content/assistance-listings for more detail", "example": "Space Technology", - } + }, ) assistance_listing_number = fields.String( + allow_none=True, metadata={ "description": "The assistance listing number, see https://sam.gov/content/assistance-listings for more detail", "example": "43.012", - } + }, ) class OpportunityV1Schema(Schema): opportunity_id = fields.Integer( - dump_only=True, metadata={"description": "The internal ID of the opportunity", "example": 12345}, ) opportunity_number = fields.String( - metadata={"description": "The funding opportunity number", "example": "ABC-123-XYZ-001"} + allow_none=True, + metadata={"description": "The funding opportunity number", "example": "ABC-123-XYZ-001"}, ) opportunity_title = fields.String( + allow_none=True, metadata={ "description": "The title of the opportunity", "example": "Research into conservation techniques", - } + }, ) agency = fields.String( - metadata={"description": "The agency who created the opportunity", "example": "US-ABC"} + allow_none=True, + metadata={"description": "The agency who created the opportunity", "example": "US-ABC"}, ) category = fields.Enum( OpportunityCategory, + allow_none=True, metadata={ "description": "The opportunity category", "example": OpportunityCategory.DISCRETIONARY, }, ) category_explanation = fields.String( + allow_none=True, metadata={ "description": "Explanation of the category when the category is 'O' (other)", "example": None, - } + }, ) opportunity_assistance_listings = fields.List( @@ -271,11 +303,58 @@ class OpportunitySearchFilterV1Schema(Schema): ) agency = fields.Nested( StrSearchSchemaBuilder("AgencyFilterV1Schema") - .with_one_of(example="US-ABC", minimum_length=2) + .with_one_of(example="USAID", minimum_length=2) .build() ) +class OpportunityFacetV1Schema(Schema): + opportunity_status = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of opportunity_status values in the full response", + "example": {"posted": 1, "forecasted": 2}, + }, + ) + applicant_type = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of applicant_type values in the full response", + "example": { + "state_governments": 3, + "county_governments": 2, + "city_or_township_governments": 1, + }, + }, + ) + funding_instrument = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of funding_instrument values in the full response", + "example": {"cooperative_agreement": 4, "grant": 3}, + }, + ) + funding_category = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of funding_category values in the full response", + "example": {"recovery_act": 2, "arts": 3, "agriculture": 5}, + }, + ) + agency = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of agency values in the full response", + "example": {"USAID": 4, "ARPAH": 3}, + }, + ) + + class OpportunitySearchRequestV1Schema(Schema): query = fields.String( metadata={ @@ -289,8 +368,9 @@ class OpportunitySearchRequestV1Schema(Schema): pagination = fields.Nested( generate_pagination_schema( - "OpportunityPaginationSchema", + "OpportunityPaginationV1Schema", [ + "relevancy", "opportunity_id", "opportunity_number", "opportunity_title", @@ -319,3 +399,8 @@ class OpportunityVersionsGetResponseV1Schema(AbstractResponseSchema): class OpportunitySearchResponseV1Schema(AbstractResponseSchema, PaginationMixinSchema): data = fields.Nested(OpportunityV1Schema(many=True)) + + facet_counts = fields.Nested( + OpportunityFacetV1Schema(), + metadata={"description": "Counts of filter/facet values in the full response"}, + ) diff --git a/api/src/api/response.py b/api/src/api/response.py index 0990dc120..f85fc0f7c 100644 --- a/api/src/api/response.py +++ b/api/src/api/response.py @@ -45,6 +45,7 @@ class ApiResponse: status_code: int = 200 pagination_info: PaginationInfo | None = None + facet_counts: dict | None = None def process_marshmallow_issues(marshmallow_issues: dict) -> list[ValidationErrorDetail]: diff --git a/api/src/api/schemas/extension/schema_fields.py b/api/src/api/schemas/extension/schema_fields.py index 97b08636d..8431c3ecd 100644 --- a/api/src/api/schemas/extension/schema_fields.py +++ b/api/src/api/schemas/extension/schema_fields.py @@ -183,6 +183,12 @@ class Raw(original_fields.Raw, MixinField): pass +class Dict(original_fields.Dict, MixinField): + error_mapping: dict[str, MarshmallowErrorContainer] = { + "invalid": MarshmallowErrorContainer(ValidationErrorType.INVALID, "Not a valid dict."), + } + + class Enum(MixinField): """ Custom field class for handling unioning together multiple Python enums into @@ -230,7 +236,7 @@ def _serialize( if value is None: return None - val = value.value + val = value return self.field._serialize(val, attr, obj, **kwargs) def _deserialize( diff --git a/api/src/app.py b/api/src/app.py index 98fea1607..ad79d1810 100644 --- a/api/src/app.py +++ b/api/src/app.py @@ -7,6 +7,8 @@ import src.adapters.db as db import src.adapters.db.flask_db as flask_db +import src.adapters.search as search +import src.adapters.search.flask_opensearch as flask_opensearch import src.api.feature_flags.feature_flag_config as feature_flag_config import src.logging import src.logging.flask_logger as flask_logger @@ -47,6 +49,7 @@ def create_app() -> APIFlask: configure_app(app) register_blueprints(app) register_index(app) + register_search_client(app) return app @@ -61,6 +64,11 @@ def register_db_client(app: APIFlask) -> None: flask_db.register_db_client(db_client, app) +def register_search_client(app: APIFlask) -> None: + search_client = search.SearchClient() + flask_opensearch.register_search_client(search_client, app) + + def configure_app(app: APIFlask) -> None: app_config = AppConfig() diff --git a/api/src/search/backend/load_opportunities_to_index.py b/api/src/search/backend/load_opportunities_to_index.py index a01357a96..630ecf616 100644 --- a/api/src/search/backend/load_opportunities_to_index.py +++ b/api/src/search/backend/load_opportunities_to_index.py @@ -9,7 +9,7 @@ import src.adapters.db as db import src.adapters.search as search -from src.api.opportunities_v0_1.opportunity_schemas import OpportunityV01Schema +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema from src.db.models.opportunity_models import CurrentOpportunitySummary, Opportunity from src.task.task import Task from src.util.datetime_util import get_now_us_eastern_datetime @@ -95,7 +95,7 @@ def fetch_opportunities(self) -> Iterator[Sequence[Opportunity]]: def load_records(self, records: Sequence[Opportunity]) -> None: logger.info("Loading batch of opportunities...") - schema = OpportunityV01Schema() + schema = OpportunityV1Schema() json_records = [] for record in records: diff --git a/api/src/search/search_config.py b/api/src/search/search_config.py new file mode 100644 index 000000000..8b7ea4f29 --- /dev/null +++ b/api/src/search/search_config.py @@ -0,0 +1,19 @@ +from pydantic import Field + +from src.util.env_config import PydanticBaseEnvConfig + + +class SearchConfig(PydanticBaseEnvConfig): + opportunity_search_index_alias: str = Field(default="opportunity-index-alias") + + +_search_config: SearchConfig | None = None + + +def get_search_config() -> SearchConfig: + global _search_config + + if _search_config is None: + _search_config = SearchConfig() + + return _search_config diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 1823bc31d..92a71344c 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -1,39 +1,147 @@ import logging +import math from typing import Sequence, Tuple from pydantic import BaseModel, Field -from src.db.models.opportunity_models import Opportunity -from src.pagination.pagination_models import PaginationInfo, PaginationParams +import src.adapters.search as search +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema +from src.pagination.pagination_models import PaginationInfo, PaginationParams, SortDirection +from src.search.search_config import get_search_config logger = logging.getLogger(__name__) +# To assist with mapping field names from our API requests +# to what they are called in the search index, this mapping +# can be used. Note that in many cases its just adjusting paths +# or for text based fields adding ".keyword" to the end to tell +# the query we want to use the raw value rather than the tokenized one +# See: https://opensearch.org/docs/latest/field-types/supported-field-types/keyword/ +REQUEST_FIELD_NAME_MAPPING = { + "opportunity_number": "opportunity_number.keyword", + "opportunity_title": "opportunity_title.keyword", + "post_date": "summary.post_date", + "close_date": "summary.close_date", + "agency_code": "agency.keyword", + "agency": "agency.keyword", + "opportunity_status": "opportunity_status.keyword", + "funding_instrument": "summary.funding_instruments.keyword", + "funding_category": "summary.funding_categories.keyword", + "applicant_type": "summary.applicant_types.keyword", +} -class SearchOpportunityFilters(BaseModel): - funding_instrument: dict | None = Field(default=None) - funding_category: dict | None = Field(default=None) - applicant_type: dict | None = Field(default=None) - opportunity_status: dict | None = Field(default=None) - agency: dict | None = Field(default=None) +SEARCH_FIELDS = [ + # Note that we do keyword for agency & opportunity number + # as we don't want to compare to a tokenized value which + # may have split on the dashes. + "agency.keyword^16", + "opportunity_title^2", + "opportunity_number.keyword^12", + "summary.summary_description", + "opportunity_assistance_listings.assistance_listing_number^10", + "opportunity_assistance_listings.program_title^4", +] + +SCHEMA = OpportunityV1Schema() class SearchOpportunityParams(BaseModel): pagination: PaginationParams query: str | None = Field(default=None) - filters: SearchOpportunityFilters | None = Field(default=None) + filters: dict | None = Field(default=None) + + +def _adjust_field_name(field: str) -> str: + return REQUEST_FIELD_NAME_MAPPING.get(field, field) + + +def _get_sort_by(pagination: PaginationParams) -> list[tuple[str, SortDirection]]: + sort_by: list[tuple[str, SortDirection]] = [] + + sort_by.append((_adjust_field_name(pagination.order_by), pagination.sort_direction)) + + # Add a secondary sort for relevancy to sort by post date (matching the sort direction) + if pagination.order_by == "relevancy": + sort_by.append((_adjust_field_name("post_date"), pagination.sort_direction)) + + return sort_by + + +def _add_search_filters(builder: search.SearchQueryBuilder, filters: dict | None) -> None: + if filters is None: + return + + for field, field_filters in filters.items(): + # one_of filters translate to an opensearch term filter + # see: https://opensearch.org/docs/latest/query-dsl/term/terms/ + one_of_filters = field_filters.get("one_of", None) + if one_of_filters: + builder.filter_terms(_adjust_field_name(field), one_of_filters) + + +def _add_aggregations(builder: search.SearchQueryBuilder) -> None: + # TODO - we'll likely want to adjust the total number of values returned, especially + # for agency as there could be hundreds of different agencies, and currently it's limited to 25. + builder.aggregation_terms("opportunity_status", _adjust_field_name("applicant_types")) + builder.aggregation_terms("applicant_type", _adjust_field_name("applicant_types")) + builder.aggregation_terms("funding_instrument", _adjust_field_name("funding_instruments")) + builder.aggregation_terms("funding_category", _adjust_field_name("funding_categories")) + builder.aggregation_terms("agency", _adjust_field_name("agency_code")) + +def _get_search_request(params: SearchOpportunityParams) -> dict: + builder = search.SearchQueryBuilder() -def search_opportunities(raw_search_params: dict) -> Tuple[Sequence[Opportunity], PaginationInfo]: + # Pagination + builder.pagination( + page_size=params.pagination.page_size, page_number=params.pagination.page_offset + ) + + # Sorting + builder.sort_by(_get_sort_by(params.pagination)) + + # Query + if params.query: + builder.simple_query(params.query, SEARCH_FIELDS) + + # Filters + _add_search_filters(builder, params.filters) + + # Aggregations / Facet / Filter Counts + _add_aggregations(builder) + + return builder.build() + + +def search_opportunities( + search_client: search.SearchClient, raw_search_params: dict +) -> Tuple[Sequence[dict], dict, PaginationInfo]: search_params = SearchOpportunityParams.model_validate(raw_search_params) + search_request = _get_search_request(search_params) + + index_alias = get_search_config().opportunity_search_index_alias + logger.info( + "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} + ) + + response = search_client.search(index_alias, search_request) + pagination_info = PaginationInfo( page_offset=search_params.pagination.page_offset, page_size=search_params.pagination.page_size, order_by=search_params.pagination.order_by, sort_direction=search_params.pagination.sort_direction, - total_records=0, - total_pages=0, + total_records=response.total_records, + total_pages=int(math.ceil(response.total_records / search_params.pagination.page_size)), ) - return [], pagination_info + # While the data returned is already JSON/dicts like we want to return + # APIFlask will try to run whatever we return through the deserializers + # which means anything that requires conversions like timestamps end up failing + # as they don't need to be converted. So, we convert everything to those types (serialize) + # so that deserialization won't fail. + records = SCHEMA.load(response.records, many=True) + + return records, response.aggregations, pagination_info diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 4b45c4f2c..6887ff0e4 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -173,13 +173,18 @@ def opportunity_index(search_client): yield index_name finally: # Try to clean up the index at the end - search_client.delete_index(index_name) + # Use a prefix which will delete the above (if it exists) + # and any that might not have been cleaned up due to issues + # in prior runs + search_client.delete_index("test-opportunity-index-*") @pytest.fixture(scope="session") -def opportunity_index_alias(search_client): +def opportunity_index_alias(search_client, monkeypatch_session): # Note we don't actually create anything, this is just a random name - return f"test-opportunity-index-alias-{uuid.uuid4().int}" + alias = f"test-opportunity-index-alias-{uuid.uuid4().int}" + monkeypatch_session.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) + return alias #################### @@ -190,7 +195,7 @@ def opportunity_index_alias(search_client): # Make app session scoped so the database connection pool is only created once # for the test session. This speeds up the tests. @pytest.fixture(scope="session") -def app(db_client) -> APIFlask: +def app(db_client, opportunity_index_alias) -> APIFlask: return app_entry.create_app() diff --git a/api/tests/src/api/opportunities_v1/conftest.py b/api/tests/src/api/opportunities_v1/conftest.py index 81668c48a..e96c14df7 100644 --- a/api/tests/src/api/opportunities_v1/conftest.py +++ b/api/tests/src/api/opportunities_v1/conftest.py @@ -28,9 +28,9 @@ def truncate_opportunities(db_session): def get_search_request( page_offset: int = 1, - page_size: int = 5, + page_size: int = 25, order_by: str = "opportunity_id", - sort_direction: str = "descending", + sort_direction: str = "ascending", query: str | None = None, funding_instrument_one_of: list[FundingInstrument] | None = None, funding_category_one_of: list[FundingCategory] | None = None, diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index 6e79419db..e3becbfb3 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -1,19 +1,764 @@ +from datetime import date + +import pytest + +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema +from src.constants.lookup_constants import ( + ApplicantType, + FundingCategory, + FundingInstrument, + OpportunityStatus, +) +from src.db.models.opportunity_models import Opportunity +from src.pagination.pagination_models import SortDirection +from src.util.dict_util import flatten_dict +from tests.conftest import BaseTestClass from tests.src.api.opportunities_v1.conftest import get_search_request +from tests.src.db.models.factories import ( + CurrentOpportunitySummaryFactory, + OpportunityAssistanceListingFactory, + OpportunityFactory, + OpportunitySummaryFactory, +) + + +def validate_search_response( + search_response, expected_results: list[Opportunity], expected_status_code: int = 200 +): + assert search_response.status_code == expected_status_code + + response_json = search_response.get_json() + + opportunities = response_json["data"] + + response_ids = [opp["opportunity_id"] for opp in opportunities] + expected_ids = [exp.opportunity_id for exp in expected_results] + + assert ( + response_ids == expected_ids + ), f"Actual opportunities:\n {'\n'.join([opp['opportunity_title'] for opp in opportunities])}" + + +def build_opp( + opportunity_title: str, + opportunity_number: str, + agency: str, + summary_description: str, + opportunity_status: OpportunityStatus, + assistance_listings: list, + applicant_types: list, + funding_instruments: list, + funding_categories: list, + post_date: date, + close_date: date | None, +) -> Opportunity: + opportunity = OpportunityFactory.build( + opportunity_title=opportunity_title, + opportunity_number=opportunity_number, + agency=agency, + opportunity_assistance_listings=[], + current_opportunity_summary=None, + ) + + for assistance_listing in assistance_listings: + opportunity.opportunity_assistance_listings.append( + OpportunityAssistanceListingFactory.build( + opportunity=opportunity, + assistance_listing_number=assistance_listing[0], + program_title=assistance_listing[1], + ) + ) + + opportunity_summary = OpportunitySummaryFactory.build( + opportunity=opportunity, + summary_description=summary_description, + applicant_types=applicant_types, + funding_instruments=funding_instruments, + funding_categories=funding_categories, + post_date=post_date, + close_date=close_date, + ) + + opportunity.current_opportunity_summary = CurrentOpportunitySummaryFactory.build( + opportunity_status=opportunity_status, + opportunity_summary=opportunity_summary, + opportunity=opportunity, + ) + + return opportunity + + +########################################## +# Opportunity scenarios for tests +# +# These try to mimic real opportunities +########################################## + +EDUCATION_AL = ("43.008", "Office of Stem Engagement (OSTEM)") +SPACE_AL = ("43.012", "Space Technology") +AERONAUTICS_AL = ("43.002", "Aeronautics") +LOC_AL = ("42.011", "Library of Congress Grants") +AMERICAN_AL = ("19.441", "ECA - American Spaces") +ECONOMIC_AL = ("11.307", "Economic Adjustment Assistance") +MANUFACTURING_AL = ("11.611", "Manufacturing Extension Partnership") + +NASA_SPACE_FELLOWSHIP = build_opp( + opportunity_title="National Space Grant College and Fellowship Program FY 2020 - 2024", + opportunity_number="NNH123ZYX", + agency="NASA", + summary_description="This Cooperative Agreement Notice is a multi-year award that aims to contribute to NASA's mission", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2020, 3, 1), + close_date=date(2027, 6, 1), +) + +NASA_INNOVATIONS = build_opp( + opportunity_title="Early Stage Innovations", + opportunity_number="NNH24-TR0N", + agency="NASA", + summary_description="The program within STMD seeks proposals from accredited U.S. universities to develop unique, disruptive, or transformational space technologies.", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[SPACE_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2019, 3, 1), + close_date=None, +) + +NASA_SUPERSONIC = build_opp( + opportunity_title="Commercial Supersonic Technology (CST) Project", + opportunity_number="NNH24-CST", + agency="NASA", + summary_description="Commercial Supersonic Technology seeks proposals for a fuel injector design concept and fabrication for testing at NASA Glenn Research Center", + opportunity_status=OpportunityStatus.CLOSED, + assistance_listings=[AERONAUTICS_AL], + applicant_types=[ApplicantType.UNRESTRICTED], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2021, 3, 1), + close_date=date(2030, 6, 1), +) + +NASA_K12_DIVERSITY = build_opp( + opportunity_title="Space Grant K-12 Inclusiveness and Diversity in STEM", + opportunity_number="NNH22ZHA", + agency="NASA", + summary_description="Expands the reach of individual Consortia to collaborate regionally on efforts that directly support middle and high school student participation in hands-on, NASA-aligned STEM activities", + opportunity_status=OpportunityStatus.ARCHIVED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2025, 3, 1), + close_date=date(2018, 6, 1), +) + +LOC_TEACHING = build_opp( + opportunity_title="Teaching with Primary Sources - New Awards for FY25-FY27", + opportunity_number="012ADV345", + agency="LOC", + summary_description="Builds student literacy, critical thinking skills, content knowledge and ability to conduct original research.", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ + ApplicantType.STATE_GOVERNMENTS, + ApplicantType.COUNTY_GOVERNMENTS, + ApplicantType.INDEPENDENT_SCHOOL_DISTRICTS, + ApplicantType.CITY_OR_TOWNSHIP_GOVERNMENTS, + ApplicantType.SPECIAL_DISTRICT_GOVERNMENTS, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2031, 3, 1), + close_date=date(2010, 6, 1), +) + +LOC_HIGHER_EDUCATION = build_opp( + opportunity_title="Of the People: Widening the Path: CCDI – Higher Education", + opportunity_number="012ADV346", + agency="LOC", + summary_description="The Library of Congress will expand the connections between the Library and diverse communities and strengthen the use of Library of Congress digital collections and digital tools", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[LOC_AL], + applicant_types=[ + ApplicantType.PRIVATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.OTHER], + post_date=date(2026, 3, 1), + close_date=None, +) + +DOS_DIGITAL_LITERACY = build_opp( + opportunity_title="American Spaces Digital Literacy and Training Program", + opportunity_number="SFOP0001234", + agency="DOS-ECA", + summary_description="An open competition to administer a new award in the field of digital and media literacy and countering disinformation", + opportunity_status=OpportunityStatus.CLOSED, + assistance_listings=[AMERICAN_AL], + applicant_types=[ + ApplicantType.OTHER, + ApplicantType.NONPROFITS_NON_HIGHER_EDUCATION_WITH_501C3, + ApplicantType.PRIVATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.OTHER], + post_date=date(2028, 3, 1), + close_date=date(2023, 6, 1), +) + +DOC_SPACE_COAST = build_opp( + opportunity_title="Space Coast RIC", + opportunity_number="SFOP0009876", + agency="DOC-EDA", + summary_description="diversification of Florida's Space Coast region", + opportunity_status=OpportunityStatus.ARCHIVED, + assistance_listings=[ECONOMIC_AL], + applicant_types=[ + ApplicantType.CITY_OR_TOWNSHIP_GOVERNMENTS, + ApplicantType.COUNTY_GOVERNMENTS, + ApplicantType.STATE_GOVERNMENTS, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT, FundingInstrument.GRANT], + funding_categories=[FundingCategory.OTHER, FundingCategory.REGIONAL_DEVELOPMENT], + post_date=date(2017, 3, 1), + close_date=date(2019, 6, 1), +) + +DOC_MANUFACTURING = build_opp( + opportunity_title="Advanced Manufacturing Jobs and Innovation Accelerator Challenge", + opportunity_number="JIAC1234AM", + agency="DOC-EDA", + summary_description="foster job creation, increase public and private investments, and enhance economic prosperity", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[ECONOMIC_AL, MANUFACTURING_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT, FundingInstrument.GRANT], + funding_categories=[ + FundingCategory.EMPLOYMENT_LABOR_AND_TRAINING, + FundingCategory.ENERGY, + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT, + ], + post_date=date(2013, 3, 1), + close_date=date(2035, 6, 1), +) + +OPPORTUNITIES = [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_SUPERSONIC, + NASA_K12_DIVERSITY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + DOC_MANUFACTURING, +] + + +def search_scenario_id_fnc(val): + if isinstance(val, dict): + return str(flatten_dict(val, separator="|")) + +class TestOpportunityRouteSearch(BaseTestClass): + @pytest.fixture(scope="class") + def setup_search_data(self, opportunity_index, opportunity_index_alias, search_client): + # Load into the search index + schema = OpportunityV1Schema() + json_records = [schema.dump(opportunity) for opportunity in OPPORTUNITIES] + search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") -def test_opportunity_route_search_200(client, api_auth_token): - req = get_search_request() + # Swap the search index alias + search_client.swap_alias_index(opportunity_index, opportunity_index_alias) - resp = client.post("/v1/opportunities/search", json=req, headers={"X-Auth": api_auth_token}) + @pytest.mark.parametrize( + "search_request,expected_results", + [ + # Opportunity ID + ( + get_search_request( + page_size=25, + page_offset=1, + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + ), + OPPORTUNITIES, + ), + ( + get_search_request( + page_size=3, + page_offset=2, + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + ), + OPPORTUNITIES[3:6], + ), + ( + get_search_request( + page_size=25, + page_offset=1, + order_by="opportunity_id", + sort_direction=SortDirection.DESCENDING, + ), + OPPORTUNITIES[::-1], + ), + # Opportunity Number + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="opportunity_number", + sort_direction=SortDirection.ASCENDING, + ), + [LOC_TEACHING, LOC_HIGHER_EDUCATION, DOC_MANUFACTURING], + ), + ( + get_search_request( + page_size=2, + page_offset=3, + order_by="opportunity_number", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], + ), + # Opportunity Title + ( + get_search_request( + page_size=4, + page_offset=2, + order_by="opportunity_title", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SPACE_FELLOWSHIP, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST, NASA_K12_DIVERSITY], + ), + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="opportunity_title", + sort_direction=SortDirection.DESCENDING, + ), + [ + LOC_TEACHING, + NASA_K12_DIVERSITY, + DOC_SPACE_COAST, + LOC_HIGHER_EDUCATION, + NASA_SPACE_FELLOWSHIP, + ], + ), + # Post Date + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Relevancy has a secondary sort of post date so should be identical. + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Close Date (note several have null values which always go to the end) + ( + get_search_request( + page_size=4, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [LOC_TEACHING, NASA_K12_DIVERSITY, DOC_SPACE_COAST, DOS_DIGITAL_LITERACY], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.DESCENDING, + ), + [DOC_MANUFACTURING, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], + ), + # close date - but check the end of the list to find the null values + ( + get_search_request( + page_size=5, + page_offset=2, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SUPERSONIC, DOC_MANUFACTURING, NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + ), + # Agency + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.ASCENDING, + ), + [ + DOC_SPACE_COAST, + DOC_MANUFACTURING, + DOS_DIGITAL_LITERACY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + ], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_sorting_and_pagination_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results) - assert resp.status_code == 200 + @pytest.mark.parametrize( + "search_request, expected_results", + [ + # Agency + (get_search_request(agency_one_of=["not an agency"]), []), + ( + get_search_request(agency_one_of=["NASA"]), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC, NASA_K12_DIVERSITY], + ), + (get_search_request(agency_one_of=["LOC"]), [LOC_TEACHING, LOC_HIGHER_EDUCATION]), + (get_search_request(agency_one_of=["DOS-ECA"]), [DOS_DIGITAL_LITERACY]), + (get_search_request(agency_one_of=["DOC-EDA"]), [DOC_SPACE_COAST, DOC_MANUFACTURING]), + ( + get_search_request( + agency_one_of=["DOC-EDA", "NASA", "LOC", "DOS-ECA", "something else"] + ), + OPPORTUNITIES, + ), + # Opportunity Status + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.POSTED]), + [NASA_SPACE_FELLOWSHIP, LOC_TEACHING, DOC_MANUFACTURING], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.FORECASTED]), + [NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.CLOSED]), + [NASA_SUPERSONIC, DOS_DIGITAL_LITERACY], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.ARCHIVED]), + [NASA_K12_DIVERSITY, DOC_SPACE_COAST], + ), + ( + get_search_request( + opportunity_status_one_of=[ + OpportunityStatus.POSTED, + OpportunityStatus.FORECASTED, + ] + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request( + opportunity_status_one_of=[ + OpportunityStatus.POSTED, + OpportunityStatus.FORECASTED, + OpportunityStatus.CLOSED, + OpportunityStatus.ARCHIVED, + ] + ), + OPPORTUNITIES, + ), + # Funding Instrument + ( + get_search_request( + funding_instrument_one_of=[FundingInstrument.COOPERATIVE_AGREEMENT] + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_K12_DIVERSITY, + LOC_TEACHING, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request(funding_instrument_one_of=[FundingInstrument.GRANT]), + [ + NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_HIGHER_EDUCATION, + DOC_SPACE_COAST, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request( + funding_instrument_one_of=[FundingInstrument.PROCUREMENT_CONTRACT] + ), + [], + ), + (get_search_request(funding_instrument_one_of=[FundingInstrument.OTHER]), []), + ( + get_search_request( + funding_instrument_one_of=[ + FundingInstrument.COOPERATIVE_AGREEMENT, + FundingInstrument.GRANT, + ] + ), + OPPORTUNITIES, + ), + # Funding Category + ( + get_search_request(funding_category_one_of=[FundingCategory.EDUCATION]), + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + ), + ( + get_search_request( + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT + ] + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.OTHER]), + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, DOC_SPACE_COAST], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.REGIONAL_DEVELOPMENT]), + [DOC_SPACE_COAST], + ), + ( + get_search_request( + funding_category_one_of=[FundingCategory.EMPLOYMENT_LABOR_AND_TRAINING] + ), + [DOC_MANUFACTURING], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.ENERGY]), + [DOC_MANUFACTURING], + ), + (get_search_request(funding_category_one_of=[FundingCategory.HOUSING]), []), + ( + get_search_request( + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT, + FundingCategory.REGIONAL_DEVELOPMENT, + ] + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], + ), + # Applicant Type + ( + get_search_request(applicant_type_one_of=[ApplicantType.OTHER]), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.UNRESTRICTED]), + [NASA_SUPERSONIC], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.STATE_GOVERNMENTS]), + [LOC_TEACHING, DOC_SPACE_COAST], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.COUNTY_GOVERNMENTS]), + [LOC_TEACHING, DOC_SPACE_COAST], + ), + ( + get_search_request( + applicant_type_one_of=[ + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION + ] + ), + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY], + ), + (get_search_request(applicant_type_one_of=[ApplicantType.INDIVIDUALS]), []), + ( + get_search_request( + applicant_type_one_of=[ + ApplicantType.STATE_GOVERNMENTS, + ApplicantType.UNRESTRICTED, + ] + ), + [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST], + ), + # Mix + ( + get_search_request( + agency_one_of=["NASA"], applicant_type_one_of=[ApplicantType.OTHER] + ), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_K12_DIVERSITY], + ), + ( + get_search_request( + funding_instrument_one_of=[ + FundingInstrument.GRANT, + FundingInstrument.PROCUREMENT_CONTRACT, + ], + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT + ], + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + ), + ( + get_search_request( + opportunity_status_one_of=[OpportunityStatus.POSTED], + applicant_type_one_of=[ApplicantType.OTHER], + ), + [NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_search_filters_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results) - # The endpoint meaningfully only returns the pagination params back - # at the moment, so just validate that for now. - resp_body = resp.get_json() - assert resp_body["pagination_info"]["page_offset"] == req["pagination"]["page_offset"] - assert resp_body["pagination_info"]["page_size"] == req["pagination"]["page_size"] - assert resp_body["pagination_info"]["sort_direction"] == req["pagination"]["sort_direction"] - assert resp_body["pagination_info"]["order_by"] == req["pagination"]["order_by"] - assert resp_body["pagination_info"]["total_records"] == 0 - assert resp_body["pagination_info"]["total_pages"] == 0 + @pytest.mark.parametrize( + "search_request, expected_results", + [ + # Note that the sorting is not relevancy for this as we intend to update the relevancy scores a bit + # and don't want to break this every time we adjust those. + ( + get_search_request( + order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="space" + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + ], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="43.008", + ), + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="012ADV*", + ), + [LOC_TEACHING, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="DOC*" + ), + [DOC_SPACE_COAST, DOC_MANUFACTURING], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="Aeronautics", + ), + [NASA_SUPERSONIC], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="literacy", + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_search_query_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + # This test isn't looking to validate opensearch behavior, just that we've connected fields properly and + # results being returned are as expected. + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results)