diff --git a/api/openapi.generated.yml b/api/openapi.generated.yml index c05f960ef..b17a53122 100644 --- a/api/openapi.generated.yml +++ b/api/openapi.generated.yml @@ -145,6 +145,61 @@ paths: application/json: schema: $ref: '#/components/schemas/OpportunitySearchRequestV1' + examples: + example1: + summary: No filters + value: + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: ascending + example2: + summary: All filters + value: + query: research + filters: + agency: + one_of: + - USAID + - ARPAH + applicant_type: + one_of: + - state_governments + - county_governments + - individuals + funding_category: + one_of: + - recovery_act + - arts + - natural_resources + funding_instrument: + one_of: + - cooperative_agreement + - grant + opportunity_status: + one_of: + - forecasted + - posted + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: descending + example3: + summary: Query & opportunity_status filters + value: + query: research + filters: + opportunity_status: + one_of: + - forecasted + - posted + pagination: + order_by: opportunity_id + page_offset: 1 + page_size: 25 + sort_direction: descending security: - ApiKeyAuth: [] /v0.1/opportunities/search: @@ -755,7 +810,7 @@ components: items: type: string minLength: 2 - example: US-ABC + example: USAID OpportunitySearchFilterV1: type: object properties: @@ -784,12 +839,13 @@ components: - object allOf: - $ref: '#/components/schemas/AgencyFilterV1' - OpportunityPagination: + OpportunityPaginationV1: type: object properties: order_by: type: string enum: + - relevancy - opportunity_id - opportunity_number - opportunity_title @@ -837,19 +893,23 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunityPagination' + - $ref: '#/components/schemas/OpportunityPaginationV1' required: - pagination OpportunityAssistanceListingV1: type: object properties: program_title: - type: string + type: + - string + - 'null' description: The name of the program, see https://sam.gov/content/assistance-listings for more detail example: Space Technology assistance_listing_number: - type: string + type: + - string + - 'null' description: The assistance listing number, see https://sam.gov/content/assistance-listings for more detail example: '43.012' @@ -857,11 +917,15 @@ components: type: object properties: summary_description: - type: string + type: + - string + - 'null' description: The summary of the opportunity example: This opportunity aims to unravel the mysteries of the universe. is_cost_sharing: - type: boolean + type: + - boolean + - 'null' description: Whether or not the opportunity has a cost sharing/matching requirement is_forecast: @@ -870,106 +934,154 @@ components: is only an estimate and not yet official example: false close_date: - type: string + type: + - string + - 'null' format: date description: The date that the opportunity will close - only set if is_forecast=False close_date_description: - type: string + type: + - string + - 'null' description: Optional details regarding the close date example: Proposals are due earlier than usual. post_date: - type: string + type: + - string + - 'null' format: date description: The date the opportunity was posted archive_date: - type: string + type: + - string + - 'null' format: date description: When the opportunity will be archived expected_number_of_awards: - type: integer + type: + - integer + - 'null' description: The number of awards the opportunity is expected to award example: 10 estimated_total_program_funding: - type: integer + type: + - integer + - 'null' description: The total program funding of the opportunity in US Dollars example: 10000000 award_floor: - type: integer + type: + - integer + - 'null' description: The minimum amount an opportunity would award example: 10000 award_ceiling: - type: integer + type: + - integer + - 'null' description: The maximum amount an opportunity would award example: 100000 additional_info_url: - type: string + type: + - string + - 'null' description: A URL to a website that can provide additional information about the opportunity example: grants.gov additional_info_url_description: - type: string + type: + - string + - 'null' description: The text to display for the additional_info_url link example: Click me for more info forecasted_post_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast forecasted_close_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the opportunity is expected to be close once posted. forecasted_close_date_description: - type: string + type: + - string + - 'null' description: Forecasted opportunity only. Optional details regarding the forecasted closed date. example: Proposals will probably be due on this date forecasted_award_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor plans to award the opportunity. forecasted_project_start_date: - type: string + type: + - string + - 'null' format: date description: Forecasted opportunity only. The date the grantor expects the award recipient should start their project fiscal_year: - type: integer + type: + - integer + - 'null' description: Forecasted opportunity only. The fiscal year the project is expected to be funded and launched funding_category_description: - type: string + type: + - string + - 'null' description: Additional information about the funding category example: Economic Support applicant_eligibility_description: - type: string + type: + - string + - 'null' description: Additional information about the types of applicants that are eligible example: All types of domestic applicants are eligible to apply agency_code: - type: string + type: + - string + - 'null' description: The agency who owns the opportunity example: US-ABC agency_name: - type: string + type: + - string + - 'null' description: The name of the agency who owns the opportunity example: US Alphabetical Basic Corp agency_phone_number: - type: string + type: + - string + - 'null' description: The phone number of the agency who owns the opportunity example: 123-456-7890 agency_contact_description: - type: string + type: + - string + - 'null' description: Information regarding contacting the agency who owns the opportunity example: For more information, reach out to Jane Smith at agency US-ABC agency_email_address: - type: string + type: + - string + - 'null' description: The contact email of the agency who owns the opportunity example: fake_email@grants.gov agency_email_address_description: - type: string + type: + - string + - 'null' description: The text for the link to the agency email address example: Click me to email the agency version_number: @@ -1046,19 +1158,24 @@ components: properties: opportunity_id: type: integer - readOnly: true description: The internal ID of the opportunity example: 12345 opportunity_number: - type: string + type: + - string + - 'null' description: The funding opportunity number example: ABC-123-XYZ-001 opportunity_title: - type: string + type: + - string + - 'null' description: The title of the opportunity example: Research into conservation techniques agency: - type: string + type: + - string + - 'null' description: The agency who created the opportunity example: US-ABC category: @@ -1073,8 +1190,12 @@ components: - other type: - string + - 'null' + - 'null' category_explanation: - type: string + type: + - string + - 'null' description: Explanation of the category when the category is 'O' (other) example: null opportunity_assistance_listings: @@ -1108,6 +1229,51 @@ components: type: string format: date-time readOnly: true + OpportunityFacetV1: + type: object + properties: + opportunity_status: + type: object + description: The counts of opportunity_status values in the full response + example: + posted: 1 + forecasted: 2 + additionalProperties: + type: integer + applicant_type: + type: object + description: The counts of applicant_type values in the full response + example: + state_governments: 3 + county_governments: 2 + city_or_township_governments: 1 + additionalProperties: + type: integer + funding_instrument: + type: object + description: The counts of funding_instrument values in the full response + example: + cooperative_agreement: 4 + grant: 3 + additionalProperties: + type: integer + funding_category: + type: object + description: The counts of funding_category values in the full response + example: + recovery_act: 2 + arts: 3 + agriculture: 5 + additionalProperties: + type: integer + agency: + type: object + description: The counts of agency values in the full response + example: + USAID: 4 + ARPAH: 3 + additionalProperties: + type: integer OpportunitySearchResponseV1: type: object properties: @@ -1128,6 +1294,12 @@ components: type: integer description: The HTTP status code example: 200 + facet_counts: + description: Counts of filter/facet values in the full response + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunityFacetV1' FundingInstrumentFilterV01: type: object properties: @@ -1257,7 +1429,7 @@ components: - object allOf: - $ref: '#/components/schemas/AgencyFilterV01' - OpportunityPagination1: + OpportunityPagination: type: object properties: order_by: @@ -1310,7 +1482,7 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunityPagination1' + - $ref: '#/components/schemas/OpportunityPagination' required: - pagination OpportunityAssistanceListingV01: diff --git a/api/src/adapters/search/__init__.py b/api/src/adapters/search/__init__.py index 6b2607a04..c44446964 100644 --- a/api/src/adapters/search/__init__.py +++ b/api/src/adapters/search/__init__.py @@ -1,4 +1,5 @@ from src.adapters.search.opensearch_client import SearchClient from src.adapters.search.opensearch_config import get_opensearch_config +from src.adapters.search.opensearch_query_builder import SearchQueryBuilder -__all__ = ["SearchClient", "get_opensearch_config"] +__all__ = ["SearchClient", "get_opensearch_config", "SearchQueryBuilder"] diff --git a/api/src/adapters/search/flask_opensearch.py b/api/src/adapters/search/flask_opensearch.py new file mode 100644 index 000000000..0fa195456 --- /dev/null +++ b/api/src/adapters/search/flask_opensearch.py @@ -0,0 +1,47 @@ +from functools import wraps +from typing import Callable, Concatenate, ParamSpec, TypeVar + +from flask import Flask, current_app + +from src.adapters.search import SearchClient + +_SEARCH_CLIENT_KEY = "search-client" + + +def register_search_client(search_client: SearchClient, app: Flask) -> None: + app.extensions[_SEARCH_CLIENT_KEY] = search_client + + +def get_search_client(app: Flask) -> SearchClient: + return app.extensions[_SEARCH_CLIENT_KEY] + + +P = ParamSpec("P") +T = TypeVar("T") + + +def with_search_client() -> Callable[[Callable[Concatenate[SearchClient, P], T]], Callable[P, T]]: + """ + Decorator for functions that need a search client. + + This decorator will return the shared search client object which + has an internal connection pool that is shared. + + Usage: + @with_search_client() + def foo(search_client: search.SearchClient): + ... + + @with_search_client() + def bar(search_client: search.SearchClient, x: int, y: int): + ... + """ + + def decorator(f: Callable[Concatenate[SearchClient, P], T]) -> Callable[P, T]: + @wraps(f) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + return f(get_search_client(current_app), *args, **kwargs) + + return wrapper + + return decorator diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py index b2e5b2bea..cb97a9c8c 100644 --- a/api/src/adapters/search/opensearch_client.py +++ b/api/src/adapters/search/opensearch_client.py @@ -15,10 +15,7 @@ "default": { "type": "custom", "filter": ["lowercase", "custom_stemmer"], - # Change tokenization to whitespace as the default is very clunky - # with a lot of our IDs that have dashes in them. - # see: https://opensearch.org/docs/latest/analyzers/tokenizers/index/ - "tokenizer": "whitespace", + "tokenizer": "standard", } }, # Change the default stemming to use snowball which handles plural diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py index d2cdd2490..4bb4484d6 100644 --- a/api/src/api/opportunities_v1/opportunity_routes.py +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -2,6 +2,8 @@ import src.adapters.db as db import src.adapters.db.flask_db as flask_db +import src.adapters.search as search +import src.adapters.search.flask_opensearch as flask_opensearch import src.api.opportunities_v1.opportunity_schemas as opportunity_schemas import src.api.response as response from src.api.opportunities_v1.opportunity_blueprint import opportunity_blueprint @@ -24,20 +26,76 @@ See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) for further details. """ +examples = { + "example1": { + "summary": "No filters", + "value": { + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "ascending", + }, + }, + }, + "example2": { + "summary": "All filters", + "value": { + "query": "research", + "filters": { + "agency": {"one_of": ["USAID", "ARPAH"]}, + "applicant_type": { + "one_of": ["state_governments", "county_governments", "individuals"] + }, + "funding_category": {"one_of": ["recovery_act", "arts", "natural_resources"]}, + "funding_instrument": {"one_of": ["cooperative_agreement", "grant"]}, + "opportunity_status": {"one_of": ["forecasted", "posted"]}, + }, + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "descending", + }, + }, + }, + "example3": { + "summary": "Query & opportunity_status filters", + "value": { + "query": "research", + "filters": { + "opportunity_status": {"one_of": ["forecasted", "posted"]}, + }, + "pagination": { + "order_by": "opportunity_id", + "page_offset": 1, + "page_size": 25, + "sort_direction": "descending", + }, + }, + }, +} + @opportunity_blueprint.post("/opportunities/search") @opportunity_blueprint.input( - opportunity_schemas.OpportunitySearchRequestV1Schema, arg_name="search_params" + opportunity_schemas.OpportunitySearchRequestV1Schema, + arg_name="search_params", + examples=examples, ) -# many=True allows us to return a list of opportunity objects -@opportunity_blueprint.output(opportunity_schemas.OpportunitySearchResponseV1Schema) +@opportunity_blueprint.output(opportunity_schemas.OpportunitySearchResponseV1Schema()) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) -def opportunity_search(search_params: dict) -> response.ApiResponse: +@flask_opensearch.with_search_client() +def opportunity_search( + search_client: search.SearchClient, search_params: dict +) -> response.ApiResponse: add_extra_data_to_current_request_logs(flatten_dict(search_params, prefix="request.body")) logger.info("POST /v1/opportunities/search") - opportunities, pagination_info = search_opportunities(search_params) + opportunities, aggregations, pagination_info = search_opportunities( + search_client, search_params + ) add_extra_data_to_current_request_logs( { @@ -48,12 +106,15 @@ def opportunity_search(search_params: dict) -> response.ApiResponse: logger.info("Successfully fetched opportunities") return response.ApiResponse( - message="Success", data=opportunities, pagination_info=pagination_info + message="Success", + data=opportunities, + facet_counts=aggregations, + pagination_info=pagination_info, ) @opportunity_blueprint.get("/opportunities/") -@opportunity_blueprint.output(opportunity_schemas.OpportunityGetResponseV1Schema) +@opportunity_blueprint.output(opportunity_schemas.OpportunityGetResponseV1Schema()) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) @flask_db.with_db_session() diff --git a/api/src/api/opportunities_v1/opportunity_schemas.py b/api/src/api/opportunities_v1/opportunity_schemas.py index 3c071f260..58a6dac36 100644 --- a/api/src/api/opportunities_v1/opportunity_schemas.py +++ b/api/src/api/opportunities_v1/opportunity_schemas.py @@ -13,15 +13,17 @@ class OpportunitySummaryV1Schema(Schema): summary_description = fields.String( + allow_none=True, metadata={ "description": "The summary of the opportunity", "example": "This opportunity aims to unravel the mysteries of the universe.", - } + }, ) is_cost_sharing = fields.Boolean( + allow_none=True, metadata={ "description": "Whether or not the opportunity has a cost sharing/matching requirement", - } + }, ) is_forecast = fields.Boolean( metadata={ @@ -31,147 +33,171 @@ class OpportunitySummaryV1Schema(Schema): ) close_date = fields.Date( + allow_none=True, metadata={ "description": "The date that the opportunity will close - only set if is_forecast=False", - } + }, ) close_date_description = fields.String( + allow_none=True, metadata={ "description": "Optional details regarding the close date", "example": "Proposals are due earlier than usual.", - } + }, ) post_date = fields.Date( + allow_none=True, metadata={ "description": "The date the opportunity was posted", - } + }, ) archive_date = fields.Date( + allow_none=True, metadata={ "description": "When the opportunity will be archived", - } + }, ) # not including unarchive date at the moment expected_number_of_awards = fields.Integer( + allow_none=True, metadata={ "description": "The number of awards the opportunity is expected to award", "example": 10, - } + }, ) estimated_total_program_funding = fields.Integer( + allow_none=True, metadata={ "description": "The total program funding of the opportunity in US Dollars", "example": 10_000_000, - } + }, ) award_floor = fields.Integer( + allow_none=True, metadata={ "description": "The minimum amount an opportunity would award", "example": 10_000, - } + }, ) award_ceiling = fields.Integer( + allow_none=True, metadata={ "description": "The maximum amount an opportunity would award", "example": 100_000, - } + }, ) additional_info_url = fields.String( + allow_none=True, metadata={ "description": "A URL to a website that can provide additional information about the opportunity", "example": "grants.gov", - } + }, ) additional_info_url_description = fields.String( + allow_none=True, metadata={ "description": "The text to display for the additional_info_url link", "example": "Click me for more info", - } + }, ) forecasted_post_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast" - } + }, ) forecasted_close_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the opportunity is expected to be close once posted." - } + }, ) forecasted_close_date_description = fields.String( + allow_none=True, metadata={ "description": "Forecasted opportunity only. Optional details regarding the forecasted closed date.", "example": "Proposals will probably be due on this date", - } + }, ) forecasted_award_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the grantor plans to award the opportunity." - } + }, ) forecasted_project_start_date = fields.Date( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The date the grantor expects the award recipient should start their project" - } + }, ) fiscal_year = fields.Integer( + allow_none=True, metadata={ "description": "Forecasted opportunity only. The fiscal year the project is expected to be funded and launched" - } + }, ) funding_category_description = fields.String( + allow_none=True, metadata={ "description": "Additional information about the funding category", "example": "Economic Support", - } + }, ) applicant_eligibility_description = fields.String( + allow_none=True, metadata={ "description": "Additional information about the types of applicants that are eligible", "example": "All types of domestic applicants are eligible to apply", - } + }, ) agency_code = fields.String( + allow_none=True, metadata={ "description": "The agency who owns the opportunity", "example": "US-ABC", - } + }, ) agency_name = fields.String( + allow_none=True, metadata={ "description": "The name of the agency who owns the opportunity", "example": "US Alphabetical Basic Corp", - } + }, ) agency_phone_number = fields.String( + allow_none=True, metadata={ "description": "The phone number of the agency who owns the opportunity", "example": "123-456-7890", - } + }, ) agency_contact_description = fields.String( + allow_none=True, metadata={ "description": "Information regarding contacting the agency who owns the opportunity", "example": "For more information, reach out to Jane Smith at agency US-ABC", - } + }, ) agency_email_address = fields.String( + allow_none=True, metadata={ "description": "The contact email of the agency who owns the opportunity", "example": "fake_email@grants.gov", - } + }, ) agency_email_address_description = fields.String( + allow_none=True, metadata={ "description": "The text for the link to the agency email address", "example": "Click me to email the agency", - } + }, ) version_number = fields.Integer( @@ -185,50 +211,56 @@ class OpportunitySummaryV1Schema(Schema): class OpportunityAssistanceListingV1Schema(Schema): program_title = fields.String( + allow_none=True, metadata={ "description": "The name of the program, see https://sam.gov/content/assistance-listings for more detail", "example": "Space Technology", - } + }, ) assistance_listing_number = fields.String( + allow_none=True, metadata={ "description": "The assistance listing number, see https://sam.gov/content/assistance-listings for more detail", "example": "43.012", - } + }, ) class OpportunityV1Schema(Schema): opportunity_id = fields.Integer( - dump_only=True, metadata={"description": "The internal ID of the opportunity", "example": 12345}, ) opportunity_number = fields.String( - metadata={"description": "The funding opportunity number", "example": "ABC-123-XYZ-001"} + allow_none=True, + metadata={"description": "The funding opportunity number", "example": "ABC-123-XYZ-001"}, ) opportunity_title = fields.String( + allow_none=True, metadata={ "description": "The title of the opportunity", "example": "Research into conservation techniques", - } + }, ) agency = fields.String( - metadata={"description": "The agency who created the opportunity", "example": "US-ABC"} + allow_none=True, + metadata={"description": "The agency who created the opportunity", "example": "US-ABC"}, ) category = fields.Enum( OpportunityCategory, + allow_none=True, metadata={ "description": "The opportunity category", "example": OpportunityCategory.DISCRETIONARY, }, ) category_explanation = fields.String( + allow_none=True, metadata={ "description": "Explanation of the category when the category is 'O' (other)", "example": None, - } + }, ) opportunity_assistance_listings = fields.List( @@ -271,11 +303,58 @@ class OpportunitySearchFilterV1Schema(Schema): ) agency = fields.Nested( StrSearchSchemaBuilder("AgencyFilterV1Schema") - .with_one_of(example="US-ABC", minimum_length=2) + .with_one_of(example="USAID", minimum_length=2) .build() ) +class OpportunityFacetV1Schema(Schema): + opportunity_status = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of opportunity_status values in the full response", + "example": {"posted": 1, "forecasted": 2}, + }, + ) + applicant_type = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of applicant_type values in the full response", + "example": { + "state_governments": 3, + "county_governments": 2, + "city_or_township_governments": 1, + }, + }, + ) + funding_instrument = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of funding_instrument values in the full response", + "example": {"cooperative_agreement": 4, "grant": 3}, + }, + ) + funding_category = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of funding_category values in the full response", + "example": {"recovery_act": 2, "arts": 3, "agriculture": 5}, + }, + ) + agency = fields.Dict( + keys=fields.String(), + values=fields.Integer(), + metadata={ + "description": "The counts of agency values in the full response", + "example": {"USAID": 4, "ARPAH": 3}, + }, + ) + + class OpportunitySearchRequestV1Schema(Schema): query = fields.String( metadata={ @@ -289,8 +368,9 @@ class OpportunitySearchRequestV1Schema(Schema): pagination = fields.Nested( generate_pagination_schema( - "OpportunityPaginationSchema", + "OpportunityPaginationV1Schema", [ + "relevancy", "opportunity_id", "opportunity_number", "opportunity_title", @@ -319,3 +399,8 @@ class OpportunityVersionsGetResponseV1Schema(AbstractResponseSchema): class OpportunitySearchResponseV1Schema(AbstractResponseSchema, PaginationMixinSchema): data = fields.Nested(OpportunityV1Schema(many=True)) + + facet_counts = fields.Nested( + OpportunityFacetV1Schema(), + metadata={"description": "Counts of filter/facet values in the full response"}, + ) diff --git a/api/src/api/response.py b/api/src/api/response.py index 0990dc120..f85fc0f7c 100644 --- a/api/src/api/response.py +++ b/api/src/api/response.py @@ -45,6 +45,7 @@ class ApiResponse: status_code: int = 200 pagination_info: PaginationInfo | None = None + facet_counts: dict | None = None def process_marshmallow_issues(marshmallow_issues: dict) -> list[ValidationErrorDetail]: diff --git a/api/src/api/schemas/extension/schema_fields.py b/api/src/api/schemas/extension/schema_fields.py index 97b08636d..8431c3ecd 100644 --- a/api/src/api/schemas/extension/schema_fields.py +++ b/api/src/api/schemas/extension/schema_fields.py @@ -183,6 +183,12 @@ class Raw(original_fields.Raw, MixinField): pass +class Dict(original_fields.Dict, MixinField): + error_mapping: dict[str, MarshmallowErrorContainer] = { + "invalid": MarshmallowErrorContainer(ValidationErrorType.INVALID, "Not a valid dict."), + } + + class Enum(MixinField): """ Custom field class for handling unioning together multiple Python enums into @@ -230,7 +236,7 @@ def _serialize( if value is None: return None - val = value.value + val = value return self.field._serialize(val, attr, obj, **kwargs) def _deserialize( diff --git a/api/src/app.py b/api/src/app.py index 98fea1607..ad79d1810 100644 --- a/api/src/app.py +++ b/api/src/app.py @@ -7,6 +7,8 @@ import src.adapters.db as db import src.adapters.db.flask_db as flask_db +import src.adapters.search as search +import src.adapters.search.flask_opensearch as flask_opensearch import src.api.feature_flags.feature_flag_config as feature_flag_config import src.logging import src.logging.flask_logger as flask_logger @@ -47,6 +49,7 @@ def create_app() -> APIFlask: configure_app(app) register_blueprints(app) register_index(app) + register_search_client(app) return app @@ -61,6 +64,11 @@ def register_db_client(app: APIFlask) -> None: flask_db.register_db_client(db_client, app) +def register_search_client(app: APIFlask) -> None: + search_client = search.SearchClient() + flask_opensearch.register_search_client(search_client, app) + + def configure_app(app: APIFlask) -> None: app_config = AppConfig() diff --git a/api/src/search/backend/load_opportunities_to_index.py b/api/src/search/backend/load_opportunities_to_index.py index a01357a96..630ecf616 100644 --- a/api/src/search/backend/load_opportunities_to_index.py +++ b/api/src/search/backend/load_opportunities_to_index.py @@ -9,7 +9,7 @@ import src.adapters.db as db import src.adapters.search as search -from src.api.opportunities_v0_1.opportunity_schemas import OpportunityV01Schema +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema from src.db.models.opportunity_models import CurrentOpportunitySummary, Opportunity from src.task.task import Task from src.util.datetime_util import get_now_us_eastern_datetime @@ -95,7 +95,7 @@ def fetch_opportunities(self) -> Iterator[Sequence[Opportunity]]: def load_records(self, records: Sequence[Opportunity]) -> None: logger.info("Loading batch of opportunities...") - schema = OpportunityV01Schema() + schema = OpportunityV1Schema() json_records = [] for record in records: diff --git a/api/src/search/search_config.py b/api/src/search/search_config.py new file mode 100644 index 000000000..8b7ea4f29 --- /dev/null +++ b/api/src/search/search_config.py @@ -0,0 +1,19 @@ +from pydantic import Field + +from src.util.env_config import PydanticBaseEnvConfig + + +class SearchConfig(PydanticBaseEnvConfig): + opportunity_search_index_alias: str = Field(default="opportunity-index-alias") + + +_search_config: SearchConfig | None = None + + +def get_search_config() -> SearchConfig: + global _search_config + + if _search_config is None: + _search_config = SearchConfig() + + return _search_config diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py index 1823bc31d..92a71344c 100644 --- a/api/src/services/opportunities_v1/search_opportunities.py +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -1,39 +1,147 @@ import logging +import math from typing import Sequence, Tuple from pydantic import BaseModel, Field -from src.db.models.opportunity_models import Opportunity -from src.pagination.pagination_models import PaginationInfo, PaginationParams +import src.adapters.search as search +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema +from src.pagination.pagination_models import PaginationInfo, PaginationParams, SortDirection +from src.search.search_config import get_search_config logger = logging.getLogger(__name__) +# To assist with mapping field names from our API requests +# to what they are called in the search index, this mapping +# can be used. Note that in many cases its just adjusting paths +# or for text based fields adding ".keyword" to the end to tell +# the query we want to use the raw value rather than the tokenized one +# See: https://opensearch.org/docs/latest/field-types/supported-field-types/keyword/ +REQUEST_FIELD_NAME_MAPPING = { + "opportunity_number": "opportunity_number.keyword", + "opportunity_title": "opportunity_title.keyword", + "post_date": "summary.post_date", + "close_date": "summary.close_date", + "agency_code": "agency.keyword", + "agency": "agency.keyword", + "opportunity_status": "opportunity_status.keyword", + "funding_instrument": "summary.funding_instruments.keyword", + "funding_category": "summary.funding_categories.keyword", + "applicant_type": "summary.applicant_types.keyword", +} -class SearchOpportunityFilters(BaseModel): - funding_instrument: dict | None = Field(default=None) - funding_category: dict | None = Field(default=None) - applicant_type: dict | None = Field(default=None) - opportunity_status: dict | None = Field(default=None) - agency: dict | None = Field(default=None) +SEARCH_FIELDS = [ + # Note that we do keyword for agency & opportunity number + # as we don't want to compare to a tokenized value which + # may have split on the dashes. + "agency.keyword^16", + "opportunity_title^2", + "opportunity_number.keyword^12", + "summary.summary_description", + "opportunity_assistance_listings.assistance_listing_number^10", + "opportunity_assistance_listings.program_title^4", +] + +SCHEMA = OpportunityV1Schema() class SearchOpportunityParams(BaseModel): pagination: PaginationParams query: str | None = Field(default=None) - filters: SearchOpportunityFilters | None = Field(default=None) + filters: dict | None = Field(default=None) + + +def _adjust_field_name(field: str) -> str: + return REQUEST_FIELD_NAME_MAPPING.get(field, field) + + +def _get_sort_by(pagination: PaginationParams) -> list[tuple[str, SortDirection]]: + sort_by: list[tuple[str, SortDirection]] = [] + + sort_by.append((_adjust_field_name(pagination.order_by), pagination.sort_direction)) + + # Add a secondary sort for relevancy to sort by post date (matching the sort direction) + if pagination.order_by == "relevancy": + sort_by.append((_adjust_field_name("post_date"), pagination.sort_direction)) + + return sort_by + + +def _add_search_filters(builder: search.SearchQueryBuilder, filters: dict | None) -> None: + if filters is None: + return + + for field, field_filters in filters.items(): + # one_of filters translate to an opensearch term filter + # see: https://opensearch.org/docs/latest/query-dsl/term/terms/ + one_of_filters = field_filters.get("one_of", None) + if one_of_filters: + builder.filter_terms(_adjust_field_name(field), one_of_filters) + + +def _add_aggregations(builder: search.SearchQueryBuilder) -> None: + # TODO - we'll likely want to adjust the total number of values returned, especially + # for agency as there could be hundreds of different agencies, and currently it's limited to 25. + builder.aggregation_terms("opportunity_status", _adjust_field_name("applicant_types")) + builder.aggregation_terms("applicant_type", _adjust_field_name("applicant_types")) + builder.aggregation_terms("funding_instrument", _adjust_field_name("funding_instruments")) + builder.aggregation_terms("funding_category", _adjust_field_name("funding_categories")) + builder.aggregation_terms("agency", _adjust_field_name("agency_code")) + +def _get_search_request(params: SearchOpportunityParams) -> dict: + builder = search.SearchQueryBuilder() -def search_opportunities(raw_search_params: dict) -> Tuple[Sequence[Opportunity], PaginationInfo]: + # Pagination + builder.pagination( + page_size=params.pagination.page_size, page_number=params.pagination.page_offset + ) + + # Sorting + builder.sort_by(_get_sort_by(params.pagination)) + + # Query + if params.query: + builder.simple_query(params.query, SEARCH_FIELDS) + + # Filters + _add_search_filters(builder, params.filters) + + # Aggregations / Facet / Filter Counts + _add_aggregations(builder) + + return builder.build() + + +def search_opportunities( + search_client: search.SearchClient, raw_search_params: dict +) -> Tuple[Sequence[dict], dict, PaginationInfo]: search_params = SearchOpportunityParams.model_validate(raw_search_params) + search_request = _get_search_request(search_params) + + index_alias = get_search_config().opportunity_search_index_alias + logger.info( + "Querying search index alias %s", index_alias, extra={"search_index_alias": index_alias} + ) + + response = search_client.search(index_alias, search_request) + pagination_info = PaginationInfo( page_offset=search_params.pagination.page_offset, page_size=search_params.pagination.page_size, order_by=search_params.pagination.order_by, sort_direction=search_params.pagination.sort_direction, - total_records=0, - total_pages=0, + total_records=response.total_records, + total_pages=int(math.ceil(response.total_records / search_params.pagination.page_size)), ) - return [], pagination_info + # While the data returned is already JSON/dicts like we want to return + # APIFlask will try to run whatever we return through the deserializers + # which means anything that requires conversions like timestamps end up failing + # as they don't need to be converted. So, we convert everything to those types (serialize) + # so that deserialization won't fail. + records = SCHEMA.load(response.records, many=True) + + return records, response.aggregations, pagination_info diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 4b45c4f2c..6887ff0e4 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -173,13 +173,18 @@ def opportunity_index(search_client): yield index_name finally: # Try to clean up the index at the end - search_client.delete_index(index_name) + # Use a prefix which will delete the above (if it exists) + # and any that might not have been cleaned up due to issues + # in prior runs + search_client.delete_index("test-opportunity-index-*") @pytest.fixture(scope="session") -def opportunity_index_alias(search_client): +def opportunity_index_alias(search_client, monkeypatch_session): # Note we don't actually create anything, this is just a random name - return f"test-opportunity-index-alias-{uuid.uuid4().int}" + alias = f"test-opportunity-index-alias-{uuid.uuid4().int}" + monkeypatch_session.setenv("OPPORTUNITY_SEARCH_INDEX_ALIAS", alias) + return alias #################### @@ -190,7 +195,7 @@ def opportunity_index_alias(search_client): # Make app session scoped so the database connection pool is only created once # for the test session. This speeds up the tests. @pytest.fixture(scope="session") -def app(db_client) -> APIFlask: +def app(db_client, opportunity_index_alias) -> APIFlask: return app_entry.create_app() diff --git a/api/tests/src/api/opportunities_v1/conftest.py b/api/tests/src/api/opportunities_v1/conftest.py index 81668c48a..e96c14df7 100644 --- a/api/tests/src/api/opportunities_v1/conftest.py +++ b/api/tests/src/api/opportunities_v1/conftest.py @@ -28,9 +28,9 @@ def truncate_opportunities(db_session): def get_search_request( page_offset: int = 1, - page_size: int = 5, + page_size: int = 25, order_by: str = "opportunity_id", - sort_direction: str = "descending", + sort_direction: str = "ascending", query: str | None = None, funding_instrument_one_of: list[FundingInstrument] | None = None, funding_category_one_of: list[FundingCategory] | None = None, diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py index 6e79419db..e3becbfb3 100644 --- a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -1,19 +1,764 @@ +from datetime import date + +import pytest + +from src.api.opportunities_v1.opportunity_schemas import OpportunityV1Schema +from src.constants.lookup_constants import ( + ApplicantType, + FundingCategory, + FundingInstrument, + OpportunityStatus, +) +from src.db.models.opportunity_models import Opportunity +from src.pagination.pagination_models import SortDirection +from src.util.dict_util import flatten_dict +from tests.conftest import BaseTestClass from tests.src.api.opportunities_v1.conftest import get_search_request +from tests.src.db.models.factories import ( + CurrentOpportunitySummaryFactory, + OpportunityAssistanceListingFactory, + OpportunityFactory, + OpportunitySummaryFactory, +) + + +def validate_search_response( + search_response, expected_results: list[Opportunity], expected_status_code: int = 200 +): + assert search_response.status_code == expected_status_code + + response_json = search_response.get_json() + + opportunities = response_json["data"] + + response_ids = [opp["opportunity_id"] for opp in opportunities] + expected_ids = [exp.opportunity_id for exp in expected_results] + + assert ( + response_ids == expected_ids + ), f"Actual opportunities:\n {'\n'.join([opp['opportunity_title'] for opp in opportunities])}" + + +def build_opp( + opportunity_title: str, + opportunity_number: str, + agency: str, + summary_description: str, + opportunity_status: OpportunityStatus, + assistance_listings: list, + applicant_types: list, + funding_instruments: list, + funding_categories: list, + post_date: date, + close_date: date | None, +) -> Opportunity: + opportunity = OpportunityFactory.build( + opportunity_title=opportunity_title, + opportunity_number=opportunity_number, + agency=agency, + opportunity_assistance_listings=[], + current_opportunity_summary=None, + ) + + for assistance_listing in assistance_listings: + opportunity.opportunity_assistance_listings.append( + OpportunityAssistanceListingFactory.build( + opportunity=opportunity, + assistance_listing_number=assistance_listing[0], + program_title=assistance_listing[1], + ) + ) + + opportunity_summary = OpportunitySummaryFactory.build( + opportunity=opportunity, + summary_description=summary_description, + applicant_types=applicant_types, + funding_instruments=funding_instruments, + funding_categories=funding_categories, + post_date=post_date, + close_date=close_date, + ) + + opportunity.current_opportunity_summary = CurrentOpportunitySummaryFactory.build( + opportunity_status=opportunity_status, + opportunity_summary=opportunity_summary, + opportunity=opportunity, + ) + + return opportunity + + +########################################## +# Opportunity scenarios for tests +# +# These try to mimic real opportunities +########################################## + +EDUCATION_AL = ("43.008", "Office of Stem Engagement (OSTEM)") +SPACE_AL = ("43.012", "Space Technology") +AERONAUTICS_AL = ("43.002", "Aeronautics") +LOC_AL = ("42.011", "Library of Congress Grants") +AMERICAN_AL = ("19.441", "ECA - American Spaces") +ECONOMIC_AL = ("11.307", "Economic Adjustment Assistance") +MANUFACTURING_AL = ("11.611", "Manufacturing Extension Partnership") + +NASA_SPACE_FELLOWSHIP = build_opp( + opportunity_title="National Space Grant College and Fellowship Program FY 2020 - 2024", + opportunity_number="NNH123ZYX", + agency="NASA", + summary_description="This Cooperative Agreement Notice is a multi-year award that aims to contribute to NASA's mission", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2020, 3, 1), + close_date=date(2027, 6, 1), +) + +NASA_INNOVATIONS = build_opp( + opportunity_title="Early Stage Innovations", + opportunity_number="NNH24-TR0N", + agency="NASA", + summary_description="The program within STMD seeks proposals from accredited U.S. universities to develop unique, disruptive, or transformational space technologies.", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[SPACE_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2019, 3, 1), + close_date=None, +) + +NASA_SUPERSONIC = build_opp( + opportunity_title="Commercial Supersonic Technology (CST) Project", + opportunity_number="NNH24-CST", + agency="NASA", + summary_description="Commercial Supersonic Technology seeks proposals for a fuel injector design concept and fabrication for testing at NASA Glenn Research Center", + opportunity_status=OpportunityStatus.CLOSED, + assistance_listings=[AERONAUTICS_AL], + applicant_types=[ApplicantType.UNRESTRICTED], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT], + post_date=date(2021, 3, 1), + close_date=date(2030, 6, 1), +) + +NASA_K12_DIVERSITY = build_opp( + opportunity_title="Space Grant K-12 Inclusiveness and Diversity in STEM", + opportunity_number="NNH22ZHA", + agency="NASA", + summary_description="Expands the reach of individual Consortia to collaborate regionally on efforts that directly support middle and high school student participation in hands-on, NASA-aligned STEM activities", + opportunity_status=OpportunityStatus.ARCHIVED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2025, 3, 1), + close_date=date(2018, 6, 1), +) + +LOC_TEACHING = build_opp( + opportunity_title="Teaching with Primary Sources - New Awards for FY25-FY27", + opportunity_number="012ADV345", + agency="LOC", + summary_description="Builds student literacy, critical thinking skills, content knowledge and ability to conduct original research.", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[EDUCATION_AL], + applicant_types=[ + ApplicantType.STATE_GOVERNMENTS, + ApplicantType.COUNTY_GOVERNMENTS, + ApplicantType.INDEPENDENT_SCHOOL_DISTRICTS, + ApplicantType.CITY_OR_TOWNSHIP_GOVERNMENTS, + ApplicantType.SPECIAL_DISTRICT_GOVERNMENTS, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.EDUCATION], + post_date=date(2031, 3, 1), + close_date=date(2010, 6, 1), +) + +LOC_HIGHER_EDUCATION = build_opp( + opportunity_title="Of the People: Widening the Path: CCDI – Higher Education", + opportunity_number="012ADV346", + agency="LOC", + summary_description="The Library of Congress will expand the connections between the Library and diverse communities and strengthen the use of Library of Congress digital collections and digital tools", + opportunity_status=OpportunityStatus.FORECASTED, + assistance_listings=[LOC_AL], + applicant_types=[ + ApplicantType.PRIVATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ], + funding_instruments=[FundingInstrument.GRANT], + funding_categories=[FundingCategory.OTHER], + post_date=date(2026, 3, 1), + close_date=None, +) + +DOS_DIGITAL_LITERACY = build_opp( + opportunity_title="American Spaces Digital Literacy and Training Program", + opportunity_number="SFOP0001234", + agency="DOS-ECA", + summary_description="An open competition to administer a new award in the field of digital and media literacy and countering disinformation", + opportunity_status=OpportunityStatus.CLOSED, + assistance_listings=[AMERICAN_AL], + applicant_types=[ + ApplicantType.OTHER, + ApplicantType.NONPROFITS_NON_HIGHER_EDUCATION_WITH_501C3, + ApplicantType.PRIVATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT], + funding_categories=[FundingCategory.OTHER], + post_date=date(2028, 3, 1), + close_date=date(2023, 6, 1), +) + +DOC_SPACE_COAST = build_opp( + opportunity_title="Space Coast RIC", + opportunity_number="SFOP0009876", + agency="DOC-EDA", + summary_description="diversification of Florida's Space Coast region", + opportunity_status=OpportunityStatus.ARCHIVED, + assistance_listings=[ECONOMIC_AL], + applicant_types=[ + ApplicantType.CITY_OR_TOWNSHIP_GOVERNMENTS, + ApplicantType.COUNTY_GOVERNMENTS, + ApplicantType.STATE_GOVERNMENTS, + ], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT, FundingInstrument.GRANT], + funding_categories=[FundingCategory.OTHER, FundingCategory.REGIONAL_DEVELOPMENT], + post_date=date(2017, 3, 1), + close_date=date(2019, 6, 1), +) + +DOC_MANUFACTURING = build_opp( + opportunity_title="Advanced Manufacturing Jobs and Innovation Accelerator Challenge", + opportunity_number="JIAC1234AM", + agency="DOC-EDA", + summary_description="foster job creation, increase public and private investments, and enhance economic prosperity", + opportunity_status=OpportunityStatus.POSTED, + assistance_listings=[ECONOMIC_AL, MANUFACTURING_AL], + applicant_types=[ApplicantType.OTHER], + funding_instruments=[FundingInstrument.COOPERATIVE_AGREEMENT, FundingInstrument.GRANT], + funding_categories=[ + FundingCategory.EMPLOYMENT_LABOR_AND_TRAINING, + FundingCategory.ENERGY, + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT, + ], + post_date=date(2013, 3, 1), + close_date=date(2035, 6, 1), +) + +OPPORTUNITIES = [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_SUPERSONIC, + NASA_K12_DIVERSITY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + DOC_MANUFACTURING, +] + + +def search_scenario_id_fnc(val): + if isinstance(val, dict): + return str(flatten_dict(val, separator="|")) + +class TestOpportunityRouteSearch(BaseTestClass): + @pytest.fixture(scope="class") + def setup_search_data(self, opportunity_index, opportunity_index_alias, search_client): + # Load into the search index + schema = OpportunityV1Schema() + json_records = [schema.dump(opportunity) for opportunity in OPPORTUNITIES] + search_client.bulk_upsert(opportunity_index, json_records, "opportunity_id") -def test_opportunity_route_search_200(client, api_auth_token): - req = get_search_request() + # Swap the search index alias + search_client.swap_alias_index(opportunity_index, opportunity_index_alias) - resp = client.post("/v1/opportunities/search", json=req, headers={"X-Auth": api_auth_token}) + @pytest.mark.parametrize( + "search_request,expected_results", + [ + # Opportunity ID + ( + get_search_request( + page_size=25, + page_offset=1, + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + ), + OPPORTUNITIES, + ), + ( + get_search_request( + page_size=3, + page_offset=2, + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + ), + OPPORTUNITIES[3:6], + ), + ( + get_search_request( + page_size=25, + page_offset=1, + order_by="opportunity_id", + sort_direction=SortDirection.DESCENDING, + ), + OPPORTUNITIES[::-1], + ), + # Opportunity Number + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="opportunity_number", + sort_direction=SortDirection.ASCENDING, + ), + [LOC_TEACHING, LOC_HIGHER_EDUCATION, DOC_MANUFACTURING], + ), + ( + get_search_request( + page_size=2, + page_offset=3, + order_by="opportunity_number", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_K12_DIVERSITY, NASA_SPACE_FELLOWSHIP], + ), + # Opportunity Title + ( + get_search_request( + page_size=4, + page_offset=2, + order_by="opportunity_title", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SPACE_FELLOWSHIP, LOC_HIGHER_EDUCATION, DOC_SPACE_COAST, NASA_K12_DIVERSITY], + ), + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="opportunity_title", + sort_direction=SortDirection.DESCENDING, + ), + [ + LOC_TEACHING, + NASA_K12_DIVERSITY, + DOC_SPACE_COAST, + LOC_HIGHER_EDUCATION, + NASA_SPACE_FELLOWSHIP, + ], + ), + # Post Date + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="post_date", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Relevancy has a secondary sort of post date so should be identical. + ( + get_search_request( + page_size=2, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.ASCENDING, + ), + [DOC_MANUFACTURING, DOC_SPACE_COAST], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + page_size=3, + page_offset=12, + order_by="relevancy", + sort_direction=SortDirection.DESCENDING, + ), + [], + ), + # Close Date (note several have null values which always go to the end) + ( + get_search_request( + page_size=4, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [LOC_TEACHING, NASA_K12_DIVERSITY, DOC_SPACE_COAST, DOS_DIGITAL_LITERACY], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="close_date", + sort_direction=SortDirection.DESCENDING, + ), + [DOC_MANUFACTURING, NASA_SUPERSONIC, NASA_SPACE_FELLOWSHIP], + ), + # close date - but check the end of the list to find the null values + ( + get_search_request( + page_size=5, + page_offset=2, + order_by="close_date", + sort_direction=SortDirection.ASCENDING, + ), + [NASA_SUPERSONIC, DOC_MANUFACTURING, NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + ), + # Agency + ( + get_search_request( + page_size=5, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.ASCENDING, + ), + [ + DOC_SPACE_COAST, + DOC_MANUFACTURING, + DOS_DIGITAL_LITERACY, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + ], + ), + ( + get_search_request( + page_size=3, + page_offset=1, + order_by="agency_code", + sort_direction=SortDirection.DESCENDING, + ), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_sorting_and_pagination_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results) - assert resp.status_code == 200 + @pytest.mark.parametrize( + "search_request, expected_results", + [ + # Agency + (get_search_request(agency_one_of=["not an agency"]), []), + ( + get_search_request(agency_one_of=["NASA"]), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_SUPERSONIC, NASA_K12_DIVERSITY], + ), + (get_search_request(agency_one_of=["LOC"]), [LOC_TEACHING, LOC_HIGHER_EDUCATION]), + (get_search_request(agency_one_of=["DOS-ECA"]), [DOS_DIGITAL_LITERACY]), + (get_search_request(agency_one_of=["DOC-EDA"]), [DOC_SPACE_COAST, DOC_MANUFACTURING]), + ( + get_search_request( + agency_one_of=["DOC-EDA", "NASA", "LOC", "DOS-ECA", "something else"] + ), + OPPORTUNITIES, + ), + # Opportunity Status + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.POSTED]), + [NASA_SPACE_FELLOWSHIP, LOC_TEACHING, DOC_MANUFACTURING], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.FORECASTED]), + [NASA_INNOVATIONS, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.CLOSED]), + [NASA_SUPERSONIC, DOS_DIGITAL_LITERACY], + ), + ( + get_search_request(opportunity_status_one_of=[OpportunityStatus.ARCHIVED]), + [NASA_K12_DIVERSITY, DOC_SPACE_COAST], + ), + ( + get_search_request( + opportunity_status_one_of=[ + OpportunityStatus.POSTED, + OpportunityStatus.FORECASTED, + ] + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + LOC_TEACHING, + LOC_HIGHER_EDUCATION, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request( + opportunity_status_one_of=[ + OpportunityStatus.POSTED, + OpportunityStatus.FORECASTED, + OpportunityStatus.CLOSED, + OpportunityStatus.ARCHIVED, + ] + ), + OPPORTUNITIES, + ), + # Funding Instrument + ( + get_search_request( + funding_instrument_one_of=[FundingInstrument.COOPERATIVE_AGREEMENT] + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_K12_DIVERSITY, + LOC_TEACHING, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request(funding_instrument_one_of=[FundingInstrument.GRANT]), + [ + NASA_INNOVATIONS, + NASA_SUPERSONIC, + LOC_HIGHER_EDUCATION, + DOC_SPACE_COAST, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request( + funding_instrument_one_of=[FundingInstrument.PROCUREMENT_CONTRACT] + ), + [], + ), + (get_search_request(funding_instrument_one_of=[FundingInstrument.OTHER]), []), + ( + get_search_request( + funding_instrument_one_of=[ + FundingInstrument.COOPERATIVE_AGREEMENT, + FundingInstrument.GRANT, + ] + ), + OPPORTUNITIES, + ), + # Funding Category + ( + get_search_request(funding_category_one_of=[FundingCategory.EDUCATION]), + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + ), + ( + get_search_request( + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT + ] + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.OTHER]), + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY, DOC_SPACE_COAST], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.REGIONAL_DEVELOPMENT]), + [DOC_SPACE_COAST], + ), + ( + get_search_request( + funding_category_one_of=[FundingCategory.EMPLOYMENT_LABOR_AND_TRAINING] + ), + [DOC_MANUFACTURING], + ), + ( + get_search_request(funding_category_one_of=[FundingCategory.ENERGY]), + [DOC_MANUFACTURING], + ), + (get_search_request(funding_category_one_of=[FundingCategory.HOUSING]), []), + ( + get_search_request( + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT, + FundingCategory.REGIONAL_DEVELOPMENT, + ] + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_SPACE_COAST, DOC_MANUFACTURING], + ), + # Applicant Type + ( + get_search_request(applicant_type_one_of=[ApplicantType.OTHER]), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, + DOC_MANUFACTURING, + ], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.UNRESTRICTED]), + [NASA_SUPERSONIC], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.STATE_GOVERNMENTS]), + [LOC_TEACHING, DOC_SPACE_COAST], + ), + ( + get_search_request(applicant_type_one_of=[ApplicantType.COUNTY_GOVERNMENTS]), + [LOC_TEACHING, DOC_SPACE_COAST], + ), + ( + get_search_request( + applicant_type_one_of=[ + ApplicantType.PUBLIC_AND_STATE_INSTITUTIONS_OF_HIGHER_EDUCATION + ] + ), + [LOC_HIGHER_EDUCATION, DOS_DIGITAL_LITERACY], + ), + (get_search_request(applicant_type_one_of=[ApplicantType.INDIVIDUALS]), []), + ( + get_search_request( + applicant_type_one_of=[ + ApplicantType.STATE_GOVERNMENTS, + ApplicantType.UNRESTRICTED, + ] + ), + [NASA_SUPERSONIC, LOC_TEACHING, DOC_SPACE_COAST], + ), + # Mix + ( + get_search_request( + agency_one_of=["NASA"], applicant_type_one_of=[ApplicantType.OTHER] + ), + [NASA_SPACE_FELLOWSHIP, NASA_INNOVATIONS, NASA_K12_DIVERSITY], + ), + ( + get_search_request( + funding_instrument_one_of=[ + FundingInstrument.GRANT, + FundingInstrument.PROCUREMENT_CONTRACT, + ], + funding_category_one_of=[ + FundingCategory.SCIENCE_TECHNOLOGY_AND_OTHER_RESEARCH_AND_DEVELOPMENT + ], + ), + [NASA_INNOVATIONS, NASA_SUPERSONIC, DOC_MANUFACTURING], + ), + ( + get_search_request( + opportunity_status_one_of=[OpportunityStatus.POSTED], + applicant_type_one_of=[ApplicantType.OTHER], + ), + [NASA_SPACE_FELLOWSHIP, DOC_MANUFACTURING], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_search_filters_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results) - # The endpoint meaningfully only returns the pagination params back - # at the moment, so just validate that for now. - resp_body = resp.get_json() - assert resp_body["pagination_info"]["page_offset"] == req["pagination"]["page_offset"] - assert resp_body["pagination_info"]["page_size"] == req["pagination"]["page_size"] - assert resp_body["pagination_info"]["sort_direction"] == req["pagination"]["sort_direction"] - assert resp_body["pagination_info"]["order_by"] == req["pagination"]["order_by"] - assert resp_body["pagination_info"]["total_records"] == 0 - assert resp_body["pagination_info"]["total_pages"] == 0 + @pytest.mark.parametrize( + "search_request, expected_results", + [ + # Note that the sorting is not relevancy for this as we intend to update the relevancy scores a bit + # and don't want to break this every time we adjust those. + ( + get_search_request( + order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="space" + ), + [ + NASA_SPACE_FELLOWSHIP, + NASA_INNOVATIONS, + NASA_K12_DIVERSITY, + DOS_DIGITAL_LITERACY, + DOC_SPACE_COAST, + ], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="43.008", + ), + [NASA_SPACE_FELLOWSHIP, NASA_K12_DIVERSITY, LOC_TEACHING], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="012ADV*", + ), + [LOC_TEACHING, LOC_HIGHER_EDUCATION], + ), + ( + get_search_request( + order_by="opportunity_id", sort_direction=SortDirection.ASCENDING, query="DOC*" + ), + [DOC_SPACE_COAST, DOC_MANUFACTURING], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="Aeronautics", + ), + [NASA_SUPERSONIC], + ), + ( + get_search_request( + order_by="opportunity_id", + sort_direction=SortDirection.ASCENDING, + query="literacy", + ), + [LOC_TEACHING, DOS_DIGITAL_LITERACY], + ), + ], + ids=search_scenario_id_fnc, + ) + def test_search_query_200( + self, client, api_auth_token, setup_search_data, search_request, expected_results + ): + # This test isn't looking to validate opensearch behavior, just that we've connected fields properly and + # results being returned are as expected. + resp = client.post( + "/v1/opportunities/search", json=search_request, headers={"X-Auth": api_auth_token} + ) + validate_search_response(resp, expected_results)