From 493fb4e65096a86facdd5e2e2f8593db5f463d5e Mon Sep 17 00:00:00 2001 From: Ryan Lewis <93001277+rylew1@users.noreply.github.com> Date: Wed, 22 May 2024 10:39:57 -0700 Subject: [PATCH 1/5] [Issue #37]: finish e2e tests (#38) ## Summary Fixes #37 ## Changes proposed - add some of the relevant tests from bug bash --- frontend/README.md | 4 +- frontend/tests/e2e/search/search.spec.ts | 79 ++++++++++++++++++- .../{searchUtil.ts => searchSpecUtil.ts} | 79 ++++++++++++++++++- 3 files changed, 155 insertions(+), 7 deletions(-) rename frontend/tests/e2e/search/{searchUtil.ts => searchSpecUtil.ts} (59%) diff --git a/frontend/README.md b/frontend/README.md index 587ce3460..2238207f7 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -144,7 +144,9 @@ To run E2E tests using VS Code: 1. Download the VS Code extension described in these [Playwright docs](https://playwright.dev/docs/running-tests#run-tests-in-vs-code) 2. Follow the [instructions](https://playwright.dev/docs/getting-started-vscode#running-tests) Playwright provides -In CI, the "Front-end Checks" workflow (`.github/workflows/ci-frontend.yml`) summary will include an "Artifacts" section where there is an attached "playwright-report". [Playwright docs](https://playwright.dev/docs/ci-intro#html-report) describe how to view HTML Report in more detail. +Playwright E2E tests run "local-to-local", requiring both the frontend and the API to be running for the tests to pass - and for the database to be seeded with data. + +In CI, the "Front-end Checks" workflow (`.github/workflows/ci-frontend-e2e.yml`) summary will include an "Artifacts" section where there is an attached "playwright-report". [Playwright docs](https://playwright.dev/docs/ci-intro#html-report) describe how to view HTML Report in more detail. ## 🤖 Type checking, linting, and formatting diff --git a/frontend/tests/e2e/search/search.spec.ts b/frontend/tests/e2e/search/search.spec.ts index 99d046961..85904bc6a 100644 --- a/frontend/tests/e2e/search/search.spec.ts +++ b/frontend/tests/e2e/search/search.spec.ts @@ -1,19 +1,25 @@ import { clickAccordionWithTitle, + clickLastPaginationPage, clickMobileNavMenu, + clickPaginationPageNumber, clickSearchNavLink, expectCheckboxIDIsChecked, expectSortBy, expectURLContainsQueryParam, fillSearchInputAndSubmit, + getFirstSearchResultTitle, + getLastSearchResultTitle, getMobileMenuButton, + getNumberOfOpportunitySearchResults, getSearchInput, hasMobileMenu, refreshPageWithCurrentURL, + selectOppositeSortOption, selectSortBy, toggleCheckboxes, - waitForSearchResultsLoaded, -} from "./searchUtil"; + waitForSearchResultsInitialLoad, +} from "./searchSpecUtil"; import { expect, test } from "@playwright/test"; test("should navigate from index to search page", async ({ page }) => { @@ -63,6 +69,7 @@ test.describe("Search page tests", () => { expectURLContainsQueryParam(page, "query", searchTerm); + // eslint-disable-next-line testing-library/prefer-screen-queries const resultsHeading = page.getByRole("heading", { name: /0 Opportunities/i, }); @@ -91,7 +98,7 @@ test.describe("Search page tests", () => { await expect(loadingIndicator).toBeVisible(); await expect(loadingIndicator).toBeHidden(); }); - test("should retain filters in a new tab", async ({ page }) => { + test("should refresh and retain filters in a new tab", async ({ page }) => { // Set all inputs, then refresh the page. Those same inputs should be // set from query params. const searchTerm = "education"; @@ -119,7 +126,7 @@ test.describe("Search page tests", () => { await selectSortBy(page, "agencyDesc"); - await waitForSearchResultsLoaded(page); + await waitForSearchResultsInitialLoad(page); await fillSearchInputAndSubmit(searchTerm, page); await toggleCheckboxes(page, statusCheckboxes, "status"); @@ -167,4 +174,68 @@ test.describe("Search page tests", () => { await expectCheckboxIDIsChecked(page, `#${checkboxID}`); } }); + + test("resets page back to 1 when choosing a filter", async ({ page }) => { + await clickPaginationPageNumber(page, 2); + + // Verify that page 1 is highlighted + let currentPageButton = page.locator(".usa-pagination__button.usa-current"); + await expect(currentPageButton).toHaveAttribute("aria-label", "Page 2"); + + // Select the 'Closed' checkbox under 'Opportunity status' + const statusCheckboxes = { + "status-closed": "closed", + }; + await toggleCheckboxes(page, statusCheckboxes, "status"); + + // Wait for the page to reload + await waitForSearchResultsInitialLoad(page); + + // Verify that page 1 is highlighted + currentPageButton = page.locator(".usa-pagination__button.usa-current"); + await expect(currentPageButton).toHaveAttribute("aria-label", "Page 1"); + + // It should not have a page query param set + expectURLContainsQueryParam(page, "page", "1", false); + }); + + test("last result becomes first result when flipping sort order", async ({ + page, + }) => { + await clickLastPaginationPage(page); + + await waitForSearchResultsInitialLoad(page); + + const lastSearchResultTitle = await getLastSearchResultTitle(page); + + await selectOppositeSortOption(page); + + const firstSearchResultTitle = await getFirstSearchResultTitle(page); + + expect(firstSearchResultTitle).toBe(lastSearchResultTitle); + }); + + test("number of results is the same with none or all opportunity status checked", async ({ + page, + }) => { + const initialNumberOfOpportunityResults = + await getNumberOfOpportunitySearchResults(page); + + // check all 4 boxes + const statusCheckboxes = { + "status-forecasted": "forecasted", + "status-posted": "posted", + "status-closed": "closed", + "status-archived": "archived", + }; + + await toggleCheckboxes(page, statusCheckboxes, "status"); + + const updatedNumberOfOpportunityResults = + await getNumberOfOpportunitySearchResults(page); + + expect(initialNumberOfOpportunityResults).toBe( + updatedNumberOfOpportunityResults, + ); + }); }); diff --git a/frontend/tests/e2e/search/searchUtil.ts b/frontend/tests/e2e/search/searchSpecUtil.ts similarity index 59% rename from frontend/tests/e2e/search/searchUtil.ts rename to frontend/tests/e2e/search/searchSpecUtil.ts index 87d066533..f54d32b15 100644 --- a/frontend/tests/e2e/search/searchUtil.ts +++ b/frontend/tests/e2e/search/searchSpecUtil.ts @@ -19,9 +19,16 @@ export function expectURLContainsQueryParam( page: Page, queryParamName: string, queryParamValue: string, + shouldContain = true, ) { const currentURL = page.url(); - expect(currentURL).toContain(`${queryParamName}=${queryParamValue}`); + const queryParam = `${queryParamName}=${queryParamValue}`; + + if (shouldContain) { + expect(currentURL).toContain(queryParam); + } else { + expect(currentURL).not.toContain(queryParam); + } } export async function waitForURLContainsQueryParam( @@ -116,7 +123,7 @@ export async function expectSortBy(page: Page, value: string) { expect(selectedValue).toBe(value); } -export async function waitForSearchResultsLoaded(page: Page) { +export async function waitForSearchResultsInitialLoad(page: Page) { // Wait for number of opportunities to show const resultsHeading = page.locator('h2:has-text("Opportunities")'); await resultsHeading.waitFor({ state: "visible", timeout: 60000 }); @@ -130,3 +137,71 @@ export async function clickAccordionWithTitle( .locator(`button.usa-accordion__button:has-text("${accordionTitle}")`) .click(); } + +export async function clickPaginationPageNumber( + page: Page, + pageNumber: number, +) { + const paginationButton = page.locator( + `button[data-testid="pagination-page-number"][aria-label="Page ${pageNumber}"]`, + ); + await paginationButton.first().click(); +} + +export async function clickLastPaginationPage(page: Page) { + const paginationButtons = page.locator("li > button"); + const count = await paginationButtons.count(); + + // must be more than 1 page + if (count > 2) { + await paginationButtons.nth(count - 2).click(); + } +} + +export async function getFirstSearchResultTitle(page: Page) { + const firstResultSelector = page.locator( + ".usa-list--unstyled > li:first-child h2 a", + ); + return await firstResultSelector.textContent(); +} + +export async function getLastSearchResultTitle(page: Page) { + const lastResultSelector = page.locator( + ".usa-list--unstyled > li:last-child h2 a", + ); + return await lastResultSelector.textContent(); +} + +// If descending, select the ascending variant +export async function selectOppositeSortOption(page: Page) { + const sortByDropdown = page.locator("#search-sort-by-select"); + const currentValue = await sortByDropdown.inputValue(); + let oppositeValue; + + if (currentValue.includes("Asc")) { + oppositeValue = currentValue.replace("Asc", "Desc"); + } else if (currentValue.includes("Desc")) { + oppositeValue = currentValue.replace("Desc", "Asc"); + } else { + throw new Error(`Unexpected sort value: ${currentValue}`); + } + + await sortByDropdown.selectOption(oppositeValue); +} + +export async function waitForLoaderToBeHidden(page: Page) { + await page.waitForSelector( + ".display-flex.flex-align-center.flex-justify-center.margin-bottom-15.margin-top-15", + { state: "hidden" }, + ); +} + +export async function getNumberOfOpportunitySearchResults(page: Page) { + await waitForLoaderToBeHidden(page); + const opportunitiesText = await page + .locator("h2.tablet-lg\\:grid-col-fill") + .textContent(); + return opportunitiesText + ? parseInt(opportunitiesText.replace(/\D/g, ""), 10) + : 0; +} From cd18a47fb821c2d1bb56fb779149d8d172ee4a48 Mon Sep 17 00:00:00 2001 From: Ryan Lewis <93001277+rylew1@users.noreply.github.com> Date: Wed, 22 May 2024 10:48:20 -0700 Subject: [PATCH 2/5] [Issue #1957]: sortby posted date desc default (#4) ## Summary Fixes #1957 ## Changes proposed - Update sortby labels and ordering --- frontend/src/app/api/SearchOpportunityAPI.ts | 10 ++++++---- .../src/components/search/SearchSortBy.tsx | 20 +++++++++---------- .../src/types/search/searchRequestTypes.ts | 8 +++++++- .../components/search/SearchSortBy.test.tsx | 6 +++--- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/frontend/src/app/api/SearchOpportunityAPI.ts b/frontend/src/app/api/SearchOpportunityAPI.ts index c515bb7bc..36fdb31ac 100644 --- a/frontend/src/app/api/SearchOpportunityAPI.ts +++ b/frontend/src/app/api/SearchOpportunityAPI.ts @@ -109,7 +109,7 @@ export default class SearchOpportunityAPI extends BaseApi { closeDate: "close_date", }; - let order_by: PaginationOrderBy = "opportunity_id"; + let order_by: PaginationOrderBy = "post_date"; if (sortby) { for (const [key, value] of Object.entries(orderByFieldLookup)) { if (sortby.startsWith(key)) { @@ -119,9 +119,11 @@ export default class SearchOpportunityAPI extends BaseApi { } } - const sort_direction: PaginationSortDirection = sortby?.endsWith("Desc") - ? "descending" - : "ascending"; + // default to descending + let sort_direction: PaginationSortDirection = "descending"; + if (sortby) { + sort_direction = sortby?.endsWith("Desc") ? "descending" : "ascending"; + } return { order_by, diff --git a/frontend/src/components/search/SearchSortBy.tsx b/frontend/src/components/search/SearchSortBy.tsx index ebd706898..5c5475534 100644 --- a/frontend/src/components/search/SearchSortBy.tsx +++ b/frontend/src/components/search/SearchSortBy.tsx @@ -7,16 +7,16 @@ type SortOption = { }; const SORT_OPTIONS: SortOption[] = [ - { label: "Opportunity Number (Ascending)", value: "opportunityNumberAsc" }, - { label: "Opportunity Number (Descending)", value: "opportunityNumberDesc" }, - { label: "Opportunity Title (Ascending)", value: "opportunityTitleAsc" }, - { label: "Opportunity Title (Descending)", value: "opportunityTitleDesc" }, - { label: "Agency (Ascending)", value: "agencyAsc" }, - { label: "Agency (Descending)", value: "agencyDesc" }, - { label: "Posted Date (Ascending)", value: "postedDateAsc" }, - { label: "Posted Date (Descending)", value: "postedDateDesc" }, - { label: "Close Date (Ascending)", value: "closeDateAsc" }, - { label: "Close Date (Descending)", value: "closeDateDesc" }, + { label: "Posted Date (newest)", value: "postedDateDesc" }, + { label: "Posted Date (oldest)", value: "postedDateAsc" }, + { label: "Close Date (newest)", value: "closeDateDesc" }, + { label: "Close Date (oldest)", value: "closeDateAsc" }, + { label: "Opportunity Title (A to Z)", value: "opportunityTitleAsc" }, + { label: "Opportunity Title (Z to A)", value: "opportunityTitleDesc" }, + { label: "Agency (A to Z)", value: "agencyAsc" }, + { label: "Agency (Z to A)", value: "agencyDesc" }, + { label: "Opportunity Number (descending)", value: "opportunityNumberDesc" }, + { label: "Opportunity Number (ascending)", value: "opportunityNumberAsc" }, ]; interface SearchSortByProps { diff --git a/frontend/src/types/search/searchRequestTypes.ts b/frontend/src/types/search/searchRequestTypes.ts index 83afcad5c..94e78635f 100644 --- a/frontend/src/types/search/searchRequestTypes.ts +++ b/frontend/src/types/search/searchRequestTypes.ts @@ -6,7 +6,13 @@ export interface SearchFilterRequestBody { funding_category?: { one_of: string[] }; } -export type PaginationOrderBy = "opportunity_id" | "opportunity_number"; +export type PaginationOrderBy = + | "opportunity_id" + | "opportunity_number" + | "opportunity_title" + | "agency_code" + | "post_date" + | "close_date"; export type PaginationSortDirection = "ascending" | "descending"; export interface PaginationRequestBody { order_by: PaginationOrderBy; diff --git a/frontend/tests/components/search/SearchSortBy.test.tsx b/frontend/tests/components/search/SearchSortBy.test.tsx index 3dff473bd..17b6f732e 100644 --- a/frontend/tests/components/search/SearchSortBy.test.tsx +++ b/frontend/tests/components/search/SearchSortBy.test.tsx @@ -12,7 +12,7 @@ jest.mock("../../../src/hooks/useSearchParamUpdater", () => ({ })); describe("SearchSortBy", () => { - const initialQueryParams = "opportunityNumberAsc"; + const initialQueryParams = "postedDateDesc"; const mockFormRef = React.createRef(); it("should not have basic accessibility issues", async () => { @@ -36,7 +36,7 @@ describe("SearchSortBy", () => { ); expect( - screen.getByDisplayValue("Opportunity Number (Ascending)"), + screen.getByDisplayValue("Posted Date (newest)"), ).toBeInTheDocument(); }); @@ -57,7 +57,7 @@ describe("SearchSortBy", () => { }); expect( - screen.getByDisplayValue("Opportunity Title (Descending)"), + screen.getByDisplayValue("Opportunity Title (Z to A)"), ).toBeInTheDocument(); expect(requestSubmitMock).toHaveBeenCalled(); From 16f708e17e6716ec852da07116d370acfd43245b Mon Sep 17 00:00:00 2001 From: Michael Chouinard <46358556+chouinar@users.noreply.github.com> Date: Wed, 22 May 2024 13:48:48 -0400 Subject: [PATCH 3/5] Upgrade dependencies for API (May 21, 2024) (#48) ### Time to review: __1 mins__ ## Changes proposed Needed to upgrade dependencies for the API for grype issue: https://github.com/navapbc/simpler-grants-gov/actions/runs/9180615894/job/25245519194?pr=47 ## Additional information As usual, just ran `poetry update` --- api/poetry.lock | 53 ++++++++++++++++++++----------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 2b372c21e..5fe4fa9e1 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -21,13 +21,13 @@ tz = ["backports.zoneinfo"] [[package]] name = "annotated-types" -version = "0.6.0" +version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" files = [ - {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, - {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] [[package]] @@ -156,17 +156,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.103" +version = "1.34.110" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.103-py3-none-any.whl", hash = "sha256:59b6499f1bb423dd99de6566a20d0a7cf1a5476824be3a792290fd86600e8365"}, - {file = "boto3-1.34.103.tar.gz", hash = "sha256:58d097241f3895c4a4c80c9e606689c6e06d77f55f9f53a4cc02dee7e03938b9"}, + {file = "boto3-1.34.110-py3-none-any.whl", hash = "sha256:2fc871b4a5090716c7a71af52c462e539529227f4d4888fd04896d5028f9cedc"}, + {file = "boto3-1.34.110.tar.gz", hash = "sha256:83ffe2273da7bdfdb480d85b0705f04e95bd110e9741f23328b7c76c03e6d53c"}, ] [package.dependencies] -botocore = ">=1.34.103,<1.35.0" +botocore = ">=1.34.110,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -175,13 +175,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.103" +version = "1.34.110" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.103-py3-none-any.whl", hash = "sha256:0330d139f18f78d38127e65361859e24ebd6a8bcba184f903c01bb999a3fa431"}, - {file = "botocore-1.34.103.tar.gz", hash = "sha256:5f07e2c7302c0a9f469dcd08b4ddac152e9f5888b12220242c20056255010939"}, + {file = "botocore-1.34.110-py3-none-any.whl", hash = "sha256:1edf3a825ec0a5edf238b2d42ad23305de11d5a71bb27d6f9a58b7e8862df1b6"}, + {file = "botocore-1.34.110.tar.gz", hash = "sha256:b2c98c40ecf0b1facb9e61ceb7dfa28e61ae2456490554a16c8dbf99f20d6a18"}, ] [package.dependencies] @@ -836,13 +836,13 @@ files = [ [[package]] name = "mako" -version = "1.3.3" +version = "1.3.5" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false python-versions = ">=3.8" files = [ - {file = "Mako-1.3.3-py3-none-any.whl", hash = "sha256:5324b88089a8978bf76d1629774fcc2f1c07b82acdf00f4c5dd8ceadfffc4b40"}, - {file = "Mako-1.3.3.tar.gz", hash = "sha256:e16c01d9ab9c11f7290eef1cfefc093fb5a45ee4a3da09e2fec2e4d1bae54e73"}, + {file = "Mako-1.3.5-py3-none-any.whl", hash = "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a"}, + {file = "Mako-1.3.5.tar.gz", hash = "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc"}, ] [package.dependencies] @@ -978,7 +978,7 @@ files = [ [package.dependencies] marshmallow = [ - {version = ">=3.13.0,<4.0"}, + {version = ">=3.13.0,<4.0", optional = true, markers = "python_version < \"3.7\" or extra != \"enum\""}, {version = ">=3.18.0,<4.0", optional = true, markers = "python_version >= \"3.7\" and extra == \"enum\""}, ] typeguard = {version = ">=2.4.1,<4.0.0", optional = true, markers = "extra == \"union\""} @@ -1141,13 +1141,13 @@ files = [ [[package]] name = "platformdirs" -version = "4.2.1" +version = "4.2.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, - {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] [package.extras] @@ -1563,7 +1563,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1571,16 +1570,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1597,7 +1588,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1605,7 +1595,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1613,13 +1602,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] From 25b0295506439ff8dacfded16269ecc21d9edc4a Mon Sep 17 00:00:00 2001 From: Michael Chouinard <46358556+chouinar@users.noreply.github.com> Date: Wed, 22 May 2024 13:58:12 -0400 Subject: [PATCH 4/5] [Issue #9] Setup opensearch locally (#39) ## Summary Fixes #9 ### Time to review: __10 mins__ ## Changes proposed Setup a search index to run locally via Docker Updated makefile to automatically initialize the index + added a script to wait for the index to start up before proceeding. Setup a very basic client for connecting to the search index (will be expanded more in subsequent PRs) Basic test / test utils to verify it is working (also will be expanded) ## Context for reviewers This is the first step in getting the search index working locally. This actually gets it running, and the client works, we just aren't doing anything meaningful with it yet besides tests. ## Additional information This doesn't yet create an index that we can use, except in the test. However, if you want to test out a search index, you can go to http://localhost:5601/app/dev_tools#/console (after running `make init`) to run some queries against the (one node) cluster. https://opensearch.org/docs/latest/getting-started/communicate/#sending-requests-in-dev-tools provides some examples of how to create + use indexes that you can follow. --- api/Makefile | 15 ++++- api/bin/wait-for-local-opensearch.sh | 31 +++++++++ api/docker-compose.yml | 37 +++++++++++ api/local.env | 9 +++ api/poetry.lock | 66 ++++++++++++++++--- api/pyproject.toml | 7 ++ api/src/adapters/search/__init__.py | 4 ++ api/src/adapters/search/opensearch_client.py | 36 ++++++++++ api/src/adapters/search/opensearch_config.py | 33 ++++++++++ api/tests/conftest.py | 29 ++++++++ api/tests/src/adapters/search/__init__.py | 0 .../src/adapters/search/test_opensearch.py | 58 ++++++++++++++++ 12 files changed, 315 insertions(+), 10 deletions(-) create mode 100755 api/bin/wait-for-local-opensearch.sh create mode 100644 api/src/adapters/search/__init__.py create mode 100644 api/src/adapters/search/opensearch_client.py create mode 100644 api/src/adapters/search/opensearch_config.py create mode 100644 api/tests/src/adapters/search/__init__.py create mode 100644 api/tests/src/adapters/search/test_opensearch.py diff --git a/api/Makefile b/api/Makefile index f2774d3a7..d5daab1d2 100644 --- a/api/Makefile +++ b/api/Makefile @@ -100,7 +100,7 @@ start-debug: run-logs: start docker-compose logs --follow --no-color $(APP_NAME) -init: build init-db +init: build init-db init-opensearch clean-volumes: ## Remove project docker volumes (which includes the DB state) docker-compose down --volumes @@ -179,6 +179,19 @@ create-erds: # Create ERD diagrams for our DB schema setup-postgres-db: ## Does any initial setup necessary for our local database to work $(PY_RUN_CMD) setup-postgres-db +################################################## +# Opensearch +################################################## + +init-opensearch: start-opensearch +# TODO - in subsequent PRs, we'll add more to this command to setup the search index locally + +start-opensearch: + docker-compose up --detach opensearch-node + docker-compose up --detach opensearch-dashboards + ./bin/wait-for-local-opensearch.sh + + ################################################## # Testing diff --git a/api/bin/wait-for-local-opensearch.sh b/api/bin/wait-for-local-opensearch.sh new file mode 100755 index 000000000..a14af8048 --- /dev/null +++ b/api/bin/wait-for-local-opensearch.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# wait-for-local-opensearch + +set -e + +# Color formatting +RED='\033[0;31m' +NO_COLOR='\033[0m' + +MAX_WAIT_TIME=30 # seconds +WAIT_TIME=0 + +# Curl the healthcheck endpoint of the local opensearch +# until it returns a success response +until curl --output /dev/null --silent http://localhost:9200/_cluster/health; +do + echo "waiting on OpenSearch to initialize..." + sleep 3 + + WAIT_TIME=$(($WAIT_TIME+3)) + if [ $WAIT_TIME -gt $MAX_WAIT_TIME ] + then + echo -e "${RED}ERROR: OpenSearch appears to not be starting up, running \"docker logs opensearch-node\" to troubleshoot.${NO_COLOR}" + docker logs opensearch-node + exit 1 + fi +done + +echo "OpenSearch is ready after ~${WAIT_TIME} seconds" + + diff --git a/api/docker-compose.yml b/api/docker-compose.yml index a364c74c3..9ec206214 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -12,6 +12,41 @@ services: volumes: - grantsdbdata:/var/lib/postgresql/data + opensearch-node: + image: opensearchproject/opensearch:latest + container_name: opensearch-node + environment: + - cluster.name=opensearch-cluster # Name the cluster + - node.name=opensearch-node # Name the node that will run in this container + - discovery.type=single-node # Nodes to look for when discovering the cluster + - bootstrap.memory_lock=true # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM + - DISABLE_INSTALL_DEMO_CONFIG=true # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch + - DISABLE_SECURITY_PLUGIN=true # Disables Security plugin + ulimits: + memlock: + soft: -1 # Set memlock to unlimited (no soft or hard limit) + hard: -1 + nofile: + soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 + hard: 65536 + volumes: + - opensearch-data:/usr/share/opensearch/data # Creates volume called opensearch-data and mounts it to the container + ports: + - 9200:9200 # REST API + - 9600:9600 # Performance Analyzer + + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest + container_name: opensearch-dashboards + ports: + - 5601:5601 # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + environment: + - 'OPENSEARCH_HOSTS=["http://opensearch-node:9200"]' + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true # disables security dashboards plugin in OpenSearch Dashboards + grants-api: build: context: . @@ -28,6 +63,8 @@ services: - .:/api depends_on: - grants-db + - opensearch-node volumes: grantsdbdata: + opensearch-data: diff --git a/api/local.env b/api/local.env index fc1c1c1a4..4ca4c86b5 100644 --- a/api/local.env +++ b/api/local.env @@ -59,6 +59,15 @@ DB_SSL_MODE=allow # could contain sensitive information. HIDE_SQL_PARAMETER_LOGS=TRUE +############################ +# Opensearch Environment Variables +############################ + +OPENSEARCH_HOST=opensearch-node +OPENSEARCH_PORT=9200 +OPENSEARCH_USE_SSL=FALSE +OPENSEARCH_VERIFY_CERTS=FALSE + ############################ # AWS Defaults ############################ diff --git a/api/poetry.lock b/api/poetry.lock index 5fe4fa9e1..017f1460e 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1106,6 +1106,30 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "opensearch-py" +version = "2.5.0" +description = "Python client for OpenSearch" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<4,>=2.7" +files = [ + {file = "opensearch-py-2.5.0.tar.gz", hash = "sha256:0dde4ac7158a717d92a8cd81964cb99705a4b80bcf9258ba195b9a9f23f5226d"}, + {file = "opensearch_py-2.5.0-py2.py3-none-any.whl", hash = "sha256:cf093a40e272b60663f20417fc1264ac724dcf1e03c1a4542a6b44835b1e6c49"}, +] + +[package.dependencies] +certifi = ">=2022.12.07" +python-dateutil = "*" +requests = ">=2.4.0,<3.0.0" +six = "*" +urllib3 = ">=1.26.18,<2" + +[package.extras] +async = ["aiohttp (>=3,<4)"] +develop = ["black", "botocore", "coverage (<8.0.0)", "jinja2", "mock", "myst-parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +docs = ["aiohttp (>=3,<4)", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +kerberos = ["requests-kerberos"] + [[package]] name = "packaging" version = "24.0" @@ -1902,6 +1926,31 @@ files = [ {file = "types_PyYAML-6.0.12.20240311-py3-none-any.whl", hash = "sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6"}, ] +[[package]] +name = "types-requests" +version = "2.31.0.1" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.31.0.1.tar.gz", hash = "sha256:3de667cffa123ce698591de0ad7db034a5317457a596eb0b4944e5a9d9e8d1ac"}, + {file = "types_requests-2.31.0.1-py3-none-any.whl", hash = "sha256:afb06ef8f25ba83d59a1d424bd7a5a939082f94b94e90ab5e6116bd2559deaa3"}, +] + +[package.dependencies] +types-urllib3 = "*" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + [[package]] name = "typing-extensions" version = "4.11.0" @@ -1941,20 +1990,19 @@ files = [ [[package]] name = "urllib3" -version = "2.2.1" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "watchdog" @@ -2050,4 +2098,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "c53875955c1b910c3d4aa1748dce786e3cfa6f507895d7ca4111391333decb13" +content-hash = "9671a2d68d2b1bc91b8ce111a7a32d08292475e0d1c4f058c33bf650349757e0" diff --git a/api/pyproject.toml b/api/pyproject.toml index f0a06b447..0f3c2f10b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -22,6 +22,7 @@ gunicorn = "^22.0.0" psycopg = { extras = ["binary"], version = "^3.1.10" } pydantic-settings = "^2.0.3" flask-cors = "^4.0.0" +opensearch-py = "^2.5.0" [tool.poetry.group.dev.dependencies] black = "^23.9.1" @@ -43,6 +44,12 @@ sadisplay = "0.4.9" ruff = "^0.4.0" debugpy = "^1.8.1" freezegun = "^1.5.0" +# This isn't the latest version of types-requests +# because otherwise it depends on urllib3 v2 but opensearch-py +# needs urlib3 v1. This should be temporary as opensearch-py +# has an unreleased change to switch to v2, so I'm guessing +# in the next few weeks we can just make this the latest? +types-requests = "2.31.0.1" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/api/src/adapters/search/__init__.py b/api/src/adapters/search/__init__.py new file mode 100644 index 000000000..166441e1d --- /dev/null +++ b/api/src/adapters/search/__init__.py @@ -0,0 +1,4 @@ +from src.adapters.search.opensearch_client import SearchClient, get_opensearch_client +from src.adapters.search.opensearch_config import get_opensearch_config + +__all__ = ["SearchClient", "get_opensearch_client", "get_opensearch_config"] diff --git a/api/src/adapters/search/opensearch_client.py b/api/src/adapters/search/opensearch_client.py new file mode 100644 index 000000000..dadcfd7c4 --- /dev/null +++ b/api/src/adapters/search/opensearch_client.py @@ -0,0 +1,36 @@ +from typing import Any + +import opensearchpy + +from src.adapters.search.opensearch_config import OpensearchConfig, get_opensearch_config + +# More configuration/setup coming in: +# TODO - https://github.com/navapbc/simpler-grants-gov/issues/13 + +# Alias the OpenSearch client so that it doesn't need to be imported everywhere +# and to make it clear it's a client +SearchClient = opensearchpy.OpenSearch + + +def get_opensearch_client( + opensearch_config: OpensearchConfig | None = None, +) -> SearchClient: + if opensearch_config is None: + opensearch_config = get_opensearch_config() + + # See: https://opensearch.org/docs/latest/clients/python-low-level/ for more details + return opensearchpy.OpenSearch(**_get_connection_parameters(opensearch_config)) + + +def _get_connection_parameters(opensearch_config: OpensearchConfig) -> dict[str, Any]: + # TODO - we'll want to add the AWS connection params here when we set that up + # See: https://opensearch.org/docs/latest/clients/python-low-level/#connecting-to-amazon-opensearch-serverless + + return dict( + hosts=[{"host": opensearch_config.host, "port": opensearch_config.port}], + http_compress=True, + use_ssl=opensearch_config.use_ssl, + verify_certs=opensearch_config.verify_certs, + ssl_assert_hostname=False, + ssl_show_warn=False, + ) diff --git a/api/src/adapters/search/opensearch_config.py b/api/src/adapters/search/opensearch_config.py new file mode 100644 index 000000000..4975feb3e --- /dev/null +++ b/api/src/adapters/search/opensearch_config.py @@ -0,0 +1,33 @@ +import logging + +from pydantic import Field +from pydantic_settings import SettingsConfigDict + +from src.util.env_config import PydanticBaseEnvConfig + +logger = logging.getLogger(__name__) + + +class OpensearchConfig(PydanticBaseEnvConfig): + model_config = SettingsConfigDict(env_prefix="OPENSEARCH_") + + host: str # OPENSEARCH_HOST + port: int # OPENSEARCH_PORT + use_ssl: bool = Field(default=True) # OPENSEARCH_USE_SSL + verify_certs: bool = Field(default=True) # OPENSEARCH_VERIFY_CERTS + + +def get_opensearch_config() -> OpensearchConfig: + opensearch_config = OpensearchConfig() + + logger.info( + "Constructed opensearch configuration", + extra={ + "host": opensearch_config.host, + "port": opensearch_config.port, + "use_ssl": opensearch_config.use_ssl, + "verify_certs": opensearch_config.verify_certs, + }, + ) + + return opensearch_config diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 928932b67..97173e9a7 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -11,6 +11,7 @@ import src.adapters.db as db import src.app as app_entry import tests.src.db.models.factories as factories +from src.adapters import search from src.constants.schema import Schemas from src.db import models from src.db.models.lookup.sync_lookup_values import sync_lookup_values @@ -143,6 +144,34 @@ def test_foreign_schema(db_schema_prefix): return f"{db_schema_prefix}{Schemas.LEGACY}" +#################### +# Opensearch Fixtures +#################### + + +@pytest.fixture(scope="session") +def search_client() -> search.SearchClient: + return search.get_opensearch_client() + + +@pytest.fixture(scope="session") +def opportunity_index(search_client): + # TODO - will adjust this in the future to use utils we'll build + # for setting up / aliasing indexes. For now, keep it simple + + # create a random index name just to make sure it won't ever conflict + # with an actual one, similar to how we create schemas for database tests + index_name = f"test_{uuid.uuid4().int}_opportunity" + + search_client.indices.create(index_name, body={}) + + try: + yield index_name + finally: + # Try to clean up the index at the end + search_client.indices.delete(index_name) + + #################### # Test App & Client #################### diff --git a/api/tests/src/adapters/search/__init__.py b/api/tests/src/adapters/search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/src/adapters/search/test_opensearch.py b/api/tests/src/adapters/search/test_opensearch.py new file mode 100644 index 000000000..490ffcb3b --- /dev/null +++ b/api/tests/src/adapters/search/test_opensearch.py @@ -0,0 +1,58 @@ +######################################## +# This is a placeholder set of tests, +# we'll evolve / change the structure +# as we continue developing this +# +# Just wanted something simple so I can verify +# the early steps of this setup are working +# before we actually have code to use +######################################## + + +def test_index_is_running(search_client, opportunity_index): + # Very simple test, will rewrite / remove later once we have something + # more meaningful to test. + + existing_indexes = search_client.cat.indices(format="json") + + found_opportunity_index = False + for index in existing_indexes: + if index["index"] == opportunity_index: + found_opportunity_index = True + break + + assert found_opportunity_index is True + + # Add a few records to the index + + record1 = { + "opportunity_id": 1, + "opportunity_title": "Research into how to make a search engine", + "opportunity_status": "posted", + } + record2 = { + "opportunity_id": 2, + "opportunity_title": "Research about words, and more words!", + "opportunity_status": "forecasted", + } + + search_client.index(index=opportunity_index, body=record1, id=1, refresh=True) + search_client.index(index=opportunity_index, body=record2, id=2, refresh=True) + + search_request = { + "query": { + "bool": { + "must": { + "simple_query_string": {"query": "research", "fields": ["opportunity_title"]} + } + } + } + } + response = search_client.search(index=opportunity_index, body=search_request) + assert response["hits"]["total"]["value"] == 2 + + filter_request = { + "query": {"bool": {"filter": [{"terms": {"opportunity_status": ["forecasted"]}}]}} + } + response = search_client.search(index=opportunity_index, body=filter_request) + assert response["hits"]["total"]["value"] == 1 From b40344d25f3e6ae65ed11012c0b24410f60e03e4 Mon Sep 17 00:00:00 2001 From: Michael Chouinard <46358556+chouinar@users.noreply.github.com> Date: Wed, 22 May 2024 14:05:43 -0400 Subject: [PATCH 5/5] [Issue #12] Setup the opportunity v1 endpoint which will be backed by the index (#44) ## Summary Fixes #12 ### Time to review: __5 mins__ ## Changes proposed Made a new set of v1 endpoints that are basically copy-pastes of the v0.1 opportunity endpoints ## Context for reviewers Some changes I want to make to the schemas wouldn't make sense without the search index (eg. adding the filter counts to the response). As we have no idea what the actual launch of the v0.1 endpoint is going to look like, I don't want to mess with any of that code or try to make a weird hacky approach that needs to account for both the DB implementation and the search index one. Also, I think we've heard that with the launch of the search index, we'll be "officially" launched, so might as well call in v1 at the same time. Other than adjusting the names of a few schemas in v0.1, I left that implementation alone and just copied the boilerplate that I'll fill out in subsequent tickets. ## Additional information The endpoint appears locally: ![Screenshot 2024-05-20 at 12 18 32 PM](https://github.com/navapbc/simpler-grants-gov/assets/46358556/86231ec1-417a-41c6-ad88-3d06bb6214e5) --------- Co-authored-by: nava-platform-bot --- api/openapi.generated.yml | 740 +++++++++++++++++- .../opportunities_v0_1/opportunity_routes.py | 8 +- .../opportunities_v0_1/opportunity_schemas.py | 26 +- api/src/api/opportunities_v1/__init__.py | 6 + .../opportunities_v1/opportunity_blueprint.py | 9 + .../opportunities_v1/opportunity_routes.py | 66 ++ .../opportunities_v1/opportunity_schemas.py | 298 +++++++ api/src/app.py | 2 + api/src/services/opportunities_v1/__init__.py | 0 .../opportunities_v1/get_opportunity.py | 24 + .../opportunities_v1/search_opportunities.py | 39 + .../test_opportunity_route_search.py | 1 - .../src/api/opportunities_v1/__init__.py | 0 .../src/api/opportunities_v1/conftest.py | 183 +++++ .../opportunities_v1/test_opportunity_auth.py | 21 + .../test_opportunity_route_get.py | 97 +++ .../test_opportunity_route_search.py | 19 + 17 files changed, 1488 insertions(+), 51 deletions(-) create mode 100644 api/src/api/opportunities_v1/__init__.py create mode 100644 api/src/api/opportunities_v1/opportunity_blueprint.py create mode 100644 api/src/api/opportunities_v1/opportunity_routes.py create mode 100644 api/src/api/opportunities_v1/opportunity_schemas.py create mode 100644 api/src/services/opportunities_v1/__init__.py create mode 100644 api/src/services/opportunities_v1/get_opportunity.py create mode 100644 api/src/services/opportunities_v1/search_opportunities.py create mode 100644 api/tests/src/api/opportunities_v1/__init__.py create mode 100644 api/tests/src/api/opportunities_v1/conftest.py create mode 100644 api/tests/src/api/opportunities_v1/test_opportunity_auth.py create mode 100644 api/tests/src/api/opportunities_v1/test_opportunity_route_get.py create mode 100644 api/tests/src/api/opportunities_v1/test_opportunity_route_search.py diff --git a/api/openapi.generated.yml b/api/openapi.generated.yml index b6b756ae0..7302ded44 100644 --- a/api/openapi.generated.yml +++ b/api/openapi.generated.yml @@ -23,6 +23,7 @@ tags: - name: Health - name: Opportunity v0 - name: Opportunity v0.1 +- name: Opportunity v1 servers: . paths: /health: @@ -204,7 +205,7 @@ paths: $ref: '#/components/schemas/OpportunitySearch' security: - ApiKeyAuth: [] - /v0.1/opportunities/search: + /v1/opportunities/search: post: parameters: [] responses: @@ -220,7 +221,7 @@ paths: data: type: array items: - $ref: '#/components/schemas/Opportunity' + $ref: '#/components/schemas/OpportunityV1' status_code: type: integer description: The HTTP status code @@ -291,6 +292,118 @@ paths: - $ref: '#/components/schemas/ValidationIssue' description: Authentication error tags: + - Opportunity v1 + summary: Opportunity Search + description: ' + + __ALPHA VERSION__ + + + This endpoint in its current form is primarily for testing and feedback. + + + Features in this endpoint are still under heavy development, and subject to + change. Not for production use. + + + See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) + for further details. + + ' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/OpportunitySearchRequestV1' + security: + - ApiKeyAuth: [] + /v0.1/opportunities/search: + post: + parameters: [] + responses: + '200': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + type: array + items: + $ref: '#/components/schemas/OpportunityV01' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: &id007 + - object + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: &id008 + - object + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Successful response + '422': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + $ref: '#/components/schemas/ErrorResponse' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: *id007 + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: *id008 + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Validation error + '401': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + $ref: '#/components/schemas/ErrorResponse' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: *id007 + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: *id008 + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Authentication error + tags: - Opportunity v0.1 summary: Opportunity Search description: ' @@ -313,7 +426,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/OpportunitySearchRequest' + $ref: '#/components/schemas/OpportunitySearchRequestV01' examples: example1: summary: No filters @@ -382,14 +495,14 @@ paths: description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: &id007 + type: &id009 - object allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: &id008 + type: &id010 - object allOf: - $ref: '#/components/schemas/ValidationIssue' @@ -410,13 +523,13 @@ paths: description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: *id007 + type: *id009 allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: *id008 + type: *id010 allOf: - $ref: '#/components/schemas/ValidationIssue' description: Authentication error @@ -436,13 +549,13 @@ paths: description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: *id007 + type: *id009 allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: *id008 + type: *id010 allOf: - $ref: '#/components/schemas/ValidationIssue' description: Not found @@ -461,6 +574,116 @@ paths: change. Not for production use. + See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) + for further details. + + ' + security: + - ApiKeyAuth: [] + /v1/opportunities/{opportunity_id}: + get: + parameters: + - in: path + name: opportunity_id + schema: + type: integer + required: true + responses: + '200': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + $ref: '#/components/schemas/OpportunityV1' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: &id011 + - object + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: &id012 + - object + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Successful response + '401': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + $ref: '#/components/schemas/ErrorResponse' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: *id011 + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: *id012 + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Authentication error + '404': + content: + application/json: + schema: + type: object + properties: + message: + type: string + description: The message to return + data: + $ref: '#/components/schemas/ErrorResponse' + status_code: + type: integer + description: The HTTP status code + pagination_info: + description: The pagination information for paginated endpoints + type: *id011 + allOf: + - $ref: '#/components/schemas/PaginationInfo' + warnings: + type: array + items: + type: *id012 + allOf: + - $ref: '#/components/schemas/ValidationIssue' + description: Not found + tags: + - Opportunity v1 + summary: Opportunity Get + description: ' + + __ALPHA VERSION__ + + + This endpoint in its current form is primarily for testing and feedback. + + + Features in this endpoint are still under heavy development, and subject to + change. Not for production use. + + See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) for further details. @@ -486,20 +709,20 @@ paths: type: string description: The message to return data: - $ref: '#/components/schemas/Opportunity' + $ref: '#/components/schemas/OpportunityV01' status_code: type: integer description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: &id009 + type: &id013 - object allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: &id010 + type: &id014 - object allOf: - $ref: '#/components/schemas/ValidationIssue' @@ -520,13 +743,13 @@ paths: description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: *id009 + type: *id013 allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: *id010 + type: *id014 allOf: - $ref: '#/components/schemas/ValidationIssue' description: Authentication error @@ -546,13 +769,13 @@ paths: description: The HTTP status code pagination_info: description: The pagination information for paginated endpoints - type: *id009 + type: *id013 allOf: - $ref: '#/components/schemas/PaginationInfo' warnings: type: array items: - type: *id010 + type: *id014 allOf: - $ref: '#/components/schemas/ValidationIssue' description: Not found @@ -768,7 +991,7 @@ components: type: string format: date-time readOnly: true - FundingInstrumentFilter: + FundingInstrumentFilterV1: type: object properties: one_of: @@ -782,7 +1005,7 @@ components: - other type: - string - FundingCategoryFilter: + FundingCategoryFilterV1: type: object properties: one_of: @@ -818,7 +1041,7 @@ components: - other type: - string - ApplicantTypeFilter: + ApplicantTypeFilterV1: type: object properties: one_of: @@ -845,7 +1068,7 @@ components: - unrestricted type: - string - OpportunityStatusFilter: + OpportunityStatusFilterV1: type: object properties: one_of: @@ -859,7 +1082,7 @@ components: - archived type: - string - AgencyFilter: + AgencyFilterV1: type: object properties: one_of: @@ -869,34 +1092,34 @@ components: type: string minLength: 2 example: US-ABC - OpportunitySearchFilter: + OpportunitySearchFilterV1: type: object properties: funding_instrument: type: - object allOf: - - $ref: '#/components/schemas/FundingInstrumentFilter' + - $ref: '#/components/schemas/FundingInstrumentFilterV1' funding_category: type: - object allOf: - - $ref: '#/components/schemas/FundingCategoryFilter' + - $ref: '#/components/schemas/FundingCategoryFilterV1' applicant_type: type: - object allOf: - - $ref: '#/components/schemas/ApplicantTypeFilter' + - $ref: '#/components/schemas/ApplicantTypeFilterV1' opportunity_status: type: - object allOf: - - $ref: '#/components/schemas/OpportunityStatusFilter' + - $ref: '#/components/schemas/OpportunityStatusFilterV1' agency: type: - object allOf: - - $ref: '#/components/schemas/AgencyFilter' + - $ref: '#/components/schemas/AgencyFilterV1' OpportunityPagination: type: object properties: @@ -932,7 +1155,7 @@ components: - page_offset - page_size - sort_direction - OpportunitySearchRequest: + OpportunitySearchRequestV1: type: object properties: query: @@ -945,7 +1168,7 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunitySearchFilter' + - $ref: '#/components/schemas/OpportunitySearchFilterV1' pagination: type: - object @@ -953,7 +1176,456 @@ components: - $ref: '#/components/schemas/OpportunityPagination' required: - pagination - OpportunityAssistanceListing: + OpportunityAssistanceListingV1: + type: object + properties: + program_title: + type: string + description: The name of the program, see https://sam.gov/content/assistance-listings + for more detail + example: Space Technology + assistance_listing_number: + type: string + description: The assistance listing number, see https://sam.gov/content/assistance-listings + for more detail + example: '43.012' + OpportunitySummaryV1: + type: object + properties: + summary_description: + type: string + description: The summary of the opportunity + example: This opportunity aims to unravel the mysteries of the universe. + is_cost_sharing: + type: boolean + description: Whether or not the opportunity has a cost sharing/matching + requirement + is_forecast: + type: boolean + description: Whether the opportunity is forecasted, that is, the information + is only an estimate and not yet official + example: false + close_date: + type: string + format: date + description: The date that the opportunity will close - only set if is_forecast=False + close_date_description: + type: string + description: Optional details regarding the close date + example: Proposals are due earlier than usual. + post_date: + type: string + format: date + description: The date the opportunity was posted + archive_date: + type: string + format: date + description: When the opportunity will be archived + expected_number_of_awards: + type: integer + description: The number of awards the opportunity is expected to award + example: 10 + estimated_total_program_funding: + type: integer + description: The total program funding of the opportunity in US Dollars + example: 10000000 + award_floor: + type: integer + description: The minimum amount an opportunity would award + example: 10000 + award_ceiling: + type: integer + description: The maximum amount an opportunity would award + example: 100000 + additional_info_url: + type: string + description: A URL to a website that can provide additional information + about the opportunity + example: grants.gov + additional_info_url_description: + type: string + description: The text to display for the additional_info_url link + example: Click me for more info + forecasted_post_date: + type: string + format: date + description: Forecasted opportunity only. The date the opportunity is expected + to be posted, and transition out of being a forecast + forecasted_close_date: + type: string + format: date + description: Forecasted opportunity only. The date the opportunity is expected + to be close once posted. + forecasted_close_date_description: + type: string + description: Forecasted opportunity only. Optional details regarding the + forecasted closed date. + example: Proposals will probably be due on this date + forecasted_award_date: + type: string + format: date + description: Forecasted opportunity only. The date the grantor plans to + award the opportunity. + forecasted_project_start_date: + type: string + format: date + description: Forecasted opportunity only. The date the grantor expects the + award recipient should start their project + fiscal_year: + type: integer + description: Forecasted opportunity only. The fiscal year the project is + expected to be funded and launched + funding_category_description: + type: string + description: Additional information about the funding category + example: Economic Support + applicant_eligibility_description: + type: string + description: Additional information about the types of applicants that are + eligible + example: All types of domestic applicants are eligible to apply + agency_code: + type: string + description: The agency who owns the opportunity + example: US-ABC + agency_name: + type: string + description: The name of the agency who owns the opportunity + example: US Alphabetical Basic Corp + agency_phone_number: + type: string + description: The phone number of the agency who owns the opportunity + example: 123-456-7890 + agency_contact_description: + type: string + description: Information regarding contacting the agency who owns the opportunity + example: For more information, reach out to Jane Smith at agency US-ABC + agency_email_address: + type: string + description: The contact email of the agency who owns the opportunity + example: fake_email@grants.gov + agency_email_address_description: + type: string + description: The text for the link to the agency email address + example: Click me to email the agency + funding_instruments: + type: array + items: + enum: + - cooperative_agreement + - grant + - procurement_contract + - other + type: + - string + funding_categories: + type: array + items: + enum: + - recovery_act + - agriculture + - arts + - business_and_commerce + - community_development + - consumer_protection + - disaster_prevention_and_relief + - education + - employment_labor_and_training + - energy + - environment + - food_and_nutrition + - health + - housing + - humanities + - infrastructure_investment_and_jobs_act + - information_and_statistics + - income_security_and_social_services + - law_justice_and_legal_services + - natural_resources + - opportunity_zone_benefits + - regional_development + - science_technology_and_other_research_and_development + - transportation + - affordable_care_act + - other + type: + - string + applicant_types: + type: array + items: + enum: + - state_governments + - county_governments + - city_or_township_governments + - special_district_governments + - independent_school_districts + - public_and_state_institutions_of_higher_education + - private_institutions_of_higher_education + - federally_recognized_native_american_tribal_governments + - other_native_american_tribal_organizations + - public_and_indian_housing_authorities + - nonprofits_non_higher_education_with_501c3 + - nonprofits_non_higher_education_without_501c3 + - individuals + - for_profit_organizations_other_than_small_businesses + - small_businesses + - other + - unrestricted + type: + - string + OpportunityV1: + type: object + properties: + opportunity_id: + type: integer + readOnly: true + description: The internal ID of the opportunity + example: 12345 + opportunity_number: + type: string + description: The funding opportunity number + example: ABC-123-XYZ-001 + opportunity_title: + type: string + description: The title of the opportunity + example: Research into conservation techniques + agency: + type: string + description: The agency who created the opportunity + example: US-ABC + category: + description: The opportunity category + example: !!python/object/apply:src.constants.lookup_constants.OpportunityCategory + - discretionary + enum: + - discretionary + - mandatory + - continuation + - earmark + - other + type: + - string + category_explanation: + type: string + description: Explanation of the category when the category is 'O' (other) + example: null + opportunity_assistance_listings: + type: array + items: + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunityAssistanceListingV1' + summary: + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunitySummaryV1' + opportunity_status: + description: The current status of the opportunity + example: !!python/object/apply:src.constants.lookup_constants.OpportunityStatus + - posted + enum: + - forecasted + - posted + - closed + - archived + type: + - string + created_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + FundingInstrumentFilterV01: + type: object + properties: + one_of: + type: array + minItems: 1 + items: + enum: + - cooperative_agreement + - grant + - procurement_contract + - other + type: + - string + FundingCategoryFilterV01: + type: object + properties: + one_of: + type: array + minItems: 1 + items: + enum: + - recovery_act + - agriculture + - arts + - business_and_commerce + - community_development + - consumer_protection + - disaster_prevention_and_relief + - education + - employment_labor_and_training + - energy + - environment + - food_and_nutrition + - health + - housing + - humanities + - infrastructure_investment_and_jobs_act + - information_and_statistics + - income_security_and_social_services + - law_justice_and_legal_services + - natural_resources + - opportunity_zone_benefits + - regional_development + - science_technology_and_other_research_and_development + - transportation + - affordable_care_act + - other + type: + - string + ApplicantTypeFilterV01: + type: object + properties: + one_of: + type: array + minItems: 1 + items: + enum: + - state_governments + - county_governments + - city_or_township_governments + - special_district_governments + - independent_school_districts + - public_and_state_institutions_of_higher_education + - private_institutions_of_higher_education + - federally_recognized_native_american_tribal_governments + - other_native_american_tribal_organizations + - public_and_indian_housing_authorities + - nonprofits_non_higher_education_with_501c3 + - nonprofits_non_higher_education_without_501c3 + - individuals + - for_profit_organizations_other_than_small_businesses + - small_businesses + - other + - unrestricted + type: + - string + OpportunityStatusFilterV01: + type: object + properties: + one_of: + type: array + minItems: 1 + items: + enum: + - forecasted + - posted + - closed + - archived + type: + - string + AgencyFilterV01: + type: object + properties: + one_of: + type: array + minItems: 1 + items: + type: string + minLength: 2 + example: US-ABC + OpportunitySearchFilterV01: + type: object + properties: + funding_instrument: + type: + - object + allOf: + - $ref: '#/components/schemas/FundingInstrumentFilterV01' + funding_category: + type: + - object + allOf: + - $ref: '#/components/schemas/FundingCategoryFilterV01' + applicant_type: + type: + - object + allOf: + - $ref: '#/components/schemas/ApplicantTypeFilterV01' + opportunity_status: + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunityStatusFilterV01' + agency: + type: + - object + allOf: + - $ref: '#/components/schemas/AgencyFilterV01' + OpportunityPagination1: + type: object + properties: + order_by: + type: string + enum: + - opportunity_id + - opportunity_number + - opportunity_title + - post_date + - close_date + - agency_code + description: The field to sort the response by + sort_direction: + description: Whether to sort the response ascending or descending + enum: + - ascending + - descending + type: + - string + page_size: + type: integer + minimum: 1 + description: The size of the page to fetch + example: 25 + page_offset: + type: integer + minimum: 1 + description: The page number to fetch, starts counting from 1 + example: 1 + required: + - order_by + - page_offset + - page_size + - sort_direction + OpportunitySearchRequestV01: + type: object + properties: + query: + type: string + minLength: 1 + maxLength: 100 + description: Query string which searches against several text fields + example: research + filters: + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunitySearchFilterV01' + pagination: + type: + - object + allOf: + - $ref: '#/components/schemas/OpportunityPagination1' + required: + - pagination + OpportunityAssistanceListingV01: type: object properties: program_title: @@ -966,7 +1638,7 @@ components: description: The assistance listing number, see https://sam.gov/content/assistance-listings for more detail example: '43.012' - OpportunitySummary: + OpportunitySummaryV01: type: object properties: summary_description: @@ -1150,7 +1822,7 @@ components: - unrestricted type: - string - Opportunity: + OpportunityV01: type: object properties: opportunity_id: @@ -1192,12 +1864,12 @@ components: type: - object allOf: - - $ref: '#/components/schemas/OpportunityAssistanceListing' + - $ref: '#/components/schemas/OpportunityAssistanceListingV01' summary: type: - object allOf: - - $ref: '#/components/schemas/OpportunitySummary' + - $ref: '#/components/schemas/OpportunitySummaryV01' opportunity_status: description: The current status of the opportunity example: !!python/object/apply:src.constants.lookup_constants.OpportunityStatus diff --git a/api/src/api/opportunities_v0_1/opportunity_routes.py b/api/src/api/opportunities_v0_1/opportunity_routes.py index a3b57f6f1..6ae77d6d0 100644 --- a/api/src/api/opportunities_v0_1/opportunity_routes.py +++ b/api/src/api/opportunities_v0_1/opportunity_routes.py @@ -62,10 +62,12 @@ @opportunity_blueprint.post("/opportunities/search") @opportunity_blueprint.input( - opportunity_schemas.OpportunitySearchRequestSchema, arg_name="search_params", examples=examples + opportunity_schemas.OpportunitySearchRequestV01Schema, + arg_name="search_params", + examples=examples, ) # many=True allows us to return a list of opportunity objects -@opportunity_blueprint.output(opportunity_schemas.OpportunitySchema(many=True)) +@opportunity_blueprint.output(opportunity_schemas.OpportunityV01Schema(many=True)) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) @flask_db.with_db_session() @@ -90,7 +92,7 @@ def opportunity_search(db_session: db.Session, search_params: dict) -> response. @opportunity_blueprint.get("/opportunities/") -@opportunity_blueprint.output(opportunity_schemas.OpportunitySchema) +@opportunity_blueprint.output(opportunity_schemas.OpportunityV01Schema) @opportunity_blueprint.auth_required(api_key_auth) @opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) @flask_db.with_db_session() diff --git a/api/src/api/opportunities_v0_1/opportunity_schemas.py b/api/src/api/opportunities_v0_1/opportunity_schemas.py index 257b05cd8..a54384660 100644 --- a/api/src/api/opportunities_v0_1/opportunity_schemas.py +++ b/api/src/api/opportunities_v0_1/opportunity_schemas.py @@ -10,7 +10,7 @@ from src.pagination.pagination_schema import generate_pagination_schema -class OpportunitySummarySchema(Schema): +class OpportunitySummaryV01Schema(Schema): summary_description = fields.String( metadata={ "description": "The summary of the opportunity", @@ -178,7 +178,7 @@ class OpportunitySummarySchema(Schema): applicant_types = fields.List(fields.Enum(ApplicantType)) -class OpportunityAssistanceListingSchema(Schema): +class OpportunityAssistanceListingV01Schema(Schema): program_title = fields.String( metadata={ "description": "The name of the program, see https://sam.gov/content/assistance-listings for more detail", @@ -193,7 +193,7 @@ class OpportunityAssistanceListingSchema(Schema): ) -class OpportunitySchema(Schema): +class OpportunityV01Schema(Schema): opportunity_id = fields.Integer( dump_only=True, metadata={"description": "The internal ID of the opportunity", "example": 12345}, @@ -227,9 +227,9 @@ class OpportunitySchema(Schema): ) opportunity_assistance_listings = fields.List( - fields.Nested(OpportunityAssistanceListingSchema()) + fields.Nested(OpportunityAssistanceListingV01Schema()) ) - summary = fields.Nested(OpportunitySummarySchema()) + summary = fields.Nested(OpportunitySummaryV01Schema()) opportunity_status = fields.Enum( OpportunityStatus, @@ -243,35 +243,35 @@ class OpportunitySchema(Schema): updated_at = fields.DateTime(dump_only=True) -class OpportunitySearchFilterSchema(Schema): +class OpportunitySearchFilterV01Schema(Schema): funding_instrument = fields.Nested( - StrSearchSchemaBuilder("FundingInstrumentFilterSchema") + StrSearchSchemaBuilder("FundingInstrumentFilterV01Schema") .with_one_of(allowed_values=FundingInstrument) .build() ) funding_category = fields.Nested( - StrSearchSchemaBuilder("FundingCategoryFilterSchema") + StrSearchSchemaBuilder("FundingCategoryFilterV01Schema") .with_one_of(allowed_values=FundingCategory) .build() ) applicant_type = fields.Nested( - StrSearchSchemaBuilder("ApplicantTypeFilterSchema") + StrSearchSchemaBuilder("ApplicantTypeFilterV01Schema") .with_one_of(allowed_values=ApplicantType) .build() ) opportunity_status = fields.Nested( - StrSearchSchemaBuilder("OpportunityStatusFilterSchema") + StrSearchSchemaBuilder("OpportunityStatusFilterV01Schema") .with_one_of(allowed_values=OpportunityStatus) .build() ) agency = fields.Nested( - StrSearchSchemaBuilder("AgencyFilterSchema") + StrSearchSchemaBuilder("AgencyFilterV01Schema") .with_one_of(example="US-ABC", minimum_length=2) .build() ) -class OpportunitySearchRequestSchema(Schema): +class OpportunitySearchRequestV01Schema(Schema): query = fields.String( metadata={ "description": "Query string which searches against several text fields", @@ -280,7 +280,7 @@ class OpportunitySearchRequestSchema(Schema): validate=[validators.Length(min=1, max=100)], ) - filters = fields.Nested(OpportunitySearchFilterSchema()) + filters = fields.Nested(OpportunitySearchFilterV01Schema()) pagination = fields.Nested( generate_pagination_schema( diff --git a/api/src/api/opportunities_v1/__init__.py b/api/src/api/opportunities_v1/__init__.py new file mode 100644 index 000000000..c757789dc --- /dev/null +++ b/api/src/api/opportunities_v1/__init__.py @@ -0,0 +1,6 @@ +from src.api.opportunities_v1.opportunity_blueprint import opportunity_blueprint + +# import opportunity_routes module to register the API routes on the blueprint +import src.api.opportunities_v1.opportunity_routes # noqa: F401 E402 isort:skip + +__all__ = ["opportunity_blueprint"] diff --git a/api/src/api/opportunities_v1/opportunity_blueprint.py b/api/src/api/opportunities_v1/opportunity_blueprint.py new file mode 100644 index 000000000..db88ee426 --- /dev/null +++ b/api/src/api/opportunities_v1/opportunity_blueprint.py @@ -0,0 +1,9 @@ +from apiflask import APIBlueprint + +opportunity_blueprint = APIBlueprint( + "opportunity_v1", + __name__, + tag="Opportunity v1", + cli_group="opportunity_v1", + url_prefix="/v1", +) diff --git a/api/src/api/opportunities_v1/opportunity_routes.py b/api/src/api/opportunities_v1/opportunity_routes.py new file mode 100644 index 000000000..0d94996b0 --- /dev/null +++ b/api/src/api/opportunities_v1/opportunity_routes.py @@ -0,0 +1,66 @@ +import logging + +import src.adapters.db as db +import src.adapters.db.flask_db as flask_db +import src.api.opportunities_v1.opportunity_schemas as opportunity_schemas +import src.api.response as response +from src.api.opportunities_v1.opportunity_blueprint import opportunity_blueprint +from src.auth.api_key_auth import api_key_auth +from src.logging.flask_logger import add_extra_data_to_current_request_logs +from src.services.opportunities_v1.get_opportunity import get_opportunity +from src.services.opportunities_v1.search_opportunities import search_opportunities +from src.util.dict_util import flatten_dict + +logger = logging.getLogger(__name__) + +# Descriptions in OpenAPI support markdown https://swagger.io/specification/ +SHARED_ALPHA_DESCRIPTION = """ +__ALPHA VERSION__ + +This endpoint in its current form is primarily for testing and feedback. + +Features in this endpoint are still under heavy development, and subject to change. Not for production use. + +See [Release Phases](https://github.com/github/roadmap?tab=readme-ov-file#release-phases) for further details. +""" + + +@opportunity_blueprint.post("/opportunities/search") +@opportunity_blueprint.input( + opportunity_schemas.OpportunitySearchRequestV1Schema, arg_name="search_params" +) +# many=True allows us to return a list of opportunity objects +@opportunity_blueprint.output(opportunity_schemas.OpportunityV1Schema(many=True)) +@opportunity_blueprint.auth_required(api_key_auth) +@opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) +def opportunity_search(search_params: dict) -> response.ApiResponse: + add_extra_data_to_current_request_logs(flatten_dict(search_params, prefix="request.body")) + logger.info("POST /v1/opportunities/search") + + opportunities, pagination_info = search_opportunities(search_params) + + add_extra_data_to_current_request_logs( + { + "response.pagination.total_pages": pagination_info.total_pages, + "response.pagination.total_records": pagination_info.total_records, + } + ) + logger.info("Successfully fetched opportunities") + + return response.ApiResponse( + message="Success", data=opportunities, pagination_info=pagination_info + ) + + +@opportunity_blueprint.get("/opportunities/") +@opportunity_blueprint.output(opportunity_schemas.OpportunityV1Schema) +@opportunity_blueprint.auth_required(api_key_auth) +@opportunity_blueprint.doc(description=SHARED_ALPHA_DESCRIPTION) +@flask_db.with_db_session() +def opportunity_get(db_session: db.Session, opportunity_id: int) -> response.ApiResponse: + add_extra_data_to_current_request_logs({"opportunity.opportunity_id": opportunity_id}) + logger.info("GET /v1/opportunities/:opportunity_id") + with db_session.begin(): + opportunity = get_opportunity(db_session, opportunity_id) + + return response.ApiResponse(message="Success", data=opportunity) diff --git a/api/src/api/opportunities_v1/opportunity_schemas.py b/api/src/api/opportunities_v1/opportunity_schemas.py new file mode 100644 index 000000000..5f72c7958 --- /dev/null +++ b/api/src/api/opportunities_v1/opportunity_schemas.py @@ -0,0 +1,298 @@ +from src.api.schemas.extension import Schema, fields, validators +from src.api.schemas.search_schema import StrSearchSchemaBuilder +from src.constants.lookup_constants import ( + ApplicantType, + FundingCategory, + FundingInstrument, + OpportunityCategory, + OpportunityStatus, +) +from src.pagination.pagination_schema import generate_pagination_schema + + +class OpportunitySummaryV1Schema(Schema): + summary_description = fields.String( + metadata={ + "description": "The summary of the opportunity", + "example": "This opportunity aims to unravel the mysteries of the universe.", + } + ) + is_cost_sharing = fields.Boolean( + metadata={ + "description": "Whether or not the opportunity has a cost sharing/matching requirement", + } + ) + is_forecast = fields.Boolean( + metadata={ + "description": "Whether the opportunity is forecasted, that is, the information is only an estimate and not yet official", + "example": False, + } + ) + + close_date = fields.Date( + metadata={ + "description": "The date that the opportunity will close - only set if is_forecast=False", + } + ) + close_date_description = fields.String( + metadata={ + "description": "Optional details regarding the close date", + "example": "Proposals are due earlier than usual.", + } + ) + + post_date = fields.Date( + metadata={ + "description": "The date the opportunity was posted", + } + ) + archive_date = fields.Date( + metadata={ + "description": "When the opportunity will be archived", + } + ) + # not including unarchive date at the moment + + expected_number_of_awards = fields.Integer( + metadata={ + "description": "The number of awards the opportunity is expected to award", + "example": 10, + } + ) + estimated_total_program_funding = fields.Integer( + metadata={ + "description": "The total program funding of the opportunity in US Dollars", + "example": 10_000_000, + } + ) + award_floor = fields.Integer( + metadata={ + "description": "The minimum amount an opportunity would award", + "example": 10_000, + } + ) + award_ceiling = fields.Integer( + metadata={ + "description": "The maximum amount an opportunity would award", + "example": 100_000, + } + ) + + additional_info_url = fields.String( + metadata={ + "description": "A URL to a website that can provide additional information about the opportunity", + "example": "grants.gov", + } + ) + additional_info_url_description = fields.String( + metadata={ + "description": "The text to display for the additional_info_url link", + "example": "Click me for more info", + } + ) + + forecasted_post_date = fields.Date( + metadata={ + "description": "Forecasted opportunity only. The date the opportunity is expected to be posted, and transition out of being a forecast" + } + ) + forecasted_close_date = fields.Date( + metadata={ + "description": "Forecasted opportunity only. The date the opportunity is expected to be close once posted." + } + ) + forecasted_close_date_description = fields.String( + metadata={ + "description": "Forecasted opportunity only. Optional details regarding the forecasted closed date.", + "example": "Proposals will probably be due on this date", + } + ) + forecasted_award_date = fields.Date( + metadata={ + "description": "Forecasted opportunity only. The date the grantor plans to award the opportunity." + } + ) + forecasted_project_start_date = fields.Date( + metadata={ + "description": "Forecasted opportunity only. The date the grantor expects the award recipient should start their project" + } + ) + fiscal_year = fields.Integer( + metadata={ + "description": "Forecasted opportunity only. The fiscal year the project is expected to be funded and launched" + } + ) + + funding_category_description = fields.String( + metadata={ + "description": "Additional information about the funding category", + "example": "Economic Support", + } + ) + applicant_eligibility_description = fields.String( + metadata={ + "description": "Additional information about the types of applicants that are eligible", + "example": "All types of domestic applicants are eligible to apply", + } + ) + + agency_code = fields.String( + metadata={ + "description": "The agency who owns the opportunity", + "example": "US-ABC", + } + ) + agency_name = fields.String( + metadata={ + "description": "The name of the agency who owns the opportunity", + "example": "US Alphabetical Basic Corp", + } + ) + agency_phone_number = fields.String( + metadata={ + "description": "The phone number of the agency who owns the opportunity", + "example": "123-456-7890", + } + ) + agency_contact_description = fields.String( + metadata={ + "description": "Information regarding contacting the agency who owns the opportunity", + "example": "For more information, reach out to Jane Smith at agency US-ABC", + } + ) + agency_email_address = fields.String( + metadata={ + "description": "The contact email of the agency who owns the opportunity", + "example": "fake_email@grants.gov", + } + ) + agency_email_address_description = fields.String( + metadata={ + "description": "The text for the link to the agency email address", + "example": "Click me to email the agency", + } + ) + + funding_instruments = fields.List(fields.Enum(FundingInstrument)) + funding_categories = fields.List(fields.Enum(FundingCategory)) + applicant_types = fields.List(fields.Enum(ApplicantType)) + + +class OpportunityAssistanceListingV1Schema(Schema): + program_title = fields.String( + metadata={ + "description": "The name of the program, see https://sam.gov/content/assistance-listings for more detail", + "example": "Space Technology", + } + ) + assistance_listing_number = fields.String( + metadata={ + "description": "The assistance listing number, see https://sam.gov/content/assistance-listings for more detail", + "example": "43.012", + } + ) + + +class OpportunityV1Schema(Schema): + opportunity_id = fields.Integer( + dump_only=True, + metadata={"description": "The internal ID of the opportunity", "example": 12345}, + ) + + opportunity_number = fields.String( + metadata={"description": "The funding opportunity number", "example": "ABC-123-XYZ-001"} + ) + opportunity_title = fields.String( + metadata={ + "description": "The title of the opportunity", + "example": "Research into conservation techniques", + } + ) + agency = fields.String( + metadata={"description": "The agency who created the opportunity", "example": "US-ABC"} + ) + + category = fields.Enum( + OpportunityCategory, + metadata={ + "description": "The opportunity category", + "example": OpportunityCategory.DISCRETIONARY, + }, + ) + category_explanation = fields.String( + metadata={ + "description": "Explanation of the category when the category is 'O' (other)", + "example": None, + } + ) + + opportunity_assistance_listings = fields.List( + fields.Nested(OpportunityAssistanceListingV1Schema()) + ) + summary = fields.Nested(OpportunitySummaryV1Schema()) + + opportunity_status = fields.Enum( + OpportunityStatus, + metadata={ + "description": "The current status of the opportunity", + "example": OpportunityStatus.POSTED, + }, + ) + + created_at = fields.DateTime(dump_only=True) + updated_at = fields.DateTime(dump_only=True) + + +class OpportunitySearchFilterV1Schema(Schema): + funding_instrument = fields.Nested( + StrSearchSchemaBuilder("FundingInstrumentFilterV1Schema") + .with_one_of(allowed_values=FundingInstrument) + .build() + ) + funding_category = fields.Nested( + StrSearchSchemaBuilder("FundingCategoryFilterV1Schema") + .with_one_of(allowed_values=FundingCategory) + .build() + ) + applicant_type = fields.Nested( + StrSearchSchemaBuilder("ApplicantTypeFilterV1Schema") + .with_one_of(allowed_values=ApplicantType) + .build() + ) + opportunity_status = fields.Nested( + StrSearchSchemaBuilder("OpportunityStatusFilterV1Schema") + .with_one_of(allowed_values=OpportunityStatus) + .build() + ) + agency = fields.Nested( + StrSearchSchemaBuilder("AgencyFilterV1Schema") + .with_one_of(example="US-ABC", minimum_length=2) + .build() + ) + + +class OpportunitySearchRequestV1Schema(Schema): + query = fields.String( + metadata={ + "description": "Query string which searches against several text fields", + "example": "research", + }, + validate=[validators.Length(min=1, max=100)], + ) + + filters = fields.Nested(OpportunitySearchFilterV1Schema()) + + pagination = fields.Nested( + generate_pagination_schema( + "OpportunityPaginationSchema", + [ + "opportunity_id", + "opportunity_number", + "opportunity_title", + "post_date", + "close_date", + "agency_code", + ], + ), + required=True, + ) diff --git a/api/src/app.py b/api/src/app.py index 8e617cce8..0d584a683 100644 --- a/api/src/app.py +++ b/api/src/app.py @@ -13,6 +13,7 @@ from src.api.healthcheck import healthcheck_blueprint from src.api.opportunities_v0 import opportunity_blueprint as opportunities_v0_blueprint from src.api.opportunities_v0_1 import opportunity_blueprint as opportunities_v0_1_blueprint +from src.api.opportunities_v1 import opportunity_blueprint as opportunities_v1_blueprint from src.api.response import restructure_error_response from src.api.schemas import response_schema from src.auth.api_key_auth import get_app_security_scheme @@ -101,6 +102,7 @@ def register_blueprints(app: APIFlask) -> None: app.register_blueprint(healthcheck_blueprint) app.register_blueprint(opportunities_v0_blueprint) app.register_blueprint(opportunities_v0_1_blueprint) + app.register_blueprint(opportunities_v1_blueprint) app.register_blueprint(data_migration_blueprint) app.register_blueprint(task_blueprint) diff --git a/api/src/services/opportunities_v1/__init__.py b/api/src/services/opportunities_v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/src/services/opportunities_v1/get_opportunity.py b/api/src/services/opportunities_v1/get_opportunity.py new file mode 100644 index 000000000..9b26cfada --- /dev/null +++ b/api/src/services/opportunities_v1/get_opportunity.py @@ -0,0 +1,24 @@ +from sqlalchemy import select +from sqlalchemy.orm import noload, selectinload + +import src.adapters.db as db +from src.api.route_utils import raise_flask_error +from src.db.models.opportunity_models import Opportunity + + +def get_opportunity(db_session: db.Session, opportunity_id: int) -> Opportunity: + opportunity: Opportunity | None = ( + db_session.execute( + select(Opportunity) + .where(Opportunity.opportunity_id == opportunity_id) + .where(Opportunity.is_draft.is_(False)) + .options(selectinload("*"), noload(Opportunity.all_opportunity_summaries)) + ) + .unique() + .scalar_one_or_none() + ) + + if opportunity is None: + raise_flask_error(404, message=f"Could not find Opportunity with ID {opportunity_id}") + + return opportunity diff --git a/api/src/services/opportunities_v1/search_opportunities.py b/api/src/services/opportunities_v1/search_opportunities.py new file mode 100644 index 000000000..1823bc31d --- /dev/null +++ b/api/src/services/opportunities_v1/search_opportunities.py @@ -0,0 +1,39 @@ +import logging +from typing import Sequence, Tuple + +from pydantic import BaseModel, Field + +from src.db.models.opportunity_models import Opportunity +from src.pagination.pagination_models import PaginationInfo, PaginationParams + +logger = logging.getLogger(__name__) + + +class SearchOpportunityFilters(BaseModel): + funding_instrument: dict | None = Field(default=None) + funding_category: dict | None = Field(default=None) + applicant_type: dict | None = Field(default=None) + opportunity_status: dict | None = Field(default=None) + agency: dict | None = Field(default=None) + + +class SearchOpportunityParams(BaseModel): + pagination: PaginationParams + + query: str | None = Field(default=None) + filters: SearchOpportunityFilters | None = Field(default=None) + + +def search_opportunities(raw_search_params: dict) -> Tuple[Sequence[Opportunity], PaginationInfo]: + search_params = SearchOpportunityParams.model_validate(raw_search_params) + + pagination_info = PaginationInfo( + page_offset=search_params.pagination.page_offset, + page_size=search_params.pagination.page_size, + order_by=search_params.pagination.order_by, + sort_direction=search_params.pagination.sort_direction, + total_records=0, + total_pages=0, + ) + + return [], pagination_info diff --git a/api/tests/src/api/opportunities_v0_1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v0_1/test_opportunity_route_search.py index 4ee7c6ba4..6529fc651 100644 --- a/api/tests/src/api/opportunities_v0_1/test_opportunity_route_search.py +++ b/api/tests/src/api/opportunities_v0_1/test_opportunity_route_search.py @@ -1121,6 +1121,5 @@ def test_opportunity_search_invalid_request_422( ) assert resp.status_code == 422 - print(resp.get_json()) response_data = resp.get_json()["errors"] assert response_data == expected_response_data diff --git a/api/tests/src/api/opportunities_v1/__init__.py b/api/tests/src/api/opportunities_v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/src/api/opportunities_v1/conftest.py b/api/tests/src/api/opportunities_v1/conftest.py new file mode 100644 index 000000000..c00490cff --- /dev/null +++ b/api/tests/src/api/opportunities_v1/conftest.py @@ -0,0 +1,183 @@ +from src.constants.lookup_constants import ( + ApplicantType, + FundingCategory, + FundingInstrument, + OpportunityStatus, +) +from src.db.models.opportunity_models import ( + Opportunity, + OpportunityAssistanceListing, + OpportunitySummary, +) + + +def get_search_request( + page_offset: int = 1, + page_size: int = 5, + order_by: str = "opportunity_id", + sort_direction: str = "descending", + query: str | None = None, + funding_instrument_one_of: list[FundingInstrument] | None = None, + funding_category_one_of: list[FundingCategory] | None = None, + applicant_type_one_of: list[ApplicantType] | None = None, + opportunity_status_one_of: list[OpportunityStatus] | None = None, + agency_one_of: list[str] | None = None, +): + req = { + "pagination": { + "page_offset": page_offset, + "page_size": page_size, + "order_by": order_by, + "sort_direction": sort_direction, + } + } + + filters = {} + + if funding_instrument_one_of is not None: + filters["funding_instrument"] = {"one_of": funding_instrument_one_of} + + if funding_category_one_of is not None: + filters["funding_category"] = {"one_of": funding_category_one_of} + + if applicant_type_one_of is not None: + filters["applicant_type"] = {"one_of": applicant_type_one_of} + + if opportunity_status_one_of is not None: + filters["opportunity_status"] = {"one_of": opportunity_status_one_of} + + if agency_one_of is not None: + filters["agency"] = {"one_of": agency_one_of} + + if len(filters) > 0: + req["filters"] = filters + + if query is not None: + req["query"] = query + + return req + + +##################################### +# Validation utils +##################################### + + +def validate_opportunity(db_opportunity: Opportunity, resp_opportunity: dict): + assert db_opportunity.opportunity_id == resp_opportunity["opportunity_id"] + assert db_opportunity.opportunity_number == resp_opportunity["opportunity_number"] + assert db_opportunity.opportunity_title == resp_opportunity["opportunity_title"] + assert db_opportunity.agency == resp_opportunity["agency"] + assert db_opportunity.category == resp_opportunity["category"] + assert db_opportunity.category_explanation == resp_opportunity["category_explanation"] + + validate_opportunity_summary(db_opportunity.summary, resp_opportunity["summary"]) + validate_assistance_listings( + db_opportunity.opportunity_assistance_listings, + resp_opportunity["opportunity_assistance_listings"], + ) + + assert db_opportunity.opportunity_status == resp_opportunity["opportunity_status"] + + +def validate_opportunity_summary(db_summary: OpportunitySummary, resp_summary: dict): + if db_summary is None: + assert resp_summary is None + return + + assert db_summary.summary_description == resp_summary["summary_description"] + assert db_summary.is_cost_sharing == resp_summary["is_cost_sharing"] + assert db_summary.is_forecast == resp_summary["is_forecast"] + assert str(db_summary.close_date) == str(resp_summary["close_date"]) + assert db_summary.close_date_description == resp_summary["close_date_description"] + assert str(db_summary.post_date) == str(resp_summary["post_date"]) + assert str(db_summary.archive_date) == str(resp_summary["archive_date"]) + assert db_summary.expected_number_of_awards == resp_summary["expected_number_of_awards"] + assert ( + db_summary.estimated_total_program_funding + == resp_summary["estimated_total_program_funding"] + ) + assert db_summary.award_floor == resp_summary["award_floor"] + assert db_summary.award_ceiling == resp_summary["award_ceiling"] + assert db_summary.additional_info_url == resp_summary["additional_info_url"] + assert ( + db_summary.additional_info_url_description + == resp_summary["additional_info_url_description"] + ) + + assert str(db_summary.forecasted_post_date) == str(resp_summary["forecasted_post_date"]) + assert str(db_summary.forecasted_close_date) == str(resp_summary["forecasted_close_date"]) + assert ( + db_summary.forecasted_close_date_description + == resp_summary["forecasted_close_date_description"] + ) + assert str(db_summary.forecasted_award_date) == str(resp_summary["forecasted_award_date"]) + assert str(db_summary.forecasted_project_start_date) == str( + resp_summary["forecasted_project_start_date"] + ) + assert db_summary.fiscal_year == resp_summary["fiscal_year"] + + assert db_summary.funding_category_description == resp_summary["funding_category_description"] + assert ( + db_summary.applicant_eligibility_description + == resp_summary["applicant_eligibility_description"] + ) + + assert db_summary.agency_code == resp_summary["agency_code"] + assert db_summary.agency_name == resp_summary["agency_name"] + assert db_summary.agency_phone_number == resp_summary["agency_phone_number"] + assert db_summary.agency_contact_description == resp_summary["agency_contact_description"] + assert db_summary.agency_email_address == resp_summary["agency_email_address"] + assert ( + db_summary.agency_email_address_description + == resp_summary["agency_email_address_description"] + ) + + assert set(db_summary.funding_instruments) == set(resp_summary["funding_instruments"]) + assert set(db_summary.funding_categories) == set(resp_summary["funding_categories"]) + assert set(db_summary.applicant_types) == set(resp_summary["applicant_types"]) + + +def validate_assistance_listings( + db_assistance_listings: list[OpportunityAssistanceListing], resp_listings: list[dict] +) -> None: + # In order to compare this list, sort them both the same and compare from there + db_assistance_listings.sort(key=lambda a: (a.assistance_listing_number, a.program_title)) + resp_listings.sort(key=lambda a: (a["assistance_listing_number"], a["program_title"])) + + assert len(db_assistance_listings) == len(resp_listings) + for db_assistance_listing, resp_listing in zip( + db_assistance_listings, resp_listings, strict=True + ): + assert ( + db_assistance_listing.assistance_listing_number + == resp_listing["assistance_listing_number"] + ) + assert db_assistance_listing.program_title == resp_listing["program_title"] + + +def validate_search_pagination( + search_response: dict, + search_request: dict, + expected_total_pages: int, + expected_total_records: int, + expected_response_record_count: int, +): + pagination_info = search_response["pagination_info"] + assert pagination_info["page_offset"] == search_request["pagination"]["page_offset"] + assert pagination_info["page_size"] == search_request["pagination"]["page_size"] + assert pagination_info["order_by"] == search_request["pagination"]["order_by"] + assert pagination_info["sort_direction"] == search_request["pagination"]["sort_direction"] + + assert pagination_info["total_pages"] == expected_total_pages + assert pagination_info["total_records"] == expected_total_records + + searched_opportunities = search_response["data"] + assert len(searched_opportunities) == expected_response_record_count + + # Verify data is sorted as expected + reverse = pagination_info["sort_direction"] == "descending" + resorted_opportunities = sorted( + searched_opportunities, key=lambda u: u[pagination_info["order_by"]], reverse=reverse + ) + assert resorted_opportunities == searched_opportunities diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_auth.py b/api/tests/src/api/opportunities_v1/test_opportunity_auth.py new file mode 100644 index 000000000..352c57bfc --- /dev/null +++ b/api/tests/src/api/opportunities_v1/test_opportunity_auth.py @@ -0,0 +1,21 @@ +import pytest + +from tests.src.api.opportunities_v1.conftest import get_search_request + + +@pytest.mark.parametrize( + "method,url,body", + [ + ("POST", "/v1/opportunities/search", get_search_request()), + ("GET", "/v1/opportunities/1", None), + ], +) +def test_opportunity_unauthorized_401(client, api_auth_token, method, url, body): + # open is just the generic method that post/get/etc. call under the hood + response = client.open(url, method=method, json=body, headers={"X-Auth": "incorrect token"}) + + assert response.status_code == 401 + assert ( + response.get_json()["message"] + == "The server could not verify that you are authorized to access the URL requested" + ) diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py new file mode 100644 index 000000000..875cddfd3 --- /dev/null +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_get.py @@ -0,0 +1,97 @@ +import pytest + +from src.db.models.opportunity_models import Opportunity +from tests.src.api.opportunities_v1.conftest import validate_opportunity +from tests.src.db.models.factories import ( + CurrentOpportunitySummaryFactory, + OpportunityFactory, + OpportunitySummaryFactory, +) + + +@pytest.fixture +def truncate_opportunities(db_session): + # Note that we can't just do db_session.query(Opportunity).delete() as the cascade deletes won't work automatically: + # https://docs.sqlalchemy.org/en/20/orm/queryguide/dml.html#orm-queryguide-update-delete-caveats + # but if we do it individually they will + opportunities = db_session.query(Opportunity).all() + for opp in opportunities: + db_session.delete(opp) + + # Force the deletes to the DB + db_session.commit() + + +##################################### +# GET opportunity tests +##################################### + + +@pytest.mark.parametrize( + "opportunity_params,opportunity_summary_params", + [ + ({}, {}), + # Only an opportunity exists, no other connected records + ( + { + "opportunity_assistance_listings": [], + }, + None, + ), + # Summary exists, but none of the list values set + ( + {}, + { + "link_funding_instruments": [], + "link_funding_categories": [], + "link_applicant_types": [], + }, + ), + # All possible values set to null/empty + # Note this uses traits on the factories to handle setting everything + ({"all_fields_null": True}, {"all_fields_null": True}), + ], +) +def test_get_opportunity_200( + client, api_auth_token, enable_factory_create, opportunity_params, opportunity_summary_params +): + # Split the setup of the opportunity from the opportunity summary to simplify the factory usage a bit + db_opportunity = OpportunityFactory.create( + **opportunity_params, current_opportunity_summary=None + ) # We'll set the current opportunity below + + if opportunity_summary_params is not None: + db_opportunity_summary = OpportunitySummaryFactory.create( + **opportunity_summary_params, opportunity=db_opportunity + ) + CurrentOpportunitySummaryFactory.create( + opportunity=db_opportunity, opportunity_summary=db_opportunity_summary + ) + + resp = client.get( + f"/v1/opportunities/{db_opportunity.opportunity_id}", headers={"X-Auth": api_auth_token} + ) + assert resp.status_code == 200 + response_data = resp.get_json()["data"] + + validate_opportunity(db_opportunity, response_data) + + +def test_get_opportunity_404_not_found(client, api_auth_token, truncate_opportunities): + resp = client.get("/v1/opportunities/1", headers={"X-Auth": api_auth_token}) + assert resp.status_code == 404 + assert resp.get_json()["message"] == "Could not find Opportunity with ID 1" + + +def test_get_opportunity_404_not_found_is_draft(client, api_auth_token, enable_factory_create): + # The endpoint won't return drafts, so this'll be a 404 despite existing + opportunity = OpportunityFactory.create(is_draft=True) + + resp = client.get( + f"/v1/opportunities/{opportunity.opportunity_id}", headers={"X-Auth": api_auth_token} + ) + assert resp.status_code == 404 + assert ( + resp.get_json()["message"] + == f"Could not find Opportunity with ID {opportunity.opportunity_id}" + ) diff --git a/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py new file mode 100644 index 000000000..6e79419db --- /dev/null +++ b/api/tests/src/api/opportunities_v1/test_opportunity_route_search.py @@ -0,0 +1,19 @@ +from tests.src.api.opportunities_v1.conftest import get_search_request + + +def test_opportunity_route_search_200(client, api_auth_token): + req = get_search_request() + + resp = client.post("/v1/opportunities/search", json=req, headers={"X-Auth": api_auth_token}) + + assert resp.status_code == 200 + + # The endpoint meaningfully only returns the pagination params back + # at the moment, so just validate that for now. + resp_body = resp.get_json() + assert resp_body["pagination_info"]["page_offset"] == req["pagination"]["page_offset"] + assert resp_body["pagination_info"]["page_size"] == req["pagination"]["page_size"] + assert resp_body["pagination_info"]["sort_direction"] == req["pagination"]["sort_direction"] + assert resp_body["pagination_info"]["order_by"] == req["pagination"]["order_by"] + assert resp_body["pagination_info"]["total_records"] == 0 + assert resp_body["pagination_info"]["total_pages"] == 0