From 85f3e0ade2c4abd7bcdd845148d8ebcbf1f56a13 Mon Sep 17 00:00:00 2001 From: Ryan Lewis <93001277+rylew1@users.noreply.github.com> Date: Tue, 14 May 2024 12:45:06 -0700 Subject: [PATCH] [Issue #1757]: E2E tests for search page (#1974) ## Summary Fixes #1757 ## Changes proposed - Add e2e tests for search page - Add e2e test (with FE and API) to its own CI job (`ci-frontend-ci.yml`) - invokes shell script to wait until API is loaded --- .github/workflows/ci-frontend-e2e.yml | 61 +++++++ .github/workflows/ci-frontend.yml | 15 +- api/.gitignore | 3 + api/bin/wait-for-api.sh | 30 ++++ frontend/.eslintrc.js | 2 + frontend/.gitignore | 6 + frontend/package-lock.json | 24 +-- frontend/package.json | 2 +- .../SearchFilterCheckbox.tsx | 2 +- frontend/tests/e2e/search/search.spec.ts | 170 ++++++++++++++++++ frontend/tests/e2e/search/searchUtil.ts | 132 ++++++++++++++ frontend/tests/playwright.config.ts | 2 + 12 files changed, 421 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/ci-frontend-e2e.yml create mode 100755 api/bin/wait-for-api.sh create mode 100644 frontend/tests/e2e/search/search.spec.ts create mode 100644 frontend/tests/e2e/search/searchUtil.ts diff --git a/.github/workflows/ci-frontend-e2e.yml b/.github/workflows/ci-frontend-e2e.yml new file mode 100644 index 000000000..1a1712993 --- /dev/null +++ b/.github/workflows/ci-frontend-e2e.yml @@ -0,0 +1,61 @@ +name: Frontend E2E Tests + +on: + workflow_call: + pull_request: + paths: + - frontend/** + - .github/workflows/ci-frontend-e2e.yml + +defaults: + run: + working-directory: ./frontend + +env: + NODE_VERSION: 18 + LOCKFILE_PATH: ./frontend/package-lock.json + PACKAGE_MANAGER: npm + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e-tests: + name: Run E2E Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: ${{ env.PACKAGE_MANAGER }} + cache-dependency-path: ${{ env.LOCKFILE_PATH }} + + - run: npm ci + + - name: Install Playwright Browsers + run: npx playwright install --with-deps + + - name: Start API Server for e2e tests + run: | + cd ../api + make init db-seed-local start & + cd ../frontend + # Ensure the API wait script is executable + chmod +x ../api/bin/wait-for-api.sh + ../api/bin/wait-for-api.sh + shell: bash + + - name: Run E2E Tests + run: npm run test:e2e + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: ./frontend/playwright-report/ + retention-days: 30 diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index a1e299d8c..67766d706 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -1,4 +1,4 @@ -name: Front-end Checks +name: Frontend Checks on: workflow_call: @@ -34,9 +34,6 @@ jobs: cache: ${{ env.PACKAGE_MANAGER }} - run: npm ci - - name: Install Playwright Browsers - run: npx playwright install --with-deps - - name: Run lint run: npm run lint @@ -58,16 +55,6 @@ jobs: skip-step: none output: comment - - name: Run e2e tests - run: npm run test:e2e - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: ./frontend/playwright-report/ - retention-days: 30 - # Confirms the front end still builds successfully check-frontend-builds: name: FE Build Check diff --git a/api/.gitignore b/api/.gitignore index ff0117b53..2b1784734 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -26,3 +26,6 @@ coverage.* # VSCode Workspace *.code-workspace .vscode + +#e2e +/test-results/ diff --git a/api/bin/wait-for-api.sh b/api/bin/wait-for-api.sh new file mode 100755 index 000000000..f68a9b13d --- /dev/null +++ b/api/bin/wait-for-api.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# wait-for-api.sh + +set -e + +# Color formatting for readability +GREEN='\033[0;32m' +RED='\033[0;31m' +NO_COLOR='\033[0m' + +MAX_WAIT_TIME=800 # seconds, adjust as necessary +WAIT_TIME=0 + +echo "Waiting for API server to become ready..." + +# Use curl to check the API server health endpoint +until curl --output /dev/null --silent --head --fail http://localhost:8080/health; +do + printf '.' + sleep 5 + + WAIT_TIME=$(($WAIT_TIME + 5)) + if [ $WAIT_TIME -gt $MAX_WAIT_TIME ] + then + echo -e "${RED}ERROR: API server did not become ready within ${MAX_WAIT_TIME} seconds.${NO_COLOR}" + exit 1 + fi +done + +echo -e "${GREEN}API server is ready after ~${WAIT_TIME} seconds.${NO_COLOR}" diff --git a/frontend/.eslintrc.js b/frontend/.eslintrc.js index 023b61c5f..280ce11e3 100644 --- a/frontend/.eslintrc.js +++ b/frontend/.eslintrc.js @@ -49,6 +49,8 @@ module.exports = { "@typescript-eslint/no-unused-vars": "error", // The usage of `any` defeats the purpose of typescript. Consider using `unknown` type instead instead. "@typescript-eslint/no-explicit-any": "error", + // Just warn since playwright tests may not use screen the way jest would + "testing-library/prefer-screen-queries": "warn", }, }, ], diff --git a/frontend/.gitignore b/frontend/.gitignore index 41bd6facb..f38f90c87 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -45,3 +45,9 @@ npm-debug.log* # uswds assets /public/uswds + +# playwright e2e +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2477fba54..68a0052ab 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -28,7 +28,7 @@ }, "devDependencies": { "@ianvs/prettier-plugin-sort-imports": "^4.0.2", - "@playwright/test": "^1.42.0", + "@playwright/test": "^1.44.0", "@storybook/addon-designs": "^7.0.1", "@storybook/addon-essentials": "^7.1.0", "@storybook/nextjs": "^7.1.0", @@ -5139,12 +5139,12 @@ } }, "node_modules/@playwright/test": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.43.0.tgz", - "integrity": "sha512-Ebw0+MCqoYflop7wVKj711ccbNlrwTBCtjY5rlbiY9kHL2bCYxq+qltK6uPsVBGGAOb033H2VO0YobcQVxoW7Q==", + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.44.0.tgz", + "integrity": "sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg==", "dev": true, "dependencies": { - "playwright": "1.43.0" + "playwright": "1.44.0" }, "bin": { "playwright": "cli.js" @@ -20767,12 +20767,12 @@ } }, "node_modules/playwright": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.0.tgz", - "integrity": "sha512-SiOKHbVjTSf6wHuGCbqrEyzlm6qvXcv7mENP+OZon1I07brfZLGdfWV0l/efAzVx7TF3Z45ov1gPEkku9q25YQ==", + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.44.0.tgz", + "integrity": "sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==", "dev": true, "dependencies": { - "playwright-core": "1.43.0" + "playwright-core": "1.44.0" }, "bin": { "playwright": "cli.js" @@ -20785,9 +20785,9 @@ } }, "node_modules/playwright-core": { - "version": "1.43.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.0.tgz", - "integrity": "sha512-iWFjyBUH97+pUFiyTqSLd8cDMMOS0r2ZYz2qEsPjH8/bX++sbIJT35MSwKnp1r/OQBAqC5XO99xFbJ9XClhf4w==", + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.44.0.tgz", + "integrity": "sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==", "dev": true, "bin": { "playwright-core": "cli.js" diff --git a/frontend/package.json b/frontend/package.json index f97eb31f5..048464bb1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@ianvs/prettier-plugin-sort-imports": "^4.0.2", - "@playwright/test": "^1.42.0", + "@playwright/test": "^1.44.0", "@storybook/addon-designs": "^7.0.1", "@storybook/addon-essentials": "^7.1.0", "@storybook/nextjs": "^7.1.0", diff --git a/frontend/src/components/search/SearchFilterAccordion/SearchFilterCheckbox.tsx b/frontend/src/components/search/SearchFilterAccordion/SearchFilterCheckbox.tsx index 4304bf2a7..027cf8864 100644 --- a/frontend/src/components/search/SearchFilterAccordion/SearchFilterCheckbox.tsx +++ b/frontend/src/components/search/SearchFilterAccordion/SearchFilterCheckbox.tsx @@ -37,7 +37,7 @@ const SearchFilterCheckbox: React.FC = ({ onChange={handleChange} disabled={!mounted} checked={option.isChecked === true} - // value={option.id} // TODO: consider poassing explicit value + // value={option.id} // TODO: consider passing explicit value /> ); }; diff --git a/frontend/tests/e2e/search/search.spec.ts b/frontend/tests/e2e/search/search.spec.ts new file mode 100644 index 000000000..99d046961 --- /dev/null +++ b/frontend/tests/e2e/search/search.spec.ts @@ -0,0 +1,170 @@ +import { + clickAccordionWithTitle, + clickMobileNavMenu, + clickSearchNavLink, + expectCheckboxIDIsChecked, + expectSortBy, + expectURLContainsQueryParam, + fillSearchInputAndSubmit, + getMobileMenuButton, + getSearchInput, + hasMobileMenu, + refreshPageWithCurrentURL, + selectSortBy, + toggleCheckboxes, + waitForSearchResultsLoaded, +} from "./searchUtil"; +import { expect, test } from "@playwright/test"; + +test("should navigate from index to search page", async ({ page }) => { + // Start from the index page with feature flag set + await page.goto("/?_ff=showSearchV0:true"); + + // Mobile chrome must first click the menu button + if (await hasMobileMenu(page)) { + const menuButton = getMobileMenuButton(page); + await clickMobileNavMenu(menuButton); + } + + await clickSearchNavLink(page); + + // Verify that the new URL is correct + expectURLContainsQueryParam(page, "status", "forecasted,posted"); + + // Verify the presence of "Search" content on the page + await expect(page.locator("h1")).toContainText( + "Search funding opportunities", + ); + + // Verify that the 'forecasted' and 'posted' are checked + await expectCheckboxIDIsChecked(page, "#status-forecasted"); + await expectCheckboxIDIsChecked(page, "#status-posted"); +}); + +test.describe("Search page tests", () => { + test.beforeEach(async ({ page }) => { + // Navigate to the search page with the feature flag set + await page.goto("/search?_ff=showSearchV0:true"); + }); + + test("should return 0 results when searching for obscure term", async ({ + page, + browserName, + }) => { + // TODO (Issue #2005): fix test for webkit + test.skip( + browserName === "webkit", + "Skipping test for WebKit due to a query param issue.", + ); + + const searchTerm = "0resultearch"; + + await fillSearchInputAndSubmit(searchTerm, page); + + expectURLContainsQueryParam(page, "query", searchTerm); + + const resultsHeading = page.getByRole("heading", { + name: /0 Opportunities/i, + }); + await expect(resultsHeading).toBeVisible(); + + await expect(page.locator("div.usa-prose h2")).toHaveText( + "Your search did not return any results.", + ); + }); + + test("should show and hide loading state", async ({ page, browserName }) => { + // TODO (Issue #2005): fix test for webkit + test.skip( + browserName === "webkit", + "Skipping test for WebKit due to a query param issue.", + ); + const searchTerm = "advanced"; + await fillSearchInputAndSubmit(searchTerm, page); + + const loadingIndicator = page.locator("text='Loading results...'"); + await expect(loadingIndicator).toBeVisible(); + await expect(loadingIndicator).toBeHidden(); + + const searchTerm2 = "agency"; + await fillSearchInputAndSubmit(searchTerm2, page); + await expect(loadingIndicator).toBeVisible(); + await expect(loadingIndicator).toBeHidden(); + }); + test("should retain filters in a new tab", async ({ page }) => { + // Set all inputs, then refresh the page. Those same inputs should be + // set from query params. + const searchTerm = "education"; + const statusCheckboxes = { + "status-forecasted": "forecasted", + "status-posted": "posted", + }; + const fundingInstrumentCheckboxes = { + "funding-instrument-cooperative_agreement": "cooperative_agreement", + "funding-instrument-grant": "grant", + }; + + const eligibilityCheckboxes = { + "eligibility-state_governments": "state_governments", + "eligibility-county_governments": "county_governments", + }; + const agencyCheckboxes = { + ARPAH: "ARPAH", + AC: "AC", + }; + const categoryCheckboxes = { + "category-recovery_act": "recovery_act", + "category-agriculture": "agriculture", + }; + + await selectSortBy(page, "agencyDesc"); + + await waitForSearchResultsLoaded(page); + await fillSearchInputAndSubmit(searchTerm, page); + await toggleCheckboxes(page, statusCheckboxes, "status"); + + await clickAccordionWithTitle(page, "Funding instrument"); + await toggleCheckboxes( + page, + fundingInstrumentCheckboxes, + "fundingInstrument", + ); + + await clickAccordionWithTitle(page, "Eligibility"); + await toggleCheckboxes(page, eligibilityCheckboxes, "eligibility"); + + await clickAccordionWithTitle(page, "Agency"); + await toggleCheckboxes(page, agencyCheckboxes, "agency"); + + await clickAccordionWithTitle(page, "Category"); + await toggleCheckboxes(page, categoryCheckboxes, "category"); + + /***********************************************************/ + /* Page refreshed should have all the same inputs selected + /***********************************************************/ + + await refreshPageWithCurrentURL(page); + + // Expect search inputs are retained in the new tab + await expectSortBy(page, "agencyDesc"); + const searchInput = getSearchInput(page); + await expect(searchInput).toHaveValue(searchTerm); + + for (const [checkboxID] of Object.entries(statusCheckboxes)) { + await expectCheckboxIDIsChecked(page, `#${checkboxID}`); + } + + for (const [checkboxID] of Object.entries(fundingInstrumentCheckboxes)) { + await expectCheckboxIDIsChecked(page, `#${checkboxID}`); + } + for (const [checkboxID] of Object.entries(eligibilityCheckboxes)) { + await expectCheckboxIDIsChecked(page, `#${checkboxID}`); + } + for (const [checkboxID] of Object.entries(agencyCheckboxes)) { + await expectCheckboxIDIsChecked(page, `#${checkboxID}`); + } + for (const [checkboxID] of Object.entries(categoryCheckboxes)) { + await expectCheckboxIDIsChecked(page, `#${checkboxID}`); + } + }); +}); diff --git a/frontend/tests/e2e/search/searchUtil.ts b/frontend/tests/e2e/search/searchUtil.ts new file mode 100644 index 000000000..87d066533 --- /dev/null +++ b/frontend/tests/e2e/search/searchUtil.ts @@ -0,0 +1,132 @@ +// ========================= +// Test Helper Functions +// ========================= + +import { Locator, Page, expect } from "@playwright/test"; + +export function getSearchInput(page: Page) { + return page.locator("#query"); +} + +export async function fillSearchInputAndSubmit(term: string, page: Page) { + const searchInput = getSearchInput(page); + await searchInput.fill(term); + await page.click(".usa-search >> button[type='submit']"); + expectURLContainsQueryParam(page, "query", term); +} + +export function expectURLContainsQueryParam( + page: Page, + queryParamName: string, + queryParamValue: string, +) { + const currentURL = page.url(); + expect(currentURL).toContain(`${queryParamName}=${queryParamValue}`); +} + +export async function waitForURLContainsQueryParam( + page: Page, + queryParamName: string, + queryParamValue: string, + timeout = 30000, // query params get set after a debounce period +) { + const endTime = Date.now() + timeout; + + while (Date.now() < endTime) { + const url = new URL(page.url()); + const params = new URLSearchParams(url.search); + const actualValue = params.get(queryParamName); + + if (actualValue === queryParamValue) { + return; + } + + await page.waitForTimeout(500); + } + + throw new Error( + `URL did not contain query parameter ${queryParamName}=${queryParamValue} within ${timeout}ms`, + ); +} + +export async function clickSearchNavLink(page: Page) { + await page.click("nav >> text=Search"); +} + +export function getMobileMenuButton(page: Page) { + return page.locator("button >> text=MENU"); +} + +export async function hasMobileMenu(page: Page) { + const menuButton = getMobileMenuButton(page); + return await menuButton.isVisible(); +} + +export async function clickMobileNavMenu(menuButton: Locator) { + await menuButton.click(); +} + +export async function expectCheckboxIDIsChecked( + page: Page, + idWithHash: string, +) { + const checkbox: Locator = page.locator(idWithHash); + await expect(checkbox).toBeChecked(); +} + +export async function toggleCheckboxes( + page: Page, + checkboxObject: Record, + queryParamName: string, +) { + let runningQueryParams = ""; + for (const [checkboxID, queryParamValue] of Object.entries(checkboxObject)) { + await toggleCheckbox(page, checkboxID); + runningQueryParams += runningQueryParams + ? `,${queryParamValue}` + : queryParamValue; + await waitForURLContainsQueryParam( + page, + queryParamName, + runningQueryParams, + ); + } +} + +export async function toggleCheckbox(page: Page, idWithoutHash: string) { + const checkBox = page.locator(`label[for=${idWithoutHash}]`); + await checkBox.isEnabled(); + await checkBox.click(); +} + +export async function refreshPageWithCurrentURL(page: Page) { + const currentURL = page.url(); + await page.goto(currentURL); // go to new url in same tab + return page; +} + +export async function selectSortBy(page: Page, sortByValue: string) { + await page.locator("#search-sort-by-select").selectOption(sortByValue); +} + +export async function expectSortBy(page: Page, value: string) { + const selectedValue = await page + .locator('select[name="search-sort-by"]') + .inputValue(); + expect(selectedValue).toBe(value); +} + +export async function waitForSearchResultsLoaded(page: Page) { + // Wait for number of opportunities to show + const resultsHeading = page.locator('h2:has-text("Opportunities")'); + await resultsHeading.waitFor({ state: "visible", timeout: 60000 }); +} + +export async function clickAccordionWithTitle( + page: Page, + accordionTitle: string, +) { + await page + .locator(`button.usa-accordion__button:has-text("${accordionTitle}")`) + .click(); +} diff --git a/frontend/tests/playwright.config.ts b/frontend/tests/playwright.config.ts index ecf694eae..9d7ddb4c0 100644 --- a/frontend/tests/playwright.config.ts +++ b/frontend/tests/playwright.config.ts @@ -28,6 +28,8 @@ export default defineConfig({ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", + screenshot: "on", + video: "on-first-retry", }, /* Configure projects for major browsers */