test(playwright): initial component testing with playwright-ct (#4779)

* chore(deps): update playwright

* text(playwright): initial playwright-ct setup

* refactor(playwright): co-locate playwright-ct with sanity package

* chore(deps): use @playwright/test in perf studio and @sanity/portable-text-editor

* refactor(playwright): simplify stories, add data-attr for activate overlay

* refactor(playwright): re-add typeWithDelay helper, temporarily disable context menu keyboard interactions

* refactor(playwright): fix e2e-ct action paths, use default delay when unbolding text

* refactor(playwright): split up megastory test, fix flakyness

* fix(playwright): fix import for validateDocument

* fix(playwright): workaround memory issue with max_old_space_size

* fix(playwright): fix report json path

* refactor(playwright): update test folder structure, add simple array input test, fix minor template rendering issues (#4797)

* fix(playwright): use correct doctype, add font-smoothing

* refactor(playwright): separate TestForm, co-locate script types, remove unnecessary exports, wrap tests in panes, add placeholder array input spec

* refactor(playwright): rename TestProvider to TestWrapper, don't accept children

* refactor(playwright): refactor test report to pr comment conversion

* chore(playwright): pin workflow deps

* fix(playwright): fix playwright cache issue in ci

* fix(playwright): fix artifact paths in workflow

* chore(playwright): setup e2e-ct workflow to use remote caching

* chore(playwright): linting

* fix(playwright): remove breaking keypresses after upgrading slate

---------

Co-authored-by: Robin Pyon <[email protected]>
Co-authored-by: Robin Pyon <[email protected]>

.github/workflows/e2e-ct.yml

@@ -0,0 +1,83 @@
+name: End-to-End Component Tests
+on:
+  # Build on pushes branches that have a PR (including drafts)
+  pull_request:
+  # Build on commits pushed to branches without a PR if it's in the allowlist
+  push:
+    branches: [next]
+jobs:
+  test:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      TURBO_TEAM: sanity-io
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-modules-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-modules-${{ env.cache-name }}-
+            ${{ runner.os }}-modules-
+            ${{ runner.os }}-
+
+      - name: Install project dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: yarn install --immutable --immutable-cache --check-cache
+
+      - name: Store Playwright's Version
+        run: |
+          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
+          echo "Playwright's Version: $PLAYWRIGHT_VERSION"
+          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
+      - name: Cache Playwright Browsers for Playwright's Version
+        id: cache-playwright-browsers
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+      - name: Build CLI
+        run: yarn build:cli # Needed for CLI tests
+      - name: Run end-to-end tests
+        run: yarn workspace sanity test:ct
+        env:
+          NODE_OPTIONS: --max_old_space_size=8192
+      - name: Get Current Job Log URL
+        uses: Tiryoh/gha-jobid-action@603885a199c331cc2f828bcebeb1c4a275f4d6e1 # v0
+        id: job_html_url
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: 'test'
+      - name: Parse playwright report
+        id: parse-playwright-report
+        if: always()
+        run: yarn workspace sanity write:playwright-report-as-pr-comment
+        env:
+          GITHUB_WORKFLOW_URL: '${{ steps.job_html_url.outputs.html_url }}'
+      - name: PR comment with report
+        uses: thollander/actions-comment-pull-request@d61db783da9abefc3437960d0cce08552c7c004f # v2
+        if: ${{ always() && steps.parse-playwright-report.conclusion == 'success' && github.event_name == 'pull_request' }}
+        with:
+          comment_tag: 'playwright-ct-report'
+          filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/results/playwright-report-pr-comment.md
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-ct-report
+          path: |
+            ${{ github.workspace }}/packages/sanity/playwright-ct/report
+            ${{ github.workspace }}/packages/sanity/playwright-ct/results
+          retention-days: 30

package.json

@@ -84,7 +84,7 @@
     "@babel/preset-react": "^7.18.6",
     "@babel/preset-typescript": "^7.18.6",
     "@optimize-lodash/rollup-plugin": "^4.0.1",
-    "@playwright/test": "^1.31.2",
+    "@playwright/test": "^1.37.0",
     "@sanity/client": "^6.4.5",
     "@sanity/pkg-utils": "^2.3.10",
     "@sanity/tsdoc": "1.0.0-alpha.38",
@@ -127,7 +127,6 @@
     "minimist": "^1.2.5",
     "npm-run-all": "^4.1.5",
     "ora": "^6.3.1",
-    "playwright": "^1.31.2",
     "prettier": "^3.0.2",
     "prettier-plugin-packagejson": "^2.4.5",
     "pretty-quick": "^3.1.3",

packages/@sanity/portable-text-editor/package.json

@@ -74,6 +74,7 @@
     "slate-react": "0.98.1"
   },
   "devDependencies": {
+    "@playwright/test": "^1.37.0",
     "@sanity/ui": "^1.7.4",
     "@portabletext/toolkit": "^2.0.4",
     "@sanity/diff-match-patch": "^3.1.1",
@@ -97,7 +98,6 @@
     "jest-environment-jsdom": "^27.3.1",
     "jest-environment-node": "^27.1.0",
     "node-ipc": "^9.2.0",
-    "playwright": "^1.31.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "rimraf": "^3.0.2",

packages/@sanity/portable-text-editor/test/setup/collaborative.jest.env.ts

@@ -1,7 +1,7 @@
+import {chromium, Browser, Page, ElementHandle} from '@playwright/test'
 import NodeEnvironment from 'jest-environment-node'
 import {isEqual} from 'lodash'
 import ipc from 'node-ipc'
-import {chromium, Browser, Page, ElementHandle} from 'playwright'
 import {PortableTextBlock} from '@sanity/types'
 import {normalizeSelection} from '../../src/utils/selection'
 import type {EditorSelection} from '../../src'

packages/sanity/.gitignore

@@ -16,3 +16,8 @@
 /cli.js
 /desk.js
 /router.js
+
+# Playwright-ct artifacts
+/playwright-ct/report
+/playwright-ct/results
+/playwright-ct/template/.cache/

packages/sanity/package.json

@@ -121,7 +121,9 @@
     "coverage": "jest --coverage",
     "prepublishOnly": "run-s write:version write:ui-version",
     "test": "pkg-utils --strict && jest",
+    "test:ct": "playwright test -c playwright-ct.config.ts",
     "watch": "pkg-utils watch --tsconfig tsconfig.lib.json",
+    "write:playwright-report-as-pr-comment": "node -r esbuild-register playwright-ct/scripts/parsePlaywrightReportJson.ts",
     "write:ui-version": "node -r esbuild-register scripts/writeRequiredUIVersion.ts",
     "write:version": "node -r esbuild-register scripts/writeVersion.ts"
   },
@@ -229,6 +231,8 @@
     "yargs": "^17.3.0"
   },
   "devDependencies": {
+    "@playwright/experimental-ct-react": "^1.37.0",
+    "@playwright/test": "^1.37.0",
     "@sanity/tsdoc": "1.0.0-alpha.38",
     "@sanity/ui-workshop": "^1.2.9",
     "@testing-library/jest-dom": "^5.16.5",

packages/sanity/playwright-ct.config.ts

@@ -0,0 +1,75 @@
+import path from 'path'
+import {defineConfig, devices} from '@playwright/experimental-ct-react'
+
+// Paths
+const TESTS_PATH = path.join(__dirname, 'playwright-ct', 'tests')
+const HTML_REPORT_PATH = path.join(__dirname, 'playwright-ct', 'report')
+const ARTIFACT_OUTPUT_PATH = path.join(__dirname, 'playwright-ct', 'results')
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: TESTS_PATH,
+
+  outputDir: ARTIFACT_OUTPUT_PATH,
+
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: [
+    ['list'],
+    ['html', {outputFolder: HTML_REPORT_PATH}],
+    [
+      'json',
+      {
+        outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'),
+      },
+    ],
+  ],
+
+  /* Maximum time one test can run for. */
+  timeout: 120 * 1000,
+  expect: {
+    // Maximum time expect() should wait for the condition to be met.
+    timeout: 20 * 1000,
+  },
+
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 40 * 1000,
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+    /* Port to use for Playwright component endpoint. */
+    ctPort: 3100,
+    /* Configure Playwright vite config */
+    /*
+    ctViteConfig: {
+      resolve: {
+        alias: {
+          '@sanity/util/content': path.join(
+            __dirname,
+            './packages/@sanity/util/src/content/index.ts'
+          ),
+        },
+      },
+    },
+    */
+    /* Where to find playwright-ct template files */
+    ctTemplateDir: './playwright-ct/template',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {name: 'chromium', use: {...devices['Desktop Chrome']}},
+    {name: 'firefox', use: {...devices['Desktop Firefox']}},
+    {name: 'webkit', use: {...devices['Desktop Safari']}},
+  ],
+})

packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts

@@ -0,0 +1,48 @@
+#!/usr/bin/env node -r esbuild-register
+
+import {writeFileSync} from 'fs'
+import {inspect} from 'util'
+import path from 'path'
+import _ from 'lodash'
+import {readJsonFile, generateOutput, groupTests} from './utils'
+
+const DEBUG = Boolean(parseInt(process.env.DEBUG || '0', 2))
+const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(path.join(__dirname, '..', 'results'))
+
+/**
+ * Summarize the Playwright report JSON into a Markdown string that can be posted as a comment to PRs
+ */
+function main() {
+  const workflowUrl = process.env.GITHUB_WORKFLOW_URL || ''
+  const jsonPath =
+    process.env.REPORT_JSON_PATH! ||
+    path.join(DEFAULT_ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json')
+  const testOutput = readJsonFile(jsonPath)
+
+  if (testOutput) {
+    const groups = groupTests(testOutput)
+    const markdownTable = generateOutput(groups, workflowUrl)
+
+    if (DEBUG) {
+      const result = inspect(groups, {
+        depth: Infinity,
+        breakLength: 80,
+        colors: true,
+      })
+      // eslint-disable-next-line no-console
+      console.log(result)
+      // eslint-disable-next-line no-console
+      console.log(markdownTable)
+    }
+
+    writeFileSync(
+      path.join(DEFAULT_ARTIFACT_OUTPUT_PATH, 'playwright-report-pr-comment.md'),
+      markdownTable,
+      'utf8',
+    )
+  }
+
+  process.stdout.write('Processing Playwright report JSON complete\n')
+}
+
+main()