feat: E2E Tests with Playwright (#3430)

* initial commit for e2e

* updated readme

* activate only stackable for all tests

* add tsconfig

* add tests for settings page and editor page

* add tests for site editor, global settings

* minor change to tests for admin and block editor

* use custom test utils

* run tests

* update playwright workflows

* add .wp-env.json

* add env

* fix env slug

* log plugins installed

* update steps, slug

* install playwright browsers

* comment out site editor tests, add reporter

* add try catch for newly installed plugin

* update admin test

* update stackable fixture

* comment on PR

* fix PR comment

* add id for upload artifact, update comment body

* update yml file

* include site editor tests

* minor update in yml file

* always update comment

* upload traces

* test script

* update comment content

* get failed tests

* use $GITHUB_OUTPUT

* update to multiline

* update output name

* fix failed tests content

* escape special characters

* use markdown to output

* run mto even on fail

* strip Ansi Escapes

* try multiple upload

* test multiple upload

* use test ids and trace file path

* update reporter

* test matrix

* update .wp-env.json and name

* update php version, remove unit tests

* continue matrix even if one job fails

* use different artifact name based on matrix

* try all test env

* remove comments, update matrix

* update php version

* update conditions, log wp and php version

* add quotes

* update php version

* run other tests

* test custom reporter

* update condition

* add report to summary

* comment site editor tests temporarily

* update report url

* update wp version

* update block editor test

* use custom way to get block attributes

* fix missing global settings for older wp versions

* add comments, update tests

* update tests

* test with existing post

* update command

* use tests-cli

* run wp-env first

* update file path

* check post content

* update file path

* update file path

* update reporter

* update locator for global settings block defaults

* check if settings are open in editor, correct typo

* update readme, global settings test

* update failing tests in premium

* update test to prevent flakiness

* make sure global block default gets updated

* update to default block page

---------

Co-authored-by: [email protected] <>

e2e/readme.md

@@ -32,8 +32,17 @@ WP_BASE_URL=http://local.local/
 WP_AUTH_STORAGE=wp-auth.json
 WP_USERNAME=admin
 WP_PASSWORD=password
+STACKABLE_SLUG={ STACKABLE SLUG }
+WP_TEST_POSTID={ POST_ID }
+WP_STK_LICENSE={ STACKABLE LICENSE }
 ```
 
+In Freemius, make sure the license key is:
+- new
+- lifetime unlimited quota
+- has an owner
+- non-blocking and white-labeled
+
 Run this command to run e2e:
 
 ```bash

e2e/tests/block-editor.spec.ts

@@ -39,6 +39,13 @@ test.describe( 'Block Editor', () => {
 		} )
 
 		await editor.selectBlocks( editor.canvas.getByLabel( 'Block: Text' ) )
+
+		const settings = page.getByLabel( 'Settings', { exact: true } )
+
+		if ( await settings.getAttribute( 'aria-pressed' ) === 'false' ) {
+			await settings.click()
+		}
+
 		await expect( page.getByLabel( 'Layout Tab' ) ).toBeVisible()
 		await expect( page.getByLabel( 'Style Tab' ) ).toBeVisible()
 		await expect( page.getByLabel( 'Advanced Tab' ) ).toBeVisible()
@@ -52,6 +59,12 @@ test.describe( 'Block Editor', () => {
 			name: 'stackable/text',
 		} )
 
+		const settings = page.getByLabel( 'Settings', { exact: true } )
+
+		if ( await settings.getAttribute( 'aria-pressed' ) === 'false' ) {
+			await settings.click()
+		}
+
 		// Add content and color to Stackable Text Block
 		await editor.canvas.locator( '[data-type="stackable/text"] > .stk-block-text > p[role="textbox"]' ).fill( 'test' )
 		await page.locator( '.stk-color-palette-control .stk-control-content > .components-dropdown > .components-button' ).first().click()

e2e/tests/global-settings.spec.ts

@@ -1,7 +1,6 @@
 import { test, expect } from 'e2e/test-utils'
-import { createColor, getRgb } from '~stackable/plugins/global-settings/colors/util'
 
-test.describe( 'Global Settigs', () => {
+test.describe( 'Global Settings', () => {
 	let pid = null
 
 	// Create Posts for testing
@@ -62,7 +61,15 @@ test.describe( 'Global Settigs', () => {
 
 		// Delete added Global Color
 		await page.getByLabel( 'Stackable Settings' ).click()
-		await panel.locator( `.ugb-global-settings-color-picker__color-indicators > div > div:nth-child(${ count }) > button.stk-global-settings-color-picker__delete-button` ).click()
+
+		page.on( 'dialog', async dialog => await dialog.accept() )
+		const deleteRequest = page.waitForResponse( response => response.url().includes( 'wp/v2/settings' ) && response.request().method() === 'POST' )
+
+		await globalColors.getByLabel( 'Delete' ).nth( count - 1 ).click()
+
+		await deleteRequest
+		const _count = ( await globalColors.evaluate( node => Array.from( node.childNodes ) ) ).length
+		expect( _count ).toBeLessThan( count )
 	} )
 
 	test( 'Global Typography Styles should be applied when adding a heading', async ( {
@@ -93,18 +100,21 @@ test.describe( 'Global Settigs', () => {
 
 		await expect( editor.canvas.locator( '[data-type="stackable/heading"] > .stk-block-heading > h2[role="textbox"]' ) ).toHaveCSS( 'font-size', '32px' )
 
-		 // Reset Global Typography Styles
-		 await page.getByLabel( 'Stackable Settings' ).click()
-		 await page.locator( '.ugb-global-settings-typography-control' ).nth( 1 ).getByRole( 'button', { name: 'Reset' } ).click()
+		// Reset Global Typography Styles
+		await page.getByLabel( 'Stackable Settings' ).click()
+
+		const resetButton = page.locator( '.ugb-global-settings-typography-control' ).nth( 1 ).getByLabel( 'Reset' )
+		const deleteRequest = page.waitForResponse( response => response.url().includes( 'wp/v2/settings' ) && response.request().method() === 'POST' )
+		await resetButton.click()
+
+		await deleteRequest
+		await expect( resetButton ).not.toBeVisible()
 	} )
 
 	test( 'When a default block is created, adding the block should have the default block\'s attributes', async ( {
 		page,
 		editor,
 	} ) => {
-		// Generate a color
-		const color = createColor()
-
 		await page.getByLabel( 'Stackable Settings' ).click()
 		await page.getByRole( 'button', { name: 'Block Defaults' } ).click()
 
@@ -116,16 +126,20 @@ test.describe( 'Global Settigs', () => {
 
 		// Set a color for the default Text Block
 		await defaultBlockPage.locator( '.stk-color-palette-control .stk-control-content > .components-dropdown > .components-button' ).first().click()
-		await defaultBlockPage.getByLabel( 'Hex color' ).fill( color )
+		await defaultBlockPage.getByLabel( 'Hex color' ).fill( 'ff0000' )
 		await defaultBlockPage.locator( '.stk-color-palette-control .stk-control-content > .components-dropdown > .components-button' ).first().click()
 
+		const updateRequest = defaultBlockPage.waitForResponse( response => response.url().includes( 'update_block_style' ) && response.request().method() === 'POST' )
+
 		// In older WP versions, the button text is 'Update' instead of 'Save'
 		if ( await defaultBlockPage.getByRole( 'button', { name: 'Save', exact: true } ).isVisible() ) {
 			await defaultBlockPage.getByRole( 'button', { name: 'Save', exact: true } ).click()
 		} else {
 			await defaultBlockPage.getByRole( 'button', { name: 'Update' } ).click()
 		}
 
+		// Make sure default block has been updated before closing the tab
+		await updateRequest
 		await defaultBlockPage.close()
 
 		// Insert a Stackable Text Block, and check if the color is the same as the one set in the default block
@@ -137,12 +151,11 @@ test.describe( 'Global Settigs', () => {
 			},
 		} )
 
-		await expect( editor.canvas.locator( '[data-type="stackable/text"] > .stk-block-text > p[role="textbox"]' ) ).toHaveCSS( 'color', `rgb(${ getRgb( color ) })` )
+		await expect( editor.canvas.locator( '[data-type="stackable/text"] > .stk-block-text > p[role="textbox"]' ) ).toHaveCSS( 'color', 'rgb(255, 0, 0)' )
 
 		// Reset Default Block
 		await page.getByLabel( 'Stackable Settings' ).click()
 		await page.getByRole( 'button', { name: 'Block Defaults' } ).click()
 		await page.locator( '.components-panel__body', { hasText: 'Block Defaults' } ).locator( '.stk-block-default-control', { hasText: /^Text$/ } ).first().getByLabel( 'Reset' ).click()
 	} )
 } )
-

src/components/sortable-picker/index.js

@@ -225,6 +225,7 @@ const LabeledItemIndicator = props => {
 				) }
 			/>
 			<Button
+				aria-label="Delete"
 				className="stk-global-settings-color-picker__delete-button"
 				icon="trash"
 				isSmall