Skip to content

Commit

Permalink
Merge branch 'develop' into t3chguy/flaky-test-chase
Browse files Browse the repository at this point in the history
  • Loading branch information
t3chguy authored Jan 14, 2025
2 parents eeab0f2 + 37f8d70 commit d537b97
Show file tree
Hide file tree
Showing 18 changed files with 63 additions and 79 deletions.
2 changes: 1 addition & 1 deletion playwright/e2e/app-loading/guest-registration.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Please see LICENSE files in the repository root for full details.
import { expect, test } from "../../element-web-test";

test.use({
synapseConfigOptions: {
synapseConfig: {
allow_guest_access: true,
},
});
Expand Down
2 changes: 1 addition & 1 deletion playwright/e2e/crypto/backups-mas.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { TestClientServerAPI } from "../csAPI";
import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";

// These tests register an account with MAS because then we go through the "normal" registration flow
// and crypto gets set up. Using the 'user' fixture create a a user an synthesizes an existing login,
// and crypto gets set up. Using the 'user' fixture create a user and synthesizes an existing login,
// which is faster but leaves us without crypto set up.
test.use(masHomeserver);
test.describe("Encryption state after registration", () => {
Expand Down
2 changes: 1 addition & 1 deletion playwright/e2e/crypto/dehydration.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ function getMemberTileByName(page: Page, name: string): Locator {

test.use({
displayName: NAME,
synapseConfigOptions: {
synapseConfig: {
experimental_features: {
msc2697_enabled: false,
msc3814_enabled: true,
Expand Down
1 change: 1 addition & 0 deletions playwright/e2e/crypto/invisible-crypto.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { bootstrapCrossSigningForClient } from "../../pages/client.ts";

/** Tests for the "invisible crypto" behaviour -- i.e., when the "exclude insecure devices" setting is enabled */
test.describe("Invisible cryptography", () => {
test.slow();
test.use({
displayName: "Alice",
botCreateOpts: { displayName: "Bob" },
Expand Down
28 changes: 6 additions & 22 deletions playwright/e2e/integration-manager/kick.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,29 +69,13 @@ async function sendActionFromIntegrationManager(
await iframe.getByRole("button", { name: "Press to send action" }).click();
}

async function clickUntilGone(page: Page, selector: string, attempt = 0) {
if (attempt === 11) {
throw new Error("clickUntilGone attempt count exceeded");
}

await page.locator(selector).last().click();

const count = await page.locator(selector).count();
if (count > 0) {
return clickUntilGone(page, selector, ++attempt);
}
}

async function expectKickedMessage(page: Page, shouldExist: boolean) {
// Expand any event summaries, we can't use a click multiple here because clicking one might de-render others
// This is quite horrible but seems the most stable way of clicking 0-N buttons,
// one at a time with a full re-evaluation after each click
await clickUntilGone(page, ".mx_GenericEventListSummary_toggle[aria-expanded=false]");

// Check for the event message (or lack thereof)
await expect(page.getByText(`${USER_DISPLAY_NAME} removed ${BOT_DISPLAY_NAME}: ${KICK_REASON}`)).toBeVisible({
visible: shouldExist,
});
await expect(async () => {
await page.locator(".mx_GenericEventListSummary_toggle[aria-expanded=false]").last().click();
await expect(page.getByText(`${USER_DISPLAY_NAME} removed ${BOT_DISPLAY_NAME}: ${KICK_REASON}`)).toBeVisible({
visible: shouldExist,
});
}).toPass();
}

test.describe("Integration Manager: Kick", () => {
Expand Down
6 changes: 1 addition & 5 deletions playwright/e2e/read-receipts/reactions-in-threads.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,7 @@ test.describe("Read receipts", { tag: "@mergequeue" }, () => {
// Given a thread exists and I have marked it as read
await util.goTo(room1);
await util.assertRead(room2);
await util.receiveMessages(room2, [
"Msg1",
msg.threadedOff("Msg1", "Reply1"),
msg.reactionTo("Reply1", "🪿"),
]);
await util.receiveMessages(room2, ["Msg1", msg.threadedOff("Msg1", "Reply1")]);
await util.assertUnread(room2, 1);
await util.markAsRead(room2);
await util.assertRead(room2);
Expand Down
2 changes: 1 addition & 1 deletion playwright/e2e/right-panel/memberlist.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ const ROOM_NAME = "Test room";
const NAME = "Alice";

test.use({
synapseConfigOptions: {
synapseConfig: {
presence: {
enabled: false,
include_offline_users_on_sync: false,
Expand Down
7 changes: 6 additions & 1 deletion playwright/e2e/timeline/timeline.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,12 @@ test.describe("Timeline", () => {
await messageEdit(page);

// Click timestamp to highlight hidden event line
await page.locator(".mx_RoomView_body .mx_EventTile_info .mx_MessageTimestamp").click();
const timestamp = page.locator(".mx_RoomView_body .mx_EventTile_info a", {
has: page.locator(".mx_MessageTimestamp"),
});
// wait for the remote echo otherwise we get an error modal due to a 404 on the /event/ API
await expect(timestamp).not.toHaveAttribute("href", /~!/);
await timestamp.locator(".mx_MessageTimestamp").click();

// should not add inline start padding to a hidden event line on IRC layout
await app.settings.setValue("layout", null, SettingLevel.DEVICE, Layout.IRC);
Expand Down
20 changes: 15 additions & 5 deletions playwright/element-web-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { expect as baseExpect, Locator, Page, ExpectMatcherState, ElementHandle } from "@playwright/test";
import {
expect as baseExpect,
Locator,
Page,
ExpectMatcherState,
ElementHandle,
PlaywrightTestArgs,
Fixtures as _Fixtures,
} from "@playwright/test";
import { sanitizeForFilePath } from "playwright-core/lib/utils";
import AxeBuilder from "@axe-core/playwright";
import _ from "lodash";
Expand All @@ -19,7 +27,7 @@ import { Crypto } from "./pages/crypto";
import { Toasts } from "./pages/toasts";
import { Bot, CreateBotOpts } from "./pages/bot";
import { Webserver } from "./plugins/webserver";
import { test as base } from "./services.ts";
import { Options, Services, test as base } from "./services.ts";

// Enable experimental service worker support
// See https://playwright.dev/docs/service-workers-experimental#how-to-enable
Expand All @@ -45,7 +53,7 @@ interface CredentialsWithDisplayName extends Credentials {
displayName: string;
}

export interface Fixtures {
export interface TestFixtures {
axe: AxeBuilder;
checkA11y: () => Promise<void>;

Expand Down Expand Up @@ -102,7 +110,9 @@ export interface Fixtures {
disablePresence: boolean;
}

export const test = base.extend<Fixtures>({
type CombinedTestFixtures = PlaywrightTestArgs & TestFixtures;
export type Fixtures = _Fixtures<CombinedTestFixtures, Services & Options, CombinedTestFixtures>;
export const test = base.extend<TestFixtures>({
context: async ({ context }, use, testInfo) => {
// We skip tests instead of using grep-invert to still surface the counts in the html report
test.skip(
Expand Down Expand Up @@ -150,7 +160,7 @@ export const test = base.extend<Fixtures>({
const displayName = testDisplayName ?? _.sample(names)!;

const credentials = await homeserver.registerUser(`user_${testInfo.testId}`, password, displayName);
console.log(`Registered test user @user:localhost with displayname ${displayName}`);
console.log(`Registered test user ${credentials.userId} with displayname ${displayName}`);

await use({
...credentials,
Expand Down
2 changes: 1 addition & 1 deletion playwright/pages/bot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ export class Bot extends Client {
return logger as unknown as Logger;
}

const logger = getLogger(`cypress bot ${credentials.userId}`);
const logger = getLogger(`bot ${credentials.userId}`);

const keys = {};

Expand Down
6 changes: 2 additions & 4 deletions playwright/plugins/homeserver/dendrite/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures } from "@playwright/test";

import { DendriteContainer, PineconeContainer } from "../../../testcontainers/dendrite.ts";
import { Services } from "../../../services.ts";
import { Fixtures } from "../../../element-web-test.ts";

export const dendriteHomeserver: Fixtures<{}, Services> = {
export const dendriteHomeserver: Fixtures = {
_homeserver: [
// eslint-disable-next-line no-empty-pattern
async ({}, use) => {
Expand Down
6 changes: 2 additions & 4 deletions playwright/plugins/homeserver/synapse/consentHomeserver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures } from "@playwright/test";
import { Fixtures } from "../../../element-web-test.ts";

import { Services } from "../../../services.ts";

export const consentHomeserver: Fixtures<{}, Services> = {
export const consentHomeserver: Fixtures = {
_homeserver: [
async ({ _homeserver: container, mailhog }, use) => {
container
Expand Down
6 changes: 2 additions & 4 deletions playwright/plugins/homeserver/synapse/emailHomeserver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures } from "@playwright/test";
import { Fixtures } from "../../../element-web-test.ts";

import { Services } from "../../../services.ts";

export const emailHomeserver: Fixtures<{}, Services> = {
export const emailHomeserver: Fixtures = {
_homeserver: [
async ({ _homeserver: container, mailhog }, use) => {
container.withConfig({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures, PlaywrightTestArgs } from "@playwright/test";
import { TestContainers } from "testcontainers";

import { Services } from "../../../services.ts";
import { OAuthServer } from "../../oauth_server";
import { Fixtures } from "../../../element-web-test.ts";

export const legacyOAuthHomeserver: Fixtures<PlaywrightTestArgs, Services, PlaywrightTestArgs> = {
export const legacyOAuthHomeserver: Fixtures = {
oAuthServer: [
// eslint-disable-next-line no-empty-pattern
async ({}, use) => {
Expand Down
8 changes: 2 additions & 6 deletions playwright/plugins/homeserver/synapse/masHomeserver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures, PlaywrightTestArgs } from "@playwright/test";

import { Services } from "../../../services.ts";
import { Fixtures as BaseFixtures } from "../../../element-web-test.ts";
import { MatrixAuthenticationServiceContainer } from "../../../testcontainers/mas.ts";
import { Fixtures } from "../../../element-web-test.ts";

type Fixture = PlaywrightTestArgs & BaseFixtures;
export const masHomeserver: Fixtures<Fixture, Services, Fixture> = {
export const masHomeserver: Fixtures = {
mas: [
async ({ _homeserver: homeserver, logger, network, postgres, mailhog }, use) => {
const config = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
Please see LICENSE files in the repository root for full details.
*/

import { Fixtures } from "@playwright/test";
import { Fixtures } from "../../../element-web-test.ts";

import { Services } from "../../../services.ts";

export const uiaLongSessionTimeoutHomeserver: Fixtures<{}, Services> = {
synapseConfigOptions: [
async ({ synapseConfigOptions }, use) => {
export const uiaLongSessionTimeoutHomeserver: Fixtures = {
synapseConfig: [
async ({ synapseConfig }, use) => {
await use({
...synapseConfigOptions,
...synapseConfig,
ui_auth: {
session_timeout: "300s",
},
Expand Down
19 changes: 10 additions & 9 deletions playwright/services.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import mailhog from "mailhog";
import { Network, StartedNetwork } from "testcontainers";
import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";

import { SynapseConfigOptions, SynapseContainer } from "./testcontainers/synapse.ts";
import { SynapseConfig, SynapseContainer } from "./testcontainers/synapse.ts";
import { Logger } from "./logger.ts";
import { StartedMatrixAuthenticationServiceContainer } from "./testcontainers/mas.ts";
import { HomeserverContainer, StartedHomeserverContainer } from "./testcontainers/HomeserverContainer.ts";
import { MailhogContainer, StartedMailhogContainer } from "./testcontainers/mailhog.ts";
import { OAuthServer } from "./plugins/oauth_server";

interface TestFixtures {
export interface TestFixtures {
mailhogClient: mailhog.API;
}

Expand All @@ -28,7 +28,7 @@ export interface Services {
postgres: StartedPostgreSqlContainer;
mailhog: StartedMailhogContainer;

synapseConfigOptions: SynapseConfigOptions;
synapseConfig: SynapseConfig;
_homeserver: HomeserverContainer<any>;
homeserver: StartedHomeserverContainer;
// Set in masHomeserver only
Expand All @@ -37,7 +37,9 @@ export interface Services {
oAuthServer?: OAuthServer;
}

export const test = base.extend<TestFixtures, Services>({
export interface Options {}

export const test = base.extend<TestFixtures, Services & Options>({
logger: [
// eslint-disable-next-line no-empty-pattern
async ({}, use) => {
Expand Down Expand Up @@ -97,11 +99,11 @@ export const test = base.extend<TestFixtures, Services>({
{ scope: "worker" },
],
mailhogClient: async ({ mailhog: container }, use) => {
await use(container.client);
await container.client.deleteAll();
await use(container.client);
},

synapseConfigOptions: [{}, { option: true, scope: "worker" }],
synapseConfig: [{}, { scope: "worker" }],
_homeserver: [
// eslint-disable-next-line no-empty-pattern
async ({}, use) => {
Expand All @@ -111,12 +113,12 @@ export const test = base.extend<TestFixtures, Services>({
{ scope: "worker" },
],
homeserver: [
async ({ logger, network, _homeserver: homeserver, synapseConfigOptions, mas }, use) => {
async ({ logger, network, _homeserver: homeserver, synapseConfig, mas }, use) => {
const container = await homeserver
.withNetwork(network)
.withNetworkAliases("homeserver")
.withLogConsumer(logger.getConsumer("synapse"))
.withConfig(synapseConfigOptions)
.withConfig(synapseConfig)
.start();

await use(container);
Expand All @@ -139,6 +141,5 @@ export const test = base.extend<TestFixtures, Services>({
await logger.onTestStarted(context);
await use(context);
await logger.onTestFinished(testInfo);
await mailhogClient.deleteAll();
},
});
8 changes: 4 additions & 4 deletions playwright/testcontainers/synapse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { Credentials } from "../plugins/homeserver";
import { deepCopy } from "../plugins/utils/object.ts";
import { HomeserverContainer, StartedHomeserverContainer } from "./HomeserverContainer.ts";

const TAG = "develop@sha256:b69222d98abe9625d46f5d3cb01683d5dc173ae339215297138392cfeec935d9";
const TAG = "develop@sha256:7be2e00da62dfbb2bad071c6d408fecb1fabf740a538d08768b9b3e0a8c45350";

const DEFAULT_CONFIG = {
server_name: "localhost",
Expand Down Expand Up @@ -138,7 +138,7 @@ const DEFAULT_CONFIG = {
},
};

export type SynapseConfigOptions = Partial<typeof DEFAULT_CONFIG>;
export type SynapseConfig = Partial<typeof DEFAULT_CONFIG>;

export class SynapseContainer extends GenericContainer implements HomeserverContainer<typeof DEFAULT_CONFIG> {
private config: typeof DEFAULT_CONFIG;
Expand Down Expand Up @@ -277,7 +277,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements

const data = await res.json();
return {
homeServer: data.home_server,
homeServer: data.home_server || data.user_id.split(":").slice(1).join(":"),
accessToken: data.access_token,
userId: data.user_id,
deviceId: data.device_id,
Expand Down Expand Up @@ -310,7 +310,7 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements
accessToken: json.access_token,
userId: json.user_id,
deviceId: json.device_id,
homeServer: json.home_server,
homeServer: json.home_server || json.user_id.split(":").slice(1).join(":"),
username: userId.slice(1).split(":")[0],
};
}
Expand Down

0 comments on commit d537b97

Please sign in to comment.