From b8c915f9c59ca291ba023f104f442b36026ba253 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 6 May 2026 12:49:10 +0200 Subject: [PATCH 001/249] docs(python): document expect.soft() (#40665) --- .../src/test-assertions-csharp-java-python.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/src/test-assertions-csharp-java-python.md b/docs/src/test-assertions-csharp-java-python.md index 9ae8a462d1bcc..4d19503115fa0 100644 --- a/docs/src/test-assertions-csharp-java-python.md +++ b/docs/src/test-assertions-csharp-java-python.md @@ -36,6 +36,28 @@ title: "Assertions" | [`method: PageAssertions.toHaveURL`] | Page has a URL | | [`method: APIResponseAssertions.toBeOK`] | Response has an OK status | +## Soft assertions +* langs: python + +By default, failed assertion will terminate test execution. Playwright also +supports *soft assertions*: failed soft assertions **do not** terminate test +execution, but mark the test as failed. + +```python +# Make a few checks that will not stop the test when failed... +expect.soft(page.get_by_test_id("status")).to_have_text("Success") +expect.soft(page.get_by_test_id("eta")).to_have_text("1 day") + +# ... and continue the test to check more things. +page.get_by_role("link", name="next page").click() +expect.soft(page.get_by_role("heading", name="Make another order")).to_be_visible() +``` + +Note that soft assertions only work with the +[`pytest-playwright`](https://pypi.org/project/pytest-playwright/) (or +[`pytest-playwright-asyncio`](https://pypi.org/project/pytest-playwright-asyncio/)) +plugin, version `0.7.3` or newer. + ## Custom Expect Message * langs: python, csharp From 9a533e12b977fca495107d24724379d5ff71ba65 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Wed, 6 May 2026 12:10:50 +0100 Subject: [PATCH 002/249] test: unflake/fix/skip a few tests (#40662) --- tests/library/browsertype-launch.spec.ts | 25 ++++++++---------------- tests/library/trace-viewer.spec.ts | 2 +- tests/page/expect-timeout.spec.ts | 2 +- tests/page/page-basic.spec.ts | 4 +++- tests/page/page-emulate-media.spec.ts | 4 +++- tests/page/page-goto.spec.ts | 8 ++++---- tests/page/page-network-response.spec.ts | 4 +++- tests/page/page-request-continue.spec.ts | 4 +++- 8 files changed, 26 insertions(+), 27 deletions(-) diff --git a/tests/library/browsertype-launch.spec.ts b/tests/library/browsertype-launch.spec.ts index a66ccf12225c7..11f123c8637af 100644 --- a/tests/library/browsertype-launch.spec.ts +++ b/tests/library/browsertype-launch.spec.ts @@ -61,15 +61,14 @@ it('should throw if page argument is passed', async ({ browserType, browserName, expect(waitError!.message).toContain('can not specify page'); }); -it('should reject if launched browser fails immediately', async ({ mode, browserType, asset, isWindows, channel }) => { +it('should reject if launched browser fails immediately', async ({ mode, browserType, asset, channel }) => { it.skip(mode.startsWith('service')); - let waitError: Error | undefined; - await browserType.launch({ executablePath: asset('dummy_bad_browser_executable.js') }).catch(e => waitError = e); + const error = await browserType.launch({ executablePath: asset('dummy_bad_browser_executable.js') }).catch(e => e); if (channel === 'webkit-wsl') - expect(waitError!.message).toContain('Cannot specify executablePath when using the \"webkit-wsl\" channel.'); + expect(error.message).toContain('Cannot specify executablePath when using the \"webkit-wsl\" channel.'); else - expect(waitError!.message).toContain(isWindows ? 'browserType.launch: spawn UNKNOWN' : 'Browser logs:'); + expect(error.message).toMatch(/browserType\.launch(.|\n)*(spawn UNKNOWN|spawn EFTYPE|Browser logs:)/gim); }); it('should reject if executable path is invalid', async ({ browserType, mode, channel }) => { @@ -92,22 +91,14 @@ it('should handle timeout', async ({ browserType, mode }) => { expect(error!.message).toContain(` pid=`); }); -it('should handle exception', async ({ browserType, mode }) => { - it.skip(mode !== 'default'); - - const e = new Error('Dummy'); - const options = { __testHookBeforeCreateBrowser: () => { throw e; }, timeout: 9000 }; - const error = await browserType.launch(options).catch(e => e); - expect(error!.message).toContain('Dummy'); -}); - -it('should report launch log', async ({ browserType, mode }) => { +it('should handle exception and report launch log', async ({ browserType, mode }) => { it.skip(mode !== 'default'); const e = new Error('Dummy'); - const options = { __testHookBeforeCreateBrowser: () => { throw e; }, timeout: 9000 }; + const options = { __testHookBeforeCreateBrowser: () => { throw e; }, timeout: 15000 }; const error = await browserType.launch(options).catch(e => e); - expect(error!.message).toContain(''); + expect(error.message).toContain('Dummy'); + expect(error.message).toContain(''); }); it('should accept objects as options', async ({ mode, browserType }) => { diff --git a/tests/library/trace-viewer.spec.ts b/tests/library/trace-viewer.spec.ts index c415e299096b8..237dd89130d55 100644 --- a/tests/library/trace-viewer.spec.ts +++ b/tests/library/trace-viewer.spec.ts @@ -200,7 +200,7 @@ test('should filter actions by text', async ({ showTraceViewer }) => { const fullCount = await traceViewer.actionTitles.count(); await filterInput.fill('Click'); await expect(traceViewer.actionTitles.filter({ hasText: 'Click' }).first()).toBeVisible(); - expect(await traceViewer.actionTitles.count()).toBeLessThan(fullCount); + await expect.poll(() => traceViewer.actionTitles.count()).toBeLessThan(fullCount); await filterInput.fill(''); await expect(traceViewer.actionTitles).toHaveCount(fullCount); diff --git a/tests/page/expect-timeout.spec.ts b/tests/page/expect-timeout.spec.ts index 6f54e295e397e..963aaa9bcd0a8 100644 --- a/tests/page/expect-timeout.spec.ts +++ b/tests/page/expect-timeout.spec.ts @@ -114,7 +114,7 @@ Call log: test('should not miss element that appears between retries before the deadline', async ({ page }) => { await page.setContent(``); await page.evaluate(() => { - setTimeout(() => { + window.builtins.setTimeout(() => { document.getElementById('target')!.style.display = 'block'; }, 1500); }); diff --git a/tests/page/page-basic.spec.ts b/tests/page/page-basic.spec.ts index e14c4a7bb7e2f..b0ba37ace0ee5 100644 --- a/tests/page/page-basic.spec.ts +++ b/tests/page/page-basic.spec.ts @@ -180,7 +180,9 @@ it('frame.press should work', async ({ page, server }) => { expect(await frame.evaluate(() => document.querySelector('textarea').value)).toBe('a'); }); -it('has navigator.webdriver set to true', async ({ page }) => { +it('has navigator.webdriver set to true', async ({ page, isAndroid }) => { + it.fixme(isAndroid); + expect(await page.evaluate(() => navigator.webdriver)).toBe(true); }); diff --git a/tests/page/page-emulate-media.spec.ts b/tests/page/page-emulate-media.spec.ts index 63c9ec0894f8f..6e6b6cbb47f8b 100644 --- a/tests/page/page-emulate-media.spec.ts +++ b/tests/page/page-emulate-media.spec.ts @@ -193,8 +193,10 @@ it('should emulate contrast ', async ({ page }) => { await expect(page).toMatchMedia('(prefers-contrast: no-preference)'); }); -it('should report hover and fine pointer for desktop', async ({ page, browserName, headless, isLinux }) => { +it('should report hover and fine pointer for desktop', async ({ page, browserName, headless, isLinux, isAndroid }) => { it.fail(browserName === 'firefox' && isLinux && headless, 'https://github.com/microsoft/playwright/issues/38835'); + it.skip(isAndroid); + await expect(page).toMatchMedia('(hover: hover)'); await expect(page).toMatchMedia('(pointer: fine)'); }); diff --git a/tests/page/page-goto.spec.ts b/tests/page/page-goto.spec.ts index a3f518665bb45..bf8e9ecf4c948 100644 --- a/tests/page/page-goto.spec.ts +++ b/tests/page/page-goto.spec.ts @@ -407,8 +407,8 @@ it('should fail when exceeding browser context navigation timeout', async ({ pag expect(error).toBeInstanceOf(playwright.errors.TimeoutError); }); -it('should fail when exceeding default maximum timeout', async ({ page, server, playwright, isAndroid }) => { - it.skip(isAndroid, 'No context per test'); +it('should fail when exceeding default maximum timeout', async ({ page, server, playwright, isAndroid, isElectron }) => { + it.skip(isAndroid || isElectron, 'No context per test'); // Hang for request to the empty.html server.setRoute('/empty.html', (req, res) => { }); @@ -423,8 +423,8 @@ it('should fail when exceeding default maximum timeout', async ({ page, server, expect(error).toBeInstanceOf(playwright.errors.TimeoutError); }); -it('should fail when exceeding browser context timeout', async ({ page, server, playwright, isAndroid }) => { - it.skip(isAndroid, 'No context per test'); +it('should fail when exceeding browser context timeout', async ({ page, server, playwright, isAndroid, isElectron }) => { + it.skip(isAndroid || isElectron, 'No context per test'); // Hang for request to the empty.html server.setRoute('/empty.html', (req, res) => { }); diff --git a/tests/page/page-network-response.spec.ts b/tests/page/page-network-response.spec.ts index a6057ba481b96..12fa8da1eca63 100644 --- a/tests/page/page-network-response.spec.ts +++ b/tests/page/page-network-response.spec.ts @@ -64,8 +64,10 @@ it('should return uncompressed text', async ({ page, server }) => { expect(await response.text()).toBe('{"foo": "bar"}\n'); }); -it('should return uncompressed text for brotli encoding', async ({ page, server, browserName }) => { +it('should return uncompressed text for brotli encoding', async ({ page, server, browserName, isAndroid }) => { it.fixme(browserName === 'firefox', 'https://github.com/microsoft/playwright/issues/39160'); + it.fixme(isAndroid, 'net::ERR_CONTENT_DECODING_FAILED'); + const text = '{"foo": "bar"}\n'; const compressed = zlib.brotliCompressSync(Buffer.from(text)); server.setRoute('/brotli.json', (req, res) => { diff --git a/tests/page/page-request-continue.spec.ts b/tests/page/page-request-continue.spec.ts index 496732a4c3732..0166f520fecad 100644 --- a/tests/page/page-request-continue.spec.ts +++ b/tests/page/page-request-continue.spec.ts @@ -139,8 +139,10 @@ it('should not allow changing protocol when overriding url', async ({ page, serv expect(error.message).toContain('New URL must have same protocol as overridden URL'); }); -it('should not throw if request was cancelled by the page', async ({ page, server, browserName }) => { +it('should not throw if request was cancelled by the page', async ({ page, server, isAndroid }) => { it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/28490' }); + it.fixme(isAndroid, 'net::ERR_CONNECTION_RESET'); + let interceptCallback; const interceptPromise = new Promise(f => interceptCallback = f); await page.route('**/data.json', route => interceptCallback(route)); From 2f1d9c062a9b2464c3c18fc13d69ab1142ed01b3 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 6 May 2026 15:35:40 +0200 Subject: [PATCH 003/249] feat(dashboard): annotate multiple screenshots (#40664) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/dashboard/src/annotateView.tsx | 221 ++++++++ packages/dashboard/src/annotationImage.ts | 108 ++++ packages/dashboard/src/annotationZip.ts | 62 +++ packages/dashboard/src/annotations.css | 314 +++++++++++ packages/dashboard/src/annotations.tsx | 235 ++------ packages/dashboard/src/dashboard.css | 12 + packages/dashboard/src/dashboard.tsx | 302 ++++++----- packages/dashboard/src/dashboardChannel.ts | 16 +- packages/dashboard/src/dashboardModel.ts | 276 +++++++--- .../src/tools/backend/devtools.ts | 32 +- .../src/tools/dashboard/dashboardApp.ts | 34 +- .../tools/dashboard/dashboardController.ts | 18 +- tests/mcp/annotate.spec.ts | 500 ++++++++++++++++++ tests/mcp/cli-fixtures.ts | 42 ++ tests/mcp/dashboard.spec.ts | 346 +----------- 15 files changed, 1703 insertions(+), 815 deletions(-) create mode 100644 packages/dashboard/src/annotateView.tsx create mode 100644 packages/dashboard/src/annotationImage.ts create mode 100644 packages/dashboard/src/annotationZip.ts create mode 100644 tests/mcp/annotate.spec.ts diff --git a/packages/dashboard/src/annotateView.tsx b/packages/dashboard/src/annotateView.tsx new file mode 100644 index 0000000000000..4eb339bf16c86 --- /dev/null +++ b/packages/dashboard/src/annotateView.tsx @@ -0,0 +1,221 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { DownloadIcon } from './icons'; +import { Annotations } from './annotations'; +import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage'; +import { ToolbarButton } from '@web/components/toolbarButton'; + +import type { Annotation, AnnotationsHandle } from './annotations'; +import type { DashboardModel, AnnotateFrame } from './dashboardModel'; + +export type AnnotateSidebarProps = { + model: DashboardModel; + session: NonNullable; + onSubmit: () => Promise | void; +}; + +export const AnnotateSidebar: React.FC = ({ model, session, onSubmit }) => { + const [submitting, setSubmitting] = React.useState(false); + const [hoveredAnnotationId, setHoveredAnnotationId] = React.useState(null); + + return ( +