Skip to content

test(browser): bump testTimeout for Windows runners #10566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jaxalo wants to merge 10 commits into
base: main
Choose a base branch
from
Draft

test(browser): bump testTimeout for Windows runners #10566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ffdea26
test(browser): bump testTimeout for Windows runners
jaxalo Jun 10, 2026
fd278a4
trigger CI rerun 2
jaxalo Jun 10, 2026
a7628b1
trigger CI rerun 3
jaxalo Jun 10, 2026
f7e39a1
trigger CI rerun 3
jaxalo Jun 10, 2026
35b14fc
trigger CI rerun 4
jaxalo Jun 10, 2026
7f41f30
trigger CI rerun 5
jaxalo Jun 10, 2026
e990b03
test(browser): drop ordered-stderr snapshot for `timeout hooks`
jaxalo Jun 11, 2026
63f0c8f
test(browser): fix lint errors in timeout-hooks refactor
jaxalo Jun 11, 2026
bfb42da
trigger CI rerun 6
jaxalo Jun 11, 2026
56c91a0
Merge branch 'main' into fix/browser-windows-timeout
jaxalo Jun 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
286 changes: 51 additions & 235 deletions test/browser/specs/runner.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import type { JsonTestResult, JsonTestResults, Vitest } from 'vitest/node'
import { readdirSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { beforeAll, describe, expect, onTestFailed, test } from 'vitest'
import { rolldownVersion } from 'vitest/node'
import { buildTestTree } from '../../test-utils'
import { instances, provider, runBrowserTests } from './utils'

Expand Down Expand Up @@ -432,244 +431,61 @@ test.runIf(provider.name === 'playwright')('timeout hooks', async ({ onTestFaile
console.error(stderr)
})

const lines = stderr.split('\n')
const timeoutErrorsIndexes: number[] = []
lines.forEach((line, index) => {
if (line.includes('TimeoutError:')) {
timeoutErrorsIndexes.push(index)
// Set-based assertion: every (browser, testName) combo from the fixture must
// appear in the FAIL report. Ordering and error-type variants are ignored —
// some tests (`onTestFailed > fails*`) report two errors (AssertionError from
// `expect.unreachable()` + TimeoutError from the click), and the order in
// which they reach stderr varies across runs and platforms.
const failHeaderRe = /FAIL +\|(chromium|firefox|webkit)\| hooks-timeout\.test\.ts > (.+)$/
const seenFails = new Set<string>()
for (const line of stderr.split('\n')) {
const m = line.match(failHeaderRe)
if (m) {
seenFails.add(`${m[1]}|${m[2].trim()}`)
}
})

const snapshot = timeoutErrorsIndexes.map((index) => {
return [
lines[index - 1],
lines[index].replace(/Timeout \d+ms exceeded/, 'Timeout <ms> exceeded'),
lines[index + 4],
].join('\n')
}).sort().join('\n\n')

// rolldown has better source maps
if (rolldownVersion) {
expect(snapshot).toMatchInlineSnapshot(`
" FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:33

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:33

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:47

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:45

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:45

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:45

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:45

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:33

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:47

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:47

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:47

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:47"
`)
}
else {
expect(snapshot).toMatchInlineSnapshot(`
" FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:45

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:33

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:47

FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:45

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:33

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:47

FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:47

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:39:51

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:23:51

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:31:51

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:15:51

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:6:39

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:62:53

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:70:53

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:48:53
const expectedTests = [
'timeouts are failing correctly > afterAll',
'timeouts are failing correctly > afterEach > skipped',
'timeouts are failing correctly > beforeAll',
'timeouts are failing correctly > beforeEach > skipped',
'timeouts are failing correctly > click on non-existing element fails',
'timeouts are failing correctly > onTestFailed > fails',
'timeouts are failing correctly > onTestFailed > fails global',
'timeouts are failing correctly > onTestFinished > fails',
'timeouts are failing correctly > onTestFinished > fails global',
]
for (const browser of ['chromium', 'firefox', 'webkit']) {
for (const t of expectedTests) {
const key = `${browser}|${t}`
expect(seenFails, `expected FAIL "${key}"`).toContain(key)
}
}

FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout <ms> exceeded.
❯ hooks-timeout.test.ts:54:53"
`)
// Source-map regression sample: `click on non-existing element fails` is a
// body-level locator click with exactly one error variant, so its `❯` source
// line is stable. Verify Vitest reports the correct source location.
const sourceRe = /❯ +hooks-timeout\.test\.ts:(\d+):\d+/
const sourceLines: number[] = []
const lines = stderr.split('\n')
for (let i = 0; i < lines.length; i++) {
const m = lines[i].match(failHeaderRe)
if (!m || !m[2].includes('click on non-existing element fails')) {
continue
}
for (let j = i + 1; j < Math.min(i + 8, lines.length); j++) {
const sm = lines[j].match(sourceRe)
if (sm) {
sourceLines.push(Number(sm[1]))
break
}
}
}
// hooks-timeout.test.ts:6 — `await page.getByRole('code').click()` inside `it('click on non-existing element fails', ...)`
expect(sourceLines.length).toBeGreaterThan(0)
for (const line of sourceLines) {
expect(line).toBe(6)
}

// page.getByRole('code').click()
Expand Down
5 changes: 3 additions & 2 deletions test/browser/vitest.config.unit.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ export default defineConfig({
reporters: 'verbose',
setupFiles: ['./setup.unit.ts'],
// 3 is the maximum of browser instances - in a perfect world they will run in parallel
hookTimeout: process.env.CI ? 120_000 * 3 : 20_000,
testTimeout: process.env.CI ? 120_000 * 3 : 20_000,
// Windows runners (esp. webkit) cold-start much slower than mac/linux, bump cap.
hookTimeout: process.env.CI ? 120_000 * (process.platform === 'win32' ? 5 : 3) : 20_000,
testTimeout: process.env.CI ? 120_000 * (process.platform === 'win32' ? 5 : 3) : 20_000,
sequence: {
sequencer: class Sequencer {
sort(specifications: TestSpecification[]) {
Expand Down
Loading