Public: anyone can read this; the crew can change it.
The test runner has a looks lane: chromium test files run only where chromium starts, named *.looks.test.ts, out of the gate budget #12506
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12496-the-test-runner-has-a-looks-lane-chromiu → main 4 commits fork
8 files
changed +81
−12
docs/ORIGINS.md +1 −1
| @@ -209,7 +209,7 @@ checking it again: the instance's `/login/oauth/authorize`, the flow's | ||
| 209 | 209 | client id and callback, a PKCE challenge. A `to` an older link still |
| 210 | 210 | carries must equal it exactly. A flow past its ten minutes says "This |
| 211 | 211 | connect link expired", with a fresh **Connect**. |
| 212 | (`test/origin-flow-page.test.ts`; `test/oauth-connect-browser.test.ts` | |
| 212 | (`test/origin-flow-page.test.ts`; `test/oauth-connect-browser.looks.test.ts` | |
| 213 | 213 | clicks Authorize and Continue in headless Chromium.) |
| 214 | 214 | |
| 215 | 215 | Discovery: `github.com` is known by name. Otherwise `GET /api/v1/version` — |
scripts/gate.ts +15 −4
| @@ -114,7 +114,6 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 114 | 114 | { file: "test/job-slices.test.ts", reason: "ticks every timed job on a busy fixture while sampling the event loop; 2.3 s a test on the gate box", issue: 1242 }, |
| 115 | 115 | { file: "test/kpis-landings-scale.test.ts", reason: "seeds a production-sized landings table and counts it two ways; 7.5 s a test on the gate box", issue: 1242 }, |
| 116 | 116 | { file: "test/live-arrive.test.ts", reason: "drops and restores a live connection in happy-dom, waiting on real timers; 2.8 s a test on the gate box", issue: 1242 }, |
| 117 | { file: "test/loading-chrome.test.ts", reason: "watches the loading indicator across pages in a real Chromium browser; 7.6 s a test on the gate box", issue: 1242 }, | |
| 118 | 117 | { file: "test/maintenance-standby.test.ts", reason: "waits out the standby's answer timeout on the real clock; 2.0 s a test on the gate box", issue: 1242 }, |
| 119 | 118 | { file: "test/make-it-yours.test.ts", reason: "forks real repositories through the action, the API and the page under a limit; 3.8 s a test on the gate box", issue: 1242 }, |
| 120 | 119 | { file: "test/map-layout-budget.test.ts", reason: "lays out a production-sized issue map to count its edge work; 5.9 s a test on the gate box", issue: 1242 }, |
| @@ -123,7 +122,6 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 123 | 122 | { file: "test/merge-queue.test.ts", reason: "lands real pulls through the merge queue, each with a git merge and a gate; 5.5 s a test on the gate box", issue: 1242 }, |
| 124 | 123 | { file: "test/model-grant.test.ts", reason: "starts stand-in agent runs under each kind of model grant and waits for them; 10.0 s a test on the gate box", issue: 1242 }, |
| 125 | 124 | { file: "test/no-polling.test.ts", reason: "walks every page the router lists, signed in as admin; 2.1 s a test on the gate box", issue: 1242 }, |
| 126 | { file: "test/oauth-connect-browser.test.ts", reason: "starts a real Chromium and walks the OAuth connect flow through two pages and a new tab; no longer waits out the 15 s load deadline", issue: 1242 }, | |
| 127 | 125 | { file: "test/origin-every-mutation.test.ts", reason: "posts to every mutating route with a foreign Origin as a signed-in user; 3.0 s a test on the gate box", issue: 1242 }, |
| 128 | 126 | { file: "test/outsiders-member.test.ts", reason: "runs a member's repository through a stand-in agent run end to end; 6.1 s a test on the gate box", issue: 1242 }, |
| 129 | 127 | { file: "test/overlap.test.ts", reason: "runs two forge processes on one data directory to see each job start once; 4.6 s a test on the gate box", issue: 1242 }, |
| @@ -260,18 +258,31 @@ export function slowRefusal( | ||
| 260 | 258 | return [...problems, `${named.join(", ")}: fix its SLOW entry in scripts/gate.ts`]; |
| 261 | 259 | } |
| 262 | 260 | |
| 263 | /** The `test/*.test.ts` files under `root`, as the gate names them. */ | |
| 264 | export function presentTests(root = "."): string[] { | |
| 261 | /** A test that drives a real Chromium: it runs in the looks lane, not the gate tiers (#12496). */ | |
| 262 | export const isLooksFile = (file: string): boolean => file.endsWith(".looks.test.ts"); | |
| 263 | ||
| 264 | function testsUnder(root: string, keep: (file: string) => boolean): string[] { | |
| 265 | 265 | try { |
| 266 | 266 | return readdirSync(`${root}/test`) |
| 267 | 267 | .filter((name) => name.endsWith(".test.ts")) |
| 268 | 268 | .map((name) => `test/${name}`) |
| 269 | .filter(keep) | |
| 269 | 270 | .sort(); |
| 270 | 271 | } catch { |
| 271 | 272 | return []; |
| 272 | 273 | } |
| 273 | 274 | } |
| 274 | 275 | |
| 276 | /** The `test/*.test.ts` files under `root`, as the gate names them: the looks lane's files aside. */ | |
| 277 | export function presentTests(root = "."): string[] { | |
| 278 | return testsUnder(root, (file) => !isLooksFile(file)); | |
| 279 | } | |
| 280 | ||
| 281 | /** The `test/*.looks.test.ts` files under `root`: the looks lane, outside the gate's budget and tiers. */ | |
| 282 | export function looksTests(root = "."): string[] { | |
| 283 | return testsUnder(root, isLooksFile); | |
| 284 | } | |
| 285 | ||
| 275 | 286 | /** What the gate runs: every present test file not excluded, in name order. */ |
| 276 | 287 | export function gateFiles( |
| 277 | 288 | present: readonly string[] = presentTests(), |
scripts/test-shards.ts +21 −4
| @@ -90,7 +90,9 @@ import { | ||
| 90 | 90 | CRASH_RERUNS, |
| 91 | 91 | type ShardResult, |
| 92 | 92 | } from "./shards.ts"; |
| 93 | import { gateFiles, gateRefusal, presentTests, SLOW, slowRefusal } from "./gate.ts"; | |
| 93 | import { gateFiles, gateRefusal, looksTests, presentTests, SLOW, slowRefusal } from "./gate.ts"; | |
| 94 | import { findChromium } from "../src/looks/capture.ts"; | |
| 95 | import { chromiumRuns } from "../test/helpers/chrome.ts"; | |
| 94 | 96 | import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, slowUnderLines, parseJunit, slowestTestLines, type Breach, type TestTime } from "./test-budget.ts"; |
| 95 | 97 | import { cgroupMemory, summaryLines } from "./gate-summary.ts"; |
| 96 | 98 | import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts"; |
| @@ -286,7 +288,20 @@ if (gate || tier === "fast") { | ||
| 286 | 288 | } |
| 287 | 289 | } |
| 288 | 290 | const GATE = gateFiles(); |
| 289 | let files = [...(gate || tier === "fast" ? GATE : []), ...split.files]; | |
| 291 | // The looks lane (#12496): `test/*.looks.test.ts` drive a real Chromium, so the | |
| 292 | // whole gate runs them only where one starts (the check box), outside the | |
| 293 | // budget; the fast tier and a box without a browser leave them out. | |
| 294 | const chromium = gate && tier === "full" ? findChromium() : undefined; | |
| 295 | const LOOKS = chromium !== undefined && chromiumRuns(chromium) ? looksTests() : []; | |
| 296 | if (gate && tier === "full") { | |
| 297 | console.log( | |
| 298 | LOOKS.length > 0 | |
| 299 | ? `test-shards: looks lane, ${LOOKS.length} file(s) with ${chromium}` | |
| 300 | : "test-shards: looks lane skipped: no Chromium that starts on this machine", | |
| 301 | ); | |
| 302 | } | |
| 303 | const LOOKS_SET = new Set(LOOKS); | |
| 304 | let files = [...(gate || tier === "fast" ? GATE : []), ...LOOKS, ...split.files]; | |
| 290 | 305 | // The fast tier's diff, for asking its base about a red (#10304). |
| 291 | 306 | let changed: string[] = []; |
| 292 | 307 | // Whether the fast tier's plan is every gated file: then its breach list is whole. |
| @@ -556,6 +571,8 @@ type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Bre | ||
| 556 | 571 | // a slow test is stopped at the pull. A slice run by hand, no tier named, |
| 557 | 572 | // names what is over and stays green. |
| 558 | 573 | const SLOW_FILES = new Set(SLOW.map((entry) => entry.file)); |
| 574 | // The looks lane has no per-test budget: a real browser's time is its own. | |
| 575 | const BUDGET_EXEMPT = new Set([...SLOW_FILES, ...LOOKS_SET]); | |
| 559 | 576 | const BUDGETS = budgetsFrom(process.env["FORGE_TEST_BUDGET_SECONDS"]); |
| 560 | 577 | const budgetBinds = (file: string): boolean => |
| 561 | 578 | (tier === "full" && (gate || tierArg.value === "full")) || |
| @@ -746,7 +763,7 @@ async function runShardIn( | ||
| 746 | 763 | slowRan = slice.map((file) => file.replace(/^\.\//, "")).filter((file) => SLOW_FILES.has(file)); |
| 747 | 764 | if (slowRan.length > 0) |
| 748 | 765 | slowOver = breaches(fileTimes, tests, new Set(), BUDGETS).filter((b) => SLOW_FILES.has(b.file)); |
| 749 | const over = breaches(fileTimes, tests, SLOW_FILES, BUDGETS); | |
| 766 | const over = breaches(fileTimes, tests, BUDGET_EXEMPT, BUDGETS); | |
| 750 | 767 | const binding = over.filter((b) => budgetBinds(b.file)); |
| 751 | 768 | const named = breachWarnings(over.filter((b) => !budgetBinds(b.file))); |
| 752 | 769 | let still = binding; |
| @@ -757,7 +774,7 @@ async function runShardIn( | ||
| 757 | 774 | const judged = retimeVerdict( |
| 758 | 775 | binding, |
| 759 | 776 | again.code === 0 |
| 760 | ? breaches(retimed.length === 1 && slice.length === 1 ? [{ file: retimed[0]!, seconds: again.seconds }] : [], againTests, SLOW_FILES, BUDGETS) | |
| 777 | ? breaches(retimed.length === 1 && slice.length === 1 ? [{ file: retimed[0]!, seconds: again.seconds }] : [], againTests, BUDGET_EXEMPT, BUDGETS) | |
| 761 | 778 | : undefined, |
| 762 | 779 | ); |
| 763 | 780 | still = judged.still; |
test/check-list.test.ts +1 −1
| @@ -13,7 +13,7 @@ import { auditGate, EXCLUDED, gateFiles, gateRefusal, SLOW, slowRefusal, treeGat | ||
| 13 | 13 | |
| 14 | 14 | const root = new URL("..", import.meta.url).pathname; |
| 15 | 15 | const present = readdirSync(`${root}test`) |
| 16 | .filter((name) => name.endsWith(".test.ts")) | |
| 16 | .filter((name) => name.endsWith(".test.ts") && !name.endsWith(".looks.test.ts")) | |
| 17 | 17 | .map((name) => `test/${name}`) |
| 18 | 18 | .sort(); |
| 19 | 19 |
test/flake.test.ts +1 −1
| @@ -188,7 +188,7 @@ describe("quarantine", () => { | ||
| 188 | 188 | gateFiles: (present?: readonly string[]) => string[]; |
| 189 | 189 | }; |
| 190 | 190 | const present = readdirSync(`${root}test`) |
| 191 | .filter((name) => name.endsWith(".test.ts")) | |
| 191 | .filter((name) => name.endsWith(".test.ts") && !name.endsWith(".looks.test.ts")) | |
| 192 | 192 | .map((name) => `test/${name}`) |
| 193 | 193 | .sort(); |
| 194 | 194 | expect(auditGate(present, mod.EXCLUDED)).toEqual([]); |
test/loading-chrome.test.ts → test/loading-chrome.looks.test.ts renamed
No content changes.
test/oauth-connect-browser.test.ts → test/oauth-connect-browser.looks.test.ts renamed
No content changes.
test/test-budget-run.test.ts +42 −1
| @@ -12,7 +12,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | ||
| 12 | 12 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 13 | 13 | import { tmpdir } from "node:os"; |
| 14 | 14 | import { join } from "node:path"; |
| 15 | import { presentTests } from "../scripts/gate.ts"; | |
| 15 | import { looksTests, presentTests } from "../scripts/gate.ts"; | |
| 16 | 16 | |
| 17 | 17 | const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname; |
| 18 | 18 | let dir = ""; |
| @@ -129,3 +129,44 @@ test("judges what the hook found", () => expect(true).toBe(true)); | ||
| 129 | 129 | expect(got.code).toBe(0); |
| 130 | 130 | }, 20_000); |
| 131 | 131 | }); |
| 132 | ||
| 133 | describe("the looks lane in the runner (#12496)", () => { | |
| 134 | const looks = "test/zz-lane.looks.test.ts"; | |
| 135 | const slow = `import { expect, test } from "bun:test"; | |
| 136 | test("a browser-slow look", () => { | |
| 137 | const end = performance.now() + 60; | |
| 138 | while (performance.now() < end) {} | |
| 139 | expect(true).toBe(true); | |
| 140 | }); | |
| 141 | `; | |
| 142 | let browser = ""; | |
| 143 | ||
| 144 | beforeAll(() => { | |
| 145 | writeFileSync(join(dir, looks), slow); | |
| 146 | // A stand-in that "starts": it prints a page, as `--dump-dom` does. | |
| 147 | browser = join(dir, "fake-chromium.sh"); | |
| 148 | writeFileSync(browser, "#!/bin/sh\necho '<html></html>'\n", { mode: 0o755 }); | |
| 149 | }); | |
| 150 | ||
| 151 | test("where Chromium starts the gate runs the looks files, outside the test budget", async () => { | |
| 152 | const got = await runner(["--gate"], { FORGE_CHROMIUM: browser, FORGE_CHECK_ONLY: looks }); | |
| 153 | expect(got.out).toContain("test-shards: looks lane, 1 file(s)"); | |
| 154 | expect(got.out).toContain("1 pass"); | |
| 155 | expect(got.out).not.toContain("(fail)"); | |
| 156 | expect(got.code).toBe(0); | |
| 157 | }, 30_000); | |
| 158 | ||
| 159 | test("where none starts the lane is skipped, named", async () => { | |
| 160 | const got = await runner(["--gate"], { FORGE_CHROMIUM: join(dir, "absent"), FORGE_CHECK_ONLY: `${looks},test/zz-budget.test.ts` }); | |
| 161 | expect(got.out).toContain("looks lane skipped: no Chromium that starts"); | |
| 162 | expect(got.out).not.toContain("a browser-slow look"); | |
| 163 | }, 30_000); | |
| 164 | ||
| 165 | test("the gate tiers leave the looks files out, and the gate audit does not count them", async () => { | |
| 166 | expect(presentTests(dir)).not.toContain(looks); | |
| 167 | expect(looksTests(dir)).toEqual([looks]); | |
| 168 | const got = await runner(["--tier", "fast", "--diff", "HEAD~1..HEAD"], { FORGE_CHROMIUM: browser }); | |
| 169 | expect(got.out).not.toContain("looks lane,"); | |
| 170 | expect(got.out).not.toContain("a browser-slow look"); | |
| 171 | }, 30_000); | |
| 172 | }); |
new above
offline