Public: anyone can read this; the crew can change it.

The test runner has a looks lane: chromium test files run only where chromium starts, named *.looks.test.ts, out of the gate budget #12506

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12496-the-test-runner-has-a-looks-lane-chromiu → main 4 commits fork

  • Merged. torque/12496-the-test-runner-has-a-looks-lane-chromiu was merged into main as b40f4f5.

Review

Sign in to review this change.
8 files changed +81 −12
docs/ORIGINS.md +1 −1
@@ -209,7 +209,7 @@ checking it again: the instance's `/login/oauth/authorize`, the flow's
209209 client id and callback, a PKCE challenge. A `to` an older link still
210210 carries must equal it exactly. A flow past its ten minutes says "This
211211 connect link expired", with a fresh **Connect**.
212 (`test/origin-flow-page.test.ts`; `test/oauth-connect-browser.test.ts`
212 (`test/origin-flow-page.test.ts`; `test/oauth-connect-browser.looks.test.ts`
213213 clicks Authorize and Continue in headless Chromium.)
214214
215215 Discovery: `github.com` is known by name. Otherwise `GET /api/v1/version` —
scripts/gate.ts +15 −4
@@ -114,7 +114,6 @@ export const SLOW: readonly Exclusion[] = [
114114 { file: "test/job-slices.test.ts", reason: "ticks every timed job on a busy fixture while sampling the event loop; 2.3 s a test on the gate box", issue: 1242 },
115115 { file: "test/kpis-landings-scale.test.ts", reason: "seeds a production-sized landings table and counts it two ways; 7.5 s a test on the gate box", issue: 1242 },
116116 { file: "test/live-arrive.test.ts", reason: "drops and restores a live connection in happy-dom, waiting on real timers; 2.8 s a test on the gate box", issue: 1242 },
117 { file: "test/loading-chrome.test.ts", reason: "watches the loading indicator across pages in a real Chromium browser; 7.6 s a test on the gate box", issue: 1242 },
118117 { file: "test/maintenance-standby.test.ts", reason: "waits out the standby's answer timeout on the real clock; 2.0 s a test on the gate box", issue: 1242 },
119118 { file: "test/make-it-yours.test.ts", reason: "forks real repositories through the action, the API and the page under a limit; 3.8 s a test on the gate box", issue: 1242 },
120119 { file: "test/map-layout-budget.test.ts", reason: "lays out a production-sized issue map to count its edge work; 5.9 s a test on the gate box", issue: 1242 },
@@ -123,7 +122,6 @@ export const SLOW: readonly Exclusion[] = [
123122 { file: "test/merge-queue.test.ts", reason: "lands real pulls through the merge queue, each with a git merge and a gate; 5.5 s a test on the gate box", issue: 1242 },
124123 { file: "test/model-grant.test.ts", reason: "starts stand-in agent runs under each kind of model grant and waits for them; 10.0 s a test on the gate box", issue: 1242 },
125124 { file: "test/no-polling.test.ts", reason: "walks every page the router lists, signed in as admin; 2.1 s a test on the gate box", issue: 1242 },
126 { file: "test/oauth-connect-browser.test.ts", reason: "starts a real Chromium and walks the OAuth connect flow through two pages and a new tab; no longer waits out the 15 s load deadline", issue: 1242 },
127125 { file: "test/origin-every-mutation.test.ts", reason: "posts to every mutating route with a foreign Origin as a signed-in user; 3.0 s a test on the gate box", issue: 1242 },
128126 { file: "test/outsiders-member.test.ts", reason: "runs a member's repository through a stand-in agent run end to end; 6.1 s a test on the gate box", issue: 1242 },
129127 { file: "test/overlap.test.ts", reason: "runs two forge processes on one data directory to see each job start once; 4.6 s a test on the gate box", issue: 1242 },
@@ -260,18 +258,31 @@ export function slowRefusal(
260258 return [...problems, `${named.join(", ")}: fix its SLOW entry in scripts/gate.ts`];
261259 }
262260
263 /** The `test/*.test.ts` files under `root`, as the gate names them. */
264 export function presentTests(root = "."): string[] {
261 /** A test that drives a real Chromium: it runs in the looks lane, not the gate tiers (#12496). */
262 export const isLooksFile = (file: string): boolean => file.endsWith(".looks.test.ts");
263
264 function testsUnder(root: string, keep: (file: string) => boolean): string[] {
265265 try {
266266 return readdirSync(`${root}/test`)
267267 .filter((name) => name.endsWith(".test.ts"))
268268 .map((name) => `test/${name}`)
269 .filter(keep)
269270 .sort();
270271 } catch {
271272 return [];
272273 }
273274 }
274275
276 /** The `test/*.test.ts` files under `root`, as the gate names them: the looks lane's files aside. */
277 export function presentTests(root = "."): string[] {
278 return testsUnder(root, (file) => !isLooksFile(file));
279 }
280
281 /** The `test/*.looks.test.ts` files under `root`: the looks lane, outside the gate's budget and tiers. */
282 export function looksTests(root = "."): string[] {
283 return testsUnder(root, isLooksFile);
284 }
285
275286 /** What the gate runs: every present test file not excluded, in name order. */
276287 export function gateFiles(
277288 present: readonly string[] = presentTests(),
scripts/test-shards.ts +21 −4
@@ -90,7 +90,9 @@ import {
9090 CRASH_RERUNS,
9191 type ShardResult,
9292 } from "./shards.ts";
93 import { gateFiles, gateRefusal, presentTests, SLOW, slowRefusal } from "./gate.ts";
93 import { gateFiles, gateRefusal, looksTests, presentTests, SLOW, slowRefusal } from "./gate.ts";
94 import { findChromium } from "../src/looks/capture.ts";
95 import { chromiumRuns } from "../test/helpers/chrome.ts";
9496 import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, slowUnderLines, parseJunit, slowestTestLines, type Breach, type TestTime } from "./test-budget.ts";
9597 import { cgroupMemory, summaryLines } from "./gate-summary.ts";
9698 import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts";
@@ -286,7 +288,20 @@ if (gate || tier === "fast") {
286288 }
287289 }
288290 const GATE = gateFiles();
289 let files = [...(gate || tier === "fast" ? GATE : []), ...split.files];
291 // The looks lane (#12496): `test/*.looks.test.ts` drive a real Chromium, so the
292 // whole gate runs them only where one starts (the check box), outside the
293 // budget; the fast tier and a box without a browser leave them out.
294 const chromium = gate && tier === "full" ? findChromium() : undefined;
295 const LOOKS = chromium !== undefined && chromiumRuns(chromium) ? looksTests() : [];
296 if (gate && tier === "full") {
297 console.log(
298 LOOKS.length > 0
299 ? `test-shards: looks lane, ${LOOKS.length} file(s) with ${chromium}`
300 : "test-shards: looks lane skipped: no Chromium that starts on this machine",
301 );
302 }
303 const LOOKS_SET = new Set(LOOKS);
304 let files = [...(gate || tier === "fast" ? GATE : []), ...LOOKS, ...split.files];
290305 // The fast tier's diff, for asking its base about a red (#10304).
291306 let changed: string[] = [];
292307 // Whether the fast tier's plan is every gated file: then its breach list is whole.
@@ -556,6 +571,8 @@ type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Bre
556571 // a slow test is stopped at the pull. A slice run by hand, no tier named,
557572 // names what is over and stays green.
558573 const SLOW_FILES = new Set(SLOW.map((entry) => entry.file));
574 // The looks lane has no per-test budget: a real browser's time is its own.
575 const BUDGET_EXEMPT = new Set([...SLOW_FILES, ...LOOKS_SET]);
559576 const BUDGETS = budgetsFrom(process.env["FORGE_TEST_BUDGET_SECONDS"]);
560577 const budgetBinds = (file: string): boolean =>
561578 (tier === "full" && (gate || tierArg.value === "full")) ||
@@ -746,7 +763,7 @@ async function runShardIn(
746763 slowRan = slice.map((file) => file.replace(/^\.\//, "")).filter((file) => SLOW_FILES.has(file));
747764 if (slowRan.length > 0)
748765 slowOver = breaches(fileTimes, tests, new Set(), BUDGETS).filter((b) => SLOW_FILES.has(b.file));
749 const over = breaches(fileTimes, tests, SLOW_FILES, BUDGETS);
766 const over = breaches(fileTimes, tests, BUDGET_EXEMPT, BUDGETS);
750767 const binding = over.filter((b) => budgetBinds(b.file));
751768 const named = breachWarnings(over.filter((b) => !budgetBinds(b.file)));
752769 let still = binding;
@@ -757,7 +774,7 @@ async function runShardIn(
757774 const judged = retimeVerdict(
758775 binding,
759776 again.code === 0
760 ? breaches(retimed.length === 1 && slice.length === 1 ? [{ file: retimed[0]!, seconds: again.seconds }] : [], againTests, SLOW_FILES, BUDGETS)
777 ? breaches(retimed.length === 1 && slice.length === 1 ? [{ file: retimed[0]!, seconds: again.seconds }] : [], againTests, BUDGET_EXEMPT, BUDGETS)
761778 : undefined,
762779 );
763780 still = judged.still;
test/check-list.test.ts +1 −1
@@ -13,7 +13,7 @@ import { auditGate, EXCLUDED, gateFiles, gateRefusal, SLOW, slowRefusal, treeGat
1313
1414 const root = new URL("..", import.meta.url).pathname;
1515 const present = readdirSync(`${root}test`)
16 .filter((name) => name.endsWith(".test.ts"))
16 .filter((name) => name.endsWith(".test.ts") && !name.endsWith(".looks.test.ts"))
1717 .map((name) => `test/${name}`)
1818 .sort();
1919
test/flake.test.ts +1 −1
@@ -188,7 +188,7 @@ describe("quarantine", () => {
188188 gateFiles: (present?: readonly string[]) => string[];
189189 };
190190 const present = readdirSync(`${root}test`)
191 .filter((name) => name.endsWith(".test.ts"))
191 .filter((name) => name.endsWith(".test.ts") && !name.endsWith(".looks.test.ts"))
192192 .map((name) => `test/${name}`)
193193 .sort();
194194 expect(auditGate(present, mod.EXCLUDED)).toEqual([]);
test/loading-chrome.test.ts → test/loading-chrome.looks.test.ts renamed
No content changes.
test/oauth-connect-browser.test.ts → test/oauth-connect-browser.looks.test.ts renamed
No content changes.
test/test-budget-run.test.ts +42 −1
@@ -12,7 +12,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
1212 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
1313 import { tmpdir } from "node:os";
1414 import { join } from "node:path";
15 import { presentTests } from "../scripts/gate.ts";
15 import { looksTests, presentTests } from "../scripts/gate.ts";
1616
1717 const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
1818 let dir = "";
@@ -129,3 +129,44 @@ test("judges what the hook found", () => expect(true).toBe(true));
129129 expect(got.code).toBe(0);
130130 }, 20_000);
131131 });
132
133 describe("the looks lane in the runner (#12496)", () => {
134 const looks = "test/zz-lane.looks.test.ts";
135 const slow = `import { expect, test } from "bun:test";
136 test("a browser-slow look", () => {
137 const end = performance.now() + 60;
138 while (performance.now() < end) {}
139 expect(true).toBe(true);
140 });
141 `;
142 let browser = "";
143
144 beforeAll(() => {
145 writeFileSync(join(dir, looks), slow);
146 // A stand-in that "starts": it prints a page, as `--dump-dom` does.
147 browser = join(dir, "fake-chromium.sh");
148 writeFileSync(browser, "#!/bin/sh\necho '<html></html>'\n", { mode: 0o755 });
149 });
150
151 test("where Chromium starts the gate runs the looks files, outside the test budget", async () => {
152 const got = await runner(["--gate"], { FORGE_CHROMIUM: browser, FORGE_CHECK_ONLY: looks });
153 expect(got.out).toContain("test-shards: looks lane, 1 file(s)");
154 expect(got.out).toContain("1 pass");
155 expect(got.out).not.toContain("(fail)");
156 expect(got.code).toBe(0);
157 }, 30_000);
158
159 test("where none starts the lane is skipped, named", async () => {
160 const got = await runner(["--gate"], { FORGE_CHROMIUM: join(dir, "absent"), FORGE_CHECK_ONLY: `${looks},test/zz-budget.test.ts` });
161 expect(got.out).toContain("looks lane skipped: no Chromium that starts");
162 expect(got.out).not.toContain("a browser-slow look");
163 }, 30_000);
164
165 test("the gate tiers leave the looks files out, and the gate audit does not count them", async () => {
166 expect(presentTests(dir)).not.toContain(looks);
167 expect(looksTests(dir)).toEqual([looks]);
168 const got = await runner(["--tier", "fast", "--diff", "HEAD~1..HEAD"], { FORGE_CHROMIUM: browser });
169 expect(got.out).not.toContain("looks lane,");
170 expect(got.out).not.toContain("a browser-slow look");
171 }, 30_000);
172 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs