Public: anyone can read this; the crew can change it.
scripts/test-budget.ts names each test file over 20 s and each test over 2 s from plain times, with no test run #12219
torque/forge:torque/12136-scripts-test-budget-ts-names-each-test-f → main 5 commits fork
2 files
changed +264
−0
scripts/test-budget.ts added +142
| @@ -0,0 +1,142 @@ | ||
| 1 | /** | |
| 2 | * The per-test budget (#1242, first half #12136): no test file over 20 s and | |
| 3 | * no single test over 2 s, unless the file is named in `SLOW` with the issue | |
| 4 | * that brings it under. Pure: plain times in seconds in, plain data and lines | |
| 5 | * out; it runs no test and reads no file. `scripts/test-shards.ts` is where | |
| 6 | * it will be called from; until then this decides, it does not enforce. | |
| 7 | * | |
| 8 | * Over the budget binds (is red) only on the fast tier and only for a test | |
| 9 | * file the pull itself changed: the full tier names breaches without failing | |
| 10 | * on them, so a slow file nobody touched cannot hold an unrelated pull. | |
| 11 | * | |
| 12 | * All times are wall seconds, as `scripts/test-times.json` keeps them. | |
| 13 | */ | |
| 14 | ||
| 15 | /** A file over this many seconds is over the budget. */ | |
| 16 | export const FILE_BUDGET_SECONDS = 20; | |
| 17 | /** A single test over this many seconds is over the budget. */ | |
| 18 | export const TEST_BUDGET_SECONDS = 2; | |
| 19 | /** How many of the slowest tests the summary names. */ | |
| 20 | export const SLOWEST_COUNT = 5; | |
| 21 | ||
| 22 | /** Why a file is let off the budget, and the issue that brings it under. */ | |
| 23 | export type SlowEntry = { issue: number; reason: string }; | |
| 24 | ||
| 25 | /** | |
| 26 | * Files let off the budget, each with the issue that brings it under. | |
| 27 | * Empty today: a file joins only with an issue that names its rewrite. | |
| 28 | */ | |
| 29 | export const SLOW: Readonly<Record<string, SlowEntry>> = {}; | |
| 30 | ||
| 31 | /** One test's time: the file it is in, its full name, wall seconds. */ | |
| 32 | export type TestTime = { file: string; name: string; seconds: number }; | |
| 33 | ||
| 34 | /** A run's times: each file's wall seconds, and each test's. */ | |
| 35 | export type RunTimes = { files: Record<string, number>; tests: TestTime[] }; | |
| 36 | ||
| 37 | /** One breach: a whole file (`test` absent) or one test in it. */ | |
| 38 | export type Breach = { | |
| 39 | file: string; | |
| 40 | test?: string; | |
| 41 | seconds: number; | |
| 42 | budget: number; | |
| 43 | /** Set when the file is in `SLOW`: the breach is set aside, not counted. */ | |
| 44 | slow?: SlowEntry; | |
| 45 | }; | |
| 46 | ||
| 47 | /** A slow test with its share of the run (0..1). */ | |
| 48 | export type Slowest = TestTime & { share: number }; | |
| 49 | ||
| 50 | export type BudgetResult = { | |
| 51 | /** Breaches that count, files first, then tests, each slowest first. */ | |
| 52 | over: Breach[]; | |
| 53 | /** Breaches in a `SLOW` file, set aside, same order. */ | |
| 54 | aside: Breach[]; | |
| 55 | /** The five slowest tests, slowest first, with their share of the run. */ | |
| 56 | slowest: Slowest[]; | |
| 57 | /** The run's total seconds: the files' sum, or the tests' when no file is timed. */ | |
| 58 | totalSeconds: number; | |
| 59 | }; | |
| 60 | ||
| 61 | function seconds(value: unknown, what: string): number { | |
| 62 | if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { | |
| 63 | throw new RangeError(`test-budget: ${what} has no time in seconds (${String(value)})`); | |
| 64 | } | |
| 65 | return value; | |
| 66 | } | |
| 67 | ||
| 68 | const bySecondsThenName = (a: { seconds: number; file: string; test?: string }, b: typeof a) => | |
| 69 | b.seconds - a.seconds || a.file.localeCompare(b.file) || (a.test ?? "").localeCompare(b.test ?? ""); | |
| 70 | ||
| 71 | /** | |
| 72 | * Each file over `FILE_BUDGET_SECONDS` and each test over | |
| 73 | * `TEST_BUDGET_SECONDS` (strictly over: a file at exactly 20 s is in), split | |
| 74 | * into those that count and those in a `slow` file; and the five slowest | |
| 75 | * tests with their share of the run. Throws on a time that is not a finite, | |
| 76 | * non-negative number, so a broken table never reads as "under budget". | |
| 77 | */ | |
| 78 | export function breaches(times: RunTimes, slow: Readonly<Record<string, SlowEntry>> = SLOW): BudgetResult { | |
| 79 | const all: Breach[] = []; | |
| 80 | let fileTotal = 0; | |
| 81 | for (const [file, value] of Object.entries(times.files)) { | |
| 82 | const s = seconds(value, file); | |
| 83 | fileTotal += s; | |
| 84 | if (s > FILE_BUDGET_SECONDS) all.push({ file, seconds: s, budget: FILE_BUDGET_SECONDS }); | |
| 85 | } | |
| 86 | let testTotal = 0; | |
| 87 | const tests = times.tests.map((t) => ({ ...t, seconds: seconds(t.seconds, `${t.file} > ${t.name}`) })); | |
| 88 | for (const t of tests) { | |
| 89 | testTotal += t.seconds; | |
| 90 | if (t.seconds > TEST_BUDGET_SECONDS) { | |
| 91 | all.push({ file: t.file, test: t.name, seconds: t.seconds, budget: TEST_BUDGET_SECONDS }); | |
| 92 | } | |
| 93 | } | |
| 94 | const files = all.filter((b) => b.test === undefined).sort(bySecondsThenName); | |
| 95 | const each = all.filter((b) => b.test !== undefined).sort(bySecondsThenName); | |
| 96 | const over: Breach[] = []; | |
| 97 | const aside: Breach[] = []; | |
| 98 | for (const b of [...files, ...each]) { | |
| 99 | const entry = Object.hasOwn(slow, b.file) ? slow[b.file] : undefined; | |
| 100 | if (entry) aside.push({ ...b, slow: entry }); | |
| 101 | else over.push(b); | |
| 102 | } | |
| 103 | const totalSeconds = Object.keys(times.files).length > 0 ? fileTotal : testTotal; | |
| 104 | const slowest = tests | |
| 105 | .sort((a, b) => b.seconds - a.seconds || a.file.localeCompare(b.file) || a.name.localeCompare(b.name)) | |
| 106 | .slice(0, SLOWEST_COUNT) | |
| 107 | .map((t) => ({ ...t, share: totalSeconds > 0 ? t.seconds / totalSeconds : 0 })); | |
| 108 | return { over, aside, slowest, totalSeconds }; | |
| 109 | } | |
| 110 | ||
| 111 | /** | |
| 112 | * Whether a breach in `file` fails the run: only on the fast tier, and only | |
| 113 | * when `file` is one of the test files the pull changed. The full tier names | |
| 114 | * breaches and stays green on them. | |
| 115 | */ | |
| 116 | export function budgetBinds(tier: "fast" | "full", changed: readonly string[], file: string): boolean { | |
| 117 | return tier === "fast" && file.endsWith(".test.ts") && changed.includes(file); | |
| 118 | } | |
| 119 | ||
| 120 | const secs = (s: number) => `${s.toFixed(1)} s`; | |
| 121 | ||
| 122 | function breachLine(b: Breach): string { | |
| 123 | const where = b.test === undefined ? b.file : `${b.file} > ${b.test}`; | |
| 124 | const why = b.slow ? ` (SLOW, #${b.slow.issue}: ${b.slow.reason})` : ""; | |
| 125 | return ` ${where}: ${secs(b.seconds)} over ${b.budget} s${why}`; | |
| 126 | } | |
| 127 | ||
| 128 | /** | |
| 129 | * The closing lines of a run: the five slowest tests with their share, then | |
| 130 | * `over the budget: N, SLOW aside: M`, then each breach named by file and | |
| 131 | * test (those that count first, then those set aside). | |
| 132 | */ | |
| 133 | export function summaryLines(result: BudgetResult): string[] { | |
| 134 | const lines = [`slowest ${result.slowest.length} tests of ${secs(result.totalSeconds)}:`]; | |
| 135 | for (const t of result.slowest) { | |
| 136 | lines.push(` ${secs(t.seconds)} ${(t.share * 100).toFixed(1)}% ${t.file} > ${t.name}`); | |
| 137 | } | |
| 138 | lines.push(`over the budget: ${result.over.length}, SLOW aside: ${result.aside.length}`); | |
| 139 | for (const b of result.over) lines.push(breachLine(b)); | |
| 140 | for (const b of result.aside) lines.push(breachLine(b)); | |
| 141 | return lines; | |
| 142 | } |
test/test-budget-run.test.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** | |
| 2 | * The per-test budget (`scripts/test-budget.ts`, #12136): from fixture times | |
| 3 | * only, a run under budget passes, a file over 20 s and a test over 2 s are | |
| 4 | * named, a `SLOW` file is set aside, and a breach binds only on the fast tier | |
| 5 | * in a test file the pull changed. Runs no test and reads no file. | |
| 6 | */ | |
| 7 | import { describe, expect, test } from "bun:test"; | |
| 8 | import { breaches, budgetBinds, SLOW, summaryLines, type RunTimes } from "../scripts/test-budget.ts"; | |
| 9 | ||
| 10 | const under: RunTimes = { | |
| 11 | files: { "test/a.test.ts": 12, "test/b.test.ts": 20 }, | |
| 12 | tests: [ | |
| 13 | { file: "test/a.test.ts", name: "reads", seconds: 1.5 }, | |
| 14 | { file: "test/a.test.ts", name: "writes", seconds: 2 }, | |
| 15 | { file: "test/b.test.ts", name: "boots", seconds: 0.5 }, | |
| 16 | ], | |
| 17 | }; | |
| 18 | ||
| 19 | const over: RunTimes = { | |
| 20 | files: { "test/a.test.ts": 12, "test/big.test.ts": 25, "test/old.test.ts": 40 }, | |
| 21 | tests: [ | |
| 22 | { file: "test/a.test.ts", name: "reads", seconds: 1 }, | |
| 23 | { file: "test/big.test.ts", name: "sleeps", seconds: 3.1 }, | |
| 24 | { file: "test/old.test.ts", name: "chromium", seconds: 7 }, | |
| 25 | { file: "test/a.test.ts", name: "small", seconds: 0.1 }, | |
| 26 | ], | |
| 27 | }; | |
| 28 | ||
| 29 | describe("breaches", () => { | |
| 30 | test("a run under budget passes: at exactly 20 s and 2 s is in", () => { | |
| 31 | const r = breaches(under, {}); | |
| 32 | expect(r.over).toEqual([]); | |
| 33 | expect(r.aside).toEqual([]); | |
| 34 | expect(summaryLines(r)).toContain("over the budget: 0, SLOW aside: 0"); | |
| 35 | }); | |
| 36 | ||
| 37 | test("a file over 20 s and a test over 2 s are named", () => { | |
| 38 | const r = breaches(over, {}); | |
| 39 | expect(r.over).toEqual([ | |
| 40 | { file: "test/old.test.ts", seconds: 40, budget: 20 }, | |
| 41 | { file: "test/big.test.ts", seconds: 25, budget: 20 }, | |
| 42 | { file: "test/old.test.ts", test: "chromium", seconds: 7, budget: 2 }, | |
| 43 | { file: "test/big.test.ts", test: "sleeps", seconds: 3.1, budget: 2 }, | |
| 44 | ]); | |
| 45 | const lines = summaryLines(r); | |
| 46 | expect(lines).toContain("over the budget: 4, SLOW aside: 0"); | |
| 47 | expect(lines).toContain(" test/big.test.ts: 25.0 s over 20 s"); | |
| 48 | expect(lines).toContain(" test/big.test.ts > sleeps: 3.1 s over 2 s"); | |
| 49 | }); | |
| 50 | ||
| 51 | test("a SLOW file is set aside with its issue", () => { | |
| 52 | const slow = { "test/old.test.ts": { issue: 1242, reason: "chromium per test" } }; | |
| 53 | const r = breaches(over, slow); | |
| 54 | expect(r.over.map((b) => `${b.file} ${b.test ?? ""}`)).toEqual(["test/big.test.ts ", "test/big.test.ts sleeps"]); | |
| 55 | expect(r.aside.map((b) => b.slow?.issue)).toEqual([1242, 1242]); | |
| 56 | const lines = summaryLines(r); | |
| 57 | expect(lines).toContain("over the budget: 2, SLOW aside: 2"); | |
| 58 | expect(lines).toContain(" test/old.test.ts > chromium: 7.0 s over 2 s (SLOW, #1242: chromium per test)"); | |
| 59 | }); | |
| 60 | ||
| 61 | test("the five slowest tests and their share of the run, by hand", () => { | |
| 62 | const tests = [0.5, 4, 1, 3, 2, 0.25].map((seconds, i) => ({ file: "test/x.test.ts", name: `t${i}`, seconds })); | |
| 63 | const r = breaches({ files: { "test/x.test.ts": 20 }, tests }, {}); | |
| 64 | // Share = test seconds / file total: | |
| 65 | // 4/20 = 20%, 3/20 = 15%, 2/20 = 10%, 1/20 = 5%, 0.5/20 = 2.5%. | |
| 66 | expect(r.totalSeconds).toBe(20); | |
| 67 | expect(r.slowest.map((t) => [t.name, t.share])).toEqual([ | |
| 68 | ["t1", 0.2], | |
| 69 | ["t3", 0.15], | |
| 70 | ["t4", 0.1], | |
| 71 | ["t2", 0.05], | |
| 72 | ["t0", 0.025], | |
| 73 | ]); | |
| 74 | const lines = summaryLines(r); | |
| 75 | expect(lines.slice(0, 3)).toEqual([ | |
| 76 | "slowest 5 tests of 20.0 s:", | |
| 77 | " 4.0 s 20.0% test/x.test.ts > t1", | |
| 78 | " 3.0 s 15.0% test/x.test.ts > t3", | |
| 79 | ]); | |
| 80 | // The slowest come before the count line. | |
| 81 | expect(lines.indexOf("over the budget: 2, SLOW aside: 0")).toBe(6); | |
| 82 | }); | |
| 83 | ||
| 84 | test("with no file timed, the share is of the tests' sum", () => { | |
| 85 | const r = breaches({ files: {}, tests: [{ file: "test/y.test.ts", name: "one", seconds: 1 }, { file: "test/y.test.ts", name: "two", seconds: 3 }] }, {}); | |
| 86 | expect(r.totalSeconds).toBe(4); | |
| 87 | expect(r.slowest.map((t) => t.share)).toEqual([0.75, 0.25]); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a time that is not a number is refused, never read as under budget", () => { | |
| 91 | expect(() => breaches({ files: { "test/a.test.ts": Number.NaN }, tests: [] }, {})).toThrow(RangeError); | |
| 92 | expect(() => breaches({ files: {}, tests: [{ file: "test/a.test.ts", name: "n", seconds: -1 }] }, {})).toThrow(RangeError); | |
| 93 | }); | |
| 94 | ||
| 95 | test("SLOW names each file with an issue and a reason", () => { | |
| 96 | for (const [file, entry] of Object.entries(SLOW)) { | |
| 97 | expect(file).toMatch(/^test\/.+\.test\.ts$/); | |
| 98 | expect(Number.isInteger(entry.issue) && entry.issue > 0).toBe(true); | |
| 99 | expect(entry.reason.length).toBeGreaterThan(0); | |
| 100 | } | |
| 101 | }); | |
| 102 | }); | |
| 103 | ||
| 104 | describe("budgetBinds", () => { | |
| 105 | const changed = ["src/x.ts", "test/big.test.ts"]; | |
| 106 | ||
| 107 | test("a fast-tier breach in a changed test file binds", () => { | |
| 108 | expect(budgetBinds("fast", changed, "test/big.test.ts")).toBe(true); | |
| 109 | }); | |
| 110 | ||
| 111 | test("a full-tier breach does not bind", () => { | |
| 112 | expect(budgetBinds("full", changed, "test/big.test.ts")).toBe(false); | |
| 113 | }); | |
| 114 | ||
| 115 | test("a fast-tier breach in an unchanged file does not bind", () => { | |
| 116 | expect(budgetBinds("fast", changed, "test/old.test.ts")).toBe(false); | |
| 117 | }); | |
| 118 | ||
| 119 | test("a changed file that is not a test file does not bind", () => { | |
| 120 | expect(budgetBinds("fast", changed, "src/x.ts")).toBe(false); | |
| 121 | }); | |
| 122 | }); |
new above
offline