Public: anyone can read this; the crew can change it.

scripts/test-budget.ts names each test file over 20 s and each test over 2 s from plain times, with no test run #12219

Closed opened this
Closed without merging

torque/forge:torque/12136-scripts-test-budget-ts-names-each-test-f → main 5 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
2 files changed +264 −0
scripts/test-budget.ts added +142
@@ -0,0 +1,142 @@
1 /**
2 * The per-test budget (#1242, first half #12136): no test file over 20 s and
3 * no single test over 2 s, unless the file is named in `SLOW` with the issue
4 * that brings it under. Pure: plain times in seconds in, plain data and lines
5 * out; it runs no test and reads no file. `scripts/test-shards.ts` is where
6 * it will be called from; until then this decides, it does not enforce.
7 *
8 * Over the budget binds (is red) only on the fast tier and only for a test
9 * file the pull itself changed: the full tier names breaches without failing
10 * on them, so a slow file nobody touched cannot hold an unrelated pull.
11 *
12 * All times are wall seconds, as `scripts/test-times.json` keeps them.
13 */
14
15 /** A file over this many seconds is over the budget. */
16 export const FILE_BUDGET_SECONDS = 20;
17 /** A single test over this many seconds is over the budget. */
18 export const TEST_BUDGET_SECONDS = 2;
19 /** How many of the slowest tests the summary names. */
20 export const SLOWEST_COUNT = 5;
21
22 /** Why a file is let off the budget, and the issue that brings it under. */
23 export type SlowEntry = { issue: number; reason: string };
24
25 /**
26 * Files let off the budget, each with the issue that brings it under.
27 * Empty today: a file joins only with an issue that names its rewrite.
28 */
29 export const SLOW: Readonly<Record<string, SlowEntry>> = {};
30
31 /** One test's time: the file it is in, its full name, wall seconds. */
32 export type TestTime = { file: string; name: string; seconds: number };
33
34 /** A run's times: each file's wall seconds, and each test's. */
35 export type RunTimes = { files: Record<string, number>; tests: TestTime[] };
36
37 /** One breach: a whole file (`test` absent) or one test in it. */
38 export type Breach = {
39 file: string;
40 test?: string;
41 seconds: number;
42 budget: number;
43 /** Set when the file is in `SLOW`: the breach is set aside, not counted. */
44 slow?: SlowEntry;
45 };
46
47 /** A slow test with its share of the run (0..1). */
48 export type Slowest = TestTime & { share: number };
49
50 export type BudgetResult = {
51 /** Breaches that count, files first, then tests, each slowest first. */
52 over: Breach[];
53 /** Breaches in a `SLOW` file, set aside, same order. */
54 aside: Breach[];
55 /** The five slowest tests, slowest first, with their share of the run. */
56 slowest: Slowest[];
57 /** The run's total seconds: the files' sum, or the tests' when no file is timed. */
58 totalSeconds: number;
59 };
60
61 function seconds(value: unknown, what: string): number {
62 if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
63 throw new RangeError(`test-budget: ${what} has no time in seconds (${String(value)})`);
64 }
65 return value;
66 }
67
68 const bySecondsThenName = (a: { seconds: number; file: string; test?: string }, b: typeof a) =>
69 b.seconds - a.seconds || a.file.localeCompare(b.file) || (a.test ?? "").localeCompare(b.test ?? "");
70
71 /**
72 * Each file over `FILE_BUDGET_SECONDS` and each test over
73 * `TEST_BUDGET_SECONDS` (strictly over: a file at exactly 20 s is in), split
74 * into those that count and those in a `slow` file; and the five slowest
75 * tests with their share of the run. Throws on a time that is not a finite,
76 * non-negative number, so a broken table never reads as "under budget".
77 */
78 export function breaches(times: RunTimes, slow: Readonly<Record<string, SlowEntry>> = SLOW): BudgetResult {
79 const all: Breach[] = [];
80 let fileTotal = 0;
81 for (const [file, value] of Object.entries(times.files)) {
82 const s = seconds(value, file);
83 fileTotal += s;
84 if (s > FILE_BUDGET_SECONDS) all.push({ file, seconds: s, budget: FILE_BUDGET_SECONDS });
85 }
86 let testTotal = 0;
87 const tests = times.tests.map((t) => ({ ...t, seconds: seconds(t.seconds, `${t.file} > ${t.name}`) }));
88 for (const t of tests) {
89 testTotal += t.seconds;
90 if (t.seconds > TEST_BUDGET_SECONDS) {
91 all.push({ file: t.file, test: t.name, seconds: t.seconds, budget: TEST_BUDGET_SECONDS });
92 }
93 }
94 const files = all.filter((b) => b.test === undefined).sort(bySecondsThenName);
95 const each = all.filter((b) => b.test !== undefined).sort(bySecondsThenName);
96 const over: Breach[] = [];
97 const aside: Breach[] = [];
98 for (const b of [...files, ...each]) {
99 const entry = Object.hasOwn(slow, b.file) ? slow[b.file] : undefined;
100 if (entry) aside.push({ ...b, slow: entry });
101 else over.push(b);
102 }
103 const totalSeconds = Object.keys(times.files).length > 0 ? fileTotal : testTotal;
104 const slowest = tests
105 .sort((a, b) => b.seconds - a.seconds || a.file.localeCompare(b.file) || a.name.localeCompare(b.name))
106 .slice(0, SLOWEST_COUNT)
107 .map((t) => ({ ...t, share: totalSeconds > 0 ? t.seconds / totalSeconds : 0 }));
108 return { over, aside, slowest, totalSeconds };
109 }
110
111 /**
112 * Whether a breach in `file` fails the run: only on the fast tier, and only
113 * when `file` is one of the test files the pull changed. The full tier names
114 * breaches and stays green on them.
115 */
116 export function budgetBinds(tier: "fast" | "full", changed: readonly string[], file: string): boolean {
117 return tier === "fast" && file.endsWith(".test.ts") && changed.includes(file);
118 }
119
120 const secs = (s: number) => `${s.toFixed(1)} s`;
121
122 function breachLine(b: Breach): string {
123 const where = b.test === undefined ? b.file : `${b.file} > ${b.test}`;
124 const why = b.slow ? ` (SLOW, #${b.slow.issue}: ${b.slow.reason})` : "";
125 return ` ${where}: ${secs(b.seconds)} over ${b.budget} s${why}`;
126 }
127
128 /**
129 * The closing lines of a run: the five slowest tests with their share, then
130 * `over the budget: N, SLOW aside: M`, then each breach named by file and
131 * test (those that count first, then those set aside).
132 */
133 export function summaryLines(result: BudgetResult): string[] {
134 const lines = [`slowest ${result.slowest.length} tests of ${secs(result.totalSeconds)}:`];
135 for (const t of result.slowest) {
136 lines.push(` ${secs(t.seconds)} ${(t.share * 100).toFixed(1)}% ${t.file} > ${t.name}`);
137 }
138 lines.push(`over the budget: ${result.over.length}, SLOW aside: ${result.aside.length}`);
139 for (const b of result.over) lines.push(breachLine(b));
140 for (const b of result.aside) lines.push(breachLine(b));
141 return lines;
142 }
test/test-budget-run.test.ts added +122
@@ -0,0 +1,122 @@
1 /**
2 * The per-test budget (`scripts/test-budget.ts`, #12136): from fixture times
3 * only, a run under budget passes, a file over 20 s and a test over 2 s are
4 * named, a `SLOW` file is set aside, and a breach binds only on the fast tier
5 * in a test file the pull changed. Runs no test and reads no file.
6 */
7 import { describe, expect, test } from "bun:test";
8 import { breaches, budgetBinds, SLOW, summaryLines, type RunTimes } from "../scripts/test-budget.ts";
9
10 const under: RunTimes = {
11 files: { "test/a.test.ts": 12, "test/b.test.ts": 20 },
12 tests: [
13 { file: "test/a.test.ts", name: "reads", seconds: 1.5 },
14 { file: "test/a.test.ts", name: "writes", seconds: 2 },
15 { file: "test/b.test.ts", name: "boots", seconds: 0.5 },
16 ],
17 };
18
19 const over: RunTimes = {
20 files: { "test/a.test.ts": 12, "test/big.test.ts": 25, "test/old.test.ts": 40 },
21 tests: [
22 { file: "test/a.test.ts", name: "reads", seconds: 1 },
23 { file: "test/big.test.ts", name: "sleeps", seconds: 3.1 },
24 { file: "test/old.test.ts", name: "chromium", seconds: 7 },
25 { file: "test/a.test.ts", name: "small", seconds: 0.1 },
26 ],
27 };
28
29 describe("breaches", () => {
30 test("a run under budget passes: at exactly 20 s and 2 s is in", () => {
31 const r = breaches(under, {});
32 expect(r.over).toEqual([]);
33 expect(r.aside).toEqual([]);
34 expect(summaryLines(r)).toContain("over the budget: 0, SLOW aside: 0");
35 });
36
37 test("a file over 20 s and a test over 2 s are named", () => {
38 const r = breaches(over, {});
39 expect(r.over).toEqual([
40 { file: "test/old.test.ts", seconds: 40, budget: 20 },
41 { file: "test/big.test.ts", seconds: 25, budget: 20 },
42 { file: "test/old.test.ts", test: "chromium", seconds: 7, budget: 2 },
43 { file: "test/big.test.ts", test: "sleeps", seconds: 3.1, budget: 2 },
44 ]);
45 const lines = summaryLines(r);
46 expect(lines).toContain("over the budget: 4, SLOW aside: 0");
47 expect(lines).toContain(" test/big.test.ts: 25.0 s over 20 s");
48 expect(lines).toContain(" test/big.test.ts > sleeps: 3.1 s over 2 s");
49 });
50
51 test("a SLOW file is set aside with its issue", () => {
52 const slow = { "test/old.test.ts": { issue: 1242, reason: "chromium per test" } };
53 const r = breaches(over, slow);
54 expect(r.over.map((b) => `${b.file} ${b.test ?? ""}`)).toEqual(["test/big.test.ts ", "test/big.test.ts sleeps"]);
55 expect(r.aside.map((b) => b.slow?.issue)).toEqual([1242, 1242]);
56 const lines = summaryLines(r);
57 expect(lines).toContain("over the budget: 2, SLOW aside: 2");
58 expect(lines).toContain(" test/old.test.ts > chromium: 7.0 s over 2 s (SLOW, #1242: chromium per test)");
59 });
60
61 test("the five slowest tests and their share of the run, by hand", () => {
62 const tests = [0.5, 4, 1, 3, 2, 0.25].map((seconds, i) => ({ file: "test/x.test.ts", name: `t${i}`, seconds }));
63 const r = breaches({ files: { "test/x.test.ts": 20 }, tests }, {});
64 // Share = test seconds / file total:
65 // 4/20 = 20%, 3/20 = 15%, 2/20 = 10%, 1/20 = 5%, 0.5/20 = 2.5%.
66 expect(r.totalSeconds).toBe(20);
67 expect(r.slowest.map((t) => [t.name, t.share])).toEqual([
68 ["t1", 0.2],
69 ["t3", 0.15],
70 ["t4", 0.1],
71 ["t2", 0.05],
72 ["t0", 0.025],
73 ]);
74 const lines = summaryLines(r);
75 expect(lines.slice(0, 3)).toEqual([
76 "slowest 5 tests of 20.0 s:",
77 " 4.0 s 20.0% test/x.test.ts > t1",
78 " 3.0 s 15.0% test/x.test.ts > t3",
79 ]);
80 // The slowest come before the count line.
81 expect(lines.indexOf("over the budget: 2, SLOW aside: 0")).toBe(6);
82 });
83
84 test("with no file timed, the share is of the tests' sum", () => {
85 const r = breaches({ files: {}, tests: [{ file: "test/y.test.ts", name: "one", seconds: 1 }, { file: "test/y.test.ts", name: "two", seconds: 3 }] }, {});
86 expect(r.totalSeconds).toBe(4);
87 expect(r.slowest.map((t) => t.share)).toEqual([0.75, 0.25]);
88 });
89
90 test("a time that is not a number is refused, never read as under budget", () => {
91 expect(() => breaches({ files: { "test/a.test.ts": Number.NaN }, tests: [] }, {})).toThrow(RangeError);
92 expect(() => breaches({ files: {}, tests: [{ file: "test/a.test.ts", name: "n", seconds: -1 }] }, {})).toThrow(RangeError);
93 });
94
95 test("SLOW names each file with an issue and a reason", () => {
96 for (const [file, entry] of Object.entries(SLOW)) {
97 expect(file).toMatch(/^test\/.+\.test\.ts$/);
98 expect(Number.isInteger(entry.issue) && entry.issue > 0).toBe(true);
99 expect(entry.reason.length).toBeGreaterThan(0);
100 }
101 });
102 });
103
104 describe("budgetBinds", () => {
105 const changed = ["src/x.ts", "test/big.test.ts"];
106
107 test("a fast-tier breach in a changed test file binds", () => {
108 expect(budgetBinds("fast", changed, "test/big.test.ts")).toBe(true);
109 });
110
111 test("a full-tier breach does not bind", () => {
112 expect(budgetBinds("full", changed, "test/big.test.ts")).toBe(false);
113 });
114
115 test("a fast-tier breach in an unchanged file does not bind", () => {
116 expect(budgetBinds("fast", changed, "test/old.test.ts")).toBe(false);
117 });
118
119 test("a changed file that is not a test file does not bind", () => {
120 expect(budgetBinds("fast", changed, "src/x.ts")).toBe(false);
121 });
122 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs