Public: anyone can read this; the crew can change it.
The check summary names the five slowest test files #15080
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/15065-the-check-summary-names-the-five-slowest → main 1 commit fork
4 files
changed +57
−17
docs/DEVELOP.md +3 −2
| @@ -255,8 +255,9 @@ cannot be brought under goes in `SLOW` in `scripts/gate.ts` with a reason and | ||
| 255 | 255 | the issue that owns it; `bun scripts/gate.ts` refuses a stale, doubled or |
| 256 | 256 | unowned entry. A green full gate writes each test's time to |
| 257 | 257 | `.forge-cache/test-times-tests.json` beside the per-file table, and its |
| 258 | summary names the five slowest tests and their share of the run, then | |
| 259 | ends with every breach, slowest first, so a log kept by its tail still says | |
| 258 | summary names the five slowest files with their seconds, each over the | |
| 259 | file budget marked (and said to be in `SLOW` when it is), the five slowest | |
| 260 | tests and their share of the run, then ends with every breach, slowest first, so a log kept by its tail still says | |
| 260 | 261 | what to fix or name in `SLOW`. |
| 261 | 262 | |
| 262 | 263 | A brand-new test file does not need that record: it joins the gate on its |
scripts/gate-summary.ts +19 −5
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The one block a timed full gate ends its log with (#2065): how many |
| 3 | * shards and why, the wall time, the cell's peak memory, the ten slowest | |
| 4 | * files, and a warning line for every file over {@link SLOW_SECONDS}. | |
| 3 | * shards and why, the wall time, the cell's peak memory, the five slowest | |
| 4 | * files with their seconds, each one over the file budget marked so | |
| 5 | * (#15065), and a warning line for every file over {@link SLOW_SECONDS}. | |
| 5 | 6 | * |
| 6 | 7 | * Plain data in, lines out. `scripts/test-shards.ts` gathers the numbers; |
| 7 | 8 | * the cgroup is read through a function handed in, so a test can say what |
| @@ -10,11 +11,12 @@ | ||
| 10 | 11 | */ |
| 11 | 12 | |
| 12 | 13 | import { GATE_SUMMARY_HEADING } from "../src/checks/summary.ts"; |
| 14 | import { FILE_BUDGET_SECONDS } from "./test-budget.ts"; | |
| 13 | 15 | |
| 14 | 16 | /** A file over this many seconds is named on its own warning line. */ |
| 15 | 17 | export const SLOW_SECONDS = 45; |
| 16 | /** How many of the slowest files the block lists. */ | |
| 17 | export const SLOWEST = 10; | |
| 18 | /** How many of the slowest files the block lists (#15065). */ | |
| 19 | export const SLOWEST = 5; | |
| 18 | 20 | |
| 19 | 21 | const GIB = 1024 ** 3; |
| 20 | 22 | /** cgroup v1 writes "no limit" as a page-rounded 2^63 − 1; anything this big is none. */ |
| @@ -77,6 +79,15 @@ export type GateSummary = { | ||
| 77 | 79 | wallSeconds: number; |
| 78 | 80 | /** Every file that ran, alone in its own `bun test`, and how long it took. */ |
| 79 | 81 | times: readonly { file: string; seconds: number }[]; |
| 82 | /** | |
| 83 | * The file budget the run held to (`budgetsFrom` in scripts/test-budget.ts); the constant when | |
| 84 | * absent. | |
| 85 | */ | |
| 86 | budget?: number; | |
| 87 | /** | |
| 88 | * Files `SLOW` in scripts/gate.ts lets run over the budget: still marked, and said to be in it. | |
| 89 | */ | |
| 90 | slow?: ReadonlySet<string>; | |
| 80 | 91 | }; |
| 81 | 92 | |
| 82 | 93 | const gib = (bytes: number): string => `${(bytes / GIB).toFixed(2)} GiB`; |
| @@ -106,13 +117,16 @@ export function summaryLines(s: GateSummary): string[] { | ||
| 106 | 117 | ? `${gib(s.memory.peak.bytes)}${s.memory.limit ? ` of ${gib(s.memory.limit.bytes)}` : ""} (${s.memory.peak.from})` |
| 107 | 118 | : "unknown here"; |
| 108 | 119 | const top = slowest.slice(0, SLOWEST); |
| 120 | const budget = s.budget ?? FILE_BUDGET_SECONDS; | |
| 121 | const mark = (t: { file: string; seconds: number }): string => | |
| 122 | t.seconds <= budget ? "" : ` — over the ${budget} s budget${s.slow?.has(t.file.replace(/^\.\//, "")) ? ", in SLOW" : ""}`; | |
| 109 | 123 | return [ |
| 110 | 124 | GATE_SUMMARY_HEADING, |
| 111 | 125 | `test-shards: shards: ${why(s)}`, |
| 112 | 126 | `test-shards: wall: ${s.wallSeconds.toFixed(1)}s`, |
| 113 | 127 | `test-shards: peak memory: ${peak}`, |
| 114 | 128 | `test-shards: slowest ${top.length} of ${s.times.length} files:`, |
| 115 | ...top.map((t) => ` ${t.seconds.toFixed(1).padStart(6)}s ${t.file}`), | |
| 129 | ...top.map((t) => ` ${t.seconds.toFixed(1).padStart(6)}s ${t.file}${mark(t)}`), | |
| 116 | 130 | ...slowest |
| 117 | 131 | .filter((t) => t.seconds > SLOW_SECONDS) |
| 118 | 132 | .map((t) => `test-shards: over ${SLOW_SECONDS} s: ${t.file} (${t.seconds.toFixed(1)}s)`), |
scripts/test-shards.ts +2
| @@ -1152,6 +1152,8 @@ if (timed) | ||
| 1152 | 1152 | memory: cgroupMemory(readLimit), |
| 1153 | 1153 | wallSeconds: wall, |
| 1154 | 1154 | times: results.map((result) => ({ file: result.files[0]!, seconds: result.seconds })), |
| 1155 | budget: BUDGETS.file, | |
| 1156 | slow: SLOW_FILES, | |
| 1155 | 1157 | }).join("\n")}\n${slowestTestLines( |
| 1156 | 1158 | results.flatMap((result) => result.tests), |
| 1157 | 1159 | results.reduce((sum, result) => sum + result.seconds, 0), |
test/gate-summary.test.ts +33 −10
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The block a timed full gate ends its log with (`scripts/gate-summary.ts`, |
| 3 | 3 | * #2065): the cgroup is read or said unknown, a file over 45 s is named on |
| 4 | * its own warning line, and fewer than ten files list what there is. | |
| 4 | * its own warning line, the five slowest files are listed with each one over | |
| 5 | * the file budget marked (#15065), and fewer than five list what there is. | |
| 5 | 6 | */ |
| 6 | 7 | import { describe, expect, test } from "bun:test"; |
| 7 | 8 | import { |
| @@ -82,7 +83,7 @@ describe("summaryLines", () => { | ||
| 82 | 83 | ); |
| 83 | 84 | }); |
| 84 | 85 | |
| 85 | test("the whole block, by hand: memory holds it under the CPUs, ten slowest of twelve", () => { | |
| 86 | test("the whole block, by hand: memory holds it under the CPUs, five slowest of twelve", () => { | |
| 86 | 87 | // 4 GiB / 512 MiB = 8 workers, fewer than 16 CPUs, SHARDS_MAX 32 or 12 files. |
| 87 | 88 | const lines = summaryLines({ |
| 88 | 89 | ...base, |
| @@ -100,20 +101,42 @@ describe("summaryLines", () => { | ||
| 100 | 101 | "test-shards: shards: 8 = least of 16 CPUs; 8 by memory, 4.00 GiB (cgroup memory.max) at 512 MiB each; SHARDS_MAX 32; 12 files — fewer than the CPUs because of memory", |
| 101 | 102 | "test-shards: wall: 312.4s", |
| 102 | 103 | "test-shards: peak memory: 3.50 GiB of 4.00 GiB (memory.peak)", |
| 103 | "test-shards: slowest 10 of 12 files:", | |
| 104 | " 22.0s test/f11.test.ts", | |
| 104 | "test-shards: slowest 5 of 12 files:", | |
| 105 | " 22.0s test/f11.test.ts — over the 20 s budget", | |
| 105 | 106 | " 20.0s test/f10.test.ts", |
| 106 | 107 | " 18.0s test/f9.test.ts", |
| 107 | 108 | " 16.0s test/f8.test.ts", |
| 108 | 109 | " 14.0s test/f7.test.ts", |
| 109 | " 12.0s test/f6.test.ts", | |
| 110 | " 10.0s test/f5.test.ts", | |
| 111 | " 8.0s test/f4.test.ts", | |
| 112 | " 6.0s test/f3.test.ts", | |
| 113 | " 4.0s test/f2.test.ts", | |
| 114 | 110 | ]); |
| 115 | 111 | }); |
| 116 | 112 | |
| 113 | test("a fake timing list: the five slowest in order, ties by name, each over its budget marked", () => { | |
| 114 | const lines = summaryLines({ | |
| 115 | ...base, | |
| 116 | budget: 10, | |
| 117 | slow: new Set(["test/slow-owned.test.ts"]), | |
| 118 | times: [ | |
| 119 | { file: "test/quick.test.ts", seconds: 0.4 }, | |
| 120 | { file: "./test/slow-owned.test.ts", seconds: 31.2 }, | |
| 121 | { file: "test/b.test.ts", seconds: 12 }, | |
| 122 | { file: "test/a.test.ts", seconds: 12 }, | |
| 123 | { file: "test/edge.test.ts", seconds: 10 }, | |
| 124 | { file: "test/mid.test.ts", seconds: 7.5 }, | |
| 125 | { file: "test/worst.test.ts", seconds: 48 }, | |
| 126 | ], | |
| 127 | }); | |
| 128 | const at = lines.indexOf("test-shards: slowest 5 of 7 files:"); | |
| 129 | expect(at).toBeGreaterThan(0); | |
| 130 | expect(lines.slice(at + 1, at + 6)).toEqual([ | |
| 131 | " 48.0s test/worst.test.ts — over the 10 s budget", | |
| 132 | " 31.2s ./test/slow-owned.test.ts — over the 10 s budget, in SLOW", | |
| 133 | " 12.0s test/a.test.ts — over the 10 s budget", | |
| 134 | " 12.0s test/b.test.ts — over the 10 s budget", | |
| 135 | " 10.0s test/edge.test.ts", | |
| 136 | ]); | |
| 137 | expect(lines.some((line) => line.includes("test/mid.test.ts"))).toBe(false); | |
| 138 | }); | |
| 139 | ||
| 117 | 140 | test("a file over 45 s is named on its own line; 45 s itself is not", () => { |
| 118 | 141 | const lines = summaryLines({ |
| 119 | 142 | ...base, |
| @@ -130,7 +153,7 @@ describe("summaryLines", () => { | ||
| 130 | 153 | ]); |
| 131 | 154 | }); |
| 132 | 155 | |
| 133 | test("fewer than ten files: every one is listed, and the heading says how many", () => { | |
| 156 | test("fewer than five files: every one is listed, and the heading says how many", () => { | |
| 134 | 157 | const lines = summaryLines({ |
| 135 | 158 | ...base, |
| 136 | 159 | files: 2, |
new above
offline