Public: anyone can read this; the crew can change it.

The check summary names the five slowest test files #15080

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/15065-the-check-summary-names-the-five-slowest → main 1 commit fork

  • Merged. mason/15065-the-check-summary-names-the-five-slowest was merged into main as f43b06c.

Review

Sign in to review this change.
4 files changed +57 −17
docs/DEVELOP.md +3 −2
@@ -255,8 +255,9 @@ cannot be brought under goes in `SLOW` in `scripts/gate.ts` with a reason and
255255 the issue that owns it; `bun scripts/gate.ts` refuses a stale, doubled or
256256 unowned entry. A green full gate writes each test's time to
257257 `.forge-cache/test-times-tests.json` beside the per-file table, and its
258 summary names the five slowest tests and their share of the run, then
259 ends with every breach, slowest first, so a log kept by its tail still says
258 summary names the five slowest files with their seconds, each over the
259 file budget marked (and said to be in `SLOW` when it is), the five slowest
260 tests and their share of the run, then ends with every breach, slowest first, so a log kept by its tail still says
260261 what to fix or name in `SLOW`.
261262
262263 A brand-new test file does not need that record: it joins the gate on its
scripts/gate-summary.ts +19 −5
@@ -1,7 +1,8 @@
11 /**
22 * The one block a timed full gate ends its log with (#2065): how many
3 * shards and why, the wall time, the cell's peak memory, the ten slowest
4 * files, and a warning line for every file over {@link SLOW_SECONDS}.
3 * shards and why, the wall time, the cell's peak memory, the five slowest
4 * files with their seconds, each one over the file budget marked so
5 * (#15065), and a warning line for every file over {@link SLOW_SECONDS}.
56 *
67 * Plain data in, lines out. `scripts/test-shards.ts` gathers the numbers;
78 * the cgroup is read through a function handed in, so a test can say what
@@ -10,11 +11,12 @@
1011 */
1112
1213 import { GATE_SUMMARY_HEADING } from "../src/checks/summary.ts";
14 import { FILE_BUDGET_SECONDS } from "./test-budget.ts";
1315
1416 /** A file over this many seconds is named on its own warning line. */
1517 export const SLOW_SECONDS = 45;
16 /** How many of the slowest files the block lists. */
17 export const SLOWEST = 10;
18 /** How many of the slowest files the block lists (#15065). */
19 export const SLOWEST = 5;
1820
1921 const GIB = 1024 ** 3;
2022 /** cgroup v1 writes "no limit" as a page-rounded 2^63 − 1; anything this big is none. */
@@ -77,6 +79,15 @@ export type GateSummary = {
7779 wallSeconds: number;
7880 /** Every file that ran, alone in its own `bun test`, and how long it took. */
7981 times: readonly { file: string; seconds: number }[];
82 /**
83 * The file budget the run held to (`budgetsFrom` in scripts/test-budget.ts); the constant when
84 * absent.
85 */
86 budget?: number;
87 /**
88 * Files `SLOW` in scripts/gate.ts lets run over the budget: still marked, and said to be in it.
89 */
90 slow?: ReadonlySet<string>;
8091 };
8192
8293 const gib = (bytes: number): string => `${(bytes / GIB).toFixed(2)} GiB`;
@@ -106,13 +117,16 @@ export function summaryLines(s: GateSummary): string[] {
106117 ? `${gib(s.memory.peak.bytes)}${s.memory.limit ? ` of ${gib(s.memory.limit.bytes)}` : ""} (${s.memory.peak.from})`
107118 : "unknown here";
108119 const top = slowest.slice(0, SLOWEST);
120 const budget = s.budget ?? FILE_BUDGET_SECONDS;
121 const mark = (t: { file: string; seconds: number }): string =>
122 t.seconds <= budget ? "" : ` — over the ${budget} s budget${s.slow?.has(t.file.replace(/^\.\//, "")) ? ", in SLOW" : ""}`;
109123 return [
110124 GATE_SUMMARY_HEADING,
111125 `test-shards: shards: ${why(s)}`,
112126 `test-shards: wall: ${s.wallSeconds.toFixed(1)}s`,
113127 `test-shards: peak memory: ${peak}`,
114128 `test-shards: slowest ${top.length} of ${s.times.length} files:`,
115 ...top.map((t) => ` ${t.seconds.toFixed(1).padStart(6)}s ${t.file}`),
129 ...top.map((t) => ` ${t.seconds.toFixed(1).padStart(6)}s ${t.file}${mark(t)}`),
116130 ...slowest
117131 .filter((t) => t.seconds > SLOW_SECONDS)
118132 .map((t) => `test-shards: over ${SLOW_SECONDS} s: ${t.file} (${t.seconds.toFixed(1)}s)`),
scripts/test-shards.ts +2
@@ -1152,6 +1152,8 @@ if (timed)
11521152 memory: cgroupMemory(readLimit),
11531153 wallSeconds: wall,
11541154 times: results.map((result) => ({ file: result.files[0]!, seconds: result.seconds })),
1155 budget: BUDGETS.file,
1156 slow: SLOW_FILES,
11551157 }).join("\n")}\n${slowestTestLines(
11561158 results.flatMap((result) => result.tests),
11571159 results.reduce((sum, result) => sum + result.seconds, 0),
test/gate-summary.test.ts +33 −10
@@ -1,7 +1,8 @@
11 /**
22 * The block a timed full gate ends its log with (`scripts/gate-summary.ts`,
33 * #2065): the cgroup is read or said unknown, a file over 45 s is named on
4 * its own warning line, and fewer than ten files list what there is.
4 * its own warning line, the five slowest files are listed with each one over
5 * the file budget marked (#15065), and fewer than five list what there is.
56 */
67 import { describe, expect, test } from "bun:test";
78 import {
@@ -82,7 +83,7 @@ describe("summaryLines", () => {
8283 );
8384 });
8485
85 test("the whole block, by hand: memory holds it under the CPUs, ten slowest of twelve", () => {
86 test("the whole block, by hand: memory holds it under the CPUs, five slowest of twelve", () => {
8687 // 4 GiB / 512 MiB = 8 workers, fewer than 16 CPUs, SHARDS_MAX 32 or 12 files.
8788 const lines = summaryLines({
8889 ...base,
@@ -100,20 +101,42 @@ describe("summaryLines", () => {
100101 "test-shards: shards: 8 = least of 16 CPUs; 8 by memory, 4.00 GiB (cgroup memory.max) at 512 MiB each; SHARDS_MAX 32; 12 files — fewer than the CPUs because of memory",
101102 "test-shards: wall: 312.4s",
102103 "test-shards: peak memory: 3.50 GiB of 4.00 GiB (memory.peak)",
103 "test-shards: slowest 10 of 12 files:",
104 " 22.0s test/f11.test.ts",
104 "test-shards: slowest 5 of 12 files:",
105 " 22.0s test/f11.test.ts — over the 20 s budget",
105106 " 20.0s test/f10.test.ts",
106107 " 18.0s test/f9.test.ts",
107108 " 16.0s test/f8.test.ts",
108109 " 14.0s test/f7.test.ts",
109 " 12.0s test/f6.test.ts",
110 " 10.0s test/f5.test.ts",
111 " 8.0s test/f4.test.ts",
112 " 6.0s test/f3.test.ts",
113 " 4.0s test/f2.test.ts",
114110 ]);
115111 });
116112
113 test("a fake timing list: the five slowest in order, ties by name, each over its budget marked", () => {
114 const lines = summaryLines({
115 ...base,
116 budget: 10,
117 slow: new Set(["test/slow-owned.test.ts"]),
118 times: [
119 { file: "test/quick.test.ts", seconds: 0.4 },
120 { file: "./test/slow-owned.test.ts", seconds: 31.2 },
121 { file: "test/b.test.ts", seconds: 12 },
122 { file: "test/a.test.ts", seconds: 12 },
123 { file: "test/edge.test.ts", seconds: 10 },
124 { file: "test/mid.test.ts", seconds: 7.5 },
125 { file: "test/worst.test.ts", seconds: 48 },
126 ],
127 });
128 const at = lines.indexOf("test-shards: slowest 5 of 7 files:");
129 expect(at).toBeGreaterThan(0);
130 expect(lines.slice(at + 1, at + 6)).toEqual([
131 " 48.0s test/worst.test.ts — over the 10 s budget",
132 " 31.2s ./test/slow-owned.test.ts — over the 10 s budget, in SLOW",
133 " 12.0s test/a.test.ts — over the 10 s budget",
134 " 12.0s test/b.test.ts — over the 10 s budget",
135 " 10.0s test/edge.test.ts",
136 ]);
137 expect(lines.some((line) => line.includes("test/mid.test.ts"))).toBe(false);
138 });
139
117140 test("a file over 45 s is named on its own line; 45 s itself is not", () => {
118141 const lines = summaryLines({
119142 ...base,
@@ -130,7 +153,7 @@ describe("summaryLines", () => {
130153 ]);
131154 });
132155
133 test("fewer than ten files: every one is listed, and the heading says how many", () => {
156 test("fewer than five files: every one is listed, and the heading says how many", () => {
134157 const lines = summaryLines({
135158 ...base,
136159 files: 2,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs