Public: anyone can read this; the crew can change it.

A test over the 2 s budget only in a loaded full-suite run is re-timed alone before main is called red #12866

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12848-a-test-over-the-2-s-budget-only-in-a-loa → main 3 commits fork 1 thread · 1 unresolved

  • Merged. torque/12848-a-test-over-the-2-s-budget-only-in-a-loa was merged into main as 0f52c99.

Review

Sign in to review this change.
4 files changed +399 −7
scripts/test-budget.ts +126 −3
@@ -36,7 +36,7 @@ export function budgetsFrom(value: string | undefined): Budgets {
3636 }
3737
3838 /** One thing over its budget: a whole file, or one test in it. */
39 export type Breach = { file: string; test?: string; seconds: number; budget: number };
39 export type Breach = { file: string; test?: string; seconds: number; budget: number; aloneRed?: true };
4040
4141 const ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
4242
@@ -99,6 +99,9 @@ export function breaches(
9999
100100 const FIX = "make it faster, or name its file in SLOW in scripts/gate.ts with an issue";
101101
102 /** Said of a breach beside other files whose re-time alone went red (#12848). */
103 const ALONE_RED = (b: Breach): string => (b.aloneRed ? " beside other files, and its file failed when run again alone, so the load is not ruled out" : "");
104
102105 /**
103106 * The report lines for `over`, grouped under each file's `path:` header the
104107 * way `bun test` writes its own, each breach a `(fail)` line: the shard's
@@ -113,8 +116,8 @@ export function breachReport(over: readonly Breach[]): string {
113116 for (const b of list)
114117 lines.push(
115118 b.test === undefined
116 ? `(fail) the whole file took ${b.seconds.toFixed(1)} s, over the ${b.budget} s file budget: ${FIX}`
117 : `(fail) ${b.test} took ${b.seconds.toFixed(2)} s, over the ${b.budget} s test budget: ${FIX}`,
119 ? `(fail) the whole file took ${b.seconds.toFixed(1)} s, over the ${b.budget} s file budget${ALONE_RED(b)}: ${FIX}`
120 : `(fail) ${b.test} took ${b.seconds.toFixed(2)} s, over the ${b.budget} s test budget${ALONE_RED(b)}: ${FIX}`,
118121 );
119122 }
120123 return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
@@ -141,6 +144,126 @@ export function retimeVerdict(
141144 };
142145 }
143146
147 /**
148 * How long, in wall-clock seconds, the re-time alone of what breached beside
149 * other files may run once the pool is done (#12848). Alone runs go one at a
outdated

A cap of 8 leaves the remaining breaches red at their loaded time, which is the false red this issue removes. Re-time them all, bounded by wall-clock time, or list them as unverified and do not file them red.

150 * time, so the cost is bounded by time, not by a count: every breaching file
151 * is timed, nearest its budget first, and a file is only started while the
152 * limit is not spent. What the limit leaves un-timed is unverified under
153 * load, named and not red: only a breach that repeats alone is a red.
154 */
155 export const RETIME_SECONDS = 60;
156
157 /**
158 * One file's run alone: green or not, its whole seconds, each test's, and
159 * whether it wrote a JUnit report at all (a test breach needs one).
160 */
161 export type AloneRun = { green: boolean; seconds: number; tests: readonly TestTime[]; report: boolean };
162
163 /** A breach beside other files that was under its budget alone: both times. */
164 export type UnderLoad = { file: string; test?: string; loaded: number; alone: number; budget: number };
165
166 /** A breach beside other files its run alone could not judge, and why. */
167 export type Unverified = { file: string; test?: string; loaded: number; budget: number; why: string };
168
169 /**
170 * Every file to re-time alone, nearest its budget first (the smallest
171 * `seconds / budget`), since a breach just over is the one a loaded box
172 * most likely made; a file appears once. The caller stops starting files
173 * when {@link RETIME_SECONDS} is spent; none is left out here.
174 */
175 export function retimeFiles(over: readonly Breach[]): string[] {
176 const nearest = [...over].sort((a, b) => a.seconds / a.budget - b.seconds / b.budget || key(a).localeCompare(key(b)));
177 return [...new Set(nearest.map((b) => b.file))];
178 }
179
180 /**
181 * The verdict on breaches found with other files running beside them, from
182 * each file's run alone (`alone`, by file; a file absent was not re-timed
183 * before the time limit ran out). Only a breach that repeats alone is red:
184 * - over the same budget alone: it stands, at its alone time;
185 * - under it alone: slow under load, not red, both times kept;
186 * - its file not re-timed, or a test breach whose alone run wrote no report
187 * or has no such test: unverified under load, named with why, not red;
188 * - the alone run went red: it stands at its loaded time, marked so its
189 * `(fail)` says the alone run failed. That is a red on its own merits (the
190 * file failed when run by itself), so the gate is red for a reason the log
191 * names, not for the load.
192 */
193 export function loadVerdict(
194 first: readonly Breach[],
195 alone: ReadonlyMap<string, AloneRun>,
196 ): { still: Breach[]; underLoad: UnderLoad[]; unverified: Unverified[] } {
197 const still: Breach[] = [];
198 const underLoad: UnderLoad[] = [];
199 const unverified: Unverified[] = [];
200 for (const b of first) {
201 const named = { file: b.file, ...(b.test === undefined ? {} : { test: b.test }) };
202 const run = alone.get(b.file);
203 if (run === undefined) {
204 unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: `not timed alone within the ${RETIME_SECONDS} s re-time limit` });
205 continue;
206 }
207 if (!run.green) {
208 still.push({ ...b, aloneRed: true });
209 continue;
210 }
211 let seconds: number | undefined = run.seconds;
212 if (b.test !== undefined) {
213 if (!run.report) {
214 unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: "its run alone wrote no JUnit report" });
215 continue;
216 }
217 seconds = run.tests.find((t) => t.test === b.test)?.seconds;
218 if (seconds === undefined) {
219 unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: "the test is missing from its run alone's JUnit report" });
220 continue;
221 }
222 }
223 if (seconds > b.budget) still.push({ ...b, seconds });
224 else underLoad.push({ ...named, loaded: b.seconds, alone: seconds, budget: b.budget });
225 }
226 return { still, underLoad, unverified };
227 }
228
229 /**
230 * The run summary's list of what was over the budget beside other files and
231 * could not be judged alone (#12848): a count, then each with its loaded
232 * time and why; none when there is nothing to name. Not red.
233 */
234 export function unverifiedLines(list: readonly Unverified[]): string[] {
235 if (list.length === 0) return [];
236 const digits = (u: Unverified): number => (u.test === undefined ? 1 : 2);
237 return [
238 `test-shards: unverified under load, over the budget beside other files and not judged alone, not red: ${list.length}`,
239 ...[...list]
240 .sort((a, b) => b.loaded - a.loaded || a.file.localeCompare(b.file))
241 .map(
242 (u) =>
243 ` ${u.loaded.toFixed(digits(u))} s under load (budget ${u.budget} s), ${u.why}: ${u.file}${u.test === undefined ? " (the whole file)" : ` > ${u.test}`}`,
244 ),
245 ];
246 }
247
248 /**
249 * The run summary's list of what was slow under load (#12848), once: a
250 * count, then each with the time beside others and the time alone; none
251 * when there is nothing to name.
252 */
253 export function underLoadLines(list: readonly UnderLoad[]): string[] {
254 if (list.length === 0) return [];
255 const digits = (u: UnderLoad): number => (u.test === undefined ? 1 : 2);
256 return [
257 `test-shards: slow under load, over the budget beside other files and under it alone, not red: ${list.length}`,
258 ...[...list]
259 .sort((a, b) => b.loaded - a.loaded || a.file.localeCompare(b.file))
260 .map(
261 (u) =>
262 ` ${u.loaded.toFixed(digits(u))} s under load, ${u.alone.toFixed(digits(u))} s alone (budget ${u.budget} s): ${u.file}${u.test === undefined ? " (the whole file)" : ` > ${u.test}`}`,
263 ),
264 ];
265 }
266
144267 /** One warning line per breach, for a run that names them without failing. */
145268 export function breachWarnings(over: readonly Breach[]): string[] {
146269 return over.map((b) =>
scripts/test-shards.ts +108 −4
@@ -49,6 +49,13 @@
4949 * Every run times each test from a JUnit report beside the console one
5050 * and holds it to the budget (`scripts/test-budget.ts`, #1242): red on the
5151 * whole gate and on a fast tier's own changed test files, named by hand.
52 * A breach found while shards ran side by side is not judged then: once
53 * every shard is done, every file with one is timed again one at a time,
54 * nearest its budget first, for at most `RETIME_SECONDS` of wall clock, and
55 * only a breach that repeats alone is a `(fail)`; one under its budget alone
56 * is listed after the tally as slow under load, with both times, and one the
57 * limit left un-timed (or whose alone run wrote no report) as unverified
58 * under load, not red (#12848).
5259 * A green timed gate also writes each test's time to
5360 * `.forge-cache/test-times-tests.json` and ends by naming the five slowest.
5461 *
@@ -94,7 +101,7 @@ import {
94101 import { gateFiles, gateRefusal, looksTests, presentTests, SLOW, slowRefusal } from "./gate.ts";
95102 import { findChromium } from "../src/looks/capture.ts";
96103 import { chromiumRuns } from "../test/helpers/chrome.ts";
97 import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, slowUnderLines, parseJunit, slowestTestLines, type Breach, type TestTime } from "./test-budget.ts";
104 import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, loadVerdict, retimeFiles, RETIME_SECONDS, slowUnderLines, parseJunit, slowestTestLines, underLoadLines, unverifiedLines, type AloneRun, type Breach, type TestTime, type UnderLoad, type Unverified } from "./test-budget.ts";
98105 import { cgroupMemory, summaryLines } from "./gate-summary.ts";
99106 import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts";
100107 import { importsReader, readImpact, scansIn } from "./tier-input.ts";
@@ -569,7 +576,9 @@ let crashReruns = 0;
569576 const RED_SIGN = /^(\(fail\)|# Unhandled error|error:)/m;
570577
571578 // `slowRan` the SLOW files of a green shard, `slowOver` those over the budget in it.
572 type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[]; slowRan: string[]; slowOver: Breach[]; retried: boolean };
579 // `pending` the binding breaches of a shard that ran beside others, held
580 // for a re-time alone once every shard is done (#12848).
581 type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[]; slowRan: string[]; slowOver: Breach[]; retried: boolean; pending: Breach[] };
573582
574583 // The budget (#1242) is red on the full tier, for every file SLOW does not
575584 // name: the whole gate, or a run that names `--tier full` itself (#12088,
@@ -761,8 +770,11 @@ async function runShardIn(
761770 // Over budget (#1242), judged on a green run: a file alone is timed whole,
762771 // every test by the report. What binds is timed once more, alone and
763772 // without coverage, and judged by that run, so one busy moment on the box
764 // is not a red; what is still over is a `(fail)` the tally counts.
773 // is not a red; what is still over is a `(fail)` the tally counts. A shard
774 // that ran beside others holds its breaches instead: timed again now, the
775 // others still run beside it, so they wait until the pool is done (#12848).
765776 let overBudget: Breach[] = [];
777 let pending: Breach[] = [];
766778 let slowRan: string[] = [];
767779 let slowOver: Breach[] = [];
768780 if (code === 0) {
@@ -774,7 +786,10 @@ async function runShardIn(
774786 const binding = over.filter((b) => budgetBinds(b.file));
775787 const named = breachWarnings(over.filter((b) => !budgetBinds(b.file)));
776788 let still = binding;
777 if (binding.length > 0) {
789 if (binding.length > 0 && !live) {
790 pending = binding;
791 still = [];
792 } else if (binding.length > 0) {
778793 const retimed = [...new Set(binding.map((b) => b.file))];
779794 const again = await run(retimed.map((file) => `./${file}`), []);
780795 const againTests = testsOfLastRun();
@@ -837,9 +852,94 @@ async function runShardIn(
837852 slowRan,
838853 slowOver,
839854 retried,
855 pending,
840856 };
841857 }
842858
859 /**
860 * One file run alone, after the pool, as its shard was (same flags, a
861 * scratch home of its own, no coverage): green or not, timed from spawn to
862 * exit, each test's time from its report. Its output is not the verdict.
863 */
864 async function timeAlone(file: string): Promise<AloneRun> {
865 const home = shardHome(process.env, tmpdir());
866 const env: Record<string, string | undefined> = { ...process.env, TMPDIR: join(home, "tmp"), FORGE_APP_DATA: join(home, "data") };
867 for (const name of ["FORGE_CHECK_SHARDS", "FORGE_CHECK_SHARD_INDEX", "FORGE_CHECK_SHARD_TOTAL", "NODE_OPTIONS"]) delete env[name];
868 const junit = join(home, "junit.xml");
869 try {
870 mkdirSync(join(home, "tmp"), { recursive: true });
871 mkdirSync(join(home, "data"), { recursive: true });
872 const report = junitAsked ? [] : ["--reporter=junit", `--reporter-outfile=${junit}`];
873 const began = performance.now();
874 const child = Bun.spawn(["bun", "test", ...flags, ...report, `./${file.replace(/^\.\//, "")}`], {
875 stdin: "ignore",
876 stdout: "ignore",
877 stderr: "ignore",
878 env,
879 });
880 children.add(child);
881 const code = await child.exited;
882 children.delete(child);
883 const seconds = (performance.now() - began) / 1000;
884 let tests: TestTime[] = [];
885 let wrote = false;
886 try {
887 tests = parseJunit(readFileSync(junit, "utf8"));
888 wrote = true;
889 } catch {
890 // no report: its test breaches are named unverified under load, with why
891 }
892 return { green: code === 0, seconds, tests, report: wrote };
893 } finally {
894 rmSync(home, { recursive: true, force: true });
895 }
896 }
897
898 /**
899 * The held breaches of shards that ran side by side, judged now that none
900 * runs beside them (#12848): every file with one re-timed one at a time,
901 * nearest its budget first, a file started only while `RETIME_SECONDS` of
902 * wall clock is not spent. Each breach still over alone is a `(fail)` on its
903 * shard; each under alone is slow under load, and each not judged alone is
904 * unverified under load: neither is a red.
905 */
906 async function judgeUnderLoad(results: Timed[]): Promise<{ underLoad: UnderLoad[]; unverified: Unverified[] }> {
907 const held = results.flatMap((result) => result.pending);
908 if (held.length === 0) return { underLoad: [], unverified: [] };
909 const files = retimeFiles(held);
910 process.stdout.write(
911 `\n── over the budget beside other files: timing ${files.length} file(s) again, alone, for at most ${RETIME_SECONDS} s: ${files.join(" ")}\n`,
912 );
913 const alone = new Map<string, AloneRun>();
914 const began = performance.now();
915 for (const file of files) {
916 if ((performance.now() - began) / 1000 >= RETIME_SECONDS) break;
917 alone.set(file, await timeAlone(file));
918 }
919 const underLoad: UnderLoad[] = [];
920 const unverified: Unverified[] = [];
921 results.forEach((result, i) => {
922 if (result.pending.length === 0) return;
923 const verdict = loadVerdict(result.pending, alone);
924 underLoad.push(...verdict.underLoad);
925 unverified.push(...verdict.unverified);
926 if (verdict.still.length === 0) {
927 results[i] = { ...result, pending: [] };
928 return;
929 }
930 const note = breachReport(verdict.still);
931 process.stdout.write(note);
932 results[i] = {
933 ...result,
934 pending: [],
935 code: 1,
936 output: `${result.output}${result.output.endsWith("\n") || result.output === "" ? "" : "\n"}${note}`,
937 overBudget: [...result.overBudget, ...verdict.still],
938 };
939 });
940 return { underLoad, unverified };
941 }
942
843943 /** Runs slices `from` up to `to`, at most `width` at once, in the slices' order. */
844944 async function pool(width: number, from: number, to: number): Promise<Timed[]> {
845945 const results: Timed[] = [];
@@ -910,11 +1010,15 @@ const results = [
9101010 // Budget files, one at a time, once the rest are done and green.
9111011 ...(await pool(1, slices.length, slices.length)),
9121012 ];
1013 // What breached beside other files is timed again alone, now, before the tally.
1014 const { underLoad, unverified } = await judgeUnderLoad(results);
9131015 const wall = (performance.now() - started) / 1000;
9141016 const ending = tally(results, wall);
9151017 for (const line of auditLines(audited, results)) console.log(line);
9161018 const spread = results.map((result) => result.seconds);
9171019 console.log(`\n${ending.lines.join("\n")}`);
1020 // Listed once, after the tally: what only a loaded box found over the budget.
1021 for (const line of [...underLoadLines(underLoad), ...unverifiedLines(unverified)]) console.log(line);
9181022 if (!record && !timed && results.length > 1)
9191023 console.log(
9201024 `Slowest shard ${Math.max(...spread).toFixed(1)}s, fastest ${Math.min(...spread).toFixed(1)}s.`,
test/test-budget-run.test.ts +69
@@ -122,6 +122,75 @@ test("judges what the hook found", () => expect(true).toBe(true));
122122 }
123123 }, 20_000);
124124
125 test("shards side by side: a breach is timed again alone after the pool, and one over alone stays red (#12848)", async () => {
126 writeFileSync(join(dir, "test/zz-quick.test.ts"), `import { expect, test } from "bun:test";\ntest("quick", () => expect(1).toBe(1));\n`);
127 try {
128 const got = await runner(["--tier", "full", "test/zz-budget.test.ts", "test/zz-quick.test.ts"], { FORGE_CHECK_SHARDS: "2" });
129 expect(got.out).toContain("── over the budget beside other files: timing 1 file(s) again, alone, for at most 60 s: test/zz-budget.test.ts\n");
130 expect(got.out).toContain("test/zz-budget.test.ts:\n(fail) spins past the budget took ");
131 expect(got.out).toContain("1 failed.");
132 expect(got.out).not.toContain("slow under load");
133 expect(got.out).not.toContain("unverified under load");
134 expect(got.code).toBe(1);
135 } finally {
136 rmSync(join(dir, "test/zz-quick.test.ts"), { force: true });
137 }
138 }, 20_000);
139
140 test("shards side by side: a test slow only beside its sibling is slow under load after the tally, not red (#12848)", async () => {
141 // zz-load-a spins past the budget only while zz-load-b's shard holds a
142 // lock; b releases it and leaves a done marker, so a's run alone, after
143 // the pool, finds no lock and is quick. Waits are in hooks (the file
144 // budget, 20 s), never in a test, and bounded.
145 const lock = JSON.stringify(join(dir, "zz-load.lock"));
146 const seen = JSON.stringify(join(dir, "zz-load.seen"));
147 const done = JSON.stringify(join(dir, "zz-load.done"));
148 const wait = `const until = async (ok: () => boolean) => { const end = performance.now() + 3000; while (!ok() && performance.now() < end) await Bun.sleep(5); }; `; // real-timer: the fixture polls a lock another bun process writes; no clock crosses processes
149 writeFileSync(
150 join(dir, "test/zz-load-a.test.ts"),
151 `import { beforeAll, expect, test } from "bun:test";
152 import { existsSync, writeFileSync } from "node:fs";
153 ${wait}
154 beforeAll(() => until(() => existsSync(${lock}) || existsSync(${done})));
155 test("slow beside its sibling", () => {
156 if (existsSync(${lock})) {
157 writeFileSync(${seen}, "");
158 const end = performance.now() + 60;
159 while (performance.now() < end) {}
160 }
161 expect(true).toBe(true);
162 });
163 `,
164 );
165 writeFileSync(
166 join(dir, "test/zz-load-b.test.ts"),
167 `import { afterAll, beforeAll, expect, test } from "bun:test";
168 import { existsSync, rmSync, writeFileSync } from "node:fs";
169 ${wait}
170 beforeAll(async () => {
171 writeFileSync(${lock}, "");
172 await until(() => existsSync(${seen}));
173 });
174 afterAll(() => {
175 rmSync(${lock}, { force: true });
176 writeFileSync(${done}, "");
177 });
178 test("holds the lock", () => expect(1).toBe(1));
179 `,
180 );
181 try {
182 const got = await runner(["--tier", "full", "test/zz-load-a.test.ts", "test/zz-load-b.test.ts"], { FORGE_CHECK_SHARDS: "2" });
183 expect(got.out).toContain("── over the budget beside other files: timing 1 file(s) again, alone, for at most 60 s: test/zz-load-a.test.ts\n");
184 expect(got.out).not.toContain("(fail)");
185 const after = got.out.slice(got.out.indexOf("Ran 2 tests"));
186 expect(after).toContain("test-shards: slow under load, over the budget beside other files and under it alone, not red: 1\n");
187 expect(after).toMatch(/\n {2}[\d.]+ s under load, [\d.]+ s alone \(budget 0\.02 s\): test\/zz-load-a\.test\.ts > slow beside its sibling\n/);
188 expect(got.code).toBe(0);
189 } finally {
190 for (const name of ["test/zz-load-a.test.ts", "test/zz-load-b.test.ts", "zz-load.lock", "zz-load.seen", "zz-load.done"]) rmSync(join(dir, name), { force: true });
191 }
192 }, 20_000);
193
125194 test("by hand the same file is named over budget and stays green", async () => {
126195 const got = await runner(["test/zz-budget.test.ts"]);
127196 expect(got.out).toContain("test-shards: over budget (not enforced here): test/zz-budget.test.ts > spins past the budget took ");
test/test-budget.test.ts +96
@@ -14,11 +14,16 @@ import {
1414 budgetsFrom,
1515 FILE_BUDGET_SECONDS,
1616 formatTestTimes,
17 loadVerdict,
1718 parseJunit,
19 RETIME_SECONDS,
20 retimeFiles,
1821 retimeVerdict,
1922 slowestTestLines,
2023 slowUnderLines,
2124 TEST_BUDGET_SECONDS,
25 underLoadLines,
26 unverifiedLines,
2227 } from "../scripts/test-budget.ts";
2328
2429 const junit = `<?xml version="1.0" encoding="UTF-8"?>
@@ -131,6 +136,97 @@ describe("retimeVerdict", () => {
131136 });
132137 });
133138
139 describe("loadVerdict: over the budget beside other files, then timed alone (#12848)", () => {
140 const capacity = { file: "test/capacity.test.ts", test: "g > admits", seconds: 2.6, budget: TEST_BUDGET_SECONDS };
141 const dispatch = { file: "test/team-dispatch.test.ts", test: "sends", seconds: 2.6, budget: TEST_BUDGET_SECONDS };
142 const green = (file: string, test: string, seconds: number) => ({ green: true, seconds: seconds + 0.1, tests: [{ file, test, seconds }], report: true });
143
144 test("2.6 s under load and 0.4 s alone is not red: it is slow under load, both times kept", () => {
145 const got = loadVerdict([capacity], new Map([[capacity.file, green(capacity.file, capacity.test, 0.4)]]));
146 expect(got.still).toEqual([]);
147 expect(got.underLoad).toEqual([{ file: capacity.file, test: capacity.test, loaded: 2.6, alone: 0.4, budget: 2 }]);
148 });
149
150 test("2.6 s alone is still red, at its alone time", () => {
151 const got = loadVerdict([dispatch], new Map([[dispatch.file, green(dispatch.file, dispatch.test, 2.6)]]));
152 expect(got.still).toEqual([{ ...dispatch, seconds: 2.6 }]);
153 expect(got.underLoad).toEqual([]);
154 // and the tally counts it, as any budget red
155 const ending = tally([{ files: [dispatch.file], code: 1, output: `Ran 1 tests across 1 file. [3.00s]\n${breachReport(got.still)}` }], 3);
156 expect(ending.code).toBe(1);
157 expect(ending.lines.filter((line) => line.startsWith("(fail) sends took 2.60 s"))).toHaveLength(1);
158 });
159
160 test("a whole file is judged by its alone run's seconds", () => {
161 const whole = { file: "test/a.test.ts", seconds: 24, budget: FILE_BUDGET_SECONDS };
162 expect(loadVerdict([whole], new Map([[whole.file, { green: true, seconds: 9, tests: [], report: true }]])).underLoad).toEqual([
163 { file: whole.file, loaded: 24, alone: 9, budget: FILE_BUDGET_SECONDS },
164 ]);
165 expect(loadVerdict([whole], new Map([[whole.file, { green: true, seconds: 21, tests: [], report: false }]])).still).toEqual([{ ...whole, seconds: 21 }]);
166 });
167
168 test("not timed alone before the time limit ran out: unverified under load, not red", () => {
169 const got = loadVerdict([capacity], new Map());
170 expect(got.still).toEqual([]);
171 expect(got.underLoad).toEqual([]);
172 expect(got.unverified).toEqual([
173 { file: capacity.file, test: capacity.test, loaded: 2.6, budget: 2, why: `not timed alone within the ${RETIME_SECONDS} s re-time limit` },
174 ]);
175 });
176
177 test("a test breach whose alone run wrote no report, or lacks the test: unverified, and the reason says which", () => {
178 const none = loadVerdict([capacity], new Map([[capacity.file, { green: true, seconds: 0.5, tests: [], report: false }]]));
179 expect(none.still).toEqual([]);
180 expect(none.unverified.map((u) => u.why)).toEqual(["its run alone wrote no JUnit report"]);
181 const missing = loadVerdict([capacity], new Map([[capacity.file, green(capacity.file, "another", 0.1)]]));
182 expect(missing.still).toEqual([]);
183 expect(missing.unverified.map((u) => u.why)).toEqual(["the test is missing from its run alone's JUnit report"]);
184 expect(unverifiedLines(none.unverified)).toEqual([
185 "test-shards: unverified under load, over the budget beside other files and not judged alone, not red: 1",
186 " 2.60 s under load (budget 2 s), its run alone wrote no JUnit report: test/capacity.test.ts > g > admits",
187 ]);
188 expect(unverifiedLines([])).toEqual([]);
189 });
190
191 test("red alone: it stands at its loaded time, and its (fail) says the alone run failed", () => {
192 const got = loadVerdict([capacity], new Map([[capacity.file, { green: false, seconds: 0.5, tests: [], report: true }]]));
193 expect(got.still).toEqual([{ ...capacity, aloneRed: true }]);
194 expect(got.unverified).toEqual([]);
195 expect(breachReport(got.still)).toBe(
196 "test/capacity.test.ts:\n(fail) g > admits took 2.60 s, over the 2 s test budget beside other files, and its file failed when run again alone, so the load is not ruled out: make it faster, or name its file in SLOW in scripts/gate.ts with an issue\n",
197 );
198 });
199
200 test("every breaching file is re-timed, nearest the budget first, each file once: no count cap", () => {
201 const over = Array.from({ length: 30 }, (_, i) => ({ file: `test/f${i}.test.ts`, test: "t", seconds: 30 - i * 0.5, budget: 2 }));
202 const files = retimeFiles([...over, { ...over[0]!, test: "u" }]);
203 expect(files).toHaveLength(30);
204 expect(files[0]).toBe("test/f29.test.ts");
205 expect(files.at(-1)).toBe("test/f0.test.ts");
206 expect(retimeFiles([])).toEqual([]);
207 expect(RETIME_SECONDS).toBe(60);
208 });
209
210 test("the summary lists each once with both times, and says nothing when none", () => {
211 expect(
212 underLoadLines([
213 { file: "test/b.test.ts", loaded: 24, alone: 9, budget: 20 },
214 { file: "test/capacity.test.ts", test: "g > admits", loaded: 2.6, alone: 0.4, budget: 2 },
215 ]),
216 ).toEqual([
217 "test-shards: slow under load, over the budget beside other files and under it alone, not red: 2",
218 " 24.0 s under load, 9.0 s alone (budget 20 s): test/b.test.ts (the whole file)",
219 " 2.60 s under load, 0.40 s alone (budget 2 s): test/capacity.test.ts > g > admits",
220 ]);
221 expect(underLoadLines([])).toEqual([]);
222 });
223
224 test("no budget is raised", () => {
225 expect(TEST_BUDGET_SECONDS).toBe(2);
226 expect(FILE_BUDGET_SECONDS).toBe(20);
227 });
228 });
229
134230 describe("breachClosing", () => {
135231 test("the count, then every breach slowest first, so a log's tail names them all", () => {
136232 const lines = breachClosing(

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs