Public: anyone can read this; the crew can change it.
A test over the 2 s budget only in a loaded full-suite run is re-timed alone before main is called red #12866
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12848-a-test-over-the-2-s-budget-only-in-a-loa → main 3 commits fork 1 thread · 1 unresolved
4 files
changed +399
−7
scripts/test-budget.ts +126 −3
| @@ -36,7 +36,7 @@ export function budgetsFrom(value: string | undefined): Budgets { | ||
| 36 | 36 | } |
| 37 | 37 | |
| 38 | 38 | /** One thing over its budget: a whole file, or one test in it. */ |
| 39 | export type Breach = { file: string; test?: string; seconds: number; budget: number }; | |
| 39 | export type Breach = { file: string; test?: string; seconds: number; budget: number; aloneRed?: true }; | |
| 40 | 40 | |
| 41 | 41 | const ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; |
| 42 | 42 | |
| @@ -99,6 +99,9 @@ export function breaches( | ||
| 99 | 99 | |
| 100 | 100 | const FIX = "make it faster, or name its file in SLOW in scripts/gate.ts with an issue"; |
| 101 | 101 | |
| 102 | /** Said of a breach beside other files whose re-time alone went red (#12848). */ | |
| 103 | const ALONE_RED = (b: Breach): string => (b.aloneRed ? " beside other files, and its file failed when run again alone, so the load is not ruled out" : ""); | |
| 104 | ||
| 102 | 105 | /** |
| 103 | 106 | * The report lines for `over`, grouped under each file's `path:` header the |
| 104 | 107 | * way `bun test` writes its own, each breach a `(fail)` line: the shard's |
| @@ -113,8 +116,8 @@ export function breachReport(over: readonly Breach[]): string { | ||
| 113 | 116 | for (const b of list) |
| 114 | 117 | lines.push( |
| 115 | 118 | b.test === undefined |
| 116 | ? `(fail) the whole file took ${b.seconds.toFixed(1)} s, over the ${b.budget} s file budget: ${FIX}` | |
| 117 | : `(fail) ${b.test} took ${b.seconds.toFixed(2)} s, over the ${b.budget} s test budget: ${FIX}`, | |
| 119 | ? `(fail) the whole file took ${b.seconds.toFixed(1)} s, over the ${b.budget} s file budget${ALONE_RED(b)}: ${FIX}` | |
| 120 | : `(fail) ${b.test} took ${b.seconds.toFixed(2)} s, over the ${b.budget} s test budget${ALONE_RED(b)}: ${FIX}`, | |
| 118 | 121 | ); |
| 119 | 122 | } |
| 120 | 123 | return lines.length === 0 ? "" : `${lines.join("\n")}\n`; |
| @@ -141,6 +144,126 @@ export function retimeVerdict( | ||
| 141 | 144 | }; |
| 142 | 145 | } |
| 143 | 146 | |
| 147 | /** | |
| 148 | * How long, in wall-clock seconds, the re-time alone of what breached beside | |
| 149 | * other files may run once the pool is done (#12848). Alone runs go one at a | |
| 150 | * time, so the cost is bounded by time, not by a count: every breaching file | |
| 151 | * is timed, nearest its budget first, and a file is only started while the | |
| 152 | * limit is not spent. What the limit leaves un-timed is unverified under | |
| 153 | * load, named and not red: only a breach that repeats alone is a red. | |
| 154 | */ | |
| 155 | export const RETIME_SECONDS = 60; | |
| 156 | ||
| 157 | /** | |
| 158 | * One file's run alone: green or not, its whole seconds, each test's, and | |
| 159 | * whether it wrote a JUnit report at all (a test breach needs one). | |
| 160 | */ | |
| 161 | export type AloneRun = { green: boolean; seconds: number; tests: readonly TestTime[]; report: boolean }; | |
| 162 | ||
| 163 | /** A breach beside other files that was under its budget alone: both times. */ | |
| 164 | export type UnderLoad = { file: string; test?: string; loaded: number; alone: number; budget: number }; | |
| 165 | ||
| 166 | /** A breach beside other files its run alone could not judge, and why. */ | |
| 167 | export type Unverified = { file: string; test?: string; loaded: number; budget: number; why: string }; | |
| 168 | ||
| 169 | /** | |
| 170 | * Every file to re-time alone, nearest its budget first (the smallest | |
| 171 | * `seconds / budget`), since a breach just over is the one a loaded box | |
| 172 | * most likely made; a file appears once. The caller stops starting files | |
| 173 | * when {@link RETIME_SECONDS} is spent; none is left out here. | |
| 174 | */ | |
| 175 | export function retimeFiles(over: readonly Breach[]): string[] { | |
| 176 | const nearest = [...over].sort((a, b) => a.seconds / a.budget - b.seconds / b.budget || key(a).localeCompare(key(b))); | |
| 177 | return [...new Set(nearest.map((b) => b.file))]; | |
| 178 | } | |
| 179 | ||
| 180 | /** | |
| 181 | * The verdict on breaches found with other files running beside them, from | |
| 182 | * each file's run alone (`alone`, by file; a file absent was not re-timed | |
| 183 | * before the time limit ran out). Only a breach that repeats alone is red: | |
| 184 | * - over the same budget alone: it stands, at its alone time; | |
| 185 | * - under it alone: slow under load, not red, both times kept; | |
| 186 | * - its file not re-timed, or a test breach whose alone run wrote no report | |
| 187 | * or has no such test: unverified under load, named with why, not red; | |
| 188 | * - the alone run went red: it stands at its loaded time, marked so its | |
| 189 | * `(fail)` says the alone run failed. That is a red on its own merits (the | |
| 190 | * file failed when run by itself), so the gate is red for a reason the log | |
| 191 | * names, not for the load. | |
| 192 | */ | |
| 193 | export function loadVerdict( | |
| 194 | first: readonly Breach[], | |
| 195 | alone: ReadonlyMap<string, AloneRun>, | |
| 196 | ): { still: Breach[]; underLoad: UnderLoad[]; unverified: Unverified[] } { | |
| 197 | const still: Breach[] = []; | |
| 198 | const underLoad: UnderLoad[] = []; | |
| 199 | const unverified: Unverified[] = []; | |
| 200 | for (const b of first) { | |
| 201 | const named = { file: b.file, ...(b.test === undefined ? {} : { test: b.test }) }; | |
| 202 | const run = alone.get(b.file); | |
| 203 | if (run === undefined) { | |
| 204 | unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: `not timed alone within the ${RETIME_SECONDS} s re-time limit` }); | |
| 205 | continue; | |
| 206 | } | |
| 207 | if (!run.green) { | |
| 208 | still.push({ ...b, aloneRed: true }); | |
| 209 | continue; | |
| 210 | } | |
| 211 | let seconds: number | undefined = run.seconds; | |
| 212 | if (b.test !== undefined) { | |
| 213 | if (!run.report) { | |
| 214 | unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: "its run alone wrote no JUnit report" }); | |
| 215 | continue; | |
| 216 | } | |
| 217 | seconds = run.tests.find((t) => t.test === b.test)?.seconds; | |
| 218 | if (seconds === undefined) { | |
| 219 | unverified.push({ ...named, loaded: b.seconds, budget: b.budget, why: "the test is missing from its run alone's JUnit report" }); | |
| 220 | continue; | |
| 221 | } | |
| 222 | } | |
| 223 | if (seconds > b.budget) still.push({ ...b, seconds }); | |
| 224 | else underLoad.push({ ...named, loaded: b.seconds, alone: seconds, budget: b.budget }); | |
| 225 | } | |
| 226 | return { still, underLoad, unverified }; | |
| 227 | } | |
| 228 | ||
| 229 | /** | |
| 230 | * The run summary's list of what was over the budget beside other files and | |
| 231 | * could not be judged alone (#12848): a count, then each with its loaded | |
| 232 | * time and why; none when there is nothing to name. Not red. | |
| 233 | */ | |
| 234 | export function unverifiedLines(list: readonly Unverified[]): string[] { | |
| 235 | if (list.length === 0) return []; | |
| 236 | const digits = (u: Unverified): number => (u.test === undefined ? 1 : 2); | |
| 237 | return [ | |
| 238 | `test-shards: unverified under load, over the budget beside other files and not judged alone, not red: ${list.length}`, | |
| 239 | ...[...list] | |
| 240 | .sort((a, b) => b.loaded - a.loaded || a.file.localeCompare(b.file)) | |
| 241 | .map( | |
| 242 | (u) => | |
| 243 | ` ${u.loaded.toFixed(digits(u))} s under load (budget ${u.budget} s), ${u.why}: ${u.file}${u.test === undefined ? " (the whole file)" : ` > ${u.test}`}`, | |
| 244 | ), | |
| 245 | ]; | |
| 246 | } | |
| 247 | ||
| 248 | /** | |
| 249 | * The run summary's list of what was slow under load (#12848), once: a | |
| 250 | * count, then each with the time beside others and the time alone; none | |
| 251 | * when there is nothing to name. | |
| 252 | */ | |
| 253 | export function underLoadLines(list: readonly UnderLoad[]): string[] { | |
| 254 | if (list.length === 0) return []; | |
| 255 | const digits = (u: UnderLoad): number => (u.test === undefined ? 1 : 2); | |
| 256 | return [ | |
| 257 | `test-shards: slow under load, over the budget beside other files and under it alone, not red: ${list.length}`, | |
| 258 | ...[...list] | |
| 259 | .sort((a, b) => b.loaded - a.loaded || a.file.localeCompare(b.file)) | |
| 260 | .map( | |
| 261 | (u) => | |
| 262 | ` ${u.loaded.toFixed(digits(u))} s under load, ${u.alone.toFixed(digits(u))} s alone (budget ${u.budget} s): ${u.file}${u.test === undefined ? " (the whole file)" : ` > ${u.test}`}`, | |
| 263 | ), | |
| 264 | ]; | |
| 265 | } | |
| 266 | ||
| 144 | 267 | /** One warning line per breach, for a run that names them without failing. */ |
| 145 | 268 | export function breachWarnings(over: readonly Breach[]): string[] { |
| 146 | 269 | return over.map((b) => |
scripts/test-shards.ts +108 −4
| @@ -49,6 +49,13 @@ | ||
| 49 | 49 | * Every run times each test from a JUnit report beside the console one |
| 50 | 50 | * and holds it to the budget (`scripts/test-budget.ts`, #1242): red on the |
| 51 | 51 | * whole gate and on a fast tier's own changed test files, named by hand. |
| 52 | * A breach found while shards ran side by side is not judged then: once | |
| 53 | * every shard is done, every file with one is timed again one at a time, | |
| 54 | * nearest its budget first, for at most `RETIME_SECONDS` of wall clock, and | |
| 55 | * only a breach that repeats alone is a `(fail)`; one under its budget alone | |
| 56 | * is listed after the tally as slow under load, with both times, and one the | |
| 57 | * limit left un-timed (or whose alone run wrote no report) as unverified | |
| 58 | * under load, not red (#12848). | |
| 52 | 59 | * A green timed gate also writes each test's time to |
| 53 | 60 | * `.forge-cache/test-times-tests.json` and ends by naming the five slowest. |
| 54 | 61 | * |
| @@ -94,7 +101,7 @@ import { | ||
| 94 | 101 | import { gateFiles, gateRefusal, looksTests, presentTests, SLOW, slowRefusal } from "./gate.ts"; |
| 95 | 102 | import { findChromium } from "../src/looks/capture.ts"; |
| 96 | 103 | import { chromiumRuns } from "../test/helpers/chrome.ts"; |
| 97 | import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, slowUnderLines, parseJunit, slowestTestLines, type Breach, type TestTime } from "./test-budget.ts"; | |
| 104 | import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, breachWarnings, formatTestTimes, loadVerdict, retimeFiles, RETIME_SECONDS, slowUnderLines, parseJunit, slowestTestLines, underLoadLines, unverifiedLines, type AloneRun, type Breach, type TestTime, type UnderLoad, type Unverified } from "./test-budget.ts"; | |
| 98 | 105 | import { cgroupMemory, summaryLines } from "./gate-summary.ts"; |
| 99 | 106 | import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts"; |
| 100 | 107 | import { importsReader, readImpact, scansIn } from "./tier-input.ts"; |
| @@ -569,7 +576,9 @@ let crashReruns = 0; | ||
| 569 | 576 | const RED_SIGN = /^(\(fail\)|# Unhandled error|error:)/m; |
| 570 | 577 | |
| 571 | 578 | // `slowRan` the SLOW files of a green shard, `slowOver` those over the budget in it. |
| 572 | type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[]; slowRan: string[]; slowOver: Breach[]; retried: boolean }; | |
| 579 | // `pending` the binding breaches of a shard that ran beside others, held | |
| 580 | // for a re-time alone once every shard is done (#12848). | |
| 581 | type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[]; slowRan: string[]; slowOver: Breach[]; retried: boolean; pending: Breach[] }; | |
| 573 | 582 | |
| 574 | 583 | // The budget (#1242) is red on the full tier, for every file SLOW does not |
| 575 | 584 | // name: the whole gate, or a run that names `--tier full` itself (#12088, |
| @@ -761,8 +770,11 @@ async function runShardIn( | ||
| 761 | 770 | // Over budget (#1242), judged on a green run: a file alone is timed whole, |
| 762 | 771 | // every test by the report. What binds is timed once more, alone and |
| 763 | 772 | // without coverage, and judged by that run, so one busy moment on the box |
| 764 | // is not a red; what is still over is a `(fail)` the tally counts. | |
| 773 | // is not a red; what is still over is a `(fail)` the tally counts. A shard | |
| 774 | // that ran beside others holds its breaches instead: timed again now, the | |
| 775 | // others still run beside it, so they wait until the pool is done (#12848). | |
| 765 | 776 | let overBudget: Breach[] = []; |
| 777 | let pending: Breach[] = []; | |
| 766 | 778 | let slowRan: string[] = []; |
| 767 | 779 | let slowOver: Breach[] = []; |
| 768 | 780 | if (code === 0) { |
| @@ -774,7 +786,10 @@ async function runShardIn( | ||
| 774 | 786 | const binding = over.filter((b) => budgetBinds(b.file)); |
| 775 | 787 | const named = breachWarnings(over.filter((b) => !budgetBinds(b.file))); |
| 776 | 788 | let still = binding; |
| 777 | if (binding.length > 0) { | |
| 789 | if (binding.length > 0 && !live) { | |
| 790 | pending = binding; | |
| 791 | still = []; | |
| 792 | } else if (binding.length > 0) { | |
| 778 | 793 | const retimed = [...new Set(binding.map((b) => b.file))]; |
| 779 | 794 | const again = await run(retimed.map((file) => `./${file}`), []); |
| 780 | 795 | const againTests = testsOfLastRun(); |
| @@ -837,9 +852,94 @@ async function runShardIn( | ||
| 837 | 852 | slowRan, |
| 838 | 853 | slowOver, |
| 839 | 854 | retried, |
| 855 | pending, | |
| 840 | 856 | }; |
| 841 | 857 | } |
| 842 | 858 | |
| 859 | /** | |
| 860 | * One file run alone, after the pool, as its shard was (same flags, a | |
| 861 | * scratch home of its own, no coverage): green or not, timed from spawn to | |
| 862 | * exit, each test's time from its report. Its output is not the verdict. | |
| 863 | */ | |
| 864 | async function timeAlone(file: string): Promise<AloneRun> { | |
| 865 | const home = shardHome(process.env, tmpdir()); | |
| 866 | const env: Record<string, string | undefined> = { ...process.env, TMPDIR: join(home, "tmp"), FORGE_APP_DATA: join(home, "data") }; | |
| 867 | for (const name of ["FORGE_CHECK_SHARDS", "FORGE_CHECK_SHARD_INDEX", "FORGE_CHECK_SHARD_TOTAL", "NODE_OPTIONS"]) delete env[name]; | |
| 868 | const junit = join(home, "junit.xml"); | |
| 869 | try { | |
| 870 | mkdirSync(join(home, "tmp"), { recursive: true }); | |
| 871 | mkdirSync(join(home, "data"), { recursive: true }); | |
| 872 | const report = junitAsked ? [] : ["--reporter=junit", `--reporter-outfile=${junit}`]; | |
| 873 | const began = performance.now(); | |
| 874 | const child = Bun.spawn(["bun", "test", ...flags, ...report, `./${file.replace(/^\.\//, "")}`], { | |
| 875 | stdin: "ignore", | |
| 876 | stdout: "ignore", | |
| 877 | stderr: "ignore", | |
| 878 | env, | |
| 879 | }); | |
| 880 | children.add(child); | |
| 881 | const code = await child.exited; | |
| 882 | children.delete(child); | |
| 883 | const seconds = (performance.now() - began) / 1000; | |
| 884 | let tests: TestTime[] = []; | |
| 885 | let wrote = false; | |
| 886 | try { | |
| 887 | tests = parseJunit(readFileSync(junit, "utf8")); | |
| 888 | wrote = true; | |
| 889 | } catch { | |
| 890 | // no report: its test breaches are named unverified under load, with why | |
| 891 | } | |
| 892 | return { green: code === 0, seconds, tests, report: wrote }; | |
| 893 | } finally { | |
| 894 | rmSync(home, { recursive: true, force: true }); | |
| 895 | } | |
| 896 | } | |
| 897 | ||
| 898 | /** | |
| 899 | * The held breaches of shards that ran side by side, judged now that none | |
| 900 | * runs beside them (#12848): every file with one re-timed one at a time, | |
| 901 | * nearest its budget first, a file started only while `RETIME_SECONDS` of | |
| 902 | * wall clock is not spent. Each breach still over alone is a `(fail)` on its | |
| 903 | * shard; each under alone is slow under load, and each not judged alone is | |
| 904 | * unverified under load: neither is a red. | |
| 905 | */ | |
| 906 | async function judgeUnderLoad(results: Timed[]): Promise<{ underLoad: UnderLoad[]; unverified: Unverified[] }> { | |
| 907 | const held = results.flatMap((result) => result.pending); | |
| 908 | if (held.length === 0) return { underLoad: [], unverified: [] }; | |
| 909 | const files = retimeFiles(held); | |
| 910 | process.stdout.write( | |
| 911 | `\n── over the budget beside other files: timing ${files.length} file(s) again, alone, for at most ${RETIME_SECONDS} s: ${files.join(" ")}\n`, | |
| 912 | ); | |
| 913 | const alone = new Map<string, AloneRun>(); | |
| 914 | const began = performance.now(); | |
| 915 | for (const file of files) { | |
| 916 | if ((performance.now() - began) / 1000 >= RETIME_SECONDS) break; | |
| 917 | alone.set(file, await timeAlone(file)); | |
| 918 | } | |
| 919 | const underLoad: UnderLoad[] = []; | |
| 920 | const unverified: Unverified[] = []; | |
| 921 | results.forEach((result, i) => { | |
| 922 | if (result.pending.length === 0) return; | |
| 923 | const verdict = loadVerdict(result.pending, alone); | |
| 924 | underLoad.push(...verdict.underLoad); | |
| 925 | unverified.push(...verdict.unverified); | |
| 926 | if (verdict.still.length === 0) { | |
| 927 | results[i] = { ...result, pending: [] }; | |
| 928 | return; | |
| 929 | } | |
| 930 | const note = breachReport(verdict.still); | |
| 931 | process.stdout.write(note); | |
| 932 | results[i] = { | |
| 933 | ...result, | |
| 934 | pending: [], | |
| 935 | code: 1, | |
| 936 | output: `${result.output}${result.output.endsWith("\n") || result.output === "" ? "" : "\n"}${note}`, | |
| 937 | overBudget: [...result.overBudget, ...verdict.still], | |
| 938 | }; | |
| 939 | }); | |
| 940 | return { underLoad, unverified }; | |
| 941 | } | |
| 942 | ||
| 843 | 943 | /** Runs slices `from` up to `to`, at most `width` at once, in the slices' order. */ |
| 844 | 944 | async function pool(width: number, from: number, to: number): Promise<Timed[]> { |
| 845 | 945 | const results: Timed[] = []; |
| @@ -910,11 +1010,15 @@ const results = [ | ||
| 910 | 1010 | // Budget files, one at a time, once the rest are done and green. |
| 911 | 1011 | ...(await pool(1, slices.length, slices.length)), |
| 912 | 1012 | ]; |
| 1013 | // What breached beside other files is timed again alone, now, before the tally. | |
| 1014 | const { underLoad, unverified } = await judgeUnderLoad(results); | |
| 913 | 1015 | const wall = (performance.now() - started) / 1000; |
| 914 | 1016 | const ending = tally(results, wall); |
| 915 | 1017 | for (const line of auditLines(audited, results)) console.log(line); |
| 916 | 1018 | const spread = results.map((result) => result.seconds); |
| 917 | 1019 | console.log(`\n${ending.lines.join("\n")}`); |
| 1020 | // Listed once, after the tally: what only a loaded box found over the budget. | |
| 1021 | for (const line of [...underLoadLines(underLoad), ...unverifiedLines(unverified)]) console.log(line); | |
| 918 | 1022 | if (!record && !timed && results.length > 1) |
| 919 | 1023 | console.log( |
| 920 | 1024 | `Slowest shard ${Math.max(...spread).toFixed(1)}s, fastest ${Math.min(...spread).toFixed(1)}s.`, |
test/test-budget-run.test.ts +69
| @@ -122,6 +122,75 @@ test("judges what the hook found", () => expect(true).toBe(true)); | ||
| 122 | 122 | } |
| 123 | 123 | }, 20_000); |
| 124 | 124 | |
| 125 | test("shards side by side: a breach is timed again alone after the pool, and one over alone stays red (#12848)", async () => { | |
| 126 | writeFileSync(join(dir, "test/zz-quick.test.ts"), `import { expect, test } from "bun:test";\ntest("quick", () => expect(1).toBe(1));\n`); | |
| 127 | try { | |
| 128 | const got = await runner(["--tier", "full", "test/zz-budget.test.ts", "test/zz-quick.test.ts"], { FORGE_CHECK_SHARDS: "2" }); | |
| 129 | expect(got.out).toContain("── over the budget beside other files: timing 1 file(s) again, alone, for at most 60 s: test/zz-budget.test.ts\n"); | |
| 130 | expect(got.out).toContain("test/zz-budget.test.ts:\n(fail) spins past the budget took "); | |
| 131 | expect(got.out).toContain("1 failed."); | |
| 132 | expect(got.out).not.toContain("slow under load"); | |
| 133 | expect(got.out).not.toContain("unverified under load"); | |
| 134 | expect(got.code).toBe(1); | |
| 135 | } finally { | |
| 136 | rmSync(join(dir, "test/zz-quick.test.ts"), { force: true }); | |
| 137 | } | |
| 138 | }, 20_000); | |
| 139 | ||
| 140 | test("shards side by side: a test slow only beside its sibling is slow under load after the tally, not red (#12848)", async () => { | |
| 141 | // zz-load-a spins past the budget only while zz-load-b's shard holds a | |
| 142 | // lock; b releases it and leaves a done marker, so a's run alone, after | |
| 143 | // the pool, finds no lock and is quick. Waits are in hooks (the file | |
| 144 | // budget, 20 s), never in a test, and bounded. | |
| 145 | const lock = JSON.stringify(join(dir, "zz-load.lock")); | |
| 146 | const seen = JSON.stringify(join(dir, "zz-load.seen")); | |
| 147 | const done = JSON.stringify(join(dir, "zz-load.done")); | |
| 148 | const wait = `const until = async (ok: () => boolean) => { const end = performance.now() + 3000; while (!ok() && performance.now() < end) await Bun.sleep(5); }; `; // real-timer: the fixture polls a lock another bun process writes; no clock crosses processes | |
| 149 | writeFileSync( | |
| 150 | join(dir, "test/zz-load-a.test.ts"), | |
| 151 | `import { beforeAll, expect, test } from "bun:test"; | |
| 152 | import { existsSync, writeFileSync } from "node:fs"; | |
| 153 | ${wait} | |
| 154 | beforeAll(() => until(() => existsSync(${lock}) || existsSync(${done}))); | |
| 155 | test("slow beside its sibling", () => { | |
| 156 | if (existsSync(${lock})) { | |
| 157 | writeFileSync(${seen}, ""); | |
| 158 | const end = performance.now() + 60; | |
| 159 | while (performance.now() < end) {} | |
| 160 | } | |
| 161 | expect(true).toBe(true); | |
| 162 | }); | |
| 163 | `, | |
| 164 | ); | |
| 165 | writeFileSync( | |
| 166 | join(dir, "test/zz-load-b.test.ts"), | |
| 167 | `import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 168 | import { existsSync, rmSync, writeFileSync } from "node:fs"; | |
| 169 | ${wait} | |
| 170 | beforeAll(async () => { | |
| 171 | writeFileSync(${lock}, ""); | |
| 172 | await until(() => existsSync(${seen})); | |
| 173 | }); | |
| 174 | afterAll(() => { | |
| 175 | rmSync(${lock}, { force: true }); | |
| 176 | writeFileSync(${done}, ""); | |
| 177 | }); | |
| 178 | test("holds the lock", () => expect(1).toBe(1)); | |
| 179 | `, | |
| 180 | ); | |
| 181 | try { | |
| 182 | const got = await runner(["--tier", "full", "test/zz-load-a.test.ts", "test/zz-load-b.test.ts"], { FORGE_CHECK_SHARDS: "2" }); | |
| 183 | expect(got.out).toContain("── over the budget beside other files: timing 1 file(s) again, alone, for at most 60 s: test/zz-load-a.test.ts\n"); | |
| 184 | expect(got.out).not.toContain("(fail)"); | |
| 185 | const after = got.out.slice(got.out.indexOf("Ran 2 tests")); | |
| 186 | expect(after).toContain("test-shards: slow under load, over the budget beside other files and under it alone, not red: 1\n"); | |
| 187 | expect(after).toMatch(/\n {2}[\d.]+ s under load, [\d.]+ s alone \(budget 0\.02 s\): test\/zz-load-a\.test\.ts > slow beside its sibling\n/); | |
| 188 | expect(got.code).toBe(0); | |
| 189 | } finally { | |
| 190 | for (const name of ["test/zz-load-a.test.ts", "test/zz-load-b.test.ts", "zz-load.lock", "zz-load.seen", "zz-load.done"]) rmSync(join(dir, name), { force: true }); | |
| 191 | } | |
| 192 | }, 20_000); | |
| 193 | ||
| 125 | 194 | test("by hand the same file is named over budget and stays green", async () => { |
| 126 | 195 | const got = await runner(["test/zz-budget.test.ts"]); |
| 127 | 196 | expect(got.out).toContain("test-shards: over budget (not enforced here): test/zz-budget.test.ts > spins past the budget took "); |
test/test-budget.test.ts +96
| @@ -14,11 +14,16 @@ import { | ||
| 14 | 14 | budgetsFrom, |
| 15 | 15 | FILE_BUDGET_SECONDS, |
| 16 | 16 | formatTestTimes, |
| 17 | loadVerdict, | |
| 17 | 18 | parseJunit, |
| 19 | RETIME_SECONDS, | |
| 20 | retimeFiles, | |
| 18 | 21 | retimeVerdict, |
| 19 | 22 | slowestTestLines, |
| 20 | 23 | slowUnderLines, |
| 21 | 24 | TEST_BUDGET_SECONDS, |
| 25 | underLoadLines, | |
| 26 | unverifiedLines, | |
| 22 | 27 | } from "../scripts/test-budget.ts"; |
| 23 | 28 | |
| 24 | 29 | const junit = `<?xml version="1.0" encoding="UTF-8"?> |
| @@ -131,6 +136,97 @@ describe("retimeVerdict", () => { | ||
| 131 | 136 | }); |
| 132 | 137 | }); |
| 133 | 138 | |
| 139 | describe("loadVerdict: over the budget beside other files, then timed alone (#12848)", () => { | |
| 140 | const capacity = { file: "test/capacity.test.ts", test: "g > admits", seconds: 2.6, budget: TEST_BUDGET_SECONDS }; | |
| 141 | const dispatch = { file: "test/team-dispatch.test.ts", test: "sends", seconds: 2.6, budget: TEST_BUDGET_SECONDS }; | |
| 142 | const green = (file: string, test: string, seconds: number) => ({ green: true, seconds: seconds + 0.1, tests: [{ file, test, seconds }], report: true }); | |
| 143 | ||
| 144 | test("2.6 s under load and 0.4 s alone is not red: it is slow under load, both times kept", () => { | |
| 145 | const got = loadVerdict([capacity], new Map([[capacity.file, green(capacity.file, capacity.test, 0.4)]])); | |
| 146 | expect(got.still).toEqual([]); | |
| 147 | expect(got.underLoad).toEqual([{ file: capacity.file, test: capacity.test, loaded: 2.6, alone: 0.4, budget: 2 }]); | |
| 148 | }); | |
| 149 | ||
| 150 | test("2.6 s alone is still red, at its alone time", () => { | |
| 151 | const got = loadVerdict([dispatch], new Map([[dispatch.file, green(dispatch.file, dispatch.test, 2.6)]])); | |
| 152 | expect(got.still).toEqual([{ ...dispatch, seconds: 2.6 }]); | |
| 153 | expect(got.underLoad).toEqual([]); | |
| 154 | // and the tally counts it, as any budget red | |
| 155 | const ending = tally([{ files: [dispatch.file], code: 1, output: `Ran 1 tests across 1 file. [3.00s]\n${breachReport(got.still)}` }], 3); | |
| 156 | expect(ending.code).toBe(1); | |
| 157 | expect(ending.lines.filter((line) => line.startsWith("(fail) sends took 2.60 s"))).toHaveLength(1); | |
| 158 | }); | |
| 159 | ||
| 160 | test("a whole file is judged by its alone run's seconds", () => { | |
| 161 | const whole = { file: "test/a.test.ts", seconds: 24, budget: FILE_BUDGET_SECONDS }; | |
| 162 | expect(loadVerdict([whole], new Map([[whole.file, { green: true, seconds: 9, tests: [], report: true }]])).underLoad).toEqual([ | |
| 163 | { file: whole.file, loaded: 24, alone: 9, budget: FILE_BUDGET_SECONDS }, | |
| 164 | ]); | |
| 165 | expect(loadVerdict([whole], new Map([[whole.file, { green: true, seconds: 21, tests: [], report: false }]])).still).toEqual([{ ...whole, seconds: 21 }]); | |
| 166 | }); | |
| 167 | ||
| 168 | test("not timed alone before the time limit ran out: unverified under load, not red", () => { | |
| 169 | const got = loadVerdict([capacity], new Map()); | |
| 170 | expect(got.still).toEqual([]); | |
| 171 | expect(got.underLoad).toEqual([]); | |
| 172 | expect(got.unverified).toEqual([ | |
| 173 | { file: capacity.file, test: capacity.test, loaded: 2.6, budget: 2, why: `not timed alone within the ${RETIME_SECONDS} s re-time limit` }, | |
| 174 | ]); | |
| 175 | }); | |
| 176 | ||
| 177 | test("a test breach whose alone run wrote no report, or lacks the test: unverified, and the reason says which", () => { | |
| 178 | const none = loadVerdict([capacity], new Map([[capacity.file, { green: true, seconds: 0.5, tests: [], report: false }]])); | |
| 179 | expect(none.still).toEqual([]); | |
| 180 | expect(none.unverified.map((u) => u.why)).toEqual(["its run alone wrote no JUnit report"]); | |
| 181 | const missing = loadVerdict([capacity], new Map([[capacity.file, green(capacity.file, "another", 0.1)]])); | |
| 182 | expect(missing.still).toEqual([]); | |
| 183 | expect(missing.unverified.map((u) => u.why)).toEqual(["the test is missing from its run alone's JUnit report"]); | |
| 184 | expect(unverifiedLines(none.unverified)).toEqual([ | |
| 185 | "test-shards: unverified under load, over the budget beside other files and not judged alone, not red: 1", | |
| 186 | " 2.60 s under load (budget 2 s), its run alone wrote no JUnit report: test/capacity.test.ts > g > admits", | |
| 187 | ]); | |
| 188 | expect(unverifiedLines([])).toEqual([]); | |
| 189 | }); | |
| 190 | ||
| 191 | test("red alone: it stands at its loaded time, and its (fail) says the alone run failed", () => { | |
| 192 | const got = loadVerdict([capacity], new Map([[capacity.file, { green: false, seconds: 0.5, tests: [], report: true }]])); | |
| 193 | expect(got.still).toEqual([{ ...capacity, aloneRed: true }]); | |
| 194 | expect(got.unverified).toEqual([]); | |
| 195 | expect(breachReport(got.still)).toBe( | |
| 196 | "test/capacity.test.ts:\n(fail) g > admits took 2.60 s, over the 2 s test budget beside other files, and its file failed when run again alone, so the load is not ruled out: make it faster, or name its file in SLOW in scripts/gate.ts with an issue\n", | |
| 197 | ); | |
| 198 | }); | |
| 199 | ||
| 200 | test("every breaching file is re-timed, nearest the budget first, each file once: no count cap", () => { | |
| 201 | const over = Array.from({ length: 30 }, (_, i) => ({ file: `test/f${i}.test.ts`, test: "t", seconds: 30 - i * 0.5, budget: 2 })); | |
| 202 | const files = retimeFiles([...over, { ...over[0]!, test: "u" }]); | |
| 203 | expect(files).toHaveLength(30); | |
| 204 | expect(files[0]).toBe("test/f29.test.ts"); | |
| 205 | expect(files.at(-1)).toBe("test/f0.test.ts"); | |
| 206 | expect(retimeFiles([])).toEqual([]); | |
| 207 | expect(RETIME_SECONDS).toBe(60); | |
| 208 | }); | |
| 209 | ||
| 210 | test("the summary lists each once with both times, and says nothing when none", () => { | |
| 211 | expect( | |
| 212 | underLoadLines([ | |
| 213 | { file: "test/b.test.ts", loaded: 24, alone: 9, budget: 20 }, | |
| 214 | { file: "test/capacity.test.ts", test: "g > admits", loaded: 2.6, alone: 0.4, budget: 2 }, | |
| 215 | ]), | |
| 216 | ).toEqual([ | |
| 217 | "test-shards: slow under load, over the budget beside other files and under it alone, not red: 2", | |
| 218 | " 24.0 s under load, 9.0 s alone (budget 20 s): test/b.test.ts (the whole file)", | |
| 219 | " 2.60 s under load, 0.40 s alone (budget 2 s): test/capacity.test.ts > g > admits", | |
| 220 | ]); | |
| 221 | expect(underLoadLines([])).toEqual([]); | |
| 222 | }); | |
| 223 | ||
| 224 | test("no budget is raised", () => { | |
| 225 | expect(TEST_BUDGET_SECONDS).toBe(2); | |
| 226 | expect(FILE_BUDGET_SECONDS).toBe(20); | |
| 227 | }); | |
| 228 | }); | |
| 229 | ||
| 134 | 230 | describe("breachClosing", () => { |
| 135 | 231 | test("the count, then every breach slowest first, so a log's tail names them all", () => { |
| 136 | 232 | const lines = breachClosing( |
new above
offline
A cap of 8 leaves the remaining breaches red at their loaded time, which is the false red this issue removes. Re-time them all, bounded by wall-clock time, or list them as unverified and do not file them red.