Public: anyone can read this; the crew can change it.

A pull's test selection says why it chose each test, and a changed doc no longer blinds it to the impact map #14940

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14845-a-pull-s-test-selection-says-why-it-chos → main 2 commits fork

  • Merged. mason/14845-a-pull-s-test-selection-says-why-it-chos was merged into main as e58dff7.

Review

Sign in to review this change.
9 files changed +422 −72
docs/DEVELOP.md +15
@@ -128,6 +128,21 @@ which only that scan's script and test read
128128 (`test/check-allow-lists.test.ts` holds each to it). With no map yet the
129129 line says `by imports (no impact map)`.
130130
131 Under the plan line the log says why it went as wide as it did (#14845):
132 `by reason: 40 quick, 12 executes, 11 repo-wide rule` (`reasonLine`), and,
133 when a changed file the map cannot see made the plan blind (every quick
134 test runs, and every test is guessed against that file), `blind … by
135 static/app.js (outside src/ and test/), src/x.ts (no test executes it)`
136 (`blindLine`). A changed doc (`.md` under `docs/` or at the root) does
137 not blind it: it selects the tests that name it, directly or through a
138 helper (`docReaders`, found as a script's readers are), and the tests the
139 map says execute a `src/` file that names it or lists its directory
140 (`src/view/docs.ts`, `src/checks/resilience.ts`). Only a doc the
141 suite's preload reads, or a manifest, chart or the Dockerfile names
142 (`AGENTS.md`, `docs/DESIGN.md` today), is still blind. A file the suite's
143 loading runs is guessed by imports, as above, but no longer runs every
144 quick test.
145
131146 Kept results (#12178). A selected file whose inputs a default-branch
132147 measuring run already proved green is not run again: the log says
133148 `kept from <commit>`. The key is the Bun, the tooling files
scripts/fast.ts +8 −3
@@ -19,9 +19,9 @@
1919
2020 import { readFileSync } from "node:fs";
2121 import { GATE_FILE, gateListAdds, handedImpact, needsFull } from "../src/checks/impact.ts";
22 import { fastSet, duration } from "../src/checks/tiers.ts";
22 import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts";
2323 import { importsReader, readImpact, scansIn } from "./tier-input.ts";
24 import { readersIn } from "./script-readers.ts";
24 import { docReadersIn, readersIn } from "./script-readers.ts";
2525 import { CHECK_CACHE } from "../src/checks/cache.ts";
2626 import { gateFiles } from "./gate.ts";
2727 import { overlayTimes, parseTimes } from "./shards.ts";
@@ -178,15 +178,20 @@ function main(): number {
178178 imports,
179179 scans: scansIn(tests, imports),
180180 readers,
181 docReaders: docReadersIn(tests),
181182 times: loadTimes(),
182183 });
183184 const out = report(plan, needsFull(changed, gateAdds, (script) => readers(script) !== undefined), args.format);
184185 process.stdout.write(out.stdout);
185186 for (const note of out.notes) console.error(note);
186 if (out.code === 0)
187 if (out.code === 0) {
187188 console.error(
188189 `fast: ${plan.files.length} of ${tests.length} files for ${changed.length} changed, expected ${duration(plan.expected)}${plan.overBudget ? " (over the 5m budget)" : ""}`,
189190 );
191 console.error(`fast: ${reasonLine(plan)}`);
192 const blind = blindLine(plan);
193 if (blind !== undefined) console.error(`fast: ${blind}`);
194 }
190195 return out.code;
191196 }
192197
scripts/script-readers.ts +31 −9
@@ -7,7 +7,7 @@
77
88 import { readdirSync, readFileSync } from "node:fs";
99 import { join } from "node:path";
10 import { scriptReaders, type ScriptTree } from "../src/checks/scans.ts";
10 import { docReaders, scriptReaders, type DocReaders, type ScriptTree } from "../src/checks/scans.ts";
1111 import { importsOf } from "../src/checks/tiers.ts";
1212
1313 const ROOT = new URL("..", import.meta.url).pathname;
@@ -46,6 +46,11 @@ function withinFiles(): string[] {
4646 return found;
4747 }
4848
49 /** The preloads `bunfig.toml` lists. */
50 function preloadsOf(read: (file: string) => string | undefined): string[] {
51 return [...(read("bunfig.toml") ?? "").matchAll(/["']\.\/([A-Za-z0-9_./-]+\.[cm]?[jt]sx?)["']/g)].map((match) => match[1]!);
52 }
53
4954 /** The scripts a manifest's check commands name, and the preloads `bunfig.toml` lists. */
5055 function ownOf(read: (file: string) => string | undefined): string[] {
5156 const found = new Set<string>();
@@ -56,12 +61,12 @@ function ownOf(read: (file: string) => string | undefined): string[] {
5661 // An unreadable manifest names no script here; a change to it runs the full tier anyway.
5762 }
5863 for (const match of JSON.stringify(check ?? {}).matchAll(/\bscripts\/[A-Za-z0-9_./-]+/g)) found.add(match[0]);
59 for (const match of (read("bunfig.toml") ?? "").matchAll(/["']\.\/([A-Za-z0-9_./-]+\.[cm]?[jt]sx?)["']/g)) found.add(match[1]!);
64 for (const preload of preloadsOf(read)) found.add(preload);
6065 return [...found];
6166 }
6267
63 /** `scriptReaders` over the working tree, for a `fastSet` input's `readers`. The tree is read at the first ask. */
64 export function readersIn(tests: readonly string[]): (script: string) => string[] | undefined {
68 /** The working tree as `scriptReaders` and `docReaders` read it, read at the first ask. */
69 function workingTree(): () => ScriptTree {
6570 let tree: ScriptTree | undefined;
6671 const texts = new Map<string, string | undefined>();
6772 const read = (file: string): string | undefined => {
@@ -89,14 +94,31 @@ export function readersIn(tests: readonly string[]): (script: string) => string[
8994 }
9095 return found;
9196 };
92 return (script) => {
93 tree ??= {
97 return () =>
98 (tree ??= {
9499 beside: BESIDE.flatMap(filesUnder),
95100 within: withinFiles(),
96101 read,
97102 imports: importsAt,
98103 own: ownOf(read),
99 };
100 return scriptReaders(script, tree, tests);
101 };
104 preloads: preloadsOf(read),
105 });
106 }
107
108 /**
109 * `scriptReaders` over the working tree, for a `fastSet` input's `readers`. The tree is read at the
110 * first ask.
111 */
112 export function readersIn(tests: readonly string[]): (script: string) => string[] | undefined {
113 const tree = workingTree();
114 return (script) => scriptReaders(script, tree(), tests);
115 }
116
117 /**
118 * `docReaders` over the working tree, for a `fastSet` input's `docReaders`. The tree is read at the
119 * first ask.
120 */
121 export function docReadersIn(tests: readonly string[]): (doc: string) => DocReaders | undefined {
122 const tree = workingTree();
123 return (doc) => docReaders(doc, tree(), tests);
102124 }
scripts/test-shards.ts +7 −2
@@ -113,9 +113,9 @@ import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, brea
113113 import { cgroupMemory, summaryLines } from "./gate-summary.ts";
114114 import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts";
115115 import { importsReader, readImpact, scansIn } from "./tier-input.ts";
116 import { readersIn } from "./script-readers.ts";
116 import { docReadersIn, readersIn } from "./script-readers.ts";
117117 import { shardHome, spaceLine } from "../src/checks/disk_full.ts";
118 import { fastSet, duration } from "../src/checks/tiers.ts";
118 import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts";
119119 import { CHECK_CACHE } from "../src/checks/cache.ts";
120120 import { parseCheck } from "../src/apps/manifest.ts";
121121 import { FLAKY_ALONE, onlyFiles } from "../src/checks/verify.ts";
@@ -373,6 +373,7 @@ if (tier === "fast") {
373373 imports,
374374 scans: scansIn(files, imports),
375375 readers: readersIn(files),
376 docReaders: docReadersIn(files),
376377 times: loadTimes(),
377378 learned: (learnedArg.value ?? "").split(",").filter((f) => f !== ""),
378379 });
@@ -380,6 +381,10 @@ if (tier === "fast") {
380381 `test-shards: fast tier, ${plan.files.length} of ${files.length} files for ${changed.length} changed, ${plan.line} ` +
381382 `(expected ${duration(plan.expected)}${plan.overBudget ? ", over the 5m budget" : ""})`,
382383 );
384 // Why it went as wide as it did, before the list (#14845).
385 console.log(`test-shards: ${reasonLine(plan)}`);
386 const blind = blindLine(plan);
387 if (blind !== undefined) console.log(`test-shards: ${blind}`);
383388 for (const file of plan.files)
384389 console.log(` ${file}: ${plan.why.get(file)}`);
385390 // A diff no test reaches (a timing table, a doc) is judged by the
src/checks/impact.ts +7
@@ -347,6 +347,13 @@ export function needsFull(
347347 */
348348 const MAPPED_KERNEL: ReadonlySet<string> = new Set(["src/kernel/vitals.ts"]);
349349
350 /**
351 * Whether a changed file is a doc `docReaders` (scans.ts) judges: Markdown under docs/ or at the
352 * root (#14845).
353 */
354 export const isDoc = (file: string): boolean =>
355 file.endsWith(".md") && (file.startsWith("docs/") || !file.includes("/"));
356
350357 export interface Reach {
351358 /** Tests whose executed files meet the diff, with the first file that did. */
352359 readonly tests: ReadonlyMap<string, string>;
src/checks/scans.ts +145 −33
@@ -179,6 +179,11 @@ export interface ScriptTree {
179179 readonly imports: (file: string) => readonly string[];
180180 /** The scripts the check's commands name (`forge.json`) and the preloads (`bunfig.toml`). */
181181 readonly own: readonly string[];
182 /**
183 * The preloads `bunfig.toml` lists, a part of `own`: every test process runs them first, so a
184 * doc one of them reads is read by every test. Absent: all of `own` counts.
185 */
186 readonly preloads?: readonly string[];
182187 }
183188
184189 /** A whole-line comment: a script named there is read by nothing. */
@@ -208,9 +213,106 @@ function namesFor(target: string): string[] {
208213 return target.startsWith("scripts/") ? [...path, `"${base}`, `'${base}`, `\`${base}`] : path;
209214 }
210215
216 /**
217 * How a file's text names a doc (`docs/**.md`, or one at the root): as
218 * {@link namesFor} names a script, plus `/<file>` (`"../AGENTS.md"`).
219 */
220 function docNamesFor(doc: string): string[] {
221 const parts = doc.split("/");
222 const base = parts[parts.length - 1]!;
223 const stem = base.replace(/\.[A-Za-z0-9]+$/, "");
224 const dir = parts[parts.length - 2];
225 return [...(dir === undefined ? [] : [`${dir}/${stem}`]), `"${base}`, `'${base}`, `\`${base}`, `/${base}`];
226 }
227
228 /**
229 * How `src/` code names the directories a doc sits in, as a whole string or
230 * one a name is joined onto (`join(root, "docs")`, `` `docs/${name}` ``):
231 * what lists a directory reads every doc in it. A test that lists one is a
232 * tree scan already ({@link treeScans}), so only `src/` is asked.
233 */
234 function docDirNamesFor(doc: string): string[] {
235 const parts = doc.split("/");
236 const names: string[] = [];
237 for (let i = 1; i < parts.length; i++)
238 for (const at of new Set([parts.slice(0, i).join("/"), parts[i - 1]!]))
239 names.push(`"${at}"`, `'${at}'`, `\`${at}\``, `"${at}/"`, `'${at}/'`, `\`${at}/\``, `\`${at}/\${`);
240 return names;
241 }
242
211243 /** Each tree's files as code, read once however many scripts a diff holds. */
212244 const codes = new WeakMap<ScriptTree, Map<string, string>>();
213245
246 /** A tree's files as code ({@link codeOf}), kept per tree. */
247 function codeIn(tree: ScriptTree): (file: string) => string {
248 let code = codes.get(tree);
249 if (code === undefined) codes.set(tree, (code = new Map()));
250 return (file) => {
251 let text = code.get(file);
252 if (text === undefined) {
253 const source = tree.read(file);
254 text = source === undefined ? "" : codeOf(file, source);
255 code.set(file, text);
256 }
257 return text;
258 };
259 }
260
261 /** The check's own: its commands' scripts and the preload, with all they import. */
262 function ownOf(tree: ScriptTree, roots: readonly string[] = tree.own): Set<string> {
263 const own = new Set<string>();
264 for (const stack = [...roots]; stack.length > 0; ) {
265 const file = stack.pop()!;
266 if (own.has(file)) continue;
267 own.add(file);
268 stack.push(...tree.imports(file));
269 }
270 return own;
271 }
272
273 /**
274 * The closure, from `target`, over "imports it or names it" among the
275 * files beside the source, with the target itself; `undefined` when one of
276 * them is the check's own.
277 */
278 function besideReaders(
279 target: string,
280 targetNames: readonly string[],
281 tree: ScriptTree,
282 codeAt: (file: string) => string,
283 own: ReadonlySet<string>,
284 ): Set<string> | undefined {
285 const readers = new Set<string>([target]);
286 const named = new Set<string>(targetNames);
287 const names = [...named];
288 // A file is asked only the names added since it was last asked: one the
289 // old names missed misses them again.
290 const asked = new Map<string, number>();
291 for (let grew = true; grew; ) {
292 grew = false;
293 for (const file of tree.beside) {
294 if (readers.has(file)) continue;
295 const from = asked.get(file) ?? 0;
296 asked.set(file, names.length);
297 if (!tree.imports(file).some((path) => readers.has(path))) {
298 const text = codeAt(file);
299 let hit = false;
300 for (let i = from; i < names.length && !hit; i++) hit = text.includes(names[i]!);
301 if (!hit) continue;
302 }
303 if (own.has(file)) return undefined;
304 readers.add(file);
305 for (const name of namesFor(file))
306 if (!named.has(name)) {
307 named.add(name);
308 names.push(name);
309 }
310 grew = true;
311 }
312 }
313 return readers;
314 }
315
214316 /**
215317 * The gate's tests that can read a changed file under `scripts/`, or
216318 * `undefined` when that cannot be bounded and the change runs the full tier
@@ -235,40 +337,11 @@ const codes = new WeakMap<ScriptTree, Map<string, string>>();
235337 * it, as they do behind every guess of the fast tier.
236338 */
237339 export function scriptReaders(script: string, tree: ScriptTree, tests: readonly string[]): string[] | undefined {
238 let code = codes.get(tree);
239 if (code === undefined) codes.set(tree, (code = new Map()));
240 const codeAt = (file: string): string => {
241 let text = code.get(file);
242 if (text === undefined) {
243 const source = tree.read(file);
244 text = source === undefined ? "" : codeOf(file, source);
245 code.set(file, text);
246 }
247 return text;
248 };
249 // The check's own: its commands' scripts and the preload, with all they import.
250 const own = new Set<string>();
251 for (const stack = [...tree.own]; stack.length > 0; ) {
252 const file = stack.pop()!;
253 if (own.has(file)) continue;
254 own.add(file);
255 stack.push(...tree.imports(file));
256 }
340 const codeAt = codeIn(tree);
341 const own = ownOf(tree);
257342 if (own.has(script)) return undefined;
258 const readers = new Set<string>([script]);
259 const names = new Set<string>(namesFor(script));
260 for (let grew = true; grew; ) {
261 grew = false;
262 for (const file of tree.beside) {
263 if (readers.has(file)) continue;
264 const text = codeAt(file);
265 if (!tree.imports(file).some((path) => readers.has(path)) && ![...names].some((name) => text.includes(name))) continue;
266 if (own.has(file)) return undefined;
267 readers.add(file);
268 for (const name of namesFor(file)) names.add(name);
269 grew = true;
270 }
271 }
343 const readers = besideReaders(script, namesFor(script), tree, codeAt, own);
344 if (readers === undefined) return undefined;
272345 const scripts = [...readers].filter((file) => file.startsWith("scripts/"));
273346 const scriptNames = scripts.flatMap(namesFor);
274347 for (const file of tree.within) {
@@ -279,3 +352,42 @@ export function scriptReaders(script: string, tree: ScriptTree, tests: readonly
279352 const gated = new Set(tests);
280353 return [...readers].filter((file) => gated.has(file)).sort();
281354 }
355
356 /** Who can read a changed doc: tests directly, and the `src/` files that name it. */
357 export interface DocReaders {
358 /** Gated tests that read the doc themselves, or through a helper beside them. */
359 readonly tests: readonly string[];
360 /** `src/` files whose code names the doc or its directory: whoever executes one can read it. */
361 readonly sources: readonly string[];
362 }
363
364 /**
365 * Who can read a changed doc (#14845), found as {@link scriptReaders}
366 * finds a script's readers, with one difference: a `src/` file that names
367 * the doc or lists its directory (src/view/docs.ts, src/checks/resilience.ts)
368 * does not unbound it, it is a source, and the impact map says which tests
369 * execute it. The check's own scripts reading a doc is no test's verdict
370 * (gate.ts reads the README for its own rule, and runs on every check), so
371 * they are followed as readers like any helper; only the suite's preload,
372 * which every test runs, unbounds one. `undefined` then, or when a file
373 * outside `src/` and beside the source names it (a manifest, a chart): the
374 * doc is judged as before, by every guess.
375 */
376 export function docReaders(doc: string, tree: ScriptTree, tests: readonly string[]): DocReaders | undefined {
377 const codeAt = codeIn(tree);
378 const own = ownOf(tree, tree.preloads ?? tree.own);
379 const names = docNamesFor(doc);
380 const readers = besideReaders(doc, names, tree, codeAt, own);
381 if (readers === undefined) return undefined;
382 const inSource = [...names, ...docDirNamesFor(doc)];
383 const sources: string[] = [];
384 for (const file of tree.within) {
385 if (!file.startsWith("src/")) {
386 if (names.some((name) => codeAt(file).includes(name))) return undefined;
387 continue;
388 }
389 if (inSource.some((name) => codeAt(file).includes(name))) sources.push(file);
390 }
391 const gated = new Set(tests);
392 return { tests: [...readers].filter((file) => gated.has(file)).sort(), sources: sources.sort() };
393 }
src/checks/tiers.ts +102 −23
@@ -11,7 +11,8 @@
1111
1212 import type { GateReason } from "./gate_reason.ts";
1313 import type { GateTests } from "./gate_tests.ts";
14 import { GATE_FILE, LOADS_ONLY, needsFull, reach, reachLine, type Impact } from "./impact.ts";
14 import { GATE_FILE, LOADS_ONLY, isDoc, needsFull, reach, reachLine, type Impact } from "./impact.ts";
15 import type { DocReaders } from "./scans.ts";
1516
1617 /** A test under this many seconds is always in the fast tier. */
1718 export const QUICK_SECONDS = 1;
@@ -148,6 +149,10 @@ export interface FastInput {
148149 readonly scans?: ReadonlyMap<string, string>;
149150 /** The tests that can read a file under scripts/ (`scriptReaders`); `undefined`: not bounded. */
150151 readonly readers?: (script: string) => readonly string[] | undefined;
152 /**
153 * Who can read a changed doc (`docReaders`, #14845); `undefined`: not bounded. Absent: no doc is.
154 */
155 readonly docReaders?: (doc: string) => DocReaders | undefined;
151156 }
152157
153158 export interface FastPlan {
@@ -159,6 +164,11 @@ export interface FastPlan {
159164 readonly overBudget: boolean;
160165 /** The line for the log: what the plan is by, and what the diff reaches. */
161166 readonly line: string;
167 /**
168 * The changed files that made the plan blind, with why (#14845): each turns on
169 * every guess and every quick test. Empty when the map saw the whole diff.
170 */
171 readonly blindBy: ReadonlyMap<string, string>;
162172 }
163173
164174 /**
@@ -178,8 +188,9 @@ export function fastSet(input: FastInput): FastPlan {
178188 (sum, file) => sum + (input.times.get(file) ?? UNKNOWN_SECONDS),
179189 0,
180190 );
181 return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS, line };
191 return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS, line, blindBy };
182192 };
193 const blindBy = new Map<string, string>();
183194 // What the map cannot bound runs everything.
184195 const readers = new Map<string, readonly string[] | undefined>();
185196 const readersOf = (script: string): readonly string[] | undefined => {
@@ -203,28 +214,46 @@ export function fastSet(input: FastInput): FastPlan {
203214 // file the map cannot say. It is judged by coverage and by the guesses.
204215 const loads = new Set(impact?.get(LOADS_ONLY) ?? []);
205216 const loadRun = diffed.filter((f) => loads.has(f));
206 // A change the map cannot see (a src file no test executes, one loading
207 // runs, or a file outside src/ and test/) is judged by the guesses for
208 // every test.
209 const blind =
210 reached !== undefined &&
211 (reached.unreached.length > 0 ||
212 loadRun.length > 0 ||
213 diffed.some(
214 (f) =>
215 !f.startsWith("src/") &&
216 !f.startsWith("test/") &&
217 !f.endsWith("test-times.json") &&
218 !f.endsWith("test-impact.json"),
219 ));
217 // A doc is read by the tests its readers name, and by whoever executes a
218 // src file that reads it (#14845); one they cannot bound is not seen.
219 const docs = new Map<string, DocReaders>();
220 for (const f of diffed) {
221 const readers = isDoc(f) ? input.docReaders?.(f) : undefined;
222 if (readers !== undefined) docs.set(f, readers);
223 }
224 const docOf = new Map<string, string>();
225 for (const [doc, readers] of docs) for (const source of readers.sources) if (!docOf.has(source)) docOf.set(source, doc);
226 // A src file that reads a doc while the suite loads: who uses what it read the map cannot say.
227 const docLoads = [...docOf.keys()].filter((source) => loads.has(source));
228 // A change the map cannot see (a src file no test executes, or a file
229 // outside src/ and test/ nothing bounds) makes the plan blind: every
230 // test is guessed against it, and every quick test runs.
231 if (reached !== undefined) {
232 for (const f of reached.unreached) blindBy.set(f, "no test executes it");
233 for (const f of diffed)
234 if (
235 !f.startsWith("src/") &&
236 !f.startsWith("test/") &&
237 !f.endsWith("test-times.json") &&
238 !f.endsWith("test-impact.json") &&
239 !docs.has(f)
240 )
241 blindBy.set(f, isDoc(f) && input.docReaders !== undefined ? "a doc its readers cannot bound" : "outside src/ and test/");
242 }
243 const blind = blindBy.size > 0;
244 // What loading runs is guessed by each test's imports, without blinding the rest (#14845).
245 const guessing = blind || loadRun.length > 0 || docLoads.length > 0;
220246 // A test the map has never seen (a new one) is judged by the guesses.
221247 const guessed = (test: string): boolean =>
222 impact === undefined || blind || !impact.has(test);
248 impact === undefined || guessing || !impact.has(test);
249 const quickAll = (test: string): boolean => impact === undefined || blind || !impact.has(test);
223250 const take = (test: string, reason: string): void => {
224251 if (tests.has(test) && !why.has(test)) why.set(test, reason);
225252 };
226253 const changed = new Set(diffed);
227254 for (const file of changed) take(file, "changed");
255 // With no map, a doc's sources are guessed for as if they had changed.
256 if (impact === undefined) for (const source of docOf.keys()) changed.add(source);
228257 // The check's own tables can break no rule: a diff of only those runs none (#3579).
229258 if (diffed.some((f) => !f.endsWith("test-times.json") && !f.endsWith("test-impact.json"))) {
230259 for (const test of RULE_TESTS) take(test, "repo-wide rule");
@@ -236,11 +265,14 @@ export function fastSet(input: FastInput): FastPlan {
236265 // for every change the map can see, by what it executes; it is guessed
237266 // only against the changes the map cannot see, so one unreached file
238267 // does not bring back every importer of the rest of the diff.
239 const unseen = new Set(
240 diffed.filter(
241 (f) => reached === undefined || reached.unreached.includes(f) || !f.startsWith("src/") || loads.has(f),
268 const unseen = new Set([
269 ...diffed.filter(
270 (f) =>
271 !docs.has(f) &&
272 (reached === undefined || reached.unreached.includes(f) || !f.startsWith("src/") || loads.has(f)),
242273 ),
243 );
274 ...docLoads,
275 ]);
244276 const lookFor = (test: string): ReadonlySet<string> =>
245277 impact === undefined || !impact.has(test) ? changed : unseen;
246278 const namesFor = (from: ReadonlySet<string>): Map<string, string> => {
@@ -258,6 +290,10 @@ export function fastSet(input: FastInput): FastPlan {
258290 if (reached)
259291 for (const [test, source] of reached.tests)
260292 take(test, test === LOADS_ONLY ? `loads ${source}` : `executes ${source}`);
293 for (const [doc, mine] of docs) for (const test of mine.tests) take(test, `reads ${doc}`);
294 if (impact && docOf.size > 0)
295 for (const [test, source] of reach([...docOf.keys()], impact).tests)
296 take(test, `reads ${docOf.get(source)!} through ${source}`);
261297 for (const test of input.tests) {
262298 if (!guessed(test)) continue;
263299 const names = lookFor(test) === changed ? allNames : unseenNames;
@@ -290,19 +326,62 @@ export function fastSet(input: FastInput): FastPlan {
290326 for (const test of input.learned ?? []) take(test, "failed on land before");
291327 for (const test of input.tests) {
292328 const seconds = input.times.get(test);
293 if (guessed(test) && seconds !== undefined && seconds < QUICK_SECONDS)
329 if (quickAll(test) && seconds !== undefined && seconds < QUICK_SECONDS)
294330 take(test, "quick");
295331 }
296332
297333 const nChanged = diffed.length;
298334 return finish(
299335 reached
300 ? `by coverage${blind ? " and imports" : ""}: ${reachLine(nChanged, reached.tests.size, reached.unreached)}` +
336 ? `by coverage${guessing ? " and imports" : ""}: ${reachLine(nChanged, reached.tests.size, reached.unreached)}` +
301337 (loadRun.length === 0 ? "" : `; the suite's loading runs ${loadRun.join(", ")}`)
302338 : "by imports (no impact map)",
303339 );
304340 }
305341
342 /** What each kind of reason a {@link FastPlan} gives starts with, and its name in the breakdown. */
343 const REASON_KINDS: readonly (readonly [string, string])[] = [
344 ["changed", "changed"],
345 ["executes ", "executes"],
346 ["loads ", "loads"],
347 ["reads ", "reads"],
348 ["quick", "quick"],
349 ["tree scan: ", "scan"],
350 ["named for ", "named"],
351 ["imports ", "imports"],
352 ["repo-wide rule", "repo-wide rule"],
353 ["page budget: ", "page budget"],
354 ["failed on land before", "learned"],
355 ["full: ", "full"],
356 ];
357
358 /**
359 * The selection by reason, for the check's log (#14845): `by reason: 40
360 * quick, 12 executes, 11 repo-wide rule`, the largest first. A pull that
361 * ran wide says which rule took it there.
362 */
363 export function reasonLine(plan: Pick<FastPlan, "files" | "why">): string {
364 const counts = new Map<string, number>();
365 for (const file of plan.files) {
366 const why = plan.why.get(file) ?? "";
367 const kind = REASON_KINDS.find(([start]) => why.startsWith(start))?.[1] ?? why;
368 counts.set(kind, (counts.get(kind) ?? 0) + 1);
369 }
370 const parts = [...counts].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([kind, n]) => `${n} ${kind}`);
371 return parts.length === 0 ? "by reason: none" : `by reason: ${parts.join(", ")}`;
372 }
373
374 /**
375 * Why the plan is blind, for the log, or `undefined` when it is not:
376 * `blind (every quick test, guesses for every test) by docs/x.md (a doc its
377 * readers cannot bound), src/y.ts (no test executes it)`.
378 */
379 export function blindLine(plan: Pick<FastPlan, "blindBy">): string | undefined {
380 if (plan.blindBy.size === 0) return undefined;
381 const by = [...plan.blindBy].map(([file, why]) => `${file} (${why})`).join(", ");
382 return `blind (every quick test, guesses for every test) by ${by}`;
383 }
384
306385 /** `220` → `3m40s`, `45` → `45s`. */
307386 export function duration(seconds: number): string {
308387 const whole = Math.max(0, Math.round(seconds));
test/check-script-readers.test.ts +40 −2
@@ -6,12 +6,16 @@
66 */
77 import { describe, expect, test } from "bun:test";
88 import { needsFull } from "../src/checks/impact.ts";
9 import { scriptReaders, type ScriptTree } from "../src/checks/scans.ts";
9 import { docReaders, scriptReaders, type ScriptTree } from "../src/checks/scans.ts";
1010 import { fastSet, importsOf } from "../src/checks/tiers.ts";
1111 import { gateFiles } from "../scripts/gate.ts";
1212 import { readersIn } from "../scripts/script-readers.ts";
1313
14 function treeOf(files: Record<string, string>, own: string[] = ["scripts/gate.ts", "test/helpers/preload.ts"]): ScriptTree {
14 function treeOf(
15 files: Record<string, string>,
16 own: string[] = ["scripts/gate.ts", "test/helpers/preload.ts"],
17 preloads?: string[],
18 ): ScriptTree {
1519 const all = Object.keys(files);
1620 return {
1721 beside: all.filter((f) => /^(test|scripts|bench)\//.test(f)),
@@ -19,6 +23,7 @@ function treeOf(files: Record<string, string>, own: string[] = ["scripts/gate.ts
1923 read: (file) => files[file],
2024 imports: (file) => (files[file] === undefined ? [] : importsOf(files[file]!, file)),
2125 own,
26 ...(preloads === undefined ? {} : { preloads }),
2227 };
2328 }
2429
@@ -91,3 +96,36 @@ describe("the tests that can read a script", () => {
9196 expect(real(script)).toBeUndefined();
9297 });
9398 });
99
100 describe("the tests that can read a doc (#14845)", () => {
101 const files: Record<string, string> = {
102 "scripts/gate.ts": 'const readme = readFileSync(join(root, "README.md"), "utf8");',
103 "test/helpers/preload.ts": 'const notes = readFileSync(join(root, "docs", "NOTES.md"), "utf8");',
104 "test/threats.test.ts": 'const text = readFileSync(join(import.meta.dir, "..", "docs", "THREATS.md"), "utf8");',
105 "test/agents.test.ts": 'const guide = Bun.file(new URL("../AGENTS.md", import.meta.url));',
106 "test/gate.test.ts": 'import { treeGate } from "../scripts/gate.ts";',
107 "test/b.test.ts": "// docs/THREATS.md is only spoken of here\nexport {};",
108 "src/view/docs.ts": 'const DOCS = join(import.meta.dir, "..", "..", "docs");',
109 "src/web/link.ts": 'export const link = "/docs/features/absorb.md";',
110 "src/x.ts": "// see docs/THREATS.md\nexport const x = 1;",
111 "Dockerfile": "COPY docs/DESIGN.md /app/docs/DESIGN.md",
112 };
113 const tree = treeOf(files, ["scripts/gate.ts", "test/helpers/preload.ts"], ["test/helpers/preload.ts"]);
114 const tests = ["test/threats.test.ts", "test/agents.test.ts", "test/gate.test.ts", "test/b.test.ts"];
115 const readers = (doc: string) => docReaders(doc, tree, tests);
116
117 test("a test naming it reads it, a comment does not, and src/ listing its directory is a source", () => {
118 expect(readers("docs/THREATS.md")).toEqual({ tests: ["test/threats.test.ts"], sources: ["src/view/docs.ts"] });
119 expect(readers("AGENTS.md")).toEqual({ tests: ["test/agents.test.ts"], sources: [] });
120 expect(readers("docs/features/absorb.md")).toEqual({
121 tests: [],
122 sources: ["src/view/docs.ts", "src/web/link.ts"],
123 });
124 });
125
126 test("the check's own script reading it is followed to its importers; the preload or a manifest unbounds it", () => {
127 expect(readers("README.md")).toEqual({ tests: ["test/gate.test.ts"], sources: [] });
128 expect(readers("docs/NOTES.md")).toBeUndefined();
129 expect(readers("docs/DESIGN.md")).toBeUndefined();
130 });
131 });
test/check-tiers.test.ts +67
@@ -21,8 +21,10 @@ import type { Repo } from "../src/core/repos.ts";
2121 import { createUser, type User } from "../src/core/users.ts";
2222 import { readRunLog } from "../src/runs/log.ts";
2323 import {
24 blindLine,
2425 duration,
2526 fastSet,
27 reasonLine,
2628 importsOf,
2729 landVerdict,
2830 PAGE_BUDGET_TESTS,
@@ -223,6 +225,71 @@ describe("the fast set, by measured coverage", () => {
223225 expect(plan.files).toContain("test/deep.test.ts");
224226 });
225227
228 test("a doc its readers bound selects them and does not blind the rest (#14845)", () => {
229 const input = {
230 ...base,
231 docReaders: (doc: string) =>
232 doc === "docs/THREATS.md" ? { tests: ["test/deep.test.ts"], sources: ["src/web/phone.ts"] } : undefined,
233 };
234 const alone = fastSet({ ...input, changed: ["src/snapshot/store.ts"] });
235 const plan = fastSet({ ...input, changed: ["src/snapshot/store.ts", "docs/THREATS.md"] });
236 expect(plan.files).toEqual([...alone.files, "test/web-phone.test.ts", "test/deep.test.ts"]);
237 // tiny is quick, but nothing is blind: it stays out.
238 expect(plan.files).not.toContain("test/tiny.test.ts");
239 expect(plan.why.get("test/deep.test.ts")).toBe("reads docs/THREATS.md");
240 expect(plan.why.get("test/web-phone.test.ts")).toBe("reads docs/THREATS.md through src/web/phone.ts");
241 expect(plan.blindBy.size).toBe(0);
242 expect(blindLine(plan)).toBeUndefined();
243 });
244
245 test("a doc nothing bounds, and any other file the map cannot see, blinds the plan and is named (#14845)", () => {
246 const unbound = fastSet({ ...base, docReaders: () => undefined, changed: ["src/snapshot/store.ts", "AGENTS.md"] });
247 expect(unbound.why.get("test/tiny.test.ts")).toBe("quick");
248 expect([...unbound.blindBy]).toEqual([["AGENTS.md", "a doc its readers cannot bound"]]);
249 // With no reader asked, a doc is a file outside src/ and test/, as before.
250 const unasked = fastSet({ ...base, changed: ["docs/THREATS.md", "src/web/x.ts", "static/app.js"] });
251 expect([...unasked.blindBy]).toEqual([
252 ["src/web/x.ts", "no test executes it"],
253 ["docs/THREATS.md", "outside src/ and test/"],
254 ["static/app.js", "outside src/ and test/"],
255 ]);
256 expect(blindLine(unasked)).toBe(
257 "blind (every quick test, guesses for every test) by src/web/x.ts (no test executes it), " +
258 "docs/THREATS.md (outside src/ and test/), static/app.js (outside src/ and test/)",
259 );
260 });
261
262 test("a file the suite's loading runs is guessed by imports but runs no quick test it does not reach (#14845)", () => {
263 const plan = fastSet({
264 ...base,
265 tests: [...base.tests, "test/suite-loads.test.ts"],
266 impact: new Map([...impact, ["test/suite-loads.test.ts", ["src/snapshot/store.ts"]]]),
267 imports: (file) => (file === "test/web-phone.test.ts" ? ["src/snapshot/store.ts"] : imports(file)),
268 changed: ["src/snapshot/store.ts"],
269 });
270 expect(plan.why.get("test/web-phone.test.ts")).toBe("imports src/snapshot/store.ts");
271 expect(plan.files).not.toContain("test/tiny.test.ts");
272 expect(plan.blindBy.size).toBe(0);
273 });
274
275 test("the log's breakdown counts the selection by reason, the largest first (#14845)", () => {
276 const plan = fastSet({ ...base, changed: ["src/snapshot/store.ts", "static/app.js"] });
277 expect(reasonLine(plan)).toBe("by reason: 2 executes, 1 quick");
278 expect(reasonLine({ files: [], why: new Map() })).toBe("by reason: none");
279 const mixed = new Map([
280 ["a", "changed"],
281 ["b", "tree scan: lists the tree"],
282 ["c", "tree scan: declares a tree scan"],
283 ["d", "page budget: src/web/layout.ts changed"],
284 ["e", "named for src/x.ts"],
285 ["f", "repo-wide rule"],
286 ["g", "reads docs/THREATS.md through src/view/docs.ts"],
287 ]);
288 expect(reasonLine({ files: [...mixed.keys()], why: mixed })).toBe(
289 "by reason: 2 scan, 1 changed, 1 named, 1 page budget, 1 reads, 1 repo-wide rule",
290 );
291 });
292
226293 test("a test the map has never seen is picked by the hop rule", () => {
227294 const hop = fastSet({
228295 ...base,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs