Public: anyone can read this; the crew can change it.
A pull's test selection says why it chose each test, and a changed doc no longer blinds it to the impact map #14940
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14845-a-pull-s-test-selection-says-why-it-chos → main 2 commits fork
9 files
changed +422
−72
docs/DEVELOP.md +15
| @@ -128,6 +128,21 @@ which only that scan's script and test read | ||
| 128 | 128 | (`test/check-allow-lists.test.ts` holds each to it). With no map yet the |
| 129 | 129 | line says `by imports (no impact map)`. |
| 130 | 130 | |
| 131 | Under the plan line the log says why it went as wide as it did (#14845): | |
| 132 | `by reason: 40 quick, 12 executes, 11 repo-wide rule` (`reasonLine`), and, | |
| 133 | when a changed file the map cannot see made the plan blind (every quick | |
| 134 | test runs, and every test is guessed against that file), `blind … by | |
| 135 | static/app.js (outside src/ and test/), src/x.ts (no test executes it)` | |
| 136 | (`blindLine`). A changed doc (`.md` under `docs/` or at the root) does | |
| 137 | not blind it: it selects the tests that name it, directly or through a | |
| 138 | helper (`docReaders`, found as a script's readers are), and the tests the | |
| 139 | map says execute a `src/` file that names it or lists its directory | |
| 140 | (`src/view/docs.ts`, `src/checks/resilience.ts`). Only a doc the | |
| 141 | suite's preload reads, or a manifest, chart or the Dockerfile names | |
| 142 | (`AGENTS.md`, `docs/DESIGN.md` today), is still blind. A file the suite's | |
| 143 | loading runs is guessed by imports, as above, but no longer runs every | |
| 144 | quick test. | |
| 145 | ||
| 131 | 146 | Kept results (#12178). A selected file whose inputs a default-branch |
| 132 | 147 | measuring run already proved green is not run again: the log says |
| 133 | 148 | `kept from <commit>`. The key is the Bun, the tooling files |
scripts/fast.ts +8 −3
| @@ -19,9 +19,9 @@ | ||
| 19 | 19 | |
| 20 | 20 | import { readFileSync } from "node:fs"; |
| 21 | 21 | import { GATE_FILE, gateListAdds, handedImpact, needsFull } from "../src/checks/impact.ts"; |
| 22 | import { fastSet, duration } from "../src/checks/tiers.ts"; | |
| 22 | import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts"; | |
| 23 | 23 | import { importsReader, readImpact, scansIn } from "./tier-input.ts"; |
| 24 | import { readersIn } from "./script-readers.ts"; | |
| 24 | import { docReadersIn, readersIn } from "./script-readers.ts"; | |
| 25 | 25 | import { CHECK_CACHE } from "../src/checks/cache.ts"; |
| 26 | 26 | import { gateFiles } from "./gate.ts"; |
| 27 | 27 | import { overlayTimes, parseTimes } from "./shards.ts"; |
| @@ -178,15 +178,20 @@ function main(): number { | ||
| 178 | 178 | imports, |
| 179 | 179 | scans: scansIn(tests, imports), |
| 180 | 180 | readers, |
| 181 | docReaders: docReadersIn(tests), | |
| 181 | 182 | times: loadTimes(), |
| 182 | 183 | }); |
| 183 | 184 | const out = report(plan, needsFull(changed, gateAdds, (script) => readers(script) !== undefined), args.format); |
| 184 | 185 | process.stdout.write(out.stdout); |
| 185 | 186 | for (const note of out.notes) console.error(note); |
| 186 | if (out.code === 0) | |
| 187 | if (out.code === 0) { | |
| 187 | 188 | console.error( |
| 188 | 189 | `fast: ${plan.files.length} of ${tests.length} files for ${changed.length} changed, expected ${duration(plan.expected)}${plan.overBudget ? " (over the 5m budget)" : ""}`, |
| 189 | 190 | ); |
| 191 | console.error(`fast: ${reasonLine(plan)}`); | |
| 192 | const blind = blindLine(plan); | |
| 193 | if (blind !== undefined) console.error(`fast: ${blind}`); | |
| 194 | } | |
| 190 | 195 | return out.code; |
| 191 | 196 | } |
| 192 | 197 |
scripts/script-readers.ts +31 −9
| @@ -7,7 +7,7 @@ | ||
| 7 | 7 | |
| 8 | 8 | import { readdirSync, readFileSync } from "node:fs"; |
| 9 | 9 | import { join } from "node:path"; |
| 10 | import { scriptReaders, type ScriptTree } from "../src/checks/scans.ts"; | |
| 10 | import { docReaders, scriptReaders, type DocReaders, type ScriptTree } from "../src/checks/scans.ts"; | |
| 11 | 11 | import { importsOf } from "../src/checks/tiers.ts"; |
| 12 | 12 | |
| 13 | 13 | const ROOT = new URL("..", import.meta.url).pathname; |
| @@ -46,6 +46,11 @@ function withinFiles(): string[] { | ||
| 46 | 46 | return found; |
| 47 | 47 | } |
| 48 | 48 | |
| 49 | /** The preloads `bunfig.toml` lists. */ | |
| 50 | function preloadsOf(read: (file: string) => string | undefined): string[] { | |
| 51 | return [...(read("bunfig.toml") ?? "").matchAll(/["']\.\/([A-Za-z0-9_./-]+\.[cm]?[jt]sx?)["']/g)].map((match) => match[1]!); | |
| 52 | } | |
| 53 | ||
| 49 | 54 | /** The scripts a manifest's check commands name, and the preloads `bunfig.toml` lists. */ |
| 50 | 55 | function ownOf(read: (file: string) => string | undefined): string[] { |
| 51 | 56 | const found = new Set<string>(); |
| @@ -56,12 +61,12 @@ function ownOf(read: (file: string) => string | undefined): string[] { | ||
| 56 | 61 | // An unreadable manifest names no script here; a change to it runs the full tier anyway. |
| 57 | 62 | } |
| 58 | 63 | for (const match of JSON.stringify(check ?? {}).matchAll(/\bscripts\/[A-Za-z0-9_./-]+/g)) found.add(match[0]); |
| 59 | for (const match of (read("bunfig.toml") ?? "").matchAll(/["']\.\/([A-Za-z0-9_./-]+\.[cm]?[jt]sx?)["']/g)) found.add(match[1]!); | |
| 64 | for (const preload of preloadsOf(read)) found.add(preload); | |
| 60 | 65 | return [...found]; |
| 61 | 66 | } |
| 62 | 67 | |
| 63 | /** `scriptReaders` over the working tree, for a `fastSet` input's `readers`. The tree is read at the first ask. */ | |
| 64 | export function readersIn(tests: readonly string[]): (script: string) => string[] | undefined { | |
| 68 | /** The working tree as `scriptReaders` and `docReaders` read it, read at the first ask. */ | |
| 69 | function workingTree(): () => ScriptTree { | |
| 65 | 70 | let tree: ScriptTree | undefined; |
| 66 | 71 | const texts = new Map<string, string | undefined>(); |
| 67 | 72 | const read = (file: string): string | undefined => { |
| @@ -89,14 +94,31 @@ export function readersIn(tests: readonly string[]): (script: string) => string[ | ||
| 89 | 94 | } |
| 90 | 95 | return found; |
| 91 | 96 | }; |
| 92 | return (script) => { | |
| 93 | tree ??= { | |
| 97 | return () => | |
| 98 | (tree ??= { | |
| 94 | 99 | beside: BESIDE.flatMap(filesUnder), |
| 95 | 100 | within: withinFiles(), |
| 96 | 101 | read, |
| 97 | 102 | imports: importsAt, |
| 98 | 103 | own: ownOf(read), |
| 99 | }; | |
| 100 | return scriptReaders(script, tree, tests); | |
| 101 | }; | |
| 104 | preloads: preloadsOf(read), | |
| 105 | }); | |
| 106 | } | |
| 107 | ||
| 108 | /** | |
| 109 | * `scriptReaders` over the working tree, for a `fastSet` input's `readers`. The tree is read at the | |
| 110 | * first ask. | |
| 111 | */ | |
| 112 | export function readersIn(tests: readonly string[]): (script: string) => string[] | undefined { | |
| 113 | const tree = workingTree(); | |
| 114 | return (script) => scriptReaders(script, tree(), tests); | |
| 115 | } | |
| 116 | ||
| 117 | /** | |
| 118 | * `docReaders` over the working tree, for a `fastSet` input's `docReaders`. The tree is read at the | |
| 119 | * first ask. | |
| 120 | */ | |
| 121 | export function docReadersIn(tests: readonly string[]): (doc: string) => DocReaders | undefined { | |
| 122 | const tree = workingTree(); | |
| 123 | return (doc) => docReaders(doc, tree(), tests); | |
| 102 | 124 | } |
scripts/test-shards.ts +7 −2
| @@ -113,9 +113,9 @@ import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, brea | ||
| 113 | 113 | import { cgroupMemory, summaryLines } from "./gate-summary.ts"; |
| 114 | 114 | import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts"; |
| 115 | 115 | import { importsReader, readImpact, scansIn } from "./tier-input.ts"; |
| 116 | import { readersIn } from "./script-readers.ts"; | |
| 116 | import { docReadersIn, readersIn } from "./script-readers.ts"; | |
| 117 | 117 | import { shardHome, spaceLine } from "../src/checks/disk_full.ts"; |
| 118 | import { fastSet, duration } from "../src/checks/tiers.ts"; | |
| 118 | import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts"; | |
| 119 | 119 | import { CHECK_CACHE } from "../src/checks/cache.ts"; |
| 120 | 120 | import { parseCheck } from "../src/apps/manifest.ts"; |
| 121 | 121 | import { FLAKY_ALONE, onlyFiles } from "../src/checks/verify.ts"; |
| @@ -373,6 +373,7 @@ if (tier === "fast") { | ||
| 373 | 373 | imports, |
| 374 | 374 | scans: scansIn(files, imports), |
| 375 | 375 | readers: readersIn(files), |
| 376 | docReaders: docReadersIn(files), | |
| 376 | 377 | times: loadTimes(), |
| 377 | 378 | learned: (learnedArg.value ?? "").split(",").filter((f) => f !== ""), |
| 378 | 379 | }); |
| @@ -380,6 +381,10 @@ if (tier === "fast") { | ||
| 380 | 381 | `test-shards: fast tier, ${plan.files.length} of ${files.length} files for ${changed.length} changed, ${plan.line} ` + |
| 381 | 382 | `(expected ${duration(plan.expected)}${plan.overBudget ? ", over the 5m budget" : ""})`, |
| 382 | 383 | ); |
| 384 | // Why it went as wide as it did, before the list (#14845). | |
| 385 | console.log(`test-shards: ${reasonLine(plan)}`); | |
| 386 | const blind = blindLine(plan); | |
| 387 | if (blind !== undefined) console.log(`test-shards: ${blind}`); | |
| 383 | 388 | for (const file of plan.files) |
| 384 | 389 | console.log(` ${file}: ${plan.why.get(file)}`); |
| 385 | 390 | // A diff no test reaches (a timing table, a doc) is judged by the |
src/checks/impact.ts +7
| @@ -347,6 +347,13 @@ export function needsFull( | ||
| 347 | 347 | */ |
| 348 | 348 | const MAPPED_KERNEL: ReadonlySet<string> = new Set(["src/kernel/vitals.ts"]); |
| 349 | 349 | |
| 350 | /** | |
| 351 | * Whether a changed file is a doc `docReaders` (scans.ts) judges: Markdown under docs/ or at the | |
| 352 | * root (#14845). | |
| 353 | */ | |
| 354 | export const isDoc = (file: string): boolean => | |
| 355 | file.endsWith(".md") && (file.startsWith("docs/") || !file.includes("/")); | |
| 356 | ||
| 350 | 357 | export interface Reach { |
| 351 | 358 | /** Tests whose executed files meet the diff, with the first file that did. */ |
| 352 | 359 | readonly tests: ReadonlyMap<string, string>; |
src/checks/scans.ts +145 −33
| @@ -179,6 +179,11 @@ export interface ScriptTree { | ||
| 179 | 179 | readonly imports: (file: string) => readonly string[]; |
| 180 | 180 | /** The scripts the check's commands name (`forge.json`) and the preloads (`bunfig.toml`). */ |
| 181 | 181 | readonly own: readonly string[]; |
| 182 | /** | |
| 183 | * The preloads `bunfig.toml` lists, a part of `own`: every test process runs them first, so a | |
| 184 | * doc one of them reads is read by every test. Absent: all of `own` counts. | |
| 185 | */ | |
| 186 | readonly preloads?: readonly string[]; | |
| 182 | 187 | } |
| 183 | 188 | |
| 184 | 189 | /** A whole-line comment: a script named there is read by nothing. */ |
| @@ -208,9 +213,106 @@ function namesFor(target: string): string[] { | ||
| 208 | 213 | return target.startsWith("scripts/") ? [...path, `"${base}`, `'${base}`, `\`${base}`] : path; |
| 209 | 214 | } |
| 210 | 215 | |
| 216 | /** | |
| 217 | * How a file's text names a doc (`docs/**.md`, or one at the root): as | |
| 218 | * {@link namesFor} names a script, plus `/<file>` (`"../AGENTS.md"`). | |
| 219 | */ | |
| 220 | function docNamesFor(doc: string): string[] { | |
| 221 | const parts = doc.split("/"); | |
| 222 | const base = parts[parts.length - 1]!; | |
| 223 | const stem = base.replace(/\.[A-Za-z0-9]+$/, ""); | |
| 224 | const dir = parts[parts.length - 2]; | |
| 225 | return [...(dir === undefined ? [] : [`${dir}/${stem}`]), `"${base}`, `'${base}`, `\`${base}`, `/${base}`]; | |
| 226 | } | |
| 227 | ||
| 228 | /** | |
| 229 | * How `src/` code names the directories a doc sits in, as a whole string or | |
| 230 | * one a name is joined onto (`join(root, "docs")`, `` `docs/${name}` ``): | |
| 231 | * what lists a directory reads every doc in it. A test that lists one is a | |
| 232 | * tree scan already ({@link treeScans}), so only `src/` is asked. | |
| 233 | */ | |
| 234 | function docDirNamesFor(doc: string): string[] { | |
| 235 | const parts = doc.split("/"); | |
| 236 | const names: string[] = []; | |
| 237 | for (let i = 1; i < parts.length; i++) | |
| 238 | for (const at of new Set([parts.slice(0, i).join("/"), parts[i - 1]!])) | |
| 239 | names.push(`"${at}"`, `'${at}'`, `\`${at}\``, `"${at}/"`, `'${at}/'`, `\`${at}/\``, `\`${at}/\${`); | |
| 240 | return names; | |
| 241 | } | |
| 242 | ||
| 211 | 243 | /** Each tree's files as code, read once however many scripts a diff holds. */ |
| 212 | 244 | const codes = new WeakMap<ScriptTree, Map<string, string>>(); |
| 213 | 245 | |
| 246 | /** A tree's files as code ({@link codeOf}), kept per tree. */ | |
| 247 | function codeIn(tree: ScriptTree): (file: string) => string { | |
| 248 | let code = codes.get(tree); | |
| 249 | if (code === undefined) codes.set(tree, (code = new Map())); | |
| 250 | return (file) => { | |
| 251 | let text = code.get(file); | |
| 252 | if (text === undefined) { | |
| 253 | const source = tree.read(file); | |
| 254 | text = source === undefined ? "" : codeOf(file, source); | |
| 255 | code.set(file, text); | |
| 256 | } | |
| 257 | return text; | |
| 258 | }; | |
| 259 | } | |
| 260 | ||
| 261 | /** The check's own: its commands' scripts and the preload, with all they import. */ | |
| 262 | function ownOf(tree: ScriptTree, roots: readonly string[] = tree.own): Set<string> { | |
| 263 | const own = new Set<string>(); | |
| 264 | for (const stack = [...roots]; stack.length > 0; ) { | |
| 265 | const file = stack.pop()!; | |
| 266 | if (own.has(file)) continue; | |
| 267 | own.add(file); | |
| 268 | stack.push(...tree.imports(file)); | |
| 269 | } | |
| 270 | return own; | |
| 271 | } | |
| 272 | ||
| 273 | /** | |
| 274 | * The closure, from `target`, over "imports it or names it" among the | |
| 275 | * files beside the source, with the target itself; `undefined` when one of | |
| 276 | * them is the check's own. | |
| 277 | */ | |
| 278 | function besideReaders( | |
| 279 | target: string, | |
| 280 | targetNames: readonly string[], | |
| 281 | tree: ScriptTree, | |
| 282 | codeAt: (file: string) => string, | |
| 283 | own: ReadonlySet<string>, | |
| 284 | ): Set<string> | undefined { | |
| 285 | const readers = new Set<string>([target]); | |
| 286 | const named = new Set<string>(targetNames); | |
| 287 | const names = [...named]; | |
| 288 | // A file is asked only the names added since it was last asked: one the | |
| 289 | // old names missed misses them again. | |
| 290 | const asked = new Map<string, number>(); | |
| 291 | for (let grew = true; grew; ) { | |
| 292 | grew = false; | |
| 293 | for (const file of tree.beside) { | |
| 294 | if (readers.has(file)) continue; | |
| 295 | const from = asked.get(file) ?? 0; | |
| 296 | asked.set(file, names.length); | |
| 297 | if (!tree.imports(file).some((path) => readers.has(path))) { | |
| 298 | const text = codeAt(file); | |
| 299 | let hit = false; | |
| 300 | for (let i = from; i < names.length && !hit; i++) hit = text.includes(names[i]!); | |
| 301 | if (!hit) continue; | |
| 302 | } | |
| 303 | if (own.has(file)) return undefined; | |
| 304 | readers.add(file); | |
| 305 | for (const name of namesFor(file)) | |
| 306 | if (!named.has(name)) { | |
| 307 | named.add(name); | |
| 308 | names.push(name); | |
| 309 | } | |
| 310 | grew = true; | |
| 311 | } | |
| 312 | } | |
| 313 | return readers; | |
| 314 | } | |
| 315 | ||
| 214 | 316 | /** |
| 215 | 317 | * The gate's tests that can read a changed file under `scripts/`, or |
| 216 | 318 | * `undefined` when that cannot be bounded and the change runs the full tier |
| @@ -235,40 +337,11 @@ const codes = new WeakMap<ScriptTree, Map<string, string>>(); | ||
| 235 | 337 | * it, as they do behind every guess of the fast tier. |
| 236 | 338 | */ |
| 237 | 339 | export function scriptReaders(script: string, tree: ScriptTree, tests: readonly string[]): string[] | undefined { |
| 238 | let code = codes.get(tree); | |
| 239 | if (code === undefined) codes.set(tree, (code = new Map())); | |
| 240 | const codeAt = (file: string): string => { | |
| 241 | let text = code.get(file); | |
| 242 | if (text === undefined) { | |
| 243 | const source = tree.read(file); | |
| 244 | text = source === undefined ? "" : codeOf(file, source); | |
| 245 | code.set(file, text); | |
| 246 | } | |
| 247 | return text; | |
| 248 | }; | |
| 249 | // The check's own: its commands' scripts and the preload, with all they import. | |
| 250 | const own = new Set<string>(); | |
| 251 | for (const stack = [...tree.own]; stack.length > 0; ) { | |
| 252 | const file = stack.pop()!; | |
| 253 | if (own.has(file)) continue; | |
| 254 | own.add(file); | |
| 255 | stack.push(...tree.imports(file)); | |
| 256 | } | |
| 340 | const codeAt = codeIn(tree); | |
| 341 | const own = ownOf(tree); | |
| 257 | 342 | if (own.has(script)) return undefined; |
| 258 | const readers = new Set<string>([script]); | |
| 259 | const names = new Set<string>(namesFor(script)); | |
| 260 | for (let grew = true; grew; ) { | |
| 261 | grew = false; | |
| 262 | for (const file of tree.beside) { | |
| 263 | if (readers.has(file)) continue; | |
| 264 | const text = codeAt(file); | |
| 265 | if (!tree.imports(file).some((path) => readers.has(path)) && ![...names].some((name) => text.includes(name))) continue; | |
| 266 | if (own.has(file)) return undefined; | |
| 267 | readers.add(file); | |
| 268 | for (const name of namesFor(file)) names.add(name); | |
| 269 | grew = true; | |
| 270 | } | |
| 271 | } | |
| 343 | const readers = besideReaders(script, namesFor(script), tree, codeAt, own); | |
| 344 | if (readers === undefined) return undefined; | |
| 272 | 345 | const scripts = [...readers].filter((file) => file.startsWith("scripts/")); |
| 273 | 346 | const scriptNames = scripts.flatMap(namesFor); |
| 274 | 347 | for (const file of tree.within) { |
| @@ -279,3 +352,42 @@ export function scriptReaders(script: string, tree: ScriptTree, tests: readonly | ||
| 279 | 352 | const gated = new Set(tests); |
| 280 | 353 | return [...readers].filter((file) => gated.has(file)).sort(); |
| 281 | 354 | } |
| 355 | ||
| 356 | /** Who can read a changed doc: tests directly, and the `src/` files that name it. */ | |
| 357 | export interface DocReaders { | |
| 358 | /** Gated tests that read the doc themselves, or through a helper beside them. */ | |
| 359 | readonly tests: readonly string[]; | |
| 360 | /** `src/` files whose code names the doc or its directory: whoever executes one can read it. */ | |
| 361 | readonly sources: readonly string[]; | |
| 362 | } | |
| 363 | ||
| 364 | /** | |
| 365 | * Who can read a changed doc (#14845), found as {@link scriptReaders} | |
| 366 | * finds a script's readers, with one difference: a `src/` file that names | |
| 367 | * the doc or lists its directory (src/view/docs.ts, src/checks/resilience.ts) | |
| 368 | * does not unbound it, it is a source, and the impact map says which tests | |
| 369 | * execute it. The check's own scripts reading a doc is no test's verdict | |
| 370 | * (gate.ts reads the README for its own rule, and runs on every check), so | |
| 371 | * they are followed as readers like any helper; only the suite's preload, | |
| 372 | * which every test runs, unbounds one. `undefined` then, or when a file | |
| 373 | * outside `src/` and beside the source names it (a manifest, a chart): the | |
| 374 | * doc is judged as before, by every guess. | |
| 375 | */ | |
| 376 | export function docReaders(doc: string, tree: ScriptTree, tests: readonly string[]): DocReaders | undefined { | |
| 377 | const codeAt = codeIn(tree); | |
| 378 | const own = ownOf(tree, tree.preloads ?? tree.own); | |
| 379 | const names = docNamesFor(doc); | |
| 380 | const readers = besideReaders(doc, names, tree, codeAt, own); | |
| 381 | if (readers === undefined) return undefined; | |
| 382 | const inSource = [...names, ...docDirNamesFor(doc)]; | |
| 383 | const sources: string[] = []; | |
| 384 | for (const file of tree.within) { | |
| 385 | if (!file.startsWith("src/")) { | |
| 386 | if (names.some((name) => codeAt(file).includes(name))) return undefined; | |
| 387 | continue; | |
| 388 | } | |
| 389 | if (inSource.some((name) => codeAt(file).includes(name))) sources.push(file); | |
| 390 | } | |
| 391 | const gated = new Set(tests); | |
| 392 | return { tests: [...readers].filter((file) => gated.has(file)).sort(), sources: sources.sort() }; | |
| 393 | } |
src/checks/tiers.ts +102 −23
| @@ -11,7 +11,8 @@ | ||
| 11 | 11 | |
| 12 | 12 | import type { GateReason } from "./gate_reason.ts"; |
| 13 | 13 | import type { GateTests } from "./gate_tests.ts"; |
| 14 | import { GATE_FILE, LOADS_ONLY, needsFull, reach, reachLine, type Impact } from "./impact.ts"; | |
| 14 | import { GATE_FILE, LOADS_ONLY, isDoc, needsFull, reach, reachLine, type Impact } from "./impact.ts"; | |
| 15 | import type { DocReaders } from "./scans.ts"; | |
| 15 | 16 | |
| 16 | 17 | /** A test under this many seconds is always in the fast tier. */ |
| 17 | 18 | export const QUICK_SECONDS = 1; |
| @@ -148,6 +149,10 @@ export interface FastInput { | ||
| 148 | 149 | readonly scans?: ReadonlyMap<string, string>; |
| 149 | 150 | /** The tests that can read a file under scripts/ (`scriptReaders`); `undefined`: not bounded. */ |
| 150 | 151 | readonly readers?: (script: string) => readonly string[] | undefined; |
| 152 | /** | |
| 153 | * Who can read a changed doc (`docReaders`, #14845); `undefined`: not bounded. Absent: no doc is. | |
| 154 | */ | |
| 155 | readonly docReaders?: (doc: string) => DocReaders | undefined; | |
| 151 | 156 | } |
| 152 | 157 | |
| 153 | 158 | export interface FastPlan { |
| @@ -159,6 +164,11 @@ export interface FastPlan { | ||
| 159 | 164 | readonly overBudget: boolean; |
| 160 | 165 | /** The line for the log: what the plan is by, and what the diff reaches. */ |
| 161 | 166 | readonly line: string; |
| 167 | /** | |
| 168 | * The changed files that made the plan blind, with why (#14845): each turns on | |
| 169 | * every guess and every quick test. Empty when the map saw the whole diff. | |
| 170 | */ | |
| 171 | readonly blindBy: ReadonlyMap<string, string>; | |
| 162 | 172 | } |
| 163 | 173 | |
| 164 | 174 | /** |
| @@ -178,8 +188,9 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 178 | 188 | (sum, file) => sum + (input.times.get(file) ?? UNKNOWN_SECONDS), |
| 179 | 189 | 0, |
| 180 | 190 | ); |
| 181 | return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS, line }; | |
| 191 | return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS, line, blindBy }; | |
| 182 | 192 | }; |
| 193 | const blindBy = new Map<string, string>(); | |
| 183 | 194 | // What the map cannot bound runs everything. |
| 184 | 195 | const readers = new Map<string, readonly string[] | undefined>(); |
| 185 | 196 | const readersOf = (script: string): readonly string[] | undefined => { |
| @@ -203,28 +214,46 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 203 | 214 | // file the map cannot say. It is judged by coverage and by the guesses. |
| 204 | 215 | const loads = new Set(impact?.get(LOADS_ONLY) ?? []); |
| 205 | 216 | const loadRun = diffed.filter((f) => loads.has(f)); |
| 206 | // A change the map cannot see (a src file no test executes, one loading | |
| 207 | // runs, or a file outside src/ and test/) is judged by the guesses for | |
| 208 | // every test. | |
| 209 | const blind = | |
| 210 | reached !== undefined && | |
| 211 | (reached.unreached.length > 0 || | |
| 212 | loadRun.length > 0 || | |
| 213 | diffed.some( | |
| 214 | (f) => | |
| 215 | !f.startsWith("src/") && | |
| 216 | !f.startsWith("test/") && | |
| 217 | !f.endsWith("test-times.json") && | |
| 218 | !f.endsWith("test-impact.json"), | |
| 219 | )); | |
| 217 | // A doc is read by the tests its readers name, and by whoever executes a | |
| 218 | // src file that reads it (#14845); one they cannot bound is not seen. | |
| 219 | const docs = new Map<string, DocReaders>(); | |
| 220 | for (const f of diffed) { | |
| 221 | const readers = isDoc(f) ? input.docReaders?.(f) : undefined; | |
| 222 | if (readers !== undefined) docs.set(f, readers); | |
| 223 | } | |
| 224 | const docOf = new Map<string, string>(); | |
| 225 | for (const [doc, readers] of docs) for (const source of readers.sources) if (!docOf.has(source)) docOf.set(source, doc); | |
| 226 | // A src file that reads a doc while the suite loads: who uses what it read the map cannot say. | |
| 227 | const docLoads = [...docOf.keys()].filter((source) => loads.has(source)); | |
| 228 | // A change the map cannot see (a src file no test executes, or a file | |
| 229 | // outside src/ and test/ nothing bounds) makes the plan blind: every | |
| 230 | // test is guessed against it, and every quick test runs. | |
| 231 | if (reached !== undefined) { | |
| 232 | for (const f of reached.unreached) blindBy.set(f, "no test executes it"); | |
| 233 | for (const f of diffed) | |
| 234 | if ( | |
| 235 | !f.startsWith("src/") && | |
| 236 | !f.startsWith("test/") && | |
| 237 | !f.endsWith("test-times.json") && | |
| 238 | !f.endsWith("test-impact.json") && | |
| 239 | !docs.has(f) | |
| 240 | ) | |
| 241 | blindBy.set(f, isDoc(f) && input.docReaders !== undefined ? "a doc its readers cannot bound" : "outside src/ and test/"); | |
| 242 | } | |
| 243 | const blind = blindBy.size > 0; | |
| 244 | // What loading runs is guessed by each test's imports, without blinding the rest (#14845). | |
| 245 | const guessing = blind || loadRun.length > 0 || docLoads.length > 0; | |
| 220 | 246 | // A test the map has never seen (a new one) is judged by the guesses. |
| 221 | 247 | const guessed = (test: string): boolean => |
| 222 | impact === undefined || blind || !impact.has(test); | |
| 248 | impact === undefined || guessing || !impact.has(test); | |
| 249 | const quickAll = (test: string): boolean => impact === undefined || blind || !impact.has(test); | |
| 223 | 250 | const take = (test: string, reason: string): void => { |
| 224 | 251 | if (tests.has(test) && !why.has(test)) why.set(test, reason); |
| 225 | 252 | }; |
| 226 | 253 | const changed = new Set(diffed); |
| 227 | 254 | for (const file of changed) take(file, "changed"); |
| 255 | // With no map, a doc's sources are guessed for as if they had changed. | |
| 256 | if (impact === undefined) for (const source of docOf.keys()) changed.add(source); | |
| 228 | 257 | // The check's own tables can break no rule: a diff of only those runs none (#3579). |
| 229 | 258 | if (diffed.some((f) => !f.endsWith("test-times.json") && !f.endsWith("test-impact.json"))) { |
| 230 | 259 | for (const test of RULE_TESTS) take(test, "repo-wide rule"); |
| @@ -236,11 +265,14 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 236 | 265 | // for every change the map can see, by what it executes; it is guessed |
| 237 | 266 | // only against the changes the map cannot see, so one unreached file |
| 238 | 267 | // does not bring back every importer of the rest of the diff. |
| 239 | const unseen = new Set( | |
| 240 | diffed.filter( | |
| 241 | (f) => reached === undefined || reached.unreached.includes(f) || !f.startsWith("src/") || loads.has(f), | |
| 268 | const unseen = new Set([ | |
| 269 | ...diffed.filter( | |
| 270 | (f) => | |
| 271 | !docs.has(f) && | |
| 272 | (reached === undefined || reached.unreached.includes(f) || !f.startsWith("src/") || loads.has(f)), | |
| 242 | 273 | ), |
| 243 | ); | |
| 274 | ...docLoads, | |
| 275 | ]); | |
| 244 | 276 | const lookFor = (test: string): ReadonlySet<string> => |
| 245 | 277 | impact === undefined || !impact.has(test) ? changed : unseen; |
| 246 | 278 | const namesFor = (from: ReadonlySet<string>): Map<string, string> => { |
| @@ -258,6 +290,10 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 258 | 290 | if (reached) |
| 259 | 291 | for (const [test, source] of reached.tests) |
| 260 | 292 | take(test, test === LOADS_ONLY ? `loads ${source}` : `executes ${source}`); |
| 293 | for (const [doc, mine] of docs) for (const test of mine.tests) take(test, `reads ${doc}`); | |
| 294 | if (impact && docOf.size > 0) | |
| 295 | for (const [test, source] of reach([...docOf.keys()], impact).tests) | |
| 296 | take(test, `reads ${docOf.get(source)!} through ${source}`); | |
| 261 | 297 | for (const test of input.tests) { |
| 262 | 298 | if (!guessed(test)) continue; |
| 263 | 299 | const names = lookFor(test) === changed ? allNames : unseenNames; |
| @@ -290,19 +326,62 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 290 | 326 | for (const test of input.learned ?? []) take(test, "failed on land before"); |
| 291 | 327 | for (const test of input.tests) { |
| 292 | 328 | const seconds = input.times.get(test); |
| 293 | if (guessed(test) && seconds !== undefined && seconds < QUICK_SECONDS) | |
| 329 | if (quickAll(test) && seconds !== undefined && seconds < QUICK_SECONDS) | |
| 294 | 330 | take(test, "quick"); |
| 295 | 331 | } |
| 296 | 332 | |
| 297 | 333 | const nChanged = diffed.length; |
| 298 | 334 | return finish( |
| 299 | 335 | reached |
| 300 | ? `by coverage${blind ? " and imports" : ""}: ${reachLine(nChanged, reached.tests.size, reached.unreached)}` + | |
| 336 | ? `by coverage${guessing ? " and imports" : ""}: ${reachLine(nChanged, reached.tests.size, reached.unreached)}` + | |
| 301 | 337 | (loadRun.length === 0 ? "" : `; the suite's loading runs ${loadRun.join(", ")}`) |
| 302 | 338 | : "by imports (no impact map)", |
| 303 | 339 | ); |
| 304 | 340 | } |
| 305 | 341 | |
| 342 | /** What each kind of reason a {@link FastPlan} gives starts with, and its name in the breakdown. */ | |
| 343 | const REASON_KINDS: readonly (readonly [string, string])[] = [ | |
| 344 | ["changed", "changed"], | |
| 345 | ["executes ", "executes"], | |
| 346 | ["loads ", "loads"], | |
| 347 | ["reads ", "reads"], | |
| 348 | ["quick", "quick"], | |
| 349 | ["tree scan: ", "scan"], | |
| 350 | ["named for ", "named"], | |
| 351 | ["imports ", "imports"], | |
| 352 | ["repo-wide rule", "repo-wide rule"], | |
| 353 | ["page budget: ", "page budget"], | |
| 354 | ["failed on land before", "learned"], | |
| 355 | ["full: ", "full"], | |
| 356 | ]; | |
| 357 | ||
| 358 | /** | |
| 359 | * The selection by reason, for the check's log (#14845): `by reason: 40 | |
| 360 | * quick, 12 executes, 11 repo-wide rule`, the largest first. A pull that | |
| 361 | * ran wide says which rule took it there. | |
| 362 | */ | |
| 363 | export function reasonLine(plan: Pick<FastPlan, "files" | "why">): string { | |
| 364 | const counts = new Map<string, number>(); | |
| 365 | for (const file of plan.files) { | |
| 366 | const why = plan.why.get(file) ?? ""; | |
| 367 | const kind = REASON_KINDS.find(([start]) => why.startsWith(start))?.[1] ?? why; | |
| 368 | counts.set(kind, (counts.get(kind) ?? 0) + 1); | |
| 369 | } | |
| 370 | const parts = [...counts].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([kind, n]) => `${n} ${kind}`); | |
| 371 | return parts.length === 0 ? "by reason: none" : `by reason: ${parts.join(", ")}`; | |
| 372 | } | |
| 373 | ||
| 374 | /** | |
| 375 | * Why the plan is blind, for the log, or `undefined` when it is not: | |
| 376 | * `blind (every quick test, guesses for every test) by docs/x.md (a doc its | |
| 377 | * readers cannot bound), src/y.ts (no test executes it)`. | |
| 378 | */ | |
| 379 | export function blindLine(plan: Pick<FastPlan, "blindBy">): string | undefined { | |
| 380 | if (plan.blindBy.size === 0) return undefined; | |
| 381 | const by = [...plan.blindBy].map(([file, why]) => `${file} (${why})`).join(", "); | |
| 382 | return `blind (every quick test, guesses for every test) by ${by}`; | |
| 383 | } | |
| 384 | ||
| 306 | 385 | /** `220` → `3m40s`, `45` → `45s`. */ |
| 307 | 386 | export function duration(seconds: number): string { |
| 308 | 387 | const whole = Math.max(0, Math.round(seconds)); |
test/check-script-readers.test.ts +40 −2
| @@ -6,12 +6,16 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | import { describe, expect, test } from "bun:test"; |
| 8 | 8 | import { needsFull } from "../src/checks/impact.ts"; |
| 9 | import { scriptReaders, type ScriptTree } from "../src/checks/scans.ts"; | |
| 9 | import { docReaders, scriptReaders, type ScriptTree } from "../src/checks/scans.ts"; | |
| 10 | 10 | import { fastSet, importsOf } from "../src/checks/tiers.ts"; |
| 11 | 11 | import { gateFiles } from "../scripts/gate.ts"; |
| 12 | 12 | import { readersIn } from "../scripts/script-readers.ts"; |
| 13 | 13 | |
| 14 | function treeOf(files: Record<string, string>, own: string[] = ["scripts/gate.ts", "test/helpers/preload.ts"]): ScriptTree { | |
| 14 | function treeOf( | |
| 15 | files: Record<string, string>, | |
| 16 | own: string[] = ["scripts/gate.ts", "test/helpers/preload.ts"], | |
| 17 | preloads?: string[], | |
| 18 | ): ScriptTree { | |
| 15 | 19 | const all = Object.keys(files); |
| 16 | 20 | return { |
| 17 | 21 | beside: all.filter((f) => /^(test|scripts|bench)\//.test(f)), |
| @@ -19,6 +23,7 @@ function treeOf(files: Record<string, string>, own: string[] = ["scripts/gate.ts | ||
| 19 | 23 | read: (file) => files[file], |
| 20 | 24 | imports: (file) => (files[file] === undefined ? [] : importsOf(files[file]!, file)), |
| 21 | 25 | own, |
| 26 | ...(preloads === undefined ? {} : { preloads }), | |
| 22 | 27 | }; |
| 23 | 28 | } |
| 24 | 29 | |
| @@ -91,3 +96,36 @@ describe("the tests that can read a script", () => { | ||
| 91 | 96 | expect(real(script)).toBeUndefined(); |
| 92 | 97 | }); |
| 93 | 98 | }); |
| 99 | ||
| 100 | describe("the tests that can read a doc (#14845)", () => { | |
| 101 | const files: Record<string, string> = { | |
| 102 | "scripts/gate.ts": 'const readme = readFileSync(join(root, "README.md"), "utf8");', | |
| 103 | "test/helpers/preload.ts": 'const notes = readFileSync(join(root, "docs", "NOTES.md"), "utf8");', | |
| 104 | "test/threats.test.ts": 'const text = readFileSync(join(import.meta.dir, "..", "docs", "THREATS.md"), "utf8");', | |
| 105 | "test/agents.test.ts": 'const guide = Bun.file(new URL("../AGENTS.md", import.meta.url));', | |
| 106 | "test/gate.test.ts": 'import { treeGate } from "../scripts/gate.ts";', | |
| 107 | "test/b.test.ts": "// docs/THREATS.md is only spoken of here\nexport {};", | |
| 108 | "src/view/docs.ts": 'const DOCS = join(import.meta.dir, "..", "..", "docs");', | |
| 109 | "src/web/link.ts": 'export const link = "/docs/features/absorb.md";', | |
| 110 | "src/x.ts": "// see docs/THREATS.md\nexport const x = 1;", | |
| 111 | "Dockerfile": "COPY docs/DESIGN.md /app/docs/DESIGN.md", | |
| 112 | }; | |
| 113 | const tree = treeOf(files, ["scripts/gate.ts", "test/helpers/preload.ts"], ["test/helpers/preload.ts"]); | |
| 114 | const tests = ["test/threats.test.ts", "test/agents.test.ts", "test/gate.test.ts", "test/b.test.ts"]; | |
| 115 | const readers = (doc: string) => docReaders(doc, tree, tests); | |
| 116 | ||
| 117 | test("a test naming it reads it, a comment does not, and src/ listing its directory is a source", () => { | |
| 118 | expect(readers("docs/THREATS.md")).toEqual({ tests: ["test/threats.test.ts"], sources: ["src/view/docs.ts"] }); | |
| 119 | expect(readers("AGENTS.md")).toEqual({ tests: ["test/agents.test.ts"], sources: [] }); | |
| 120 | expect(readers("docs/features/absorb.md")).toEqual({ | |
| 121 | tests: [], | |
| 122 | sources: ["src/view/docs.ts", "src/web/link.ts"], | |
| 123 | }); | |
| 124 | }); | |
| 125 | ||
| 126 | test("the check's own script reading it is followed to its importers; the preload or a manifest unbounds it", () => { | |
| 127 | expect(readers("README.md")).toEqual({ tests: ["test/gate.test.ts"], sources: [] }); | |
| 128 | expect(readers("docs/NOTES.md")).toBeUndefined(); | |
| 129 | expect(readers("docs/DESIGN.md")).toBeUndefined(); | |
| 130 | }); | |
| 131 | }); |
test/check-tiers.test.ts +67
| @@ -21,8 +21,10 @@ import type { Repo } from "../src/core/repos.ts"; | ||
| 21 | 21 | import { createUser, type User } from "../src/core/users.ts"; |
| 22 | 22 | import { readRunLog } from "../src/runs/log.ts"; |
| 23 | 23 | import { |
| 24 | blindLine, | |
| 24 | 25 | duration, |
| 25 | 26 | fastSet, |
| 27 | reasonLine, | |
| 26 | 28 | importsOf, |
| 27 | 29 | landVerdict, |
| 28 | 30 | PAGE_BUDGET_TESTS, |
| @@ -223,6 +225,71 @@ describe("the fast set, by measured coverage", () => { | ||
| 223 | 225 | expect(plan.files).toContain("test/deep.test.ts"); |
| 224 | 226 | }); |
| 225 | 227 | |
| 228 | test("a doc its readers bound selects them and does not blind the rest (#14845)", () => { | |
| 229 | const input = { | |
| 230 | ...base, | |
| 231 | docReaders: (doc: string) => | |
| 232 | doc === "docs/THREATS.md" ? { tests: ["test/deep.test.ts"], sources: ["src/web/phone.ts"] } : undefined, | |
| 233 | }; | |
| 234 | const alone = fastSet({ ...input, changed: ["src/snapshot/store.ts"] }); | |
| 235 | const plan = fastSet({ ...input, changed: ["src/snapshot/store.ts", "docs/THREATS.md"] }); | |
| 236 | expect(plan.files).toEqual([...alone.files, "test/web-phone.test.ts", "test/deep.test.ts"]); | |
| 237 | // tiny is quick, but nothing is blind: it stays out. | |
| 238 | expect(plan.files).not.toContain("test/tiny.test.ts"); | |
| 239 | expect(plan.why.get("test/deep.test.ts")).toBe("reads docs/THREATS.md"); | |
| 240 | expect(plan.why.get("test/web-phone.test.ts")).toBe("reads docs/THREATS.md through src/web/phone.ts"); | |
| 241 | expect(plan.blindBy.size).toBe(0); | |
| 242 | expect(blindLine(plan)).toBeUndefined(); | |
| 243 | }); | |
| 244 | ||
| 245 | test("a doc nothing bounds, and any other file the map cannot see, blinds the plan and is named (#14845)", () => { | |
| 246 | const unbound = fastSet({ ...base, docReaders: () => undefined, changed: ["src/snapshot/store.ts", "AGENTS.md"] }); | |
| 247 | expect(unbound.why.get("test/tiny.test.ts")).toBe("quick"); | |
| 248 | expect([...unbound.blindBy]).toEqual([["AGENTS.md", "a doc its readers cannot bound"]]); | |
| 249 | // With no reader asked, a doc is a file outside src/ and test/, as before. | |
| 250 | const unasked = fastSet({ ...base, changed: ["docs/THREATS.md", "src/web/x.ts", "static/app.js"] }); | |
| 251 | expect([...unasked.blindBy]).toEqual([ | |
| 252 | ["src/web/x.ts", "no test executes it"], | |
| 253 | ["docs/THREATS.md", "outside src/ and test/"], | |
| 254 | ["static/app.js", "outside src/ and test/"], | |
| 255 | ]); | |
| 256 | expect(blindLine(unasked)).toBe( | |
| 257 | "blind (every quick test, guesses for every test) by src/web/x.ts (no test executes it), " + | |
| 258 | "docs/THREATS.md (outside src/ and test/), static/app.js (outside src/ and test/)", | |
| 259 | ); | |
| 260 | }); | |
| 261 | ||
| 262 | test("a file the suite's loading runs is guessed by imports but runs no quick test it does not reach (#14845)", () => { | |
| 263 | const plan = fastSet({ | |
| 264 | ...base, | |
| 265 | tests: [...base.tests, "test/suite-loads.test.ts"], | |
| 266 | impact: new Map([...impact, ["test/suite-loads.test.ts", ["src/snapshot/store.ts"]]]), | |
| 267 | imports: (file) => (file === "test/web-phone.test.ts" ? ["src/snapshot/store.ts"] : imports(file)), | |
| 268 | changed: ["src/snapshot/store.ts"], | |
| 269 | }); | |
| 270 | expect(plan.why.get("test/web-phone.test.ts")).toBe("imports src/snapshot/store.ts"); | |
| 271 | expect(plan.files).not.toContain("test/tiny.test.ts"); | |
| 272 | expect(plan.blindBy.size).toBe(0); | |
| 273 | }); | |
| 274 | ||
| 275 | test("the log's breakdown counts the selection by reason, the largest first (#14845)", () => { | |
| 276 | const plan = fastSet({ ...base, changed: ["src/snapshot/store.ts", "static/app.js"] }); | |
| 277 | expect(reasonLine(plan)).toBe("by reason: 2 executes, 1 quick"); | |
| 278 | expect(reasonLine({ files: [], why: new Map() })).toBe("by reason: none"); | |
| 279 | const mixed = new Map([ | |
| 280 | ["a", "changed"], | |
| 281 | ["b", "tree scan: lists the tree"], | |
| 282 | ["c", "tree scan: declares a tree scan"], | |
| 283 | ["d", "page budget: src/web/layout.ts changed"], | |
| 284 | ["e", "named for src/x.ts"], | |
| 285 | ["f", "repo-wide rule"], | |
| 286 | ["g", "reads docs/THREATS.md through src/view/docs.ts"], | |
| 287 | ]); | |
| 288 | expect(reasonLine({ files: [...mixed.keys()], why: mixed })).toBe( | |
| 289 | "by reason: 2 scan, 1 changed, 1 named, 1 page budget, 1 reads, 1 repo-wide rule", | |
| 290 | ); | |
| 291 | }); | |
| 292 | ||
| 226 | 293 | test("a test the map has never seen is picked by the hop rule", () => { |
| 227 | 294 | const hop = fastSet({ |
| 228 | 295 | ...base, |
new above
offline