Public: anyone can read this; the crew can change it.
Test what the change reaches, by measured coverage, not by guessing: a per-test impact map from the full run drives the fast tier #1441
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1279-test-what-the-change-reaches-by-measured → main 5 commits fork
6 files
changed +366
−13
README.md +9
| @@ -481,6 +481,15 @@ shard's output is written whole as it finishes; the log ends with every | ||
| 481 | 481 | failing test from every shard, or on green with one line — how many tests, |
| 482 | 482 | across how many files, in how long, then the slowest and fastest shard. |
| 483 | 483 | |
| 484 | The fast tier picks by measurement. The full gate runs each file under | |
| 485 | Bun's coverage (`--impact`) and writes `scripts/test-impact.json`: test | |
| 486 | file → the `src/**` files it executed. A diff intersected with it is the | |
| 487 | set of tests the change reaches; the check line says `N changed files | |
| 488 | reach M tests; src/x.ts reaches none` (a coverage gap). Tests the map has | |
| 489 | never seen fall back to the import-hop guess, and a change to `forge.json`, | |
| 490 | `package.json`, `scripts/**` or `src/kernel/**` runs everything. Until a | |
| 491 | release commits the map, the fast tier guesses by imports as before. | |
| 492 | ||
| 484 | 493 | Files go to shards by how long they take, not how big they are: longest |
| 485 | 494 | first onto the least-loaded shard, weighed from `scripts/test-times.json` |
| 486 | 495 | (file → seconds, from one recorded run). A file the table does not name |
forge.json +1 −1
| @@ -15,7 +15,7 @@ | ||
| 15 | 15 | }, |
| 16 | 16 | "check": { |
| 17 | 17 | "fast": "node --max-old-space-size=1024 ./node_modules/typescript/lib/tsc.js --noEmit && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"", |
| 18 | "run": "node --max-old-space-size=1024 ./node_modules/typescript/lib/tsc.js --noEmit && bun scripts/test-shards.ts --timeout 60000 --gate", | |
| 18 | "run": "node --max-old-space-size=1024 ./node_modules/typescript/lib/tsc.js --noEmit && bun scripts/test-shards.ts --timeout 60000 --gate --impact", | |
| 19 | 19 | "timeout": 7200, |
| 20 | 20 | "limits": { |
| 21 | 21 | "memory": 8192, |
scripts/test-shards.ts +53 −5
| @@ -31,8 +31,9 @@ | ||
| 31 | 31 | * table is rewritten from those times when every file is green. |
| 32 | 32 | */ |
| 33 | 33 | |
| 34 | import { availableParallelism } from "node:os"; | |
| 35 | import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; | |
| 34 | import { availableParallelism, tmpdir } from "node:os"; | |
| 35 | import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; | |
| 36 | import { join } from "node:path"; | |
| 36 | 37 | import { |
| 37 | 38 | formatTimes, |
| 38 | 39 | parseShardSpec, |
| @@ -46,13 +47,16 @@ import { | ||
| 46 | 47 | type ShardResult, |
| 47 | 48 | } from "./shards.ts"; |
| 48 | 49 | import { auditGate, EXCLUDED, GATE } from "./gate.ts"; |
| 50 | import { executedFiles, formatImpact, mergeImpact, parseImpact } from "../src/checks/impact.ts"; | |
| 49 | 51 | import { fastSet, importsOf, duration } from "../src/checks/tiers.ts"; |
| 50 | 52 | |
| 51 | 53 | const TIMES = new URL("./test-times.json", import.meta.url).pathname; |
| 54 | const IMPACT = new URL("./test-impact.json", import.meta.url).pathname; | |
| 52 | 55 | |
| 53 | 56 | const argv = process.argv.slice(2); |
| 54 | 57 | const record = argv.includes("--record"); |
| 55 | 58 | const gate = argv.includes("--gate"); |
| 59 | const measure = argv.includes("--impact"); | |
| 56 | 60 | |
| 57 | 61 | /** Pull `--shard N/M` or `--shard=N/M` out of argv so it never reaches bun test. */ |
| 58 | 62 | function takeShardArg(raw: string[]): { |
| @@ -112,7 +116,7 @@ function takeValue( | ||
| 112 | 116 | } |
| 113 | 117 | |
| 114 | 118 | const tierArg = takeValue( |
| 115 | argv.filter((arg) => arg !== "--record" && arg !== "--gate"), | |
| 119 | argv.filter((arg) => arg !== "--record" && arg !== "--gate" && arg !== "--impact"), | |
| 116 | 120 | "--tier", |
| 117 | 121 | ); |
| 118 | 122 | const diffArg = takeValue(tierArg.rest, "--diff"); |
| @@ -227,7 +231,16 @@ if (tier === "fast") { | ||
| 227 | 231 | } |
| 228 | 232 | return found; |
| 229 | 233 | }; |
| 234 | let impact: Map<string, string[]> | undefined; | |
| 235 | try { | |
| 236 | const parsed = parseImpact(readFileSync(IMPACT, "utf8")); | |
| 237 | if (parsed.ok) impact = parsed.value; | |
| 238 | else console.log(`test-shards: scripts/test-impact.json refused (${parsed.error}); guessing by imports`); | |
| 239 | } catch { | |
| 240 | // no map yet: the import guess stands | |
| 241 | } | |
| 230 | 242 | const plan = fastSet({ |
| 243 | ...(impact === undefined ? {} : { impact }), | |
| 231 | 244 | changed, |
| 232 | 245 | tests: files, |
| 233 | 246 | imports, |
| @@ -235,7 +248,7 @@ if (tier === "fast") { | ||
| 235 | 248 | learned: (learnedArg.value ?? "").split(",").filter((f) => f !== ""), |
| 236 | 249 | }); |
| 237 | 250 | console.log( |
| 238 | `test-shards: fast tier, ${plan.files.length} of ${files.length} files for ${changed.length} changed ` + | |
| 251 | `test-shards: fast tier, ${plan.files.length} of ${files.length} files for ${changed.length} changed, ${plan.line} ` + | |
| 239 | 252 | `(expected ${duration(plan.expected)}${plan.overBudget ? ", over the 5m budget" : ""})`, |
| 240 | 253 | ); |
| 241 | 254 | for (const file of plan.files) |
| @@ -358,6 +371,9 @@ async function drain( | ||
| 358 | 371 | take(decoder.decode()); |
| 359 | 372 | } |
| 360 | 373 | |
| 374 | const measured = new Map<string, string[]>(); | |
| 375 | const unmeasured = new Set<string>(); | |
| 376 | ||
| 361 | 377 | type Timed = ShardResult & { seconds: number }; |
| 362 | 378 | |
| 363 | 379 | /** |
| @@ -390,7 +406,13 @@ async function runShard( | ||
| 390 | 406 | // to bun test shards — a leftover max-old-space-size races the check |
| 391 | 407 | // cell's memory cgroup and the kernel SIGKILLs the gate (exit 137). |
| 392 | 408 | delete childEnv["NODE_OPTIONS"]; |
| 393 | const child = Bun.spawn(["bun", "test", ...flags, ...paths], { | |
| 409 | // Measuring: one file per process, its coverage in a directory of its own. | |
| 410 | const covDir = | |
| 411 | measure && slice.length === 1 ? mkdtempSync(join(tmpdir(), "impact-")) : undefined; | |
| 412 | const cover = covDir | |
| 413 | ? ["--coverage", "--coverage-reporter=lcov", `--coverage-dir=${covDir}`] | |
| 414 | : []; | |
| 415 | const child = Bun.spawn(["bun", "test", ...flags, ...cover, ...paths], { | |
| 394 | 416 | stdin: "ignore", |
| 395 | 417 | stdout: "pipe", |
| 396 | 418 | stderr: "pipe", |
| @@ -420,6 +442,14 @@ async function runShard( | ||
| 420 | 442 | ? `${text.endsWith("\n") || text === "" ? "" : "\n"}── shard ${where} done (exit ${code}, ${seconds}s)\n` |
| 421 | 443 | : `\n── shard ${where} (exit ${code}, ${seconds}s): ${slice.join(" ")}\n${text}${text.endsWith("\n") ? "" : "\n"}`, |
| 422 | 444 | ); |
| 445 | if (covDir) { | |
| 446 | try { | |
| 447 | measured.set(slice[0]!, executedFiles(readFileSync(join(covDir, "lcov.info"), "utf8"))); | |
| 448 | } catch { | |
| 449 | unmeasured.add(slice[0]!); | |
| 450 | } | |
| 451 | rmSync(covDir, { recursive: true, force: true }); | |
| 452 | } | |
| 423 | 453 | return { |
| 424 | 454 | files: slice, |
| 425 | 455 | code, |
| @@ -487,4 +517,22 @@ if (record) { | ||
| 487 | 517 | ); |
| 488 | 518 | } |
| 489 | 519 | } |
| 520 | if (measure) { | |
| 521 | if (ending.code !== 0 || unmeasured.size > 0 || !(record || serialFiles)) { | |
| 522 | console.log("test-shards: not writing scripts/test-impact.json: the run was not green and measured, one file at a time"); | |
| 523 | } else { | |
| 524 | // A partial run refreshes what it ran and forgets nothing else. | |
| 525 | let old = new Map<string, string[]>(); | |
| 526 | try { | |
| 527 | const parsed = parseImpact(readFileSync(IMPACT, "utf8")); | |
| 528 | if (parsed.ok) old = parsed.value; | |
| 529 | else console.log(`test-shards: scripts/test-impact.json refused (${parsed.error}); writing a fresh map`); | |
| 530 | } catch (error) { | |
| 531 | if ((error as NodeJS.ErrnoException).code !== "ENOENT") | |
| 532 | console.log(`test-shards: scripts/test-impact.json unreadable (${String(error)}); writing a fresh map`); | |
| 533 | } | |
| 534 | writeFileSync(IMPACT, formatImpact(mergeImpact(old, measured, GATE))); | |
| 535 | console.log(`test-shards: wrote ${measured.size} test impacts to scripts/test-impact.json`); | |
| 536 | } | |
| 537 | } | |
| 490 | 538 | process.exit(ending.code); |
src/checks/impact.ts added +120
| @@ -0,0 +1,120 @@ | ||
| 1 | /** | |
| 2 | * The impact map (#1279): for each test file, the `src/**` files it | |
| 3 | * executed, as measured by Bun's coverage in the full run. A pull's diff | |
| 4 | * intersected with it is the set of tests the change can reach. | |
| 5 | * | |
| 6 | * Plain data in, plain data out. `scripts/test-shards.ts` runs the | |
| 7 | * coverage and reads the files; this module parses, formats and decides | |
| 8 | * what the map cannot bound. | |
| 9 | */ | |
| 10 | ||
| 11 | export type Impact = ReadonlyMap<string, readonly string[]>; | |
| 12 | ||
| 13 | const TEST = /^test\/[A-Za-z0-9_.-]+\.test\.[cm]?[jt]sx?$/; | |
| 14 | const SOURCE = /^src\/[A-Za-z0-9_./-]+$/; | |
| 15 | ||
| 16 | /** The `src/**` files an lcov report shows executed (a line hit at least once). */ | |
| 17 | export function executedFiles(lcov: string): string[] { | |
| 18 | const found = new Set<string>(); | |
| 19 | let file: string | undefined; | |
| 20 | let hit = false; | |
| 21 | const end = (): void => { | |
| 22 | if (file !== undefined && hit && SOURCE.test(file)) found.add(file); | |
| 23 | file = undefined; | |
| 24 | hit = false; | |
| 25 | }; | |
| 26 | for (const line of lcov.split("\n")) { | |
| 27 | if (line.startsWith("SF:")) { | |
| 28 | end(); | |
| 29 | file = line.slice(3).trim().replace(/^\.\//, ""); | |
| 30 | } else if (line.startsWith("DA:")) { | |
| 31 | const count = Number(line.slice(3).split(",")[1]); | |
| 32 | if (count > 0) hit = true; | |
| 33 | } else if (line.startsWith("end_of_record")) end(); | |
| 34 | } | |
| 35 | end(); | |
| 36 | return [...found].sort(); | |
| 37 | } | |
| 38 | ||
| 39 | export function parseImpact( | |
| 40 | text: string, | |
| 41 | ): { ok: true; value: Map<string, string[]> } | { ok: false; error: string } { | |
| 42 | let data: unknown; | |
| 43 | try { | |
| 44 | data = JSON.parse(text); | |
| 45 | } catch { | |
| 46 | return { ok: false, error: "not JSON" }; | |
| 47 | } | |
| 48 | if (typeof data !== "object" || data === null || Array.isArray(data)) | |
| 49 | return { ok: false, error: "not an object of test file to source files" }; | |
| 50 | const map = new Map<string, string[]>(); | |
| 51 | for (const [test, files] of Object.entries(data)) { | |
| 52 | if (!TEST.test(test)) return { ok: false, error: `${JSON.stringify(test)} is not a test file` }; | |
| 53 | if (!Array.isArray(files) || !files.every((f) => typeof f === "string" && SOURCE.test(f))) | |
| 54 | return { ok: false, error: `${test}: not a list of src files` }; | |
| 55 | map.set(test, files as string[]); | |
| 56 | } | |
| 57 | return { ok: true, value: map }; | |
| 58 | } | |
| 59 | ||
| 60 | export function formatImpact(map: ReadonlyMap<string, readonly string[]>): string { | |
| 61 | const entries = [...map].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); | |
| 62 | if (entries.length === 0) return "{}\n"; | |
| 63 | const lines = entries.map( | |
| 64 | ([test, files]) => ` ${JSON.stringify(test)}: ${JSON.stringify([...files].sort())}`, | |
| 65 | ); | |
| 66 | return `{\n${lines.join(",\n")}\n}\n`; | |
| 67 | } | |
| 68 | ||
| 69 | /** What the map cannot bound: manifests, scripts, the kernel. */ | |
| 70 | export function needsFull(changed: readonly string[]): string | undefined { | |
| 71 | return changed.find( | |
| 72 | (f) => | |
| 73 | f === "forge.json" || | |
| 74 | f === "package.json" || | |
| 75 | (f.startsWith("scripts/") && !f.endsWith("test-times.json") && !f.endsWith("test-impact.json")) || | |
| 76 | f.startsWith("src/kernel/"), | |
| 77 | ); | |
| 78 | } | |
| 79 | ||
| 80 | export interface Reach { | |
| 81 | /** Tests whose executed files meet the diff, with the first file that did. */ | |
| 82 | readonly tests: ReadonlyMap<string, string>; | |
| 83 | /** Changed `src/**` files no mapped test executes. */ | |
| 84 | readonly unreached: readonly string[]; | |
| 85 | } | |
| 86 | ||
| 87 | export function reach(changed: readonly string[], impact: Impact): Reach { | |
| 88 | const tests = new Map<string, string>(); | |
| 89 | const hit = new Set<string>(); | |
| 90 | const sources = changed.filter((f) => SOURCE.test(f)); | |
| 91 | for (const [test, files] of impact) { | |
| 92 | const set = new Set(files); | |
| 93 | const first = sources.find((f) => set.has(f)); | |
| 94 | if (first !== undefined) tests.set(test, first); | |
| 95 | for (const f of sources) if (set.has(f)) hit.add(f); | |
| 96 | } | |
| 97 | return { tests, unreached: sources.filter((f) => !hit.has(f)) }; | |
| 98 | } | |
| 99 | ||
| 100 | /** `3 changed files reach 14 tests; src/web/x.ts reaches none` */ | |
| 101 | export function reachLine(changed: number, tests: number, unreached: readonly string[]): string { | |
| 102 | const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? "" : "s"}`; | |
| 103 | const head = `${plural(changed, "changed file")} reach${changed === 1 ? "es" : ""} ${plural(tests, "test")}`; | |
| 104 | return unreached.length === 0 ? head : `${head}; ${unreached.join(", ")} reaches none`; | |
| 105 | } | |
| 106 | ||
| 107 | /** | |
| 108 | * The map to write: the old one refreshed by what this run measured, and | |
| 109 | * every test that left the gate forgotten. | |
| 110 | */ | |
| 111 | export function mergeImpact( | |
| 112 | old: ReadonlyMap<string, readonly string[]>, | |
| 113 | measured: ReadonlyMap<string, readonly string[]>, | |
| 114 | gate: readonly string[], | |
| 115 | ): Map<string, string[]> { | |
| 116 | const merged = new Map<string, string[]>(); | |
| 117 | for (const [test, files] of [...old, ...measured]) | |
| 118 | if (gate.includes(test)) merged.set(test, [...files]); | |
| 119 | return merged; | |
| 120 | } |
src/checks/tiers.ts +48 −7
| @@ -9,6 +9,8 @@ | ||
| 9 | 9 | * only decides. |
| 10 | 10 | */ |
| 11 | 11 | |
| 12 | import { needsFull, reach, reachLine, type Impact } from "./impact.ts"; | |
| 13 | ||
| 12 | 14 | /** A test under this many seconds is always in the fast tier. */ |
| 13 | 15 | export const QUICK_SECONDS = 1; |
| 14 | 16 | |
| @@ -75,6 +77,8 @@ export interface FastInput { | ||
| 75 | 77 | readonly times: ReadonlyMap<string, number>; |
| 76 | 78 | /** Tests a red full gate on land taught this pull. */ |
| 77 | 79 | readonly learned?: readonly string[]; |
| 80 | /** Measured: the `src/**` files each test executed (`scripts/test-impact.json`). */ | |
| 81 | readonly impact?: Impact; | |
| 78 | 82 | } |
| 79 | 83 | |
| 80 | 84 | export interface FastPlan { |
| @@ -84,6 +88,8 @@ export interface FastPlan { | ||
| 84 | 88 | /** Expected seconds, from the times table (a test it lacks counts 5). */ |
| 85 | 89 | readonly expected: number; |
| 86 | 90 | readonly overBudget: boolean; |
| 91 | /** The line for the log: what the plan is by, and what the diff reaches. */ | |
| 92 | readonly line: string; | |
| 87 | 93 | } |
| 88 | 94 | |
| 89 | 95 | /** |
| @@ -96,6 +102,37 @@ export interface FastPlan { | ||
| 96 | 102 | export function fastSet(input: FastInput): FastPlan { |
| 97 | 103 | const tests = new Set(input.tests); |
| 98 | 104 | const why = new Map<string, string>(); |
| 105 | const finish = (line: string): FastPlan => { | |
| 106 | const files = input.tests.filter((test) => why.has(test)); | |
| 107 | const expected = files.reduce( | |
| 108 | (sum, file) => sum + (input.times.get(file) ?? UNKNOWN_SECONDS), | |
| 109 | 0, | |
| 110 | ); | |
| 111 | return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS, line }; | |
| 112 | }; | |
| 113 | // What the map cannot bound runs everything. | |
| 114 | const unbounded = needsFull(input.changed); | |
| 115 | if (unbounded !== undefined) { | |
| 116 | for (const test of input.tests) why.set(test, `full: ${unbounded} changed`); | |
| 117 | return finish(`full tier, ${unbounded} changed`); | |
| 118 | } | |
| 119 | const impact = input.impact !== undefined && input.impact.size > 0 ? input.impact : undefined; | |
| 120 | const reached = impact && reach(input.changed, impact); | |
| 121 | // A change the map cannot see (a src file no test executes, or a file | |
| 122 | // outside src/ and test/) is judged by the guesses for every test. | |
| 123 | const blind = | |
| 124 | reached !== undefined && | |
| 125 | (reached.unreached.length > 0 || | |
| 126 | input.changed.some( | |
| 127 | (f) => | |
| 128 | !f.startsWith("src/") && | |
| 129 | !f.startsWith("test/") && | |
| 130 | !f.endsWith("test-times.json") && | |
| 131 | !f.endsWith("test-impact.json"), | |
| 132 | )); | |
| 133 | // A test the map has never seen (a new one) is judged by the guesses. | |
| 134 | const guessed = (test: string): boolean => | |
| 135 | impact === undefined || blind || !impact.has(test); | |
| 99 | 136 | const take = (test: string, reason: string): void => { |
| 100 | 137 | if (tests.has(test) && !why.has(test)) why.set(test, reason); |
| 101 | 138 | }; |
| @@ -106,13 +143,17 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 106 | 143 | const names = new Map<string, string>(); |
| 107 | 144 | for (const source of sources) |
| 108 | 145 | for (const name of namesOf(source)) names.set(name, source); |
| 146 | if (reached) | |
| 147 | for (const [test, source] of reached.tests) | |
| 148 | take(test, `executes ${source}`); | |
| 109 | 149 | for (const test of input.tests) { |
| 150 | if (!guessed(test)) continue; | |
| 110 | 151 | const hit = names.get(stemOf(test)); |
| 111 | 152 | if (hit !== undefined) take(test, `named for ${hit}`); |
| 112 | 153 | } |
| 113 | 154 | |
| 114 | 155 | for (const test of input.tests) { |
| 115 | if (why.has(test)) continue; | |
| 156 | if (why.has(test) || !guessed(test)) continue; | |
| 116 | 157 | let frontier: readonly string[] = input.imports(test); |
| 117 | 158 | const seen = new Set<string>(); |
| 118 | 159 | for (let depth = 0; depth <= IMPORT_DEPTH && frontier.length > 0; depth++) { |
| @@ -134,16 +175,16 @@ export function fastSet(input: FastInput): FastPlan { | ||
| 134 | 175 | for (const test of input.learned ?? []) take(test, "failed on land before"); |
| 135 | 176 | for (const test of input.tests) { |
| 136 | 177 | const seconds = input.times.get(test); |
| 137 | if (seconds !== undefined && seconds < QUICK_SECONDS) | |
| 178 | if (guessed(test) && seconds !== undefined && seconds < QUICK_SECONDS) | |
| 138 | 179 | take(test, "quick"); |
| 139 | 180 | } |
| 140 | 181 | |
| 141 | const files = input.tests.filter((test) => why.has(test)); | |
| 142 | const expected = files.reduce( | |
| 143 | (sum, file) => sum + (input.times.get(file) ?? UNKNOWN_SECONDS), | |
| 144 | 0, | |
| 182 | const nChanged = input.changed.length; | |
| 183 | return finish( | |
| 184 | reached | |
| 185 | ? `by coverage${blind ? " and imports" : ""}: ${reachLine(nChanged, reached.tests.size, reached.unreached)}` | |
| 186 | : "by imports (no impact map)", | |
| 145 | 187 | ); |
| 146 | return { files, why, expected, overBudget: expected > FAST_BUDGET_SECONDS }; | |
| 147 | 188 | } |
| 148 | 189 | |
| 149 | 190 | /** `220` → `3m40s`, `45` → `45s`. */ |
test/check-tiers.test.ts +135
| @@ -24,6 +24,13 @@ import { | ||
| 24 | 24 | landVerdict, |
| 25 | 25 | tierLine, |
| 26 | 26 | } from "../src/checks/tiers.ts"; |
| 27 | import { | |
| 28 | executedFiles, | |
| 29 | formatImpact, | |
| 30 | mergeImpact, | |
| 31 | parseImpact, | |
| 32 | reach, | |
| 33 | } from "../src/checks/impact.ts"; | |
| 27 | 34 | import { parseCheck } from "../src/apps/manifest.ts"; |
| 28 | 35 | |
| 29 | 36 | const tests = [ |
| @@ -102,6 +109,134 @@ describe("the fast set", () => { | ||
| 102 | 109 | }); |
| 103 | 110 | }); |
| 104 | 111 | |
| 112 | describe("the fast set, by measured coverage", () => { | |
| 113 | const impact = new Map([ | |
| 114 | ["test/snapshot-store.test.ts", ["src/snapshot/store.ts"]], | |
| 115 | ["test/snapshot.test.ts", ["src/snapshot/service.ts", "src/snapshot/store.ts"]], | |
| 116 | ["test/web-phone.test.ts", ["src/web/phone.ts"]], | |
| 117 | ["test/deep.test.ts", ["src/a.ts"]], | |
| 118 | ["test/tiny.test.ts", ["src/tiny.ts"]], | |
| 119 | ]); | |
| 120 | const base = { tests: [...tests, "test/new.test.ts"], imports, times, impact }; | |
| 121 | ||
| 122 | test("a diff to one source file picks exactly its tests", () => { | |
| 123 | const plan = fastSet({ ...base, changed: ["src/snapshot/store.ts"] }); | |
| 124 | expect(plan.files).toEqual([ | |
| 125 | "test/snapshot-store.test.ts", | |
| 126 | "test/snapshot.test.ts", | |
| 127 | ]); | |
| 128 | expect(plan.why.get("test/snapshot.test.ts")).toBe("executes src/snapshot/store.ts"); | |
| 129 | expect(plan.line).toBe("by coverage: 1 changed file reaches 2 tests"); | |
| 130 | }); | |
| 131 | ||
| 132 | test("a changed file no test executes is said, and judged by the guesses", () => { | |
| 133 | const plan = fastSet({ ...base, changed: ["src/web/phone.ts", "src/web/x.ts"] }); | |
| 134 | expect(plan.files).toEqual(["test/web-phone.test.ts", "test/tiny.test.ts"]); | |
| 135 | expect(plan.line).toBe( | |
| 136 | "by coverage and imports: 2 changed files reach 1 test; src/web/x.ts reaches none", | |
| 137 | ); | |
| 138 | }); | |
| 139 | ||
| 140 | test("an unreached source file is found by the name guess, not left to run nothing", () => { | |
| 141 | const plan = fastSet({ ...base, changed: ["src/snapshot/extra.ts"], tests: [...tests, "test/snapshot-extra.test.ts"] }); | |
| 142 | expect(plan.files).toContain("test/snapshot-extra.test.ts"); | |
| 143 | expect(plan.why.get("test/snapshot-extra.test.ts")).toBe("named for src/snapshot/extra.ts"); | |
| 144 | }); | |
| 145 | ||
| 146 | test("a non-src file the map cannot see turns the guesses on", () => { | |
| 147 | const plan = fastSet({ | |
| 148 | ...base, | |
| 149 | changed: ["static/app.js"], | |
| 150 | imports: (file) => (file === "test/deep.test.ts" ? ["static/app.js"] : imports(file)), | |
| 151 | }); | |
| 152 | expect(plan.files).toContain("test/deep.test.ts"); | |
| 153 | }); | |
| 154 | ||
| 155 | test("a test the map has never seen is picked by the hop rule", () => { | |
| 156 | const hop = fastSet({ | |
| 157 | ...base, | |
| 158 | changed: ["src/c.ts"], | |
| 159 | imports: (file) => (file === "test/new.test.ts" ? ["src/a.ts"] : imports(file)), | |
| 160 | }); | |
| 161 | expect(hop.files).toContain("test/new.test.ts"); | |
| 162 | expect(hop.why.get("test/new.test.ts")).toBe("imports src/c.ts"); | |
| 163 | }); | |
| 164 | ||
| 165 | test("a kernel, manifest or scripts change runs full", () => { | |
| 166 | for (const file of ["src/kernel/db.ts", "forge.json", "package.json", "scripts/gate.ts"]) { | |
| 167 | const plan = fastSet({ ...base, changed: [file] }); | |
| 168 | expect(plan.files).toEqual(base.tests); | |
| 169 | expect(plan.line).toContain("full tier"); | |
| 170 | } | |
| 171 | expect(fastSet({ ...base, changed: ["scripts/test-times.json"] }).files).toEqual([]); | |
| 172 | }); | |
| 173 | ||
| 174 | test("lcov: only src files with a line hit count as executed", () => { | |
| 175 | const lcov = "SF:src/a.ts\nDA:1,0\nend_of_record\nSF:src/b.ts\nDA:1,3\nend_of_record\nSF:test/x.test.ts\nDA:1,1\nend_of_record\n"; | |
| 176 | expect(executedFiles(lcov)).toEqual(["src/b.ts"]); | |
| 177 | }); | |
| 178 | }); | |
| 179 | ||
| 180 | describe("the impact map", () => { | |
| 181 | test("parseImpact accepts a map and rejects bad shapes and paths", () => { | |
| 182 | const ok = parseImpact('{"test/a.test.ts":["src/a.ts"]}'); | |
| 183 | expect(ok).toEqual({ ok: true, value: new Map([["test/a.test.ts", ["src/a.ts"]]]) }); | |
| 184 | for (const bad of [ | |
| 185 | "nope", | |
| 186 | "[]", | |
| 187 | "null", | |
| 188 | '{"test/a.test.ts":"src/a.ts"}', | |
| 189 | '{"test/a.test.ts":[1]}', | |
| 190 | '{"test/a.test.ts":["../etc/passwd"]}', | |
| 191 | '{"test/a.test.ts":["scripts/x.ts"]}', | |
| 192 | '{"src/a.ts":["src/a.ts"]}', | |
| 193 | '{"test/../a.test.ts":[]}', | |
| 194 | ]) | |
| 195 | expect(parseImpact(bad).ok).toBe(false); | |
| 196 | }); | |
| 197 | ||
| 198 | test("formatImpact sorts and round-trips", () => { | |
| 199 | const map = new Map([ | |
| 200 | ["test/b.test.ts", ["src/z.ts", "src/a.ts"]], | |
| 201 | ["test/a.test.ts", ["src/a.ts"]], | |
| 202 | ]); | |
| 203 | const text = formatImpact(map); | |
| 204 | expect(text.indexOf("test/a.test.ts")).toBeLessThan(text.indexOf("test/b.test.ts")); | |
| 205 | expect(text).toContain('["src/a.ts","src/z.ts"]'); | |
| 206 | const back = parseImpact(text); | |
| 207 | if (!back.ok) throw new Error(back.error); | |
| 208 | expect(formatImpact(back.value)).toBe(text); | |
| 209 | expect(formatImpact(new Map())).toBe("{}\n"); | |
| 210 | }); | |
| 211 | ||
| 212 | test("reach names the first file that meets the diff and the unreached", () => { | |
| 213 | const map = new Map([["test/a.test.ts", ["src/a.ts", "src/b.ts"]]]); | |
| 214 | const got = reach(["src/b.ts", "src/c.ts", "README.md"], map); | |
| 215 | expect(got.tests).toEqual(new Map([["test/a.test.ts", "src/b.ts"]])); | |
| 216 | expect(got.unreached).toEqual(["src/c.ts"]); | |
| 217 | }); | |
| 218 | ||
| 219 | test("mergeImpact refreshes what was measured and drops tests that left the gate", () => { | |
| 220 | const old = new Map([ | |
| 221 | ["test/a.test.ts", ["src/old.ts"]], | |
| 222 | ["test/b.test.ts", ["src/b.ts"]], | |
| 223 | ["test/gone.test.ts", ["src/g.ts"]], | |
| 224 | ]); | |
| 225 | const measured = new Map([ | |
| 226 | ["test/a.test.ts", ["src/new.ts"]], | |
| 227 | ["test/c.test.ts", ["src/c.ts"]], | |
| 228 | ]); | |
| 229 | const merged = mergeImpact(old, measured, ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"]); | |
| 230 | expect(merged).toEqual( | |
| 231 | new Map([ | |
| 232 | ["test/a.test.ts", ["src/new.ts"]], | |
| 233 | ["test/b.test.ts", ["src/b.ts"]], | |
| 234 | ["test/c.test.ts", ["src/c.ts"]], | |
| 235 | ]), | |
| 236 | ); | |
| 237 | }); | |
| 238 | }); | |
| 239 | ||
| 105 | 240 | describe("importsOf", () => { |
| 106 | 241 | test("resolves relative specifiers and leaves packages out", () => { |
| 107 | 242 | const source = `import { a } from "../src/snapshot/store.ts";\nimport x from "node:fs";\nconst m = await import("./helper.ts");`; |
new above
offline