trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns18g rEventsFlowsAPI

A check never redoes work whose inputs did not change: results are kept per t… #12224

Merged masoncrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/12178-a-check-never-redoes-work-whose-inputs-d → main 4 commits fork 2 threads · 2 unresolved

  • Merged. mason/12178-a-check-never-redoes-work-whose-inputs-d was merged into main as e36361f.

Review

Sign in to review this change.
9 files changed +612 −2
.gitignore +4
@@ -24,6 +24,10 @@ typesafe-api-key
2424 # what a check keeps for the next one (src/checks/cache.ts)
2525 /.forge-cache/
2626
27 # what a check hands between runs: the impact map and the kept results, never committed
28 /.forge-impact.json
29 /.forge-kept.json
30
2731 # empty stubs an agent sandbox mounts over home-directory config; never the
2832 # forge's own files, and never committed (#1148)
2933 /.bash_profile
docs/DEVELOP.md +10
@@ -104,6 +104,16 @@ map has never seen fall back to the import-hop guess, and a change to
104104 `forge.json`, `package.json`, `scripts/**` or `src/kernel/**` runs
105105 everything. With no map yet the line says `by imports (no impact map)`.
106106
107 Kept results (#12178). A selected file whose inputs a default-branch
108 measuring run already proved green is not run again: the log says
109 `kept from <commit>`. The key is the Bun, the tooling files
110 (`KEPT_TOOLING`: the runner scripts, `forge.json`, the preload among
111 them), the test, the files its impact entry names and its imports. Only a
112 file green on its first try is kept (never one a retry made green), only
113 the default branch's run hands results back (`$FORGE_CHECK_KEPT`,
114 `src/checks/kept-store.ts`), a test that opens the tree is never kept, and
115 one kept file in ten is run anyway (`sampled`) until an audit exists.
116
107117 A test that reads the whole tree — every test file, every migration, the
108118 README — is reached by no diff, yet any file can break it, so every fast
109119 tier runs it whatever else the diff holds (#7138) — a diff of only the
scripts/kept-input.ts added +102
@@ -0,0 +1,102 @@
1 /**
2 * What a test file reads, as hashes, for `src/checks/kept.ts`: the file
3 * itself, the `src/**` files its impact entry names, the files it imports
4 * (through any depth, inside the repository), and the tooling files. The
5 * one reading for the run that keeps a result and the run that looks one up.
6 */
7
8 import { readFileSync } from "node:fs";
9 import { join } from "node:path";
10 import { sha256 } from "../src/kernel/hash.ts";
11 import {
12 KEPT_TOOLING,
13 findsItsPlace,
14 keptKey,
15 readsTheTree,
16 sampled,
17 splitKept,
18 type Kept,
19 type KeptEntry,
20 type KeptSplit,
21 } from "../src/checks/kept.ts";
22
23 const MOST_FILES = 400;
24
25 function text(root: string, path: string): string | undefined {
26 try {
27 return readFileSync(join(root, path), "utf8");
28 } catch {
29 return undefined;
30 }
31 }
32
33 /** The key of a test file, or `undefined` when it is never kept (no impact entry, reads the tree, an import unreadable); `root` is the checkout. */
34 export function keyOfTest(
35 test: string,
36 impact: ReadonlyMap<string, readonly string[]>,
37 imports: (file: string) => readonly string[],
38 root: string = ".",
39 ): string | undefined {
40 const entry = impact.get(test);
41 if (entry === undefined) return undefined;
42 const seen = new Map<string, string>();
43 const queue = [test, ...entry];
44 while (queue.length > 0) {
45 const file = queue.pop()!;
46 if (seen.has(file)) continue;
47 const source = text(root, file);
48 if (source === undefined) {
49 // A named source file gone is a changed input; an import guess that is no file is no input.
50 if (entry.includes(file) || file === test) seen.set(file, "absent");
51 continue;
52 }
53 // A test, or a helper it imports, that opens the tree reads what no entry names; a source
54 // file that finds its own place may do the same.
55 if (file.startsWith("src/") ? findsItsPlace(source) : readsTheTree(source)) return undefined;
56 seen.set(file, sha256(source));
57 if (seen.size > MOST_FILES) return undefined;
58 for (const next of imports(file)) if (!seen.has(next)) queue.push(next);
59 }
60 for (const file of KEPT_TOOLING) seen.set(file, sha256(text(root, file) ?? "absent"));
61 return keptKey({ bun: Bun.version, platform: process.platform, test, files: [...seen] });
62 }
63
64 /** What the fast tier runs and what it leaves, given the kept results. */
65 export function answerKept(x: {
66 readonly files: readonly string[];
67 readonly kept: Kept;
68 readonly impact: ReadonlyMap<string, readonly string[]>;
69 readonly imports: (file: string) => readonly string[];
70 readonly commit: string;
71 readonly root?: string;
72 }): KeptSplit {
73 return splitKept(
74 x.files,
75 x.kept,
76 (file) => keyOfTest(file, x.impact, x.imports, x.root),
77 (file) => sampled(file, x.commit),
78 );
79 }
80
81 /**
82 * The results a measuring run keeps: a file that ran alone, with coverage, green on its
83 * first try. A file a retry made green is never kept.
84 */
85 export function freshKept(x: {
86 readonly results: readonly { readonly files: readonly string[]; readonly code: number; readonly retried: boolean }[];
87 readonly measured: ReadonlySet<string>;
88 readonly impact: ReadonlyMap<string, readonly string[]>;
89 readonly imports: (file: string) => readonly string[];
90 readonly commit: string;
91 readonly root?: string;
92 }): Map<string, KeptEntry> {
93 const fresh = new Map<string, KeptEntry>();
94 for (const result of x.results) {
95 if (result.code !== 0 || result.retried || result.files.length !== 1) continue;
96 const test = result.files[0]!.replace(/^\.\//, "");
97 if (!x.measured.has(test)) continue;
98 const key = keyOfTest(test, x.impact, x.imports, x.root);
99 if (key !== undefined) fresh.set(key, { test, commit: x.commit });
100 }
101 return fresh;
102 }
scripts/test-shards.ts +51 −2
@@ -99,6 +99,8 @@ import { CHECK_CACHE } from "../src/checks/cache.ts";
9999 import { parseCheck } from "../src/apps/manifest.ts";
100100 import { FLAKY_ALONE, onlyFiles } from "../src/checks/verify.ts";
101101 import { crashFlakeLine } from "../src/checks/flake.ts";
102 import { formatKept, keptLine, mergeKept, parseKept, type KeptEntry } from "../src/checks/kept.ts";
103 import { answerKept, freshKept } from "./kept-input.ts";
102104 import { askBase, baseVerdict, type OnBase } from "../src/checks/base_red.ts";
103105
104106 const TIMES = new URL("./test-times.json", import.meta.url).pathname;
@@ -108,6 +110,8 @@ const TIMES = new URL("./test-times.json", import.meta.url).pathname;
108110 const handed = handedImpact(process.env["FORGE_CHECK_IMPACT"]);
109111 const IMPACT = handed ?? new URL("./test-impact.json", import.meta.url).pathname;
110112 const IMPACT_NAME = handed ?? "scripts/test-impact.json";
113 // Test results kept by what each reads (`src/checks/kept.ts`): handed over like the map, by name.
114 const KEPT = handedImpact(process.env["FORGE_CHECK_KEPT"]);
111115 // What the forge keeps between checks (`src/checks/cache.ts`): the full
112116 // gate's own per-file times, fresher than the committed table.
113117 const CACHED_TIMES = `${CHECK_CACHE}/test-times.json`;
@@ -326,6 +330,31 @@ if (tier === "fast") {
326330 process.exit(0);
327331 }
328332 files = plan.files;
333 // A file whose inputs a default-branch run already proved green is not run again.
334 if (KEPT !== undefined && impact !== undefined) {
335 let kept: ReturnType<typeof parseKept> | undefined;
336 try {
337 kept = parseKept(readFileSync(KEPT, "utf8"));
338 } catch {
339 // none yet
340 }
341 if (kept?.ok) {
342 const split = answerKept({
343 files,
344 kept: kept.value,
345 impact,
346 imports,
347 commit: process.env["FORGE_COMMIT"] ?? "",
348 });
349 for (const [file, commit] of split.kept) console.log(keptLine(file, commit));
350 console.log(`test-shards: ${split.kept.size} of ${files.length} files kept, ${split.run.length} to run`);
351 files = split.run;
352 if (files.length === 0) {
353 console.log("test-shards: every selected file is kept; nothing to run");
354 process.exit(0);
355 }
356 }
357 }
329358 }
330359
331360 const only = onlyFiles(process.env["FORGE_CHECK_ONLY"]);
@@ -486,7 +515,7 @@ let crashReruns = 0;
486515 /** What a live first run shows once it has gone red: held until its end. */
487516 const RED_SIGN = /^(\(fail\)|# Unhandled error|error:)/m;
488517
489 type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[] };
518 type Timed = ShardResult & { seconds: number; tests: TestTime[]; overBudget: Breach[]; retried: boolean };
490519
491520 // The budget (#1242) is red on the whole gate, for every file SLOW does not
492521 // name, and on a fast tier for the test files the pull itself changed: a
@@ -626,6 +655,7 @@ async function runShardIn(
626655 // Green the second time: the crash is a flake, named for the check to record.
627656 if (first.code === 0) process.stderr.write(`${crashFlakeLine(where, quoted)}\n`);
628657 }
658 let retried = crashed;
629659 let code = first.code;
630660 let output = quoted === "" ? first.output : [quoted, ...first.output];
631661 let lastSeconds = first.seconds;
@@ -638,6 +668,7 @@ async function runShardIn(
638668 rmSync(covDir, { recursive: true, force: true });
639669 mkdirSync(covDir);
640670 }
671 retried = true;
641672 const again = await run(paths, cover, through);
642673 lastSeconds = again.seconds;
643674 tests = testsOfLastRun();
@@ -661,6 +692,7 @@ async function runShardIn(
661692 }
662693 const verdict = besideOthers(red, alone, RETRY_CAP);
663694 if (verdict.line !== null) {
695 retried = true;
664696 const note = `${output.join("").endsWith("\n") ? "" : "\n"}test-shards: ${verdict.line}\n`;
665697 output.push(note);
666698 through?.(note);
@@ -734,6 +766,7 @@ async function runShardIn(
734766 seconds: (performance.now() - started) / 1000,
735767 tests,
736768 overBudget,
769 retried,
737770 };
738771 }
739772
@@ -814,8 +847,24 @@ if (measure) {
814847 if ((error as NodeJS.ErrnoException).code !== "ENOENT")
815848 console.log(`test-shards: ${IMPACT_NAME} unreadable (${String(error)}); writing a fresh map`);
816849 }
817 writeFileSync(IMPACT, formatImpact(mergeImpact(old, measured, GATE)));
850 const merged = mergeImpact(old, measured, GATE);
851 writeFileSync(IMPACT, formatImpact(merged));
818852 console.log(`test-shards: wrote ${measured.size} test impacts to ${IMPACT_NAME}`);
853 // Each file that ran green with coverage keeps its result under what it reads.
854 const commit = process.env["FORGE_COMMIT"] ?? "";
855 if (KEPT !== undefined && /^[0-9a-f]{7,64}$/.test(commit)) {
856 const imports = importsReader();
857 const fresh = freshKept({ results, measured: new Set(measured.keys()), impact: merged, imports, commit });
858 let old: Map<string, KeptEntry> = new Map();
859 try {
860 const parsed = parseKept(readFileSync(KEPT, "utf8"));
861 if (parsed.ok) old = parsed.value;
862 } catch {
863 // none yet
864 }
865 writeFileSync(KEPT, formatKept(mergeKept(old, fresh)));
866 console.log(`test-shards: kept ${fresh.size} green results in ${KEPT}`);
867 }
819868 }
820869 }
821870 // The timed full gate is the check on the cell itself: its log ends with
src/checks/kept-store.ts added +71
@@ -0,0 +1,71 @@
1 /**
2 * The kept test results beside the forge (#12178), as the impact map is
3 * (`impact-store.ts`): every check's checkout is seeded with them, and
4 * only a run of the default branch's own measuring gate hands them back.
5 * A pull's code never fills what another check reads. What comes back is
6 * repository code's output: size-capped and parsed by `parseKept`.
7 */
8
9 import { lstatSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
10 import { join } from "node:path";
11 import { formatKept, parseKept } from "./kept.ts";
12
13 /** Where in the checkout the results are handed over, both ways. */
14 export const KEPT_FILE = ".forge-kept.json";
15
16 const MAX_BYTES = 2 * 1024 * 1024;
17
18 const storeOf = (dataDir: string, repoId: string): string =>
19 join(dataDir, "checks", "kept", `${repoId.replace(/[^A-Za-z0-9_-]/g, "_")}.json`);
20
21 function readValid(path: string): string | undefined {
22 try {
23 if (statSync(path).size > MAX_BYTES) return undefined;
24 const text = readFileSync(path, "utf8");
25 const parsed = parseKept(text);
26 return parsed.ok && parsed.value.size > 0 ? formatKept(parsed.value) : undefined;
27 } catch {
28 return undefined;
29 }
30 }
31
32 /** Copies the kept results into a checkout; how many, or 0 for none. */
33 export function seedKept(dataDir: string, repoId: string, checkout: string): number {
34 const text = readValid(storeOf(dataDir, repoId));
35 if (text === undefined) return 0;
36 const path = join(checkout, KEPT_FILE);
37 rmSync(path, { force: true, recursive: true });
38 writeFileSync(path, text, { flag: "wx" });
39 return Object.keys(JSON.parse(text)).length;
40 }
41
42 /**
43 * Keeps what a run of the default branch left in its checkout; the log line, or `undefined`
44 * when it left none or the run is not the default branch's (`byDefaultBranch`).
45 */
46 export function keepKept(
47 dataDir: string,
48 repoId: string,
49 checkout: string,
50 byDefaultBranch: boolean,
51 ): string | undefined {
52 if (!byDefaultBranch) return undefined;
53 const path = join(checkout, KEPT_FILE);
54 let size: number;
55 try {
56 const stat = lstatSync(path);
57 if (!stat.isFile()) return "kept results not kept: not a plain file";
58 size = stat.size;
59 } catch {
60 return undefined;
61 }
62 if (size > MAX_BYTES) return `kept results not kept: ${size} bytes is over ${MAX_BYTES}`;
63 const parsed = parseKept(readFileSync(path, "utf8"));
64 if (!parsed.ok) return `kept results not kept: ${parsed.error}`;
65 if (parsed.value.size === 0) return undefined;
66 mkdirSync(join(dataDir, "checks", "kept"), { recursive: true });
67 const partial = `${storeOf(dataDir, repoId)}.${process.pid}.tmp`;
68 writeFileSync(partial, formatKept(parsed.value));
69 renameSync(partial, storeOf(dataDir, repoId));
70 return `kept results kept: ${parsed.value.size} tests`;
71 }
src/checks/kept.ts added +165
@@ -0,0 +1,165 @@
1 /**
junipercrew

Retried-green results must not be kept, and the key should include the runner scripts (see KEPT_TOOLING).
2 * A test file's result, kept by what it reads (#12178). The fast tier ran
3 * every selected test again each time main moved, though most had read
4 * nothing the move touched. A green test file of the default branch's own
5 * measuring run is remembered under a key: the Bun that ran it, the files
6 * the gate's tooling reads, the test file's own text, the source files its
7 * impact entry says it executed, and the files it imports. A pull's fast
8 * tier that finds the same key does not run the file again and says where
9 * the result was kept from.
10 *
11 * Only a run that recorded coverage keeps a result (the key needs the
12 * impact entry), only a file that ran green on its first try is kept (never
13 * one a retry made green), only the default branch's run writes, a sample
14 * of kept files runs anyway, and a test that
15 * reads the tree or starts processes is never kept: its inputs are not
16 * the files named here. The tier that lands a change never skips a test.
17 *
18 * Plain data in, plain data out; `scripts/kept-input.ts` reads the files.
19 */
20
21 import { sha256 } from "../kernel/hash.ts";
22
23 /** Most results one repository keeps. */
24 export const KEPT_MAX = 4000;
25
26 /** What the tooling reads besides the files named: a change to any is a new key. */
27 export const KEPT_TOOLING: readonly string[] = [
28 "package.json",
29 "bun.lock",
30 "bunfig.toml",
31 "tsconfig.json",
32 "forge.json",
33 "test/helpers/preload.ts",
34 "scripts/test-shards.ts",
35 "scripts/kept-input.ts",
36 "scripts/tier-input.ts",
37 "src/checks/kept.ts",
38 ];
39
40 /** One in this many kept files is run anyway, until an audit of kept results exists. */
41 export const SAMPLE_EVERY = 10;
42
43 /**
44 * Whether `file` is in the sample run again at `commit`: the same answer for the same pair, a
45 * new one next commit.
46 */
47 export function sampled(file: string, commit: string, every: number = SAMPLE_EVERY): boolean {
48 return parseInt(sha256(`${commit}\0${file}`).slice(0, 8), 16) % every === 0;
49 }
50
51 export interface KeptEntry {
52 readonly test: string;
53 /** The commit whose run proved it. */
54 readonly commit: string;
55 }
56
57 export type Kept = ReadonlyMap<string, KeptEntry>;
58
59 /** One result's key; `files` is each input's path and the hash of its text. */
60 export function keptKey(x: {
61 readonly bun: string;
62 readonly platform: string;
63 readonly test: string;
64 readonly files: readonly (readonly [string, string])[];
65 }): string {
66 return sha256(
67 JSON.stringify([x.bun, x.platform, x.test, [...x.files].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))]),
68 );
69 }
70
71 /**
72 * A test whose source reads files from the tree, starts a process or finds
73 * its own place: what it reads is not what its impact entry names.
74 */
75 const OPENS_THE_TREE =
76 /\b(readFileSync|readdirSync|readFile|readdir|Bun\.file|Bun\.Glob|new Glob|statSync|existsSync|Bun\.spawn|Bun\.spawnSync|Bun\.\$|child_process|execSync|spawnSync)\b|import\.meta\.(dir|dirname|url|path|file)|process\.cwd\(|from\s*["']bun["']/;
77
78 export const readsTheTree = (source: string): boolean => OPENS_THE_TREE.test(source);
79
80 /** A source file that finds its own place in the tree can read what no impact entry names. */
81 const FINDS_ITS_PLACE = /import\.meta\.(dir|dirname|url|path|file)|process\.cwd\(/;
82
83 export const findsItsPlace = (source: string): boolean => FINDS_ITS_PLACE.test(source);
84
85 const KEY = /^[0-9a-f]{64}$/;
86 const COMMIT = /^[0-9a-f]{7,64}$/;
87 const TEST = /^test\/[A-Za-z0-9_.-]+\.test\.[cm]?[jt]sx?$/;
88
89 /** The kept results in `text`, or why it is refused. */
90 export function parseKept(text: string): { ok: true; value: Map<string, KeptEntry> } | { ok: false; error: string } {
91 let data: unknown;
92 try {
93 data = JSON.parse(text);
94 } catch {
95 return { ok: false, error: "not JSON" };
96 }
97 if (typeof data !== "object" || data === null || Array.isArray(data))
98 return { ok: false, error: "not an object of key to result" };
99 const map = new Map<string, KeptEntry>();
100 for (const [key, entry] of Object.entries(data)) {
101 const e = entry as { test?: unknown; commit?: unknown } | null;
102 if (
103 !KEY.test(key) ||
104 typeof e !== "object" ||
105 e === null ||
106 typeof e.test !== "string" ||
107 !TEST.test(e.test) ||
108 typeof e.commit !== "string" ||
109 !COMMIT.test(e.commit)
110 )
111 return { ok: false, error: `${key.slice(0, 20)}: not a kept result` };
112 map.set(key, { test: e.test, commit: e.commit });
113 }
114 return { ok: true, value: map };
115 }
116
117 export function formatKept(kept: Kept): string {
118 return JSON.stringify(Object.fromEntries(kept)) + "\n";
119 }
120
121 /**
122 * The results to write: the old ones, then this run's, the oldest dropped
123 * past the bound. A test that ran green now replaces nothing: keys are
124 * content, so an old key is never wrong, only unused.
125 */
126 export function mergeKept(old: Kept, fresh: Kept): Map<string, KeptEntry> {
127 const merged = new Map(old);
128 for (const [key, entry] of fresh) {
129 merged.delete(key);
130 merged.set(key, entry);
131 }
132 while (merged.size > KEPT_MAX) merged.delete(merged.keys().next().value!);
133 return merged;
134 }
135
136 export interface KeptSplit {
137 readonly run: string[];
138 /** The tests not run, each with the commit that proved it. */
139 readonly kept: Map<string, string>;
140 }
141
142 /**
143 * Which of `files` the kept results already answer; `keyOf` is `undefined` for a test never
144 * kept, and `rerun` names a kept file to run anyway (the sample).
145 */
146 export function splitKept(
147 files: readonly string[],
148 kept: Kept,
149 keyOf: (file: string) => string | undefined,
150 rerun: (file: string) => boolean = () => false,
151 ): KeptSplit {
152 const run: string[] = [];
153 const answered = new Map<string, string>();
154 for (const file of files) {
155 const key = keyOf(file);
156 const entry = key === undefined ? undefined : kept.get(key);
157 if (entry === undefined || rerun(file)) run.push(file);
158 else answered.set(file, entry.commit);
159 }
160 return { run, kept: answered };
161 }
162
163 /** The log line for a test not run. */
164 export const keptLine = (file: string, commit: string): string =>
165 ` ${file}: kept from ${commit.slice(0, 7)}: nothing it reads changed`;
src/checks/service.ts +14
@@ -74,6 +74,7 @@ import { ASKED, redMain, unheld } from "./red_main.ts";
7474 import { proposeRevert } from "./revert_red.ts";
7575 import { rebasesFirst } from "./finishing_first.ts";
7676 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
77 import { KEPT_FILE, keepKept, seedKept } from "./kept-store.ts";
7778 import { afterVerify, closeOnGreen } from "./culprit.ts";
7879 import { redDetail, reportGreen, reportRed } from "./report.ts";
7980 import {
@@ -1274,6 +1275,13 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
12741275 } catch (error) {
12751276 write(`impact map not seeded: ${String(error)}`);
12761277 }
1278 // The fast tier skips a test file whose inputs a default-branch run already proved green.
1279 try {
1280 const seeded = seedKept(app.config.dataDir, repo.id, dir);
1281 if (seeded > 0) write(`kept results: ${seeded} tests, from ${repo.defaultBranch}'s measuring runs`);
1282 } catch (error) {
1283 write(`kept results not seeded: ${String(error)}`);
1284 }
12771285 // The check is repository code: it runs in a cell of its own
12781286 // (`apps/contain.ts`), its checkout and a home its only places to write,
12791287 // the forge, the registry and what it declared its only way out.
@@ -1334,6 +1342,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
13341342 FORGE_CHECK: run.id,
13351343 FORGE_COMMIT: x.commitOid,
13361344 FORGE_CHECK_IMPACT: IMPACT_FILE,
1345 FORGE_CHECK_KEPT: KEPT_FILE,
13371346 ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }),
13381347 // The cell's CPU limit, which `availableParallelism()` may not
13391348 // report inside it; the shard count follows this (#2257).
@@ -1575,6 +1584,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
15751584 try {
15761585 const kept = keepImpact(app.config.dataDir, repo.id, dir);
15771586 if (kept !== undefined) write(kept);
1587 const results = keepKept(app.config.dataDir, repo.id, dir, true);
1588 if (results !== undefined) write(results);
15781589 } catch (error) {
15791590 write(`impact map not kept: ${String(error)}`);
15801591 }
@@ -1599,6 +1610,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
15991610 try {
16001611 const kept = keepImpact(app.config.dataDir, repo.id, dir);
16011612 if (kept !== undefined) write(kept);
1613 // Only the default branch's own run hands results back: a pull's tree could write any.
1614 const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true);
1615 if (results !== undefined) write(results);
16021616 } catch (error) {
16031617 write(`impact map not kept: ${String(error)}`);
16041618 }
src/kernel/destroy_reach.ts +1
@@ -69,6 +69,7 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
6969 "src/checks/cache.ts:restoreCache",
7070 "src/checks/cache.ts:keepCache",
7171 "src/checks/impact-store.ts:seedImpact",
72 "src/checks/kept-store.ts:seedKept",
7273 "src/checks/install.ts:prefillInstallCache",
7374 "src/checks/install.ts:holdFile",
7475 "src/checks/install_evict.ts:evictInstallCache",
test/check-kept.test.ts added +194
@@ -0,0 +1,194 @@
1 import { afterEach, describe, expect, test } from "bun:test";
2 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { formatKept, keptKey, mergeKept, parseKept, readsTheTree, splitKept, sampled, KEPT_TOOLING, KEPT_MAX } from "../src/checks/kept.ts";
6 import { KEPT_FILE, keepKept, seedKept } from "../src/checks/kept-store.ts";
7 import { answerKept, freshKept, keyOfTest } from "../scripts/kept-input.ts";
8
9 const base = { bun: "1.2.3", platform: "linux", test: "test/a.test.ts", files: [["src/a.ts", "h1"], ["test/a.test.ts", "h2"]] as [string, string][] };
10 const key = keptKey(base);
11 const commit = "a".repeat(40);
12
13 describe("a test file's kept result (#12178)", () => {
14 test("the key is the same for the same inputs in any order and differs when any input does", () => {
15 expect(keptKey({ ...base, files: [...base.files].reverse() })).toBe(key);
16 expect(keptKey({ ...base, files: [["src/a.ts", "h9"], ["test/a.test.ts", "h2"]] })).not.toBe(key);
17 expect(keptKey({ ...base, bun: "1.2.4" })).not.toBe(key);
18 expect(keptKey({ ...base, files: [...base.files, ["src/b.ts", "h3"]] })).not.toBe(key);
19 });
20
21 test("a kept key answers; a changed input or an unkeyed test runs", () => {
22 const kept = new Map([[key, { test: base.test, commit }]]);
23 const keys: Record<string, string | undefined> = { "test/a.test.ts": key, "test/b.test.ts": "0".repeat(64), "test/c.test.ts": undefined };
24 const split = splitKept(Object.keys(keys), kept, (f) => keys[f]);
25 expect(split.run).toEqual(["test/b.test.ts", "test/c.test.ts"]);
26 expect([...split.kept]).toEqual([["test/a.test.ts", commit]]);
27 });
28
29 test("the file round-trips and refuses what is not a result", () => {
30 const kept = new Map([[key, { test: base.test, commit }]]);
31 const parsed = parseKept(formatKept(kept));
32 expect(parsed.ok && [...parsed.value]).toEqual([...kept]);
33 expect(parseKept("[]").ok).toBe(false);
34 expect(parseKept(JSON.stringify({ [key]: { test: "../x", commit } })).ok).toBe(false);
35 expect(parseKept(JSON.stringify({ nope: { test: base.test, commit } })).ok).toBe(false);
36 });
37
38 test("the oldest results go past the bound", () => {
39 const old = new Map<string, { test: string; commit: string }>();
40 for (let i = 0; i < KEPT_MAX; i++) old.set(String(i).padStart(64, "0"), { test: base.test, commit });
41 const merged = mergeKept(old, new Map([[key, { test: base.test, commit }]]));
42 expect(merged.size).toBe(KEPT_MAX);
43 expect(merged.has(key)).toBe(true);
44 expect(merged.has("0".repeat(64))).toBe(false);
45 });
46
47 test("a test that opens the tree is recognised", () => {
48 expect(readsTheTree('readFileSync("README.md")')).toBe(true);
49 expect(readsTheTree("expect(1).toBe(1)")).toBe(false);
50 });
51 });
52
53 describe("what a test file reads, hashed", () => {
54 const dirs: string[] = [];
55 let root = ".";
56 afterEach(() => {
57 for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
58 });
59 function tree(files: Record<string, string>): void {
60 const dir = mkdtempSync(join(tmpdir(), "kept-"));
61 dirs.push(dir);
62 for (const [path, text] of Object.entries(files)) {
63 mkdirSync(join(dir, path, ".."), { recursive: true });
64 writeFileSync(join(dir, path), text);
65 }
66 root = dir;
67 }
68 const impact = new Map([["test/a.test.ts", ["src/a.ts"]]]);
69 const files = { "test/a.test.ts": "test('x',()=>{})", "src/a.ts": "export const a = 1;", "package.json": "{}" };
70 const imports = (): string[] => [];
71
72 test("a change to a file the test executes, to the test, or to the tooling is a new key; any other file is not", () => {
73 tree(files);
74 const before = keyOfTest("test/a.test.ts", impact, imports, root);
75 expect(before).toBeDefined();
76 writeFileSync(join(root, "src/other.ts"), "export const o = 1;");
77 expect(keyOfTest("test/a.test.ts", impact, imports, root)).toBe(before);
78 for (const [path, text] of [["src/a.ts", "export const a = 2;"], ["test/a.test.ts", "test('y',()=>{})"], ["package.json", '{"a":1}']] as const) {
79 tree(files);
80 writeFileSync(join(root, path), text);
81 expect(keyOfTest("test/a.test.ts", impact, imports, root)).not.toBe(before);
82 }
83 });
84
85 test("a test with no impact entry, one that reads the tree, or whose source finds its place is never kept", () => {
86 tree(files);
87 expect(keyOfTest("test/b.test.ts", impact, imports, root)).toBeUndefined();
88 writeFileSync(join(root, "test/a.test.ts"), 'import { readFileSync } from "node:fs";');
89 expect(keyOfTest("test/a.test.ts", impact, imports, root)).toBeUndefined();
90 tree({ ...files, "src/a.ts": "export const d = import.meta.dir;" });
91 expect(keyOfTest("test/a.test.ts", impact, imports, root)).toBeUndefined();
92 });
93
94 test("a change to the runner, forge.json or the preload is a new key", () => {
95 for (const path of ["scripts/test-shards.ts", "scripts/kept-input.ts", "forge.json", "test/helpers/preload.ts"])
96 expect(KEPT_TOOLING).toContain(path);
97 tree(files);
98 const before = keyOfTest("test/a.test.ts", impact, imports, root);
99 mkdirSync(join(root, "scripts"));
100 writeFileSync(join(root, "scripts/test-shards.ts"), "// changed");
101 expect(keyOfTest("test/a.test.ts", impact, imports, root)).not.toBe(before);
102 });
103
104 test("a helper the test imports is an input", () => {
105 tree({ ...files, "test/help.ts": "export const h = 1;" });
106 const via = (file: string): string[] => (file === "test/a.test.ts" ? ["test/help.ts"] : []);
107 const before = keyOfTest("test/a.test.ts", impact, via, root);
108 writeFileSync(join(root, "test/help.ts"), "export const h = 2;");
109 expect(keyOfTest("test/a.test.ts", impact, via, root)).not.toBe(before);
110 });
111 });
112
113 describe("the fast tier's use of kept results", () => {
114 const dirs: string[] = [];
115 afterEach(() => {
116 for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
117 });
118 const dir = mkdtempSync(join(tmpdir(), "kept-wire-"));
119 dirs.push(dir);
120 mkdirSync(join(dir, "test"), { recursive: true });
121 mkdirSync(join(dir, "src"), { recursive: true });
122 const names = ["a", "b", "c"].map((n) => `test/${n}.test.ts`);
123 for (const n of ["a", "b", "c"]) {
124 writeFileSync(join(dir, `test/${n}.test.ts`), `test('${n}',()=>{})`);
125 writeFileSync(join(dir, `src/${n}.ts`), `export const ${n} = 1;`);
126 }
127 const impact = new Map(names.map((t, i) => [t, [`src/${"abc"[i]}.ts`]] as const));
128 const imports = (): string[] => [];
129 const measured = new Set(names);
130 const green = (file: string, retried = false) => ({ files: [`./${file}`], code: 0, retried });
131
132 test("a green first-try file is kept, answers the next run, and a changed input runs", () => {
133 const fresh = freshKept({ results: names.map((n) => green(n)), measured, impact, imports, commit, root: dir });
134 expect(fresh.size).toBe(3);
135 // every file sampled out of the way: pick a commit where none of them is sampled
136 let salt = "b".repeat(40);
137 for (let i = 0; names.some((n) => sampled(n, salt)); i++) salt = i.toString(16).padStart(40, "0");
138 const answer = () => answerKept({ files: names, kept: fresh, impact, imports, commit: salt, root: dir });
139 expect(answer().run).toEqual([]);
140 expect([...answer().kept.keys()]).toEqual(names);
141 writeFileSync(join(dir, "src/b.ts"), "export const b = 2;");
142 expect(answer().run).toEqual(["test/b.test.ts"]);
143 writeFileSync(join(dir, "src/b.ts"), "export const b = 1;");
144 });
145
146 test("a file a retry made green, a red one, one run beside others or never measured is not kept", () => {
147 const fresh = freshKept({
148 results: [green(names[0]!, true), { files: [`./${names[1]!}`], code: 1, retried: false }, { files: names, code: 0, retried: false }],
149 measured,
150 impact,
151 imports,
152 commit,
153 root: dir,
154 });
155 expect(fresh.size).toBe(0);
156 expect(freshKept({ results: [green(names[0]!)], measured: new Set(), impact, imports, commit, root: dir }).size).toBe(0);
157 });
158
159 test("a sample of kept files runs anyway, the same one for the same commit", () => {
160 const hits = Array.from({ length: 400 }, (_, i) => sampled(`test/f${i}.test.ts`, commit)).filter(Boolean).length;
161 expect(hits).toBeGreaterThan(10);
162 expect(hits).toBeLessThan(90);
163 expect(sampled("test/a.test.ts", commit)).toBe(sampled("test/a.test.ts", commit));
164 const kept = new Map([[key, { test: base.test, commit }]]);
165 expect(splitKept(["test/a.test.ts"], kept, () => key, () => true).run).toEqual(["test/a.test.ts"]);
166 });
167 });
168
169 describe("the kept results beside the forge", () => {
170 test("the file handed between checks is ignored, as the impact map is", () => {
171 const ignored = readFileSync(join(import.meta.dir, "..", ".gitignore"), "utf8").split("\n");
172 expect(ignored).toContain(`/${KEPT_FILE}`);
173 expect(ignored).toContain("/.forge-impact.json");
174 });
175
176 test("seed what was kept; refuse what is not a result; hand back only a valid file", () => {
177 const data = mkdtempSync(join(tmpdir(), "kept-data-"));
178 const first = mkdtempSync(join(tmpdir(), "kept-co-"));
179 const second = mkdtempSync(join(tmpdir(), "kept-co-"));
180 try {
181 expect(seedKept(data, "r1", first)).toBe(0);
182 writeFileSync(join(first, KEPT_FILE), "nonsense");
183 expect(keepKept(data, "r1", first, true)).toContain("not kept");
184 writeFileSync(join(first, KEPT_FILE), formatKept(new Map([[key, { test: base.test, commit }]])));
185 expect(keepKept(data, "r1", first, false)).toBeUndefined();
186 expect(seedKept(data, "r1", second)).toBe(0);
187 expect(keepKept(data, "r1", first, true)).toContain("1 tests");
188 expect(seedKept(data, "r1", second)).toBe(1);
189 expect(seedKept(data, "r2", second)).toBe(0);
190 } finally {
191 for (const dir of [data, first, second]) rmSync(dir, { recursive: true, force: true });
192 }
193 });
194 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs