Public: anyone can read this; the crew can change it.

The fast tier's import guess uses Bun's transpiler, and logs what bun test --changed would pick beside it #14947

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/14941-the-fast-tier-s-import-guess-uses-bun-s → main 8 commits fork 3 threads · 3 unresolved

  • Merged. torque/14941-the-fast-tier-s-import-guess-uses-bun-s was merged into main as 4cfcb7f.

Review

Sign in to review this change.
7 files changed +261 −8
scripts/changed-pick.ts added +77
@@ -0,0 +1,77 @@
1 /**
2 * What `bun test --changed=<base>` would pick for this tree, beside the fast
3 * tier's pick, as one line of the check log (#14941). It only reports: a
4 * failure here is a word in the log, never a red check.
5 *
6 * `--changed` has no list-only mode: a run whose `-t` filter matches no test
7 * still opens every file it picked, so the junit report names them.
8 */
9
10 import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { comparisonLine, junitFiles, runsAll } from "../src/checks/changed_compare.ts";
14
15 /** A revision git cannot read as a flag. */
16 const REV = /^[A-Za-z0-9_][A-Za-z0-9_./@^~-]*$/;
17 const NEEDLE = "NO-TEST-IS-NAMED-THIS";
18 /** Cut sharply: it runs beside the plan, and a slow answer costs nothing but the word. */
19 export const LIMIT_MS = 15_000;
20
21 /** What `bun test --changed` answered: its files, "all", or why there is no answer. */
22 export type Outcome =
23 | { readonly kind: "all" }
24 | { readonly kind: "files"; readonly files: readonly string[] }
25 | { readonly kind: "unavailable"; readonly why: string };
26
27 /** The spawn seam: argv and options in, the process out (`Bun.spawn`'s shape, as far as used). */
28 export type Spawn = (
29 cmd: string[],
outdated

changedLine is untested; add tests for the REV guard and the runsAll path (both spawn nothing).

30 opts: { stdout: "ignore"; stderr: "ignore"; env: Record<string, string | undefined> },
31 ) => { readonly exited: Promise<number>; kill(): void };
32
33 /**
34 * Start `bun test --changed=<base>` now and answer later: await the result after
35 * the plan, so the comparison overlaps it. Spawns nothing for a bad revision or a
36 * diff `--changed` cannot judge.
37 */
38 export async function startChanged(
outdated

Serial 60s spawn on every fast run: overlap it with the plan (Bun.spawn) or shorten the limit.

39 base: string,
40 changed: readonly string[],
41 spawn: Spawn = Bun.spawn as unknown as Spawn,
42 limitMs: number = LIMIT_MS,
43 ): Promise<Outcome> {
44 if (!REV.test(base)) return { kind: "unavailable", why: "the base is not a revision" };
45 if (changed.some(runsAll)) return { kind: "all" };
46 const scratch = mkdtempSync(join(process.env["TMPDIR"] ?? tmpdir(), "changed-"));
47 let timer: ReturnType<typeof setTimeout> | undefined;
outdated

Swallowed error: say in the line why it was unavailable (timeout, no report, spawn failure).

48 try {
49 const report = join(scratch, "junit.xml");
50 const proc = spawn(
51 ["bun", "test", `--changed=${base}`, "-t", NEEDLE, "--reporter=junit", `--reporter-outfile=${report}`],
52 { stdout: "ignore", stderr: "ignore", env: { ...process.env, FORGE_TEST_SOCKET: "port" } },
53 );
54 const timedOut = new Promise<"timeout">((resolve) => {
55 timer = setTimeout(() => resolve("timeout"), limitMs);
56 });
57 if ((await Promise.race([proc.exited, timedOut])) === "timeout") {
58 proc.kill();
59 return { kind: "unavailable", why: `timed out after ${limitMs / 1000}s` };
60 }
61 if (!existsSync(report)) return { kind: "unavailable", why: "no report written" };
62 return { kind: "files", files: junitFiles(readFileSync(report, "utf8")) };
63 } catch (e) {
64 return { kind: "unavailable", why: `spawn failed: ${e instanceof Error ? e.message : String(e)}` };
65 } finally {
66 if (timer !== undefined) clearTimeout(timer);
67 rmSync(scratch, { recursive: true, force: true });
68 }
69 }
70
71 /** The line for the log: the map's `fast` picks against the outcome, among the gate's `tests`. */
72 export function changedLine(outcome: Outcome, tests: readonly string[], fast: readonly string[]): string {
73 if (outcome.kind === "all") return comparisonLine(fast, tests, true, tests.length);
74 if (outcome.kind === "unavailable") return comparisonLine(fast, undefined, false, tests.length, outcome.why);
75 const gate = new Set(tests);
76 return comparisonLine(fast, outcome.files.filter((f) => gate.has(f)), false, tests.length);
77 }
scripts/test-shards.ts +5
@@ -113,6 +113,7 @@ import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, brea
113113 import { cgroupMemory, summaryLines } from "./gate-summary.ts";
114114 import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts";
115115 import { importsReader, readImpact, scansIn } from "./tier-input.ts";
116 import { changedLine, startChanged } from "./changed-pick.ts";
116117 import { docReadersIn, readersIn } from "./script-readers.ts";
117118 import { shardHome, spaceLine } from "../src/checks/disk_full.ts";
118119 import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts";
@@ -363,6 +364,8 @@ if (tier === "fast") {
363364 );
364365 if (gateDiff.exitCode === 0) gateAdds = gateListAdds(gateDiff.stdout.toString());
365366 }
367 // Started before the plan work and awaited after it, so a slow comparison costs nothing (#14941).
368 const picked = startChanged(diffArg.value!.split("..")[0]!, changed);
366369 const imports = importsReader();
367370 const impact = readImpact(IMPACT, IMPACT_NAME, (line) => console.log(`test-shards: ${line}`));
368371 const plan = fastSet({
@@ -387,6 +390,8 @@ if (tier === "fast") {
387390 if (blind !== undefined) console.log(`test-shards: ${blind}`);
388391 for (const file of plan.files)
389392 console.log(` ${file}: ${plan.why.get(file)}`);
393 // Beside the map's choice, never acting on it: what `bun test --changed` would pick (#14941).
394 console.log(changedLine(await picked, files, plan.files));
390395 // A diff no test reaches (a timing table, a doc) is judged by the
391396 // type check alone: green, not a usage error.
392397 if (plan.files.length === 0) {
src/checks/changed_compare.ts +27
@@ -93,3 +93,30 @@ export function verdict(rows: readonly Row[]): Verdict {
9393 : "neither";
9494 return { fastMissed, bunMissed, fastTotal, bunTotal, compared: both.length, use };
9595 }
96
97 /**
98 * The check log's line comparing the map's pick with what `bun test
99 * --changed` picked, which is logged and acted on by nothing (#14941).
100 * `bun` is `undefined` when the flag's run could not be read; `all` is a
101 * diff `--changed` cannot judge (`runsAll`), which it answers by running
102 * everything.
103 */
104 export function comparisonLine(
105 fast: readonly string[],
106 bun: readonly string[] | undefined,
107 all: boolean,
108 of: number,
109 why = "bun test --changed gave no report",
110 ): string {
111 if (bun === undefined) return `test-shards: --changed comparison: unavailable (${why})`;
112 const mine = new Set(fast);
113 const theirs = new Set(bun);
114 const only = (a: Iterable<string>, b: ReadonlySet<string>): string[] => [...a].filter((f) => !b.has(f)).sort();
115 const names = (files: readonly string[]): string => (files.length === 0 ? "none" : files.slice(0, 5).join(", ") + (files.length > 5 ? `, +${files.length - 5} more` : ""));
116 const looser = only(theirs, mine);
117 const tighter = only(mine, theirs);
118 return (
119 `test-shards: --changed comparison: the map ${mine.size} of ${of}, --changed ${all ? `all ${of} (the diff holds a file no test imports)` : `${theirs.size}`}` +
120 (all ? "" : `; only --changed: ${names(looser)}; only the map: ${names(tighter)}`)
121 );
122 }
src/checks/tiers.ts +43 −6
@@ -98,12 +98,8 @@ function normalise(path: string): string | undefined {
9898 return out.join("/");
9999 }
100100
101 /**
102 * The repository files a source file imports by a relative specifier, as
103 * repository-relative paths. Packages and `node:`/`bun:` modules are not
104 * the repository's and are left out.
105 */
106 export function importsOf(source: string, file: string): string[] {
101 /** The regex reading, for source the transpiler cannot parse (a fragment, a half-edited file). */
102 function importsByPattern(source: string, file: string): string[] {
107103 const found = new Set<string>();
108104 const pattern =
109105 /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\brequire\s*\(\s*)["'](\.{1,2}\/[^"'\n]*)["']/g;
@@ -114,6 +110,47 @@ export function importsOf(source: string, file: string): string[] {
114110 return [...found];
115111 }
116112
113 // One transpiler per loader, made on first use (a fresh one per call costs
114 // more than the parse); a lazy getter keeps no collection that is written to.
115 const scanners = {
116 ts: lazy("ts"),
117 tsx: lazy("tsx"),
118 };
119
120 function lazy(loader: "ts" | "tsx"): () => InstanceType<typeof Bun.Transpiler> {
121 let made: InstanceType<typeof Bun.Transpiler> | undefined;
122 return () => (made ??= new Bun.Transpiler({ loader }));
123 }
124
125 /**
126 * The repository files a source file imports by a relative specifier, as
127 * repository-relative paths. Packages and `node:`/`bun:` modules are not
128 * the repository's and are left out.
129 *
130 * Read by Bun's own transpiler (`Bun.Transpiler.scanImports`), which is
131 * the parse a run does: a type-only import is erased before it runs, so it
132 * is not a dependency and is left out; a specifier inside a comment or a
133 * string is not an import. tsconfig.json declares no `paths` or `baseUrl`,
134 * so a relative specifier is the only way into the repository. Source that
135 * does not parse is read by pattern instead.
136 */
137 export function importsOf(source: string, file: string): string[] {
138 const loader = /\.[cm]?[jt]sx$/.test(file) ? "tsx" : "ts";
139 let scanned: { path: string }[];
140 try {
141 scanned = scanners[loader]().scanImports(source);
142 } catch {
143 return importsByPattern(source, file);
144 }
145 const found = new Set<string>();
146 for (const { path: specifier } of scanned) {
147 if (!/^\.{1,2}\//.test(specifier)) continue;
148 const path = normalise(`${dirname(file)}/${specifier}`);
149 if (path !== undefined) found.add(path);
150 }
151 return [...found];
152 }
153
117154 /** The words a test file's name shares with a source path: `src/a/b.ts` → a-b, b, a. */
118155 function namesOf(path: string): string[] {
119156 const parts = path.replace(/\.[cm]?[jt]sx?$/, "").split("/");
test/changed-compare.test.ts +17 −2
@@ -1,5 +1,5 @@
1 import { expect, test } from "bun:test";
2 import { junitFiles, missed, redFiles, runsAll, verdict } from "../src/checks/changed_compare.ts";
1 import { describe, expect, test } from "bun:test";
2 import { comparisonLine, junitFiles, missed, redFiles, runsAll, verdict } from "../src/checks/changed_compare.ts";
33
44 test("a failed test belongs to the file header printed before it", () => {
55 const log = [
@@ -47,3 +47,18 @@ test("the verdict keeps a pick that missed nothing, the smaller of two such", ()
4747 expect(verdict([row(["a"], ["b"], ["a", "b"])]).use).toBe("neither");
4848 expect(verdict([row(undefined, ["a"], ["b"])]).compared).toBe(0);
4949 });
50
51 describe("comparisonLine", () => {
52 const tests = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
53 test("names what each pick has the other lacks", () => {
54 expect(comparisonLine(["test/a.test.ts", "test/b.test.ts"], ["test/b.test.ts", "test/c.test.ts"], false, 3)).toBe(
55 "test-shards: --changed comparison: the map 2 of 3, --changed 2; only --changed: test/c.test.ts; only the map: test/a.test.ts",
56 );
57 });
58 test("a diff --changed cannot judge is all of the gate", () => {
59 expect(comparisonLine(["test/a.test.ts"], tests, true, 3)).toContain("--changed all 3");
60 });
61 test("an unreadable run is said, not guessed", () => {
62 expect(comparisonLine(["test/a.test.ts"], undefined, false, 3)).toContain("unavailable");
63 });
64 });
test/changed-pick.test.ts added +67
@@ -0,0 +1,67 @@
1 import { describe, expect, test } from "bun:test";
2 import { writeFileSync } from "node:fs";
3 import { changedLine, startChanged, type Spawn } from "../scripts/changed-pick.ts";
4
5 const tests = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
6 const never: Spawn = () => {
7 throw new Error("must not spawn");
8 };
9 const reportOf = (...names: string[]): string => names.map((n) => `<testsuite name="${n}">`).join("");
10 /** A spawn that writes the junit report named in argv, then exits. */
11 const writing = (xml: string): Spawn => (cmd) => {
12 writeFileSync(cmd.find((a) => a.startsWith("--reporter-outfile="))!.split("=")[1]!, xml);
13 return { exited: Promise.resolve(0), kill() {} };
14 };
15
16 describe("startChanged", () => {
17 test("a base that is not a revision spawns nothing and is unavailable", async () => {
18 for (const base of ["-x", "", "--help", "a b"])
19 expect(await startChanged(base, ["src/a.ts"], never)).toEqual({ kind: "unavailable", why: "the base is not a revision" });
20 });
21
22 test("a file no test imports answers all without spawning", async () => {
23 expect(await startChanged("main", ["scripts/x.ts"], never)).toEqual({ kind: "all" });
24 expect(await startChanged("main", ["forge.json"], never)).toEqual({ kind: "all" });
25 });
26
27 test("the junit report's test files are the answer", async () => {
28 const out = await startChanged("main", ["src/a.ts"], writing(reportOf("test/a.test.ts", "other")));
29 expect(out).toEqual({ kind: "files", files: ["test/a.test.ts"] });
30 });
31
32 test("no report is said so", async () => {
33 const out = await startChanged("main", ["src/a.ts"], () => ({ exited: Promise.resolve(1), kill() {} }));
34 expect(out).toEqual({ kind: "unavailable", why: "no report written" });
35 });
36
37 test("a hung run is killed and reported as a timeout", async () => {
38 let killed = false;
39 const hung: Spawn = () => ({ exited: new Promise<number>(() => {}), kill: () => (killed = true) });
40 const out = await startChanged("main", ["src/a.ts"], hung, 20);
41 expect(killed).toBe(true);
42 expect(out.kind === "unavailable" && out.why.startsWith("timed out")).toBe(true);
43 });
44
45 test("a spawn that throws gives its reason", async () => {
46 const out = await startChanged("main", ["src/a.ts"], () => {
47 throw new Error("no bun");
48 });
49 expect(out).toEqual({ kind: "unavailable", why: "spawn failed: no bun" });
50 });
51 });
52
53 describe("changedLine", () => {
54 test("all names the whole gate", () => {
55 expect(changedLine({ kind: "all" }, tests, ["test/a.test.ts"])).toContain("--changed all 3");
56 });
57
58 test("unavailable carries its reason", () => {
59 const line = changedLine({ kind: "unavailable", why: "timed out after 15s" }, tests, []);
60 expect(line).toContain("unavailable (timed out after 15s)");
61 });
62
63 test("files outside the gate list are dropped", () => {
64 const line = changedLine({ kind: "files", files: ["test/a.test.ts", "test/zzz.test.ts"] }, tests, ["test/b.test.ts"]);
65 expect(line).toContain("only --changed: test/a.test.ts; only the map: test/b.test.ts");
66 });
67 });
test/check-tiers.test.ts +25
@@ -598,6 +598,31 @@ describe("importsOf", () => {
598598 });
599599 });
600600
601 describe("importsOf by the transpiler", () => {
602 test("leaves out a type-only import, a comment and a string, and keeps the rest", () => {
603 const source = [
604 `import type { T } from "./types.ts";`,
605 `import { type U } from "./only-types.ts";`,
606 `import { v } from "./value.ts";`,
607 `import "./side-effect.ts";`,
608 `export * from "../up.ts";`,
609 `// import { c } from "./commented.ts";`,
610 `const s = 'import x from "./quoted.ts"';`,
611 `const r = require("./required.ts");`,
612 ].join("\n");
613 expect(importsOf(source, "src/a/b.ts").sort()).toEqual([
614 "src/a/required.ts",
615 "src/a/side-effect.ts",
616 "src/a/value.ts",
617 "src/up.ts",
618 ]);
619 });
620
621 test("source that does not parse is still read by pattern", () => {
622 expect(importsOf(`from "./x.ts"`, "src/a.ts")).toEqual(["src/x.ts"]);
623 });
624 });
625
601626 describe("the checks line", () => {
602627 test("words", () => {
603628 expect(tierLine("green", 220)).toBe(

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs