Public: anyone can read this; the crew can change it.
The fast tier's import guess uses Bun's transpiler, and logs what bun test --changed would pick beside it #14947
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14941-the-fast-tier-s-import-guess-uses-bun-s → main 8 commits fork 3 threads · 3 unresolved
7 files
changed +261
−8
scripts/changed-pick.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | /** | |
| 2 | * What `bun test --changed=<base>` would pick for this tree, beside the fast | |
| 3 | * tier's pick, as one line of the check log (#14941). It only reports: a | |
| 4 | * failure here is a word in the log, never a red check. | |
| 5 | * | |
| 6 | * `--changed` has no list-only mode: a run whose `-t` filter matches no test | |
| 7 | * still opens every file it picked, so the junit report names them. | |
| 8 | */ | |
| 9 | ||
| 10 | import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { comparisonLine, junitFiles, runsAll } from "../src/checks/changed_compare.ts"; | |
| 14 | ||
| 15 | /** A revision git cannot read as a flag. */ | |
| 16 | const REV = /^[A-Za-z0-9_][A-Za-z0-9_./@^~-]*$/; | |
| 17 | const NEEDLE = "NO-TEST-IS-NAMED-THIS"; | |
| 18 | /** Cut sharply: it runs beside the plan, and a slow answer costs nothing but the word. */ | |
| 19 | export const LIMIT_MS = 15_000; | |
| 20 | ||
| 21 | /** What `bun test --changed` answered: its files, "all", or why there is no answer. */ | |
| 22 | export type Outcome = | |
| 23 | | { readonly kind: "all" } | |
| 24 | | { readonly kind: "files"; readonly files: readonly string[] } | |
| 25 | | { readonly kind: "unavailable"; readonly why: string }; | |
| 26 | ||
| 27 | /** The spawn seam: argv and options in, the process out (`Bun.spawn`'s shape, as far as used). */ | |
| 28 | export type Spawn = ( | |
| 29 | cmd: string[], | |
| 30 | opts: { stdout: "ignore"; stderr: "ignore"; env: Record<string, string | undefined> }, | |
| 31 | ) => { readonly exited: Promise<number>; kill(): void }; | |
| 32 | ||
| 33 | /** | |
| 34 | * Start `bun test --changed=<base>` now and answer later: await the result after | |
| 35 | * the plan, so the comparison overlaps it. Spawns nothing for a bad revision or a | |
| 36 | * diff `--changed` cannot judge. | |
| 37 | */ | |
| 38 | export async function startChanged( | |
outdated
Serial 60s spawn on every fast run: overlap it with the plan (Bun.spawn) or shorten the limit. |
||
| 39 | base: string, | |
| 40 | changed: readonly string[], | |
| 41 | spawn: Spawn = Bun.spawn as unknown as Spawn, | |
| 42 | limitMs: number = LIMIT_MS, | |
| 43 | ): Promise<Outcome> { | |
| 44 | if (!REV.test(base)) return { kind: "unavailable", why: "the base is not a revision" }; | |
| 45 | if (changed.some(runsAll)) return { kind: "all" }; | |
| 46 | const scratch = mkdtempSync(join(process.env["TMPDIR"] ?? tmpdir(), "changed-")); | |
| 47 | let timer: ReturnType<typeof setTimeout> | undefined; | |
outdated
Swallowed error: say in the line why it was unavailable (timeout, no report, spawn failure). |
||
| 48 | try { | |
| 49 | const report = join(scratch, "junit.xml"); | |
| 50 | const proc = spawn( | |
| 51 | ["bun", "test", `--changed=${base}`, "-t", NEEDLE, "--reporter=junit", `--reporter-outfile=${report}`], | |
| 52 | { stdout: "ignore", stderr: "ignore", env: { ...process.env, FORGE_TEST_SOCKET: "port" } }, | |
| 53 | ); | |
| 54 | const timedOut = new Promise<"timeout">((resolve) => { | |
| 55 | timer = setTimeout(() => resolve("timeout"), limitMs); | |
| 56 | }); | |
| 57 | if ((await Promise.race([proc.exited, timedOut])) === "timeout") { | |
| 58 | proc.kill(); | |
| 59 | return { kind: "unavailable", why: `timed out after ${limitMs / 1000}s` }; | |
| 60 | } | |
| 61 | if (!existsSync(report)) return { kind: "unavailable", why: "no report written" }; | |
| 62 | return { kind: "files", files: junitFiles(readFileSync(report, "utf8")) }; | |
| 63 | } catch (e) { | |
| 64 | return { kind: "unavailable", why: `spawn failed: ${e instanceof Error ? e.message : String(e)}` }; | |
| 65 | } finally { | |
| 66 | if (timer !== undefined) clearTimeout(timer); | |
| 67 | rmSync(scratch, { recursive: true, force: true }); | |
| 68 | } | |
| 69 | } | |
| 70 | ||
| 71 | /** The line for the log: the map's `fast` picks against the outcome, among the gate's `tests`. */ | |
| 72 | export function changedLine(outcome: Outcome, tests: readonly string[], fast: readonly string[]): string { | |
| 73 | if (outcome.kind === "all") return comparisonLine(fast, tests, true, tests.length); | |
| 74 | if (outcome.kind === "unavailable") return comparisonLine(fast, undefined, false, tests.length, outcome.why); | |
| 75 | const gate = new Set(tests); | |
| 76 | return comparisonLine(fast, outcome.files.filter((f) => gate.has(f)), false, tests.length); | |
| 77 | } |
scripts/test-shards.ts +5
| @@ -113,6 +113,7 @@ import { breaches, breachReport, budgetsFrom, retimeVerdict, breachClosing, brea | ||
| 113 | 113 | import { cgroupMemory, summaryLines } from "./gate-summary.ts"; |
| 114 | 114 | import { GATE_FILE, LOADS_ONLY, coverOf, executedFiles, formatImpact, gateListAdds, handedImpact, mergeImpact, packImpact, usedFiles, type Cover } from "../src/checks/impact.ts"; |
| 115 | 115 | import { importsReader, readImpact, scansIn } from "./tier-input.ts"; |
| 116 | import { changedLine, startChanged } from "./changed-pick.ts"; | |
| 116 | 117 | import { docReadersIn, readersIn } from "./script-readers.ts"; |
| 117 | 118 | import { shardHome, spaceLine } from "../src/checks/disk_full.ts"; |
| 118 | 119 | import { blindLine, fastSet, duration, reasonLine } from "../src/checks/tiers.ts"; |
| @@ -363,6 +364,8 @@ if (tier === "fast") { | ||
| 363 | 364 | ); |
| 364 | 365 | if (gateDiff.exitCode === 0) gateAdds = gateListAdds(gateDiff.stdout.toString()); |
| 365 | 366 | } |
| 367 | // Started before the plan work and awaited after it, so a slow comparison costs nothing (#14941). | |
| 368 | const picked = startChanged(diffArg.value!.split("..")[0]!, changed); | |
| 366 | 369 | const imports = importsReader(); |
| 367 | 370 | const impact = readImpact(IMPACT, IMPACT_NAME, (line) => console.log(`test-shards: ${line}`)); |
| 368 | 371 | const plan = fastSet({ |
| @@ -387,6 +390,8 @@ if (tier === "fast") { | ||
| 387 | 390 | if (blind !== undefined) console.log(`test-shards: ${blind}`); |
| 388 | 391 | for (const file of plan.files) |
| 389 | 392 | console.log(` ${file}: ${plan.why.get(file)}`); |
| 393 | // Beside the map's choice, never acting on it: what `bun test --changed` would pick (#14941). | |
| 394 | console.log(changedLine(await picked, files, plan.files)); | |
| 390 | 395 | // A diff no test reaches (a timing table, a doc) is judged by the |
| 391 | 396 | // type check alone: green, not a usage error. |
| 392 | 397 | if (plan.files.length === 0) { |
src/checks/changed_compare.ts +27
| @@ -93,3 +93,30 @@ export function verdict(rows: readonly Row[]): Verdict { | ||
| 93 | 93 | : "neither"; |
| 94 | 94 | return { fastMissed, bunMissed, fastTotal, bunTotal, compared: both.length, use }; |
| 95 | 95 | } |
| 96 | ||
| 97 | /** | |
| 98 | * The check log's line comparing the map's pick with what `bun test | |
| 99 | * --changed` picked, which is logged and acted on by nothing (#14941). | |
| 100 | * `bun` is `undefined` when the flag's run could not be read; `all` is a | |
| 101 | * diff `--changed` cannot judge (`runsAll`), which it answers by running | |
| 102 | * everything. | |
| 103 | */ | |
| 104 | export function comparisonLine( | |
| 105 | fast: readonly string[], | |
| 106 | bun: readonly string[] | undefined, | |
| 107 | all: boolean, | |
| 108 | of: number, | |
| 109 | why = "bun test --changed gave no report", | |
| 110 | ): string { | |
| 111 | if (bun === undefined) return `test-shards: --changed comparison: unavailable (${why})`; | |
| 112 | const mine = new Set(fast); | |
| 113 | const theirs = new Set(bun); | |
| 114 | const only = (a: Iterable<string>, b: ReadonlySet<string>): string[] => [...a].filter((f) => !b.has(f)).sort(); | |
| 115 | const names = (files: readonly string[]): string => (files.length === 0 ? "none" : files.slice(0, 5).join(", ") + (files.length > 5 ? `, +${files.length - 5} more` : "")); | |
| 116 | const looser = only(theirs, mine); | |
| 117 | const tighter = only(mine, theirs); | |
| 118 | return ( | |
| 119 | `test-shards: --changed comparison: the map ${mine.size} of ${of}, --changed ${all ? `all ${of} (the diff holds a file no test imports)` : `${theirs.size}`}` + | |
| 120 | (all ? "" : `; only --changed: ${names(looser)}; only the map: ${names(tighter)}`) | |
| 121 | ); | |
| 122 | } |
src/checks/tiers.ts +43 −6
| @@ -98,12 +98,8 @@ function normalise(path: string): string | undefined { | ||
| 98 | 98 | return out.join("/"); |
| 99 | 99 | } |
| 100 | 100 | |
| 101 | /** | |
| 102 | * The repository files a source file imports by a relative specifier, as | |
| 103 | * repository-relative paths. Packages and `node:`/`bun:` modules are not | |
| 104 | * the repository's and are left out. | |
| 105 | */ | |
| 106 | export function importsOf(source: string, file: string): string[] { | |
| 101 | /** The regex reading, for source the transpiler cannot parse (a fragment, a half-edited file). */ | |
| 102 | function importsByPattern(source: string, file: string): string[] { | |
| 107 | 103 | const found = new Set<string>(); |
| 108 | 104 | const pattern = |
| 109 | 105 | /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\brequire\s*\(\s*)["'](\.{1,2}\/[^"'\n]*)["']/g; |
| @@ -114,6 +110,47 @@ export function importsOf(source: string, file: string): string[] { | ||
| 114 | 110 | return [...found]; |
| 115 | 111 | } |
| 116 | 112 | |
| 113 | // One transpiler per loader, made on first use (a fresh one per call costs | |
| 114 | // more than the parse); a lazy getter keeps no collection that is written to. | |
| 115 | const scanners = { | |
| 116 | ts: lazy("ts"), | |
| 117 | tsx: lazy("tsx"), | |
| 118 | }; | |
| 119 | ||
| 120 | function lazy(loader: "ts" | "tsx"): () => InstanceType<typeof Bun.Transpiler> { | |
| 121 | let made: InstanceType<typeof Bun.Transpiler> | undefined; | |
| 122 | return () => (made ??= new Bun.Transpiler({ loader })); | |
| 123 | } | |
| 124 | ||
| 125 | /** | |
| 126 | * The repository files a source file imports by a relative specifier, as | |
| 127 | * repository-relative paths. Packages and `node:`/`bun:` modules are not | |
| 128 | * the repository's and are left out. | |
| 129 | * | |
| 130 | * Read by Bun's own transpiler (`Bun.Transpiler.scanImports`), which is | |
| 131 | * the parse a run does: a type-only import is erased before it runs, so it | |
| 132 | * is not a dependency and is left out; a specifier inside a comment or a | |
| 133 | * string is not an import. tsconfig.json declares no `paths` or `baseUrl`, | |
| 134 | * so a relative specifier is the only way into the repository. Source that | |
| 135 | * does not parse is read by pattern instead. | |
| 136 | */ | |
| 137 | export function importsOf(source: string, file: string): string[] { | |
| 138 | const loader = /\.[cm]?[jt]sx$/.test(file) ? "tsx" : "ts"; | |
| 139 | let scanned: { path: string }[]; | |
| 140 | try { | |
| 141 | scanned = scanners[loader]().scanImports(source); | |
| 142 | } catch { | |
| 143 | return importsByPattern(source, file); | |
| 144 | } | |
| 145 | const found = new Set<string>(); | |
| 146 | for (const { path: specifier } of scanned) { | |
| 147 | if (!/^\.{1,2}\//.test(specifier)) continue; | |
| 148 | const path = normalise(`${dirname(file)}/${specifier}`); | |
| 149 | if (path !== undefined) found.add(path); | |
| 150 | } | |
| 151 | return [...found]; | |
| 152 | } | |
| 153 | ||
| 117 | 154 | /** The words a test file's name shares with a source path: `src/a/b.ts` → a-b, b, a. */ |
| 118 | 155 | function namesOf(path: string): string[] { |
| 119 | 156 | const parts = path.replace(/\.[cm]?[jt]sx?$/, "").split("/"); |
test/changed-compare.test.ts +17 −2
| @@ -1,5 +1,5 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { junitFiles, missed, redFiles, runsAll, verdict } from "../src/checks/changed_compare.ts"; | |
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { comparisonLine, junitFiles, missed, redFiles, runsAll, verdict } from "../src/checks/changed_compare.ts"; | |
| 3 | 3 | |
| 4 | 4 | test("a failed test belongs to the file header printed before it", () => { |
| 5 | 5 | const log = [ |
| @@ -47,3 +47,18 @@ test("the verdict keeps a pick that missed nothing, the smaller of two such", () | ||
| 47 | 47 | expect(verdict([row(["a"], ["b"], ["a", "b"])]).use).toBe("neither"); |
| 48 | 48 | expect(verdict([row(undefined, ["a"], ["b"])]).compared).toBe(0); |
| 49 | 49 | }); |
| 50 | ||
| 51 | describe("comparisonLine", () => { | |
| 52 | const tests = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"]; | |
| 53 | test("names what each pick has the other lacks", () => { | |
| 54 | expect(comparisonLine(["test/a.test.ts", "test/b.test.ts"], ["test/b.test.ts", "test/c.test.ts"], false, 3)).toBe( | |
| 55 | "test-shards: --changed comparison: the map 2 of 3, --changed 2; only --changed: test/c.test.ts; only the map: test/a.test.ts", | |
| 56 | ); | |
| 57 | }); | |
| 58 | test("a diff --changed cannot judge is all of the gate", () => { | |
| 59 | expect(comparisonLine(["test/a.test.ts"], tests, true, 3)).toContain("--changed all 3"); | |
| 60 | }); | |
| 61 | test("an unreadable run is said, not guessed", () => { | |
| 62 | expect(comparisonLine(["test/a.test.ts"], undefined, false, 3)).toContain("unavailable"); | |
| 63 | }); | |
| 64 | }); |
test/changed-pick.test.ts added +67
| @@ -0,0 +1,67 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { writeFileSync } from "node:fs"; | |
| 3 | import { changedLine, startChanged, type Spawn } from "../scripts/changed-pick.ts"; | |
| 4 | ||
| 5 | const tests = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"]; | |
| 6 | const never: Spawn = () => { | |
| 7 | throw new Error("must not spawn"); | |
| 8 | }; | |
| 9 | const reportOf = (...names: string[]): string => names.map((n) => `<testsuite name="${n}">`).join(""); | |
| 10 | /** A spawn that writes the junit report named in argv, then exits. */ | |
| 11 | const writing = (xml: string): Spawn => (cmd) => { | |
| 12 | writeFileSync(cmd.find((a) => a.startsWith("--reporter-outfile="))!.split("=")[1]!, xml); | |
| 13 | return { exited: Promise.resolve(0), kill() {} }; | |
| 14 | }; | |
| 15 | ||
| 16 | describe("startChanged", () => { | |
| 17 | test("a base that is not a revision spawns nothing and is unavailable", async () => { | |
| 18 | for (const base of ["-x", "", "--help", "a b"]) | |
| 19 | expect(await startChanged(base, ["src/a.ts"], never)).toEqual({ kind: "unavailable", why: "the base is not a revision" }); | |
| 20 | }); | |
| 21 | ||
| 22 | test("a file no test imports answers all without spawning", async () => { | |
| 23 | expect(await startChanged("main", ["scripts/x.ts"], never)).toEqual({ kind: "all" }); | |
| 24 | expect(await startChanged("main", ["forge.json"], never)).toEqual({ kind: "all" }); | |
| 25 | }); | |
| 26 | ||
| 27 | test("the junit report's test files are the answer", async () => { | |
| 28 | const out = await startChanged("main", ["src/a.ts"], writing(reportOf("test/a.test.ts", "other"))); | |
| 29 | expect(out).toEqual({ kind: "files", files: ["test/a.test.ts"] }); | |
| 30 | }); | |
| 31 | ||
| 32 | test("no report is said so", async () => { | |
| 33 | const out = await startChanged("main", ["src/a.ts"], () => ({ exited: Promise.resolve(1), kill() {} })); | |
| 34 | expect(out).toEqual({ kind: "unavailable", why: "no report written" }); | |
| 35 | }); | |
| 36 | ||
| 37 | test("a hung run is killed and reported as a timeout", async () => { | |
| 38 | let killed = false; | |
| 39 | const hung: Spawn = () => ({ exited: new Promise<number>(() => {}), kill: () => (killed = true) }); | |
| 40 | const out = await startChanged("main", ["src/a.ts"], hung, 20); | |
| 41 | expect(killed).toBe(true); | |
| 42 | expect(out.kind === "unavailable" && out.why.startsWith("timed out")).toBe(true); | |
| 43 | }); | |
| 44 | ||
| 45 | test("a spawn that throws gives its reason", async () => { | |
| 46 | const out = await startChanged("main", ["src/a.ts"], () => { | |
| 47 | throw new Error("no bun"); | |
| 48 | }); | |
| 49 | expect(out).toEqual({ kind: "unavailable", why: "spawn failed: no bun" }); | |
| 50 | }); | |
| 51 | }); | |
| 52 | ||
| 53 | describe("changedLine", () => { | |
| 54 | test("all names the whole gate", () => { | |
| 55 | expect(changedLine({ kind: "all" }, tests, ["test/a.test.ts"])).toContain("--changed all 3"); | |
| 56 | }); | |
| 57 | ||
| 58 | test("unavailable carries its reason", () => { | |
| 59 | const line = changedLine({ kind: "unavailable", why: "timed out after 15s" }, tests, []); | |
| 60 | expect(line).toContain("unavailable (timed out after 15s)"); | |
| 61 | }); | |
| 62 | ||
| 63 | test("files outside the gate list are dropped", () => { | |
| 64 | const line = changedLine({ kind: "files", files: ["test/a.test.ts", "test/zzz.test.ts"] }, tests, ["test/b.test.ts"]); | |
| 65 | expect(line).toContain("only --changed: test/a.test.ts; only the map: test/b.test.ts"); | |
| 66 | }); | |
| 67 | }); |
test/check-tiers.test.ts +25
| @@ -598,6 +598,31 @@ describe("importsOf", () => { | ||
| 598 | 598 | }); |
| 599 | 599 | }); |
| 600 | 600 | |
| 601 | describe("importsOf by the transpiler", () => { | |
| 602 | test("leaves out a type-only import, a comment and a string, and keeps the rest", () => { | |
| 603 | const source = [ | |
| 604 | `import type { T } from "./types.ts";`, | |
| 605 | `import { type U } from "./only-types.ts";`, | |
| 606 | `import { v } from "./value.ts";`, | |
| 607 | `import "./side-effect.ts";`, | |
| 608 | `export * from "../up.ts";`, | |
| 609 | `// import { c } from "./commented.ts";`, | |
| 610 | `const s = 'import x from "./quoted.ts"';`, | |
| 611 | `const r = require("./required.ts");`, | |
| 612 | ].join("\n"); | |
| 613 | expect(importsOf(source, "src/a/b.ts").sort()).toEqual([ | |
| 614 | "src/a/required.ts", | |
| 615 | "src/a/side-effect.ts", | |
| 616 | "src/a/value.ts", | |
| 617 | "src/up.ts", | |
| 618 | ]); | |
| 619 | }); | |
| 620 | ||
| 621 | test("source that does not parse is still read by pattern", () => { | |
| 622 | expect(importsOf(`from "./x.ts"`, "src/a.ts")).toEqual(["src/x.ts"]); | |
| 623 | }); | |
| 624 | }); | |
| 625 | ||
| 601 | 626 | describe("the checks line", () => { |
| 602 | 627 | test("words", () => { |
| 603 | 628 | expect(tierLine("green", 220)).toBe( |
new above
offline
changedLineis untested; add tests for the REV guard and the runsAll path (both spawn nothing).