Public: anyone can read this; the crew can change it.
A check never redoes work whose inputs did not change: results are kept per t… #12352
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12178-a-check-never-redoes-work-whose-inputs-d → main 19 commits fork 2 threads · 2 unresolved
5 files
changed +87
−5
scripts/shards.ts +22
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | * is disjoint and covering, and that one red shard makes the run red. |
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | import { auditLine } from "../src/checks/kept.ts"; | |
| 10 | 11 | import { isSummaryLine, nameUnnamed } from "../src/checks/bun-test.ts"; |
| 11 | 12 | |
| 12 | 13 | /** |
| @@ -423,6 +424,27 @@ function unnamedLine(n: number, result: ShardResult): string { | ||
| 423 | 424 | return `shard ${n} exited ${result.code} without naming a test: ${result.files.join(" ")}`; |
| 424 | 425 | } |
| 425 | 426 | |
| 427 | /** | |
| 428 | * The audit lines for kept files that ran anyway, each judged by its own file: red only when | |
| 429 | * the shard's report names a failing test in it, never by the shard's exit code, which a | |
| 430 | * red shard-mate sets too. A red shard that names no failing test is the file's own red only | |
| 431 | * when the file ran alone; among others the audit says it cannot tell. | |
| 432 | */ | |
| 433 | export function auditLines(audited: ReadonlyMap<string, string>, results: readonly ShardResult[]): string[] { | |
| 434 | const bare = (file: string): string => file.replace(/^\.\//, ""); | |
| 435 | const lines: string[] = []; | |
| 436 | for (const [file, commit] of audited) { | |
| 437 | const ran = results.find((result) => result.files.some((f) => bare(f) === file)); | |
| 438 | if (ran === undefined) continue; | |
| 439 | const red = failingFiles(ran.output).map(bare); | |
| 440 | if (red.includes(file)) lines.push(auditLine(file, commit, false)); | |
| 441 | else if (ran.code === 0 || red.length > 0) lines.push(auditLine(file, commit, true)); | |
| 442 | else if (ran.files.length === 1) lines.push(auditLine(file, commit, false)); | |
| 443 | else lines.push(` ${file}: audit cannot tell: its shard went red without naming a failing test`); | |
| 444 | } | |
| 445 | return lines; | |
| 446 | } | |
| 447 | ||
| 426 | 448 | /** |
| 427 | 449 | * Reads every shard's `bun test` output into one ending. Green: one line, |
| 428 | 450 | * how many tests across how many files and how long. Red: every failing |
scripts/test-shards.ts +6
| @@ -82,6 +82,7 @@ import { | ||
| 82 | 82 | tally, |
| 83 | 83 | weigher, |
| 84 | 84 | besideOthers, |
| 85 | auditLines, | |
| 85 | 86 | failingFiles, |
| 86 | 87 | failingTests, |
| 87 | 88 | onlyTestFailures, |
| @@ -303,6 +304,9 @@ if (gate && tier === "full") { | ||
| 303 | 304 | } |
| 304 | 305 | const LOOKS_SET = new Set(LOOKS); |
| 305 | 306 | let files = [...(gate || tier === "fast" ? GATE : []), ...LOOKS, ...split.files]; |
| 307 | // Kept files that run anyway as an audit, with the commit that proved each: filled by the fast | |
| 308 | // tier's kept answer below, read by the audit lines once the shards are done. | |
| 309 | const audited = new Map<string, string>(); | |
| 306 | 310 | // The fast tier's diff, for asking its base about a red (#10304). |
| 307 | 311 | let changed: string[] = []; |
| 308 | 312 | // Whether the fast tier's plan is every gated file: then its breach list is whole. |
| @@ -370,6 +374,7 @@ if (tier === "fast") { | ||
| 370 | 374 | }); |
| 371 | 375 | for (const [file, commit] of split.kept) console.log(keptLine(file, commit)); |
| 372 | 376 | console.log(`test-shards: ${split.kept.size} of ${files.length} files kept, ${split.run.length} to run`); |
| 377 | for (const [file, commit] of split.audited) audited.set(file, commit); | |
| 373 | 378 | files = split.run; |
| 374 | 379 | if (files.length === 0) { |
| 375 | 380 | console.log("test-shards: every selected file is kept; nothing to run"); |
| @@ -907,6 +912,7 @@ const results = [ | ||
| 907 | 912 | ]; |
| 908 | 913 | const wall = (performance.now() - started) / 1000; |
| 909 | 914 | const ending = tally(results, wall); |
| 915 | for (const line of auditLines(audited, results)) console.log(line); | |
| 910 | 916 | const spread = results.map((result) => result.seconds); |
| 911 | 917 | console.log(`\n${ending.lines.join("\n")}`); |
| 912 | 918 | if (!record && !timed && results.length > 1) |
src/checks/kept.ts +18 −3
| @@ -137,6 +137,8 @@ export interface KeptSplit { | ||
| 137 | 137 | readonly run: string[]; |
| 138 | 138 | /** The tests not run, each with the commit that proved it. */ |
| 139 | 139 | readonly kept: Map<string, string>; |
| 140 | /** Files run anyway as an audit, each with the commit that proved it. */ | |
| 141 | readonly audited: Map<string, string>; | |
| 140 | 142 | } |
| 141 | 143 | |
| 142 | 144 | /** |
| @@ -151,15 +153,28 @@ export function splitKept( | ||
| 151 | 153 | ): KeptSplit { |
| 152 | 154 | const run: string[] = []; |
| 153 | 155 | const answered = new Map<string, string>(); |
| 156 | const audited = new Map<string, string>(); | |
| 154 | 157 | for (const file of files) { |
| 155 | 158 | const key = keyOf(file); |
| 156 | 159 | const entry = key === undefined ? undefined : kept.get(key); |
| 157 | if (entry === undefined || rerun(file)) run.push(file); | |
| 158 | else answered.set(file, entry.commit); | |
| 160 | if (entry === undefined) run.push(file); | |
| 161 | else if (rerun(file)) { | |
| 162 | run.push(file); | |
| 163 | audited.set(file, entry.commit); | |
| 164 | } else answered.set(file, entry.commit); | |
| 159 | 165 | } |
| 160 | return { run, kept: answered }; | |
| 166 | return { run, kept: answered, audited }; | |
| 161 | 167 | } |
| 162 | 168 | |
| 163 | 169 | /** The log line for a test not run. */ |
| 164 | 170 | export const keptLine = (file: string, commit: string): string => |
| 165 | 171 | ` ${file}: kept from ${commit.slice(0, 7)}: nothing it reads changed`; |
| 172 | ||
| 173 | /** | |
| 174 | * The log line for an audited file once it has run: a kept result and a real one on the same | |
| 175 | * inputs either agree or they do not. A disagreement means the key missed an input. | |
| 176 | */ | |
| 177 | export const auditLine = (file: string, commit: string, green: boolean): string => | |
| 178 | green | |
| 179 | ? ` ${file}: audit agrees with the result kept from ${commit.slice(0, 7)}` | |
| 180 | : ` ${file}: audit DISAGREES: kept green from ${commit.slice(0, 7)} on the same inputs, red now`; |
test/check-kept.test.ts +8 −2
| @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"; | ||
| 2 | 2 | import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 3 | 3 | import { tmpdir } from "node:os"; |
| 4 | 4 | import { join } from "node:path"; |
| 5 | import { formatKept, keptKey, mergeKept, parseKept, readsTheTree, splitKept, sampled, KEPT_TOOLING, KEPT_MAX } from "../src/checks/kept.ts"; | |
| 5 | import { auditLine, formatKept, keptKey, mergeKept, parseKept, readsTheTree, splitKept, sampled, KEPT_TOOLING, KEPT_MAX } from "../src/checks/kept.ts"; | |
| 6 | 6 | import { KEPT_FILE, keepKept, seedKept } from "../src/checks/kept-store.ts"; |
| 7 | 7 | import { answerKept, freshKept, keyOfTest } from "../scripts/kept-input.ts"; |
| 8 | 8 | |
| @@ -162,7 +162,13 @@ describe("the fast tier's use of kept results", () => { | ||
| 162 | 162 | expect(hits).toBeLessThan(90); |
| 163 | 163 | expect(sampled("test/a.test.ts", commit)).toBe(sampled("test/a.test.ts", commit)); |
| 164 | 164 | const kept = new Map([[key, { test: base.test, commit }]]); |
| 165 | expect(splitKept(["test/a.test.ts"], kept, () => key, () => true).run).toEqual(["test/a.test.ts"]); | |
| 165 | const audit = splitKept(["test/a.test.ts"], kept, () => key, () => true); | |
| 166 | expect(audit.run).toEqual(["test/a.test.ts"]); | |
| 167 | expect([...audit.audited]).toEqual([["test/a.test.ts", commit]]); | |
| 168 | expect(audit.kept.size).toBe(0); | |
| 169 | expect(splitKept(["test/a.test.ts"], kept, () => key).audited.size).toBe(0); | |
| 170 | expect(auditLine("test/a.test.ts", commit, true)).toContain("agrees"); | |
| 171 | expect(auditLine("test/a.test.ts", commit, false)).toContain("DISAGREES"); | |
| 166 | 172 | }); |
| 167 | 173 | }); |
| 168 | 174 |
test/shards.test.ts +33
| @@ -11,6 +11,7 @@ import { describe, expect, test } from "bun:test"; | ||
| 11 | 11 | import { readFileSync } from "node:fs"; |
| 12 | 12 | import { resolveLimits } from "../src/apps/contain.ts"; |
| 13 | 13 | import { |
| 14 | auditLines, | |
| 14 | 15 | SECONDS_PER_BYTE_DEFAULT, |
| 15 | 16 | SHARDS_MAX, |
| 16 | 17 | besideOthers, |
| @@ -803,3 +804,35 @@ describe("this file", () => { | ||
| 803 | 804 | expect(source.includes("Bun." + "spawn")).toBe(false); |
| 804 | 805 | }); |
| 805 | 806 | }); |
| 807 | ||
| 808 | describe("the audit of a kept file, judged by its own file", () => { | |
| 809 | const commit = "abcdef1234567"; | |
| 810 | const audited = new Map([ | |
| 811 | ["test/a.test.ts", commit], | |
| 812 | ["test/b.test.ts", commit], | |
| 813 | ]); | |
| 814 | test("a red shard-mate does not make a green audited file disagree", () => { | |
| 815 | const output = `test/a.test.ts:\n(pass) fine\n\ntest/c.test.ts:\n(fail) broken [1.00ms]\n`; | |
| 816 | const lines = auditLines(audited, [{ files: ["./test/a.test.ts", "./test/c.test.ts"], code: 1, output }]); | |
| 817 | expect(lines).toEqual([" test/a.test.ts: audit agrees with the result kept from abcdef1"]); | |
| 818 | }); | |
| 819 | ||
| 820 | test("a failing test in the audited file itself is the disagreement", () => { | |
| 821 | const output = `test/a.test.ts:\n(fail) broken [1.00ms]\n\ntest/b.test.ts:\n(pass) fine\n`; | |
| 822 | const lines = auditLines(audited, [{ files: ["test/a.test.ts", "test/b.test.ts"], code: 1, output }]); | |
| 823 | expect(lines.map((l) => l.includes("DISAGREES"))).toEqual([true, false]); | |
| 824 | expect(lines[1]).toContain("agrees"); | |
| 825 | }); | |
| 826 | ||
| 827 | test("a file that crashed alone is red; among others the audit says it cannot tell", () => { | |
| 828 | expect(auditLines(audited, [{ files: ["test/a.test.ts"], code: 1, output: "error: cannot load" }])[0]).toContain( | |
| 829 | "DISAGREES", | |
| 830 | ); | |
| 831 | const mixed = auditLines(audited, [{ files: ["test/a.test.ts", "test/b.test.ts"], code: 1, output: "crash" }]); | |
| 832 | expect(mixed.every((l) => l.includes("cannot tell") && !l.includes("DISAGREES"))).toBe(true); | |
| 833 | }); | |
| 834 | ||
| 835 | test("a file that did not run says nothing", () => { | |
| 836 | expect(auditLines(audited, [])).toEqual([]); | |
| 837 | }); | |
| 838 | }); |
new above
offline