Public: anyone can read this; the crew can change it.

A check never redoes work whose inputs did not change: results are kept per t… #12352

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12178-a-check-never-redoes-work-whose-inputs-d → main 19 commits fork 2 threads · 2 unresolved

  • Merged. mason/12178-a-check-never-redoes-work-whose-inputs-d was merged into main as 5baf89d.

Review

Sign in to review this change.
5 files changed +87 −5
scripts/shards.ts +22
@@ -7,6 +7,7 @@
77 * is disjoint and covering, and that one red shard makes the run red.
88 */
99
10 import { auditLine } from "../src/checks/kept.ts";
1011 import { isSummaryLine, nameUnnamed } from "../src/checks/bun-test.ts";
1112
1213 /**
@@ -423,6 +424,27 @@ function unnamedLine(n: number, result: ShardResult): string {
423424 return `shard ${n} exited ${result.code} without naming a test: ${result.files.join(" ")}`;
424425 }
425426
427 /**
428 * The audit lines for kept files that ran anyway, each judged by its own file: red only when
429 * the shard's report names a failing test in it, never by the shard's exit code, which a
430 * red shard-mate sets too. A red shard that names no failing test is the file's own red only
431 * when the file ran alone; among others the audit says it cannot tell.
432 */
433 export function auditLines(audited: ReadonlyMap<string, string>, results: readonly ShardResult[]): string[] {
434 const bare = (file: string): string => file.replace(/^\.\//, "");
435 const lines: string[] = [];
436 for (const [file, commit] of audited) {
437 const ran = results.find((result) => result.files.some((f) => bare(f) === file));
438 if (ran === undefined) continue;
439 const red = failingFiles(ran.output).map(bare);
440 if (red.includes(file)) lines.push(auditLine(file, commit, false));
441 else if (ran.code === 0 || red.length > 0) lines.push(auditLine(file, commit, true));
442 else if (ran.files.length === 1) lines.push(auditLine(file, commit, false));
443 else lines.push(` ${file}: audit cannot tell: its shard went red without naming a failing test`);
444 }
445 return lines;
446 }
447
426448 /**
427449 * Reads every shard's `bun test` output into one ending. Green: one line,
428450 * how many tests across how many files and how long. Red: every failing
scripts/test-shards.ts +6
@@ -82,6 +82,7 @@ import {
8282 tally,
8383 weigher,
8484 besideOthers,
85 auditLines,
8586 failingFiles,
8687 failingTests,
8788 onlyTestFailures,
@@ -303,6 +304,9 @@ if (gate && tier === "full") {
303304 }
304305 const LOOKS_SET = new Set(LOOKS);
305306 let files = [...(gate || tier === "fast" ? GATE : []), ...LOOKS, ...split.files];
307 // Kept files that run anyway as an audit, with the commit that proved each: filled by the fast
308 // tier's kept answer below, read by the audit lines once the shards are done.
309 const audited = new Map<string, string>();
306310 // The fast tier's diff, for asking its base about a red (#10304).
307311 let changed: string[] = [];
308312 // Whether the fast tier's plan is every gated file: then its breach list is whole.
@@ -370,6 +374,7 @@ if (tier === "fast") {
370374 });
371375 for (const [file, commit] of split.kept) console.log(keptLine(file, commit));
372376 console.log(`test-shards: ${split.kept.size} of ${files.length} files kept, ${split.run.length} to run`);
377 for (const [file, commit] of split.audited) audited.set(file, commit);
373378 files = split.run;
374379 if (files.length === 0) {
375380 console.log("test-shards: every selected file is kept; nothing to run");
@@ -907,6 +912,7 @@ const results = [
907912 ];
908913 const wall = (performance.now() - started) / 1000;
909914 const ending = tally(results, wall);
915 for (const line of auditLines(audited, results)) console.log(line);
910916 const spread = results.map((result) => result.seconds);
911917 console.log(`\n${ending.lines.join("\n")}`);
912918 if (!record && !timed && results.length > 1)
src/checks/kept.ts +18 −3
@@ -137,6 +137,8 @@ export interface KeptSplit {
137137 readonly run: string[];
138138 /** The tests not run, each with the commit that proved it. */
139139 readonly kept: Map<string, string>;
140 /** Files run anyway as an audit, each with the commit that proved it. */
141 readonly audited: Map<string, string>;
140142 }
141143
142144 /**
@@ -151,15 +153,28 @@ export function splitKept(
151153 ): KeptSplit {
152154 const run: string[] = [];
153155 const answered = new Map<string, string>();
156 const audited = new Map<string, string>();
154157 for (const file of files) {
155158 const key = keyOf(file);
156159 const entry = key === undefined ? undefined : kept.get(key);
157 if (entry === undefined || rerun(file)) run.push(file);
158 else answered.set(file, entry.commit);
160 if (entry === undefined) run.push(file);
161 else if (rerun(file)) {
162 run.push(file);
163 audited.set(file, entry.commit);
164 } else answered.set(file, entry.commit);
159165 }
160 return { run, kept: answered };
166 return { run, kept: answered, audited };
161167 }
162168
163169 /** The log line for a test not run. */
164170 export const keptLine = (file: string, commit: string): string =>
165171 ` ${file}: kept from ${commit.slice(0, 7)}: nothing it reads changed`;
172
173 /**
174 * The log line for an audited file once it has run: a kept result and a real one on the same
175 * inputs either agree or they do not. A disagreement means the key missed an input.
176 */
177 export const auditLine = (file: string, commit: string, green: boolean): string =>
178 green
179 ? ` ${file}: audit agrees with the result kept from ${commit.slice(0, 7)}`
180 : ` ${file}: audit DISAGREES: kept green from ${commit.slice(0, 7)} on the same inputs, red now`;
test/check-kept.test.ts +8 −2
@@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test";
22 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
33 import { tmpdir } from "node:os";
44 import { join } from "node:path";
5 import { formatKept, keptKey, mergeKept, parseKept, readsTheTree, splitKept, sampled, KEPT_TOOLING, KEPT_MAX } from "../src/checks/kept.ts";
5 import { auditLine, formatKept, keptKey, mergeKept, parseKept, readsTheTree, splitKept, sampled, KEPT_TOOLING, KEPT_MAX } from "../src/checks/kept.ts";
66 import { KEPT_FILE, keepKept, seedKept } from "../src/checks/kept-store.ts";
77 import { answerKept, freshKept, keyOfTest } from "../scripts/kept-input.ts";
88
@@ -162,7 +162,13 @@ describe("the fast tier's use of kept results", () => {
162162 expect(hits).toBeLessThan(90);
163163 expect(sampled("test/a.test.ts", commit)).toBe(sampled("test/a.test.ts", commit));
164164 const kept = new Map([[key, { test: base.test, commit }]]);
165 expect(splitKept(["test/a.test.ts"], kept, () => key, () => true).run).toEqual(["test/a.test.ts"]);
165 const audit = splitKept(["test/a.test.ts"], kept, () => key, () => true);
166 expect(audit.run).toEqual(["test/a.test.ts"]);
167 expect([...audit.audited]).toEqual([["test/a.test.ts", commit]]);
168 expect(audit.kept.size).toBe(0);
169 expect(splitKept(["test/a.test.ts"], kept, () => key).audited.size).toBe(0);
170 expect(auditLine("test/a.test.ts", commit, true)).toContain("agrees");
171 expect(auditLine("test/a.test.ts", commit, false)).toContain("DISAGREES");
166172 });
167173 });
168174
test/shards.test.ts +33
@@ -11,6 +11,7 @@ import { describe, expect, test } from "bun:test";
1111 import { readFileSync } from "node:fs";
1212 import { resolveLimits } from "../src/apps/contain.ts";
1313 import {
14 auditLines,
1415 SECONDS_PER_BYTE_DEFAULT,
1516 SHARDS_MAX,
1617 besideOthers,
@@ -803,3 +804,35 @@ describe("this file", () => {
803804 expect(source.includes("Bun." + "spawn")).toBe(false);
804805 });
805806 });
807
808 describe("the audit of a kept file, judged by its own file", () => {
809 const commit = "abcdef1234567";
810 const audited = new Map([
811 ["test/a.test.ts", commit],
812 ["test/b.test.ts", commit],
813 ]);
814 test("a red shard-mate does not make a green audited file disagree", () => {
815 const output = `test/a.test.ts:\n(pass) fine\n\ntest/c.test.ts:\n(fail) broken [1.00ms]\n`;
816 const lines = auditLines(audited, [{ files: ["./test/a.test.ts", "./test/c.test.ts"], code: 1, output }]);
817 expect(lines).toEqual([" test/a.test.ts: audit agrees with the result kept from abcdef1"]);
818 });
819
820 test("a failing test in the audited file itself is the disagreement", () => {
821 const output = `test/a.test.ts:\n(fail) broken [1.00ms]\n\ntest/b.test.ts:\n(pass) fine\n`;
822 const lines = auditLines(audited, [{ files: ["test/a.test.ts", "test/b.test.ts"], code: 1, output }]);
823 expect(lines.map((l) => l.includes("DISAGREES"))).toEqual([true, false]);
824 expect(lines[1]).toContain("agrees");
825 });
826
827 test("a file that crashed alone is red; among others the audit says it cannot tell", () => {
828 expect(auditLines(audited, [{ files: ["test/a.test.ts"], code: 1, output: "error: cannot load" }])[0]).toContain(
829 "DISAGREES",
830 );
831 const mixed = auditLines(audited, [{ files: ["test/a.test.ts", "test/b.test.ts"], code: 1, output: "crash" }]);
832 expect(mixed.every((l) => l.includes("cannot tell") && !l.includes("DISAGREES"))).toBe(true);
833 });
834
835 test("a file that did not run says nothing", () => {
836 expect(auditLines(audited, [])).toEqual([]);
837 });
838 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs