Public: anyone can read this; the crew can change it.

A check's failure that belongs to no test is named with the error it threw, not "(unnamed)" alone #5728

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/5727-a-check-s-failure-that-belongs-to-no-tes → main 3 commits fork 1 thread · 1 unresolved

  • Merged. scout/5727-a-check-s-failure-that-belongs-to-no-tes was merged into main as c0c2022.

Review

Sign in to review this change.
3 files changed +80 −7
src/checks/bun-test.ts +36 −5
@@ -21,32 +21,63 @@ const HEADER = /^(\S+\.test\.[cm]?[jt]sx?):$/;
2121 const UNNAMED_FAIL = /^\(fail\) ((?:.* > )?)\(unnamed\)/;
2222 /** bun's own caret line under a hook that timed out, printed right after its `(fail)`. */
2323 const HOOK_TIMEOUT = /^\s*\^ a (\S+) hook timed out/;
24 /** bun prints what a hook or stray promise threw just above its `(fail)`. */
25 const THROWN = /^(?:[A-Za-z]*Error|error): .+/;
26 const RESULT_LINE = /^\((?:fail|pass)\) /;
27 const MAX_THROWN = 120;
2428 /** The tail that ends a file's report; bun's recap of `(fail)` lines can follow, headerless. */
2529 const SUMMARY = /^\s*\d+ pass$|^\d+ tests? failed:$/;
2630
31 /**
32 * The first error bun printed since the previous result line, printable only,
33 * with any directory prefix of a path dropped so no machine path is shown.
34 */
35 function thrownSince(lines: readonly string[], from: number, to: number): string {
36 for (let j = from; j < to; j++) {
37 if (!THROWN.test(lines[j]!)) continue;
38 const text = lines[j]!
39 // eslint-disable-next-line no-control-regex
40 .replace(/[\u0000-\u001f\u007f-\u009f]/g, "")
41 .replace(/(?<![\w.])\/(?:[\w.@-]+\/)+/g, "")
42 .trim();
43 return `: ${text.slice(0, MAX_THROWN)}`;
44 }
45 return "";
46 }
47
2748 /**
2849 * bun labels a failure that belongs to no test — a hook that timed out or
2950 * threw — `(unnamed)`, so every such red looked like one. Each is renamed
3051 * for the file whose header precedes it, and for the hook when bun's caret
3152 * line under it says one: `a.test.ts > beforeEach/afterEach (timeout)`.
32 * A hook that threw keeps bun's own describe prefix: `a.test.ts > s > (unnamed)`.
53 * A hook that threw keeps bun's own describe prefix and gains the error bun
54 * printed above it: `a.test.ts > s > (unnamed): error: database is closed`.
3355 * Once the summary starts no file is current until the next header, so a
3456 * recap's `(fail)` lines are never given the last file's name.
3557 */
3658 export function nameUnnamed(lines: readonly string[]): string[] {
3759 let file: string | undefined;
60 let segment = 0;
3861 return lines.map((line, i) => {
3962 const header = HEADER.exec(line);
4063 if (header) {
4164 file = header[1];
65 segment = i + 1;
4266 return line;
4367 }
4468 if (SUMMARY.test(line)) file = undefined;
4569 const unnamed = UNNAMED_FAIL.exec(line);
46 if (!unnamed || file === undefined) return line;
47 const timeout = HOOK_TIMEOUT.exec(lines[i + 1] ?? "");
48 const what = timeout ? `${timeout[1]} (timeout)` : `${unnamed[1]}(unnamed)`;
49 return `(fail) ${file} > ${what}${line.slice(unnamed[0].length)}`;
70 if (RESULT_LINE.test(line)) {
71 const from = segment;
72 segment = i + 1;
73 if (!unnamed || file === undefined) return line;
74 const timeout = HOOK_TIMEOUT.exec(lines[i + 1] ?? "");
75 const what = timeout
76 ? `${timeout[1]} (timeout)`
77 : `${unnamed[1]}(unnamed)${thrownSince(lines, from, i)}`;
78 return `(fail) ${file} > ${what}${line.slice(unnamed[0].length)}`;
79 }
80 return line;
5081 });
5182 }
5283
src/checks/shared_red.ts +1 −1
@@ -88,7 +88,7 @@ const PULLS_PER_TEST = `
8888 WHERE run.id >= ?1 AND run.finished_at >= ?2
8989 AND run.kind = 'check' AND run.issue_id IS NOT NULL AND run.state = 'failed'
9090 AND (?3 IS NULL OR run.repo_id = ?3)
91 AND n.type = 'text' AND n.value <> ?4
91 AND n.type = 'text' AND instr(n.value, ?4) = 0

This widens the exclusion from the exact string to any name containing the marker, a behaviour change to shared-red counting. No test fails if it is reverted: the shared-red tests only cover nameUnnamed output. Add one that goes through pullsPerTest/the shared-red summary.

9292 GROUP BY run.repo_id, n.value`;
9393
9494 function pullsPerTest(db: Database, now: number, repoId: string | null): Row[] {
test/shared-red.test.ts +43 −1
@@ -68,6 +68,13 @@ describe("a red many pulls share", () => {
6868 expect(sharedTests(db)).toEqual([]);
6969 });
7070
71 test("a named (unnamed) failure, with the error it threw, is no test either (#5727)", () => {
72 const name = "a.test.ts > (unnamed): error: x";
73 for (const i of ["i1", "i2", "i3"]) red(i, [name]);
74 expect(shared(red("i4", [name]))).toBe(false);
75 expect(sharedTests(db)).toEqual([]);
76 });
77
7178 test("a gate on the tree is each pull's own red, however many trip it (#2868)", () => {
7279 const gate = "design check: nothing added outside the system (#2678) > the tree is no worse than its baseline";
7380 for (const i of ["i1", "i2", "i3"]) red(i, [gate]);
@@ -192,7 +199,7 @@ describe("a failure outside a named test names its file and hook (#4582)", () =>
192199 const failure = summariseCheckFailure(captured, "check exited with code 1");
193200 expect(failure.detail?.names).toEqual([
194201 "a.test.ts > beforeEach/afterEach (timeout)",
195 "c.test.ts > s > (unnamed)",
202 "c.test.ts > s > (unnamed): error: late",
196203 "(unnamed)",
197204 ]);
198205 expect(failure.summary).toContain("a.test.ts > beforeEach/afterEach (timeout)");
@@ -208,3 +215,38 @@ describe("a failure outside a named test names its file and hook (#4582)", () =>
208215 expect(lines).toContain("(fail) a.test.ts > beforeEach/afterEach (timeout) [50.82ms] [shard 1]");
209216 });
210217 });
218
219 const one = (body: string[]) => nameUnnamed(["a.test.ts:", ...body]).at(-1);
220
221 describe("a failure outside a test names what it threw (#5727)", () => {
222 test("a thrown error is appended", () => {
223 expect(
224 one(["error: database is closed", " at x (a.test.ts:1:1)", "(fail) (unnamed) [1.00ms]"]),
225 ).toBe("(fail) a.test.ts > (unnamed): error: database is closed [1.00ms]");
226 });
227
228 test("a hook timeout is named as before", () => {
229 const lines = nameUnnamed([
230 "a.test.ts:",
231 "error: late",
232 "(fail) (unnamed) [50ms]",
233 " ^ a beforeAll hook timed out for this test.",
234 ]);
235 expect(lines[2]).toBe("(fail) a.test.ts > beforeAll (timeout) [50ms]");
236 });
237
238 test("no error line keeps today's name; an earlier test's error is not borrowed", () => {
239 expect(one(["(fail) (unnamed) [1ms]"])).toBe("(fail) a.test.ts > (unnamed) [1ms]");
240 expect(
241 one(["error: other", "(fail) t [1ms]", "(fail) s > (unnamed) [1ms]"]),
242 ).toBe("(fail) a.test.ts > s > (unnamed) [1ms]");
243 });
244
245 test("a long error is cut at 120, control characters and paths dropped", () => {
246 const got = one([`error: ${"x".repeat(200)}`, "(fail) (unnamed)"])!;
247 expect(got).toBe(`(fail) a.test.ts > (unnamed): error: ${"x".repeat(113)}`);
248 expect(one(["error: bad\u001b /home/u/proj/f.ts", "(fail) (unnamed)"])).toBe(
249 "(fail) a.test.ts > (unnamed): error: bad f.ts",
250 );
251 });
252 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs