trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns15g rEventsFlows

Sentinel: red:01m23vygyffx5hg2f5vm8r74sr was degraded 96 minutes today: trevato/forge: main is red at 931899b — "a private repository is linked only to who may read it" fails; 3 pull checks w… #6643

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/6614-sentinel-red-01m23vygyffx5hg2f5vm8r74sr → main 11 commits fork 7 threads · 7 unresolved

  • Merged. scout/6614-sentinel-red-01m23vygyffx5hg2f5vm8r74sr was merged into main as ef9a70f.

Review

Sign in to review this change.
15 files changed +590 −10
docs/features/check-flake.md added +22
@@ -0,0 +1,22 @@
1 # Flaky tests on main (#6643)
2
3 A default-branch check that fails **one or two named tests** re-runs only those
4 tests once, alone (`bun test <file> -t <name>`), on the same commit in the same
5 cell, before main counts as red (`src/checks/flake.ts`, wired in
6 `checks/service.ts`).
7
8 - They pass alone: the check is **green**, so no red-main hold (`checks/red_main.ts`)
9 starts. Each is a row in `check_flake`; the check's log says
10 `flaky: … recorded, main stays green`.
11 - One fails alone too, or the red was a build error, a hook, a timeout or more
12 than two tests: **red, unchanged**.
13 - A test that flakes **twice in 7 days** gets one issue (`Flaky test: <name>`,
14 `Files:` its file). An open issue by that title is not filed again.
15 - The week's list is `flakes` in the vitals (the health answer for an admin):
16 `repo`, `test`, `file`, `count`, `lastCommit`, `last`. A site admin reads it as
17 "Flaky this week: …" under the forge notes on the home board and in the terminal's
18 dashboard strip. It never degrades the forge.
19
20 Left out: pull checks (they keep failing as before, shared with main by name),
21 cluster Jobs (the re-run needs the check's own cell), and hook or `(unnamed)`
22 failures, which have no name to select by.
src/actions/health.ts +7
@@ -16,6 +16,7 @@
1616 * One answer for the API (`GET /api/v1/health`) and the MCP `health` tool.
1717 */
1818
19 import { flakeLine, flakeList, flakeNeeds } from "../checks/flake.ts";
1920 import type { App } from "../app.ts";
2021 import { appsRuntime } from "../apps/runtime.ts";
2122 import { listApps } from "../core/apps.ts";
@@ -95,6 +96,7 @@ export function forgeNeeds(app: App, actor: User | undefined): string[] | undefi
9596 ...(rollout === null ? [] : [rollout]),
9697 ...h.incidents.map((i) => `${i.key}: ${i.message}`),
9798 ...h.degraded.map(needLine),
99 ...flakeNeeds(h.flakes),
98100 ];
99101 }
100102
@@ -107,6 +109,11 @@ export function needLine(d: Degraded, now: number = Date.now()): string {
107109 : `${d.sentence} Since ${sinceWords(d.since, now)}.`;
108110 }
109111
112 /** The week's flaky tests as one line for a site admin; `null` for anyone else, and when none flaked. */
113 export function forgeFlakes(app: App, actor: User | undefined): string | null {
junipercrew outdated

forgeVitals(app) runs the full vitals (all queries) just for one line; pass in the vitals the caller already has or call flakeList directly.
114 return actor?.isAdmin === true ? flakeLine(flakeList(app.db, Date.now())) : null;
115 }
116
110117 /**
111118 * The check lane as one line for a site admin, as the board's forge strip
112119 * prints it; `null` for anyone else, and when no check is queued.
src/checks/flake.ts added +239
@@ -0,0 +1,239 @@
1 /**
2 * A default-branch check that fails at most two tests re-runs those tests
3 * once, alone, before main counts as red (#6643). Passing alone is a flake:
4 * the check is green, the flake is recorded, and a test that flakes twice in
5 * a week gets one issue for its file. A test that fails alone too, or a red
6 * that is not a pair of named tests (a build error, a timeout), stays red.
7 *
8 * Plain lines in, commands and rows out. The names come from the repository's
9 * own output, so each is checked before it reaches a shell, and quoted there.
10 */
11
12 import type { Database } from "bun:sqlite";
13 import { createIssue } from "../core/issues.ts";
14 import type { Repo } from "../core/repos.ts";
15 import type { EventBus } from "../core/events.ts";
16 import { id } from "../kernel/id.ts";
17 import type { CheckDetail } from "../core/runs.ts";
18
19 /** The most failures that are re-run; more is a broken main, not a flake. */
20 export const MAX_RERUN = 2;
21
22 /** A test that flakes this often in {@link WINDOW_MS} is filed. */
23 export const FILE_AT = 2;
24 export const WINDOW_MS = 7 * 24 * 3_600_000;
25
26 const HEADER = /^(\S+\.test\.[cm]?[jt]sx?):$/;
27 const FAIL = /^\(fail\) (.+?)(?: \[[0-9.]+m?s\])?(?: \[shard [0-9]+\])?$/;
28 const PATH = /^[\w@./+-]{1,200}$/;
29 // eslint-disable-next-line no-control-regex
30 const CONTROL = /[\u0000-\u001f\u007f-\u009f]/;
31 const NAME_MAX = 300;
32
33 export interface Flaky {
34 readonly file: string;
35 readonly name: string;
36 }
37
38 /**
39 * The tests a failed check can re-run alone: at most {@link MAX_RERUN}
40 * failures, every one named with the file whose report printed it. Anything
41 * else (a hook, an unnamed failure, more failures) is undefined — red.
42 */
43 export function rerunnable(
44 lines: readonly string[],
45 detail: CheckDetail | undefined,
46 ): Flaky[] | undefined {
47 if (detail === undefined || detail.failed < 1 || detail.failed > MAX_RERUN) return undefined;
48 if (detail.names.length !== detail.failed) return undefined;
49 const found = new Map<string, string>();
50 let file: string | undefined;
51 for (const line of lines) {
52 const header = HEADER.exec(line);
53 if (header) {
54 file = header[1];
55 continue;
56 }
57 const name = FAIL.exec(line)?.[1];
58 if (name !== undefined && detail.names.includes(name) && !found.has(name) && file !== undefined)
59 found.set(name, file);
60 }
61 const out: Flaky[] = [];
62 for (const name of detail.names) {
63 const at = found.get(name);
64 if (
65 at === undefined ||
66 !PATH.test(at) ||
67 name.length > NAME_MAX ||
68 CONTROL.test(name) ||
69 name.includes("(unnamed)") ||
70 name.includes("(timeout)")
71 )
72 return undefined;
73 out.push({ file: at, name: name });
74 }
75 return out;
76 }
77
78 const escapeRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
79 const quote = (s: string): string => `'${s.replace(/'/g, `'\\''`)}'`;
80
81 /**
82 * `bun test` of the one test, anchored so a sibling that contains its name is
83 * not run: bun joins a describe's names with a space.
84 */
85 export function rerunCommand(test: Flaky): string {
86 const pattern = `^${escapeRegex(test.name.split(" > ").join(" "))}$`;
87 return `bun test ${quote(test.file)} -t ${quote(pattern)}`;
88 }
89
90 /**
91 * Whether the re-run's own output says exactly one test ran and none failed:
92 * a pattern that matched nothing exits 0.
93 */
94 export function passedAlone(lines: readonly string[]): boolean {
95 let pass = 0;
96 let fail = 0;
97 for (const line of lines) {
98 const p = /^\s*(\d+) pass$/.exec(line);
99 if (p) pass += Number(p[1]);
100 const f = /^\s*(\d+) fail$/.exec(line);
101 if (f) fail += Number(f[1]);
102 }
103 return pass === 1 && fail === 0;
104 }
105
106 export const flakeTitle = (name: string): string => `Flaky test: ${name.slice(0, 200)}`;
107
108 /**
109 * Record each flake; a test that has now flaked {@link FILE_AT} times in the
110 * week gets an issue for its file unless one by its name is open. Returns the
111 * issue numbers opened.
112 */
113 export function recordFlakes(
114 db: Database,
115 bus: EventBus,
116 repo: Pick<Repo, "id" | "ownerId">,
117 tests: readonly Flaky[],
118 ref: { runId: string; commit: string },
119 now = Date.now(),
120 ): number[] {
121 const opened: number[] = [];
122 for (const t of tests) {
123 db.query(
124 "INSERT INTO check_flake (id, repo_id, run_id, commit_oid, test, file, at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
125 ).run(id(), repo.id, ref.runId, ref.commit, t.name, t.file, new Date(now).toISOString());
126 const since = new Date(now - WINDOW_MS).toISOString();
127 const count = db
128 .query<{ n: number }, [string, string, string]>(
129 "SELECT COUNT(*) AS n FROM check_flake WHERE repo_id = ?1 AND test = ?2 AND at >= ?3",
130 )
131 .get(repo.id, t.name, since)!.n;
132 // No issue without a file: the "Files:" line is what routes it to a builder.
133 if (count < FILE_AT || t.file === "") continue;
134 const title = flakeTitle(t.name);
135 const open = db
136 .query<{ id: string }, [string, string]>(
137 "SELECT id FROM issue WHERE repo_id = ?1 AND title = ?2 AND state = 'open' LIMIT 1",
138 )
139 .get(repo.id, title);
140 if (open !== null) continue;
141 const made = createIssue(db, {
142 repoId: repo.id,
143 authorId: repo.ownerId,
144 title,
145 body:
146 `_Filed by the check runner from a check log, not written by a person._\n\n` +
147 `\`${t.name}\` in \`${t.file}\` failed in main's check and passed when run alone, ${count} times in 7 days ` +
148 `(latest: check run ${ref.runId}, ${ref.commit.slice(0, 7)}). Main stayed green each time. ` +
149 `Find what it depends on: time, load, order, or shared state.\n\nFiles: ${t.file}`,
150 });
151 if (!made.ok) continue;
152 bus.append({
153 kind: "issue.opened",
154 repoId: repo.id,
155 actorId: repo.ownerId,
156 payload: { number: made.value.number },
157 });
158 opened.push(made.value.number);
159 }
160 return opened;
161 }
162
163 /** What a red queue verdict says of its tests: each failing one with its file, and the files it learned. */
164 export interface QueueRed {
165 readonly tests?: { readonly failing: readonly { readonly name: string; readonly file?: string }[] };
166 readonly learn: readonly string[];
167 }
168
169 /**
170 * Record the tests a shorter queue prefix failed that a longer green prefix
171 * held. A failing test carries the file the gate saw it run in; without one,
172 * the file is known only when the verdicts learned exactly one, and with
173 * none the flake is kept without a file and no issue is filed for it.
174 */
175 export function recordQueueFlakes(
176 db: Database,
177 bus: EventBus,
178 repo: Pick<Repo, "id" | "ownerId">,
179 reds: readonly QueueRed[],
180 tip: string,
181 now = Date.now(),
182 ): number[] {
183 const failing = reds.flatMap((r) => r.tests?.failing ?? []);
184 if (failing.length === 0) return [];
185 const files = [...new Set(reds.flatMap((r) => r.learn))];
186 const only = files.length === 1 && PATH.test(files[0]!) ? files[0]! : "";
187 const seen = new Set<string>();
188 const tests = failing
189 .filter((t) => t.name.length <= NAME_MAX && !CONTROL.test(t.name))
190 .map((t) => ({ file: t.file !== undefined && PATH.test(t.file) ? t.file : only, name: t.name }))
191 .filter((t) => !seen.has(`${t.file}\0${t.name}`) && seen.add(`${t.file}\0${t.name}`));
192 return recordFlakes(db, bus, repo, tests, { runId: `gate-${tip.slice(0, 12)}`, commit: tip }, now);
193 }
194
195 export interface FlakeRow {
196 readonly repo: string;
197 readonly test: string;
198 readonly file: string;
199 readonly count: number;
200 readonly lastCommit: string;
201 readonly last: string;
202 }
203
204 /** A test that flakes this often in the week needs a person, not a note. */
205 export const NEEDS_AT = 3;
206
207 /** The tests that flaked {@link NEEDS_AT} times or more, one need each for the attention list. */
208 export function flakeNeeds(rows: readonly FlakeRow[]): string[] {
209 return rows
210 .filter((r) => r.count >= NEEDS_AT)
211 .map((r) => `flaky: ${r.test} failed and passed alone ${r.count} times this week (${r.repo})`);
212 }
213
214 /** The flake list as one line for the board and the terminal; null when the week had none. */
215 export function flakeLine(rows: readonly FlakeRow[], shown = 3): string | null {
216 if (rows.length === 0) return null;
217 const top = rows.slice(0, shown).map((r) => `${r.test} ×${r.count} (${r.repo})`);
218 const more = rows.length - top.length;
219 return `${top.join("; ")}${more > 0 ? ` and ${more} more` : ""}`;
220 }
221
222 /** The week's flake list, most frequent first: one query, read by the vitals. */
223 export function flakeList(db: Database, now: number, limit = 20): FlakeRow[] {
224 return db
225 .query<FlakeRow, [string, number]>(
226 `SELECT u.username || '/' || r.name AS repo, f.test AS test, f.file AS file,
227 COUNT(*) AS count, MAX(f.at) AS last,
228 (SELECT commit_oid FROM check_flake g
229 WHERE g.repo_id = f.repo_id AND g.test = f.test ORDER BY g.at DESC LIMIT 1) AS lastCommit
230 FROM check_flake f
231 JOIN repo r ON r.id = f.repo_id
232 JOIN user u ON u.id = r.owner_id
233 WHERE f.at >= ?1
234 GROUP BY f.repo_id, f.test
235 ORDER BY count DESC, last DESC
236 LIMIT ?2`,
237 )
238 .all(new Date(now - WINDOW_MS).toISOString(), limit);
239 }
src/checks/queue.ts +31 −6
@@ -180,6 +180,11 @@ export interface QueueOptions {
180180 * kept beyond this process (#6427).
181181 */
182182 readonly onBatch?: (repoId: string, red: boolean) => void;
183 /**
184 * A shorter prefix was red and a longer one holding it green: the red was
185 * a flake, and its verdicts are handed here to be recorded.
186 */
187 readonly onFlake?: (repoId: string, reds: readonly Red[], tip: string) => void;
183188 /** Where the line's gates and red marks outlive this process (#6611). */
184189 readonly store?: QueueStore;
185190 /**
@@ -975,7 +980,12 @@ async function drain(line: Line, gate: GateCandidate, options: QueueOptions): Pr
975980 ahead.shift();
976981 let left: Waiting[];
977982 try {
978 left = await settleHead({ line, gated: searching, room, onBatch: options.onBatch }, head, verdict, ahead);
983 left = await settleHead(
984 { line, gated: searching, room, onBatch: options.onBatch, onFlake: options.onFlake },
985 head,
986 verdict,
987 ahead,
988 );
979989 } catch (error) {
980990 log.warn("merge queue round failed", { error: String(error) });
981991 for (const w of head.steps) w.settle(false);
@@ -1293,6 +1303,7 @@ interface Search {
12931303 readonly gated: Gated;
12941304 readonly room: Room;
12951305 readonly onBatch: ((repoId: string, red: boolean) => void) | undefined;
1306 readonly onFlake: ((repoId: string, reds: readonly Red[], tip: string) => void) | undefined;
12961307 }
12971308
12981309 /**
@@ -1344,7 +1355,9 @@ async function round(
13441355 * top of it. All green, the culprit is in the last run. The runs after the
13451356 * red one wait, in line order, for the next candidate. No green verdict is
13461357 * thrown away: a disjoint segment gated on the base alone was void once
1347 * another landed (#6353).
1358 * another landed (#6353). A longer prefix green lands whole, even when a
1359 * shorter one was red: that red was a flake, reported to `onFlake`, and
1360 * every pull in the green prefix is cleared as a suspect.
13481361 *
13491362 * However few slots are free, a search gates at least √n − 1 prefixes, so
13501363 * the red run holds at most √n steps (#6674): a red batch blocks its line
@@ -1393,10 +1406,22 @@ async function judged(
13931406 for (const w of steps) w.settle(false);
13941407 return [];
13951408 }
1396 // The first red prefix, the whole when every gated one was green.
1397 const found = verdicts.findIndex((v) => !v.lands);
1398 const bad = found < 0 ? prefixes.length : found;
1399 const known = bad < prefixes.length ? { treeOid: prefixes[bad]!.treeOid, red: verdicts[bad] as Red } : { treeOid: candidate.treeOid, red };
1409 // The prefix after the longest green one, the whole when every gated one
1410 // was green. A red before that green is a flake: the green holds it.
1411 const bad = verdicts.findLastIndex((v) => v.lands) + 1;
1412 const flaked = verdicts.slice(0, bad).filter((v): v is Red => !v.lands && "reopen" in v);
1413 if (flaked.length > 0) {
1414 for (const w of steps.slice(0, ends[bad - 1]!)) search.line.suspects.delete(w.entry.pull.id);
1415 try {
1416 search.onFlake?.(search.line.repoId, flaked, prefixes[bad - 1]!.tipOid);
1417 } catch (error) {
1418 log.warn("flake not kept", { repo: search.line.repoId, error: String(error) });
1419 }
1420 }
1421 const known =
1422 bad < prefixes.length
1423 ? { treeOid: prefixes[bad]!.treeOid, red: verdicts[bad] as Red }
1424 : { treeOid: candidate.treeOid, red };
14001425 const green = bad > 0 ? ends[bad - 1]! : 0;
14011426 if (green > 0) await land(prefixes[bad - 1]!, steps.slice(0, green));
14021427 const now = [...landed, ...steps.slice(0, green).map((w) => w.entry)];
src/checks/service.ts +46
@@ -55,6 +55,8 @@ import { giveHistory } from "./history.ts";
5555 import { readLearned } from "./learned.ts";
5656 import { keepCache, restoreCache } from "./cache.ts";
5757 import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts";
58 import { parseBunTestFailure } from "./bun-test.ts";
59 import { passedAlone, recordFlakes, rerunCommand, rerunnable } from "./flake.ts";
5860 import { withDisk } from "./disk_red.ts";
5961 import { ASKED, redMain, unheld } from "./red_main.ts";
6062 import { proposeRevert } from "./revert_red.ts";
@@ -205,6 +207,9 @@ export const PULL_RESERVE = 2;
205207 */
206208 export const LAND_SETTLE_MS = 30_000;
207209
210 /** The longest a failed test is given to pass alone, in seconds. */
211 const RERUN_TIMEOUT_S = 300;
212
208213 /** How often queued checks another forge left are looked for. */
209214 const TICK_MS = 5_000;
210215
@@ -700,6 +705,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
700705 keep: run.issueId === null && run.env === repo.defaultBranch,
701706 keepImpact: pull === undefined && run.env === repo.defaultBranch,
702707 fileOrderDependent: run.issueId === null && run.env === repo.defaultBranch,
708 isMain: run.issueId === null && run.env === repo.defaultBranch,
703709 write,
704710 signal,
705711 });
@@ -722,6 +728,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
722728 readonly keepImpact?: boolean;
723729 /** The default branch's own run: only its log may open issues. */
724730 readonly fileOrderDependent?: boolean;
731 /** The default branch's own run: a failure of one or two tests is re-run alone before it counts as red. */
732 readonly isMain?: boolean;
725733 readonly write: (line: string) => void;
726734 readonly signal: AbortSignal;
727735 }
@@ -1038,6 +1046,44 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
10381046 } catch (error) {
10391047 write(`order-dependent tests not filed: ${String(error)}`);
junipercrew outdated

The condition reuses fileOrderDependent, a flag that means 'default-branch check' only by coincidence. Name it for what it says here.
10401048 }
1049 // Main's red costs the forge every pull's check (#6643): one or two
1050 // failed tests are re-run alone, on this commit in this cell, before
1051 // main counts as red. Passing alone is a flake: green, and recorded.
1052 if (!ran.ok && job === undefined && x.isMain === true) {
1053 try {
1054 const tests = rerunnable(checkLines, parseBunTestFailure(checkLines));
1055 if (tests !== undefined) {
1056 write(`re-running ${tests.length} failed test(s) alone`);
1057 let alone = true;
1058 for (const test of tests) {
1059 const out: string[] = [];
1060 const again = await runStep("flake", rerunCommand(test), {
1061 cwd: dir,
1062 env,
1063 log: (line) => {
1064 out.push(line);
1065 write(line);
1066 },
1067 signal,
1068 timeoutMs: Math.min(manifest.timeout, RERUN_TIMEOUT_S) * 1000,
1069 cell,
1070 });
1071 if (!again.ok || !passedAlone(out)) {
1072 alone = false;
1073 write(`"${test.name}" fails alone too: main is red`);
1074 break;
1075 }
1076 }
1077 if (alone) {
1078 recordFlakes(app.db, app.bus, repo, tests, { runId: run.id, commit: run.commitOid });
1079 write(`flaky: ${tests.map((t) => `"${t.name}"`).join(", ")} failed in the full run and passed alone; recorded, main stays green`);
1080 ran = { ok: true, value: undefined };
1081 }
1082 }
1083 } catch (error) {
1084 write(`flake re-run failed: ${String(error)}`);
1085 }
1086 }
10411087 if (!ran.ok) {
10421088 const { summary, detail } = summariseCheckFailure(
10431089 checkLines,
src/core/migrations/20261026190000-check-flake.ts added +15
@@ -0,0 +1,15 @@
1 // One row per test a default-branch check saw fail and then pass when run
2 // alone (#6643, checks/flake.ts): the commit, the run, the test and its file.
3 // The flake list is a count of these over a week. Additive.
4 export default `
5 CREATE TABLE check_flake (
6 id TEXT PRIMARY KEY,
7 repo_id TEXT NOT NULL REFERENCES repo(id) ON DELETE CASCADE,
8 run_id TEXT NOT NULL,
9 commit_oid TEXT NOT NULL DEFAULT '',
10 test TEXT NOT NULL,
11 file TEXT NOT NULL,
12 at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
13 );
14 CREATE INDEX check_flake_test ON check_flake(repo_id, test, at);
15 `;
src/crew/autonomy.ts +5
@@ -129,6 +129,7 @@ import { crewOffInPreview } from "./preview.ts";
129129 import { approvedOnHead, finishApproved, finishFields } from "./finishing.ts";
130130 import { recheckRedApproved } from "./red_approved.ts";
131131 import { keepBatch } from "../flow/red_batches.ts";
132 import { recordQueueFlakes } from "../checks/flake.ts";
132133 import { readVitals, vitalsMisses, vitalsWords } from "../flow/vitals.ts";
133134 import { moveRequested, tallyRequested, tallyWords } from "./requested.ts";
134135 import { reviewersOf, sendToReview, sweepReviews } from "./review.ts";
@@ -2100,6 +2101,10 @@ function queueOptions(app: App, repo: Repo): QueueOptions {
21002101 batch: queueBatch(app.db),
21012102 room: () => checksFor(app)?.spareSlots?.() ?? 0,
21022103 onBatch: (repoId, red) => keepBatch(app.db, repoId, red),
2104 onFlake: (repoId, reds, tip) => {
2105 const owner = getRepoById(app.db, repoId);
2106 if (owner !== undefined) recordQueueFlakes(app.db, app.bus, owner, reds, tip);
2107 },
21032108 store: queueStore(app.db),
21042109 adopt: (record, signal) => adoptGate(app, repo, record, signal),
21052110 };
src/kernel/vitals.ts +4
@@ -36,6 +36,7 @@ import { DOMAINS } from "./domains.ts";
3636 import { domainDrillNeeds } from "./domain_drill.ts";
3737 import { freeChecks, heldChecks, redHref, redSentence } from "../checks/red_main.ts";
3838 import { diskRedSentence, diskReds } from "../checks/disk_red.ts";
39 import { flakeList, type FlakeRow } from "../checks/flake.ts";
3940 import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
4041
4142 /** A red in a repository other than the forge's own is not fixable from here: say so, so it reads as known. */
@@ -106,6 +107,8 @@ export interface Vitals {
106107 }[];
107108 readonly apps: { serving: number; asleep: number; previews: number };
108109 readonly errors: { lastHour: number; latest: ErrorLine[] };
110 /** Tests that failed in main's check and passed alone this week: a list, never a need (#6643). */
111 readonly flakes: FlakeRow[];
109112 readonly at: string;
110113 }
111114
@@ -510,6 +513,7 @@ export function vitals(input: VitalsInput): Vitals {
510513 stuck,
511514 apps,
512515 errors: { lastHour: seen.lastHour, latest: seen.latest },
516 flakes: flakeList(input.db, now),
513517 at,
514518 };
515519 }
src/tui/screens/dashboard.ts +5 −1
@@ -16,7 +16,7 @@
1616
1717 import { protectionNotice } from "../../kernel/protection.ts";
1818 import { holdWords, plannerHold } from "../../crew/autonomy.ts";
19 import { forgeLane, forgeNeeds } from "../../actions/health.ts";
19 import { forgeFlakes, forgeLane, forgeNeeds } from "../../actions/health.ts";
2020 import { ownerWaits, waitLine, withMore } from "../../actions/owner_waits.ts";
2121 import { forgeHealth } from "../../kernel/guard.ts";
2222 import { coveredByVerdict, forgeVerdict, verdictLine } from "../../kernel/guard_verdict.ts";
@@ -196,6 +196,8 @@ export function dashboardScreen(f: Session, ctx: Ctx): Screen {
196196 let clientLive = new Map<string, string>();
197197 /** The check lane, as the board's strip says it; null when nothing is queued. */
198198 let lane: string | null = null;
199 /** The week's flaky tests, one line; null when none flaked. */
200 let flaky: string | null = null;
199201 /** What the decider did today, the board's Today line; null on a quiet day or for a reader. */
200202 let decider: string | null = null;
201203 /** Home's three figures, as the web's row says them; empty unless an admin looks (they count every repository). */
@@ -300,6 +302,7 @@ export function dashboardScreen(f: Session, ctx: Ctx): Screen {
300302 const facts = f.user === undefined ? new Set<string>() : ownerFactRepoIds(f.db, all, f.user);
301303 waits = ownerWaits(f.db, all.filter((r) => facts.has(r.id))).map(waitLine);
302304 lane = forgeLane(f.app, f.user);
305 flaky = forgeFlakes(f.app, f.user);
303306 decider = deciderLine(f.app, f.user);
304307 const commit = running().commit;
305308 figures =
@@ -964,6 +967,7 @@ export function dashboardScreen(f: Session, ctx: Ctx): Screen {
964967 if (forge !== undefined) {
965968 const lines = forge.length === 0 ? ["All fine"] : withMore(forge, 3);
966969 if (lane !== null) lines.push(`Lane: ${lane}`);
970 if (flaky !== null) lines.push(`Flaky this week: ${flaky}`);
967971 const [strip, rest] = split(body, "v", [len(lines.length), fill(1)]);
968972 lines.forEach((line, i) =>
969973 s.text(
src/web/pages/board.ts +3
@@ -47,6 +47,7 @@ import { spendView } from "./spend.ts";
4747 import type { HandContext } from "../../crew/handover.ts";
4848 import { forgeHealth, rolloutLine } from "../../kernel/guard.ts";
4949 import { coveredByVerdict, forgeVerdict, verdictLine } from "../../kernel/guard_verdict.ts";
50 import { flakeLine } from "../../checks/flake.ts";
5051 import { laneLine } from "../../kernel/vitals.ts";
5152 import { getIssue, getIssuesByIds, type Issue } from "../../core/issues.ts";
5253 import { looksForApps } from "../../core/looks.ts";
@@ -393,6 +394,7 @@ function forgeNotes(ctx: Ctx): { notes: Note[]; ok: boolean } | null {
393394 const others = fh.incidents.filter((i) => !said(`${i.key}: ${i.message}`));
394395 const ok = v.ok && fh.ok && verdict === undefined;
395396 const lane = laneLine(v.lane);
397 const flaky = flakeLine(v.flakes);
396398 const sentinel = sentinelNotice(ctx.app.db);
397399 const need = (key: string, line: Html): Note => ({ key, need: true, line });
398400 const notes: Note[] = [
@@ -412,6 +414,7 @@ function forgeNotes(ctx: Ctx): { notes: Note[]; ok: boolean } | null {
412414 need(`degraded:${d.cause}`, html`<a data-need href="${d.href}">${dot("attn")} ${needLine(d)}</a>`),
413415 )),
414416 ...(lane === null ? [] : [{ key: "lane", need: false, line: html`<span>Lane: ${lane}</span>` }]),
417 ...(flaky === null ? [] : [{ key: "flakes", need: false, line: html`<span data-flakes>Flaky this week: ${flaky}</span>` }]),
junipercrew

Plain span with no destination. Link it to the filed Flaky test issue (an <a>) so it is reachable in two taps; keep data-flakes.
415418 ];
416419 return { notes, ok };
417420 }
test/board.test.ts +12
@@ -490,6 +490,18 @@ describe("the board's forge strip", () => {
490490 expect(other).not.toContain("stuck.");
491491 });
492492
493 test("the home lists the week's flaky tests, to an admin only", async () => {
494 expect(await board("root")).not.toContain("Flaky this week");
495 const made = await createRepo(app, admin, { name: "flaky" });
496 if (!made.ok) throw new Error(String(made.error));
497 app.db
498 .query("INSERT INTO check_flake (id, repo_id, run_id, commit_oid, test, file) VALUES ('f1', ?1, 'r1', 'abc1234', 'grp > flaky', 't/a.test.ts')")
499 .run(made.value.id);
500 expect(await board("root")).toContain("Flaky this week: grp &gt; flaky ×1 (root/flaky)");
501 expect(await board("bob")).not.toContain("Flaky this week");
502 app.db.query("DELETE FROM check_flake").run();
503 });
504
493505 test("the home says the lane when checks are queued, to an admin only", async () => {
494506 app.db.query("UPDATE run SET state = 'cancelled' WHERE state IN ('queued','running')").run();
495507 const idle = await board("root");
test/check-flake.test.ts added +162
@@ -0,0 +1,162 @@
1 import { afterAll, beforeAll, expect, test } from "bun:test";
2 import { mkdtemp, rm } from "node:fs/promises";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
6 import { createApp, type App } from "../src/app.ts";
7 import { redMain } from "../src/checks/red_main.ts";
8 import { flakeList, flakeNeeds, passedAlone, recordFlakes, recordQueueFlakes, rerunCommand, rerunnable } from "../src/checks/flake.ts";
9 import { startChecks } from "../src/checks/service.ts";
10 import { parseBunTestFailure } from "../src/checks/bun-test.ts";
11 import { listRuns } from "../src/core/runs.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { readRunLog } from "../src/runs/log.ts";
14 import { quietLogs } from "./helpers/globals.ts";
15
16 quietLogs();
17
18 const report = (names: string[], file = "t/a.test.ts"): string[] => [
19 `${file}:`,
20 ...names.map((n) => `(fail) ${n} [1.00ms]`),
21 ` 0 pass`,
22 ` ${names.length} fail`,
23 ];
24
25 test("one or two named failures can be re-run; more, or a hook, cannot", () => {
26 const one = report(["grp > flaky"]);
27 expect(rerunnable(one, parseBunTestFailure(one))).toEqual([{ file: "t/a.test.ts", name: "grp > flaky" }]);
28 const three = report(["a", "b", "c"]);
29 expect(rerunnable(three, parseBunTestFailure(three))).toBeUndefined();
30 const hook = ["t/a.test.ts:", "(fail) t/a.test.ts > beforeEach (timeout)", " 1 pass", " 1 fail"];
31 expect(rerunnable(hook, parseBunTestFailure(hook))).toBeUndefined();
32 expect(rerunnable(["tsc failed"], undefined)).toBeUndefined();
33 });
34
35 test("the re-run command quotes the name and anchors it to one test", () => {
36 const cmd = rerunCommand({ file: "t/a.test.ts", name: "grp (x) > it's $(rm) a+b" });
37 expect(cmd).toBe(`bun test 't/a.test.ts' -t '^grp \\(x\\) it'\\''s \\$\\(rm\\) a\\+b$'`);
38 });
39
40 test("a re-run that matched nothing or failed is not a pass", () => {
41 expect(passedAlone([" 1 pass", " 0 fail"])).toBe(true);
42 expect(passedAlone(['error: regex "x" matched 0 tests.'])).toBe(false);
43 expect(passedAlone([" 0 pass", " 1 fail"])).toBe(false);
44 // A pattern that caught a sibling too is not the one test passing.
45 expect(passedAlone([" 2 pass", " 0 fail"])).toBe(false);
46 });
47
48 let dir: string;
49 let app: App;
50 let alice: Awaited<ReturnType<typeof makeUser>>;
51 async function makeUser() {
52 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
53 if (!u.ok) throw new Error(u.error);
54 return u.value;
55 }
56 beforeAll(async () => {
57 dir = await mkdtemp(join(tmpdir(), "flake-"));
58 app = createApp({ dataDir: dir, port: 0, host: "forge.test" });
59 alice = await makeUser();
60 });
61 afterAll(async () => {
62 await rm(dir, { recursive: true, force: true });
63 });
64
65 test("two flakes in a week file one issue for the test; a third adds none; the list counts them", async () => {
66 const made = await createRepo(app, alice, { name: "list", visibility: "public" });
67 if (!made.ok) throw new Error("repo");
68 const repo = made.value;
69 const t = [{ file: "t/a.test.ts", name: "grp > flaky" }];
70 const issues = () => app.db.query<{ title: string; body: string }, [string]>("SELECT title, body FROM issue WHERE repo_id = ?1").all(repo.id);
71 expect(recordFlakes(app.db, app.bus, repo, t, { runId: "r1", commit: "aaaaaaa1" })).toEqual([]);
72 expect(issues().length).toBe(0);
73 expect(recordFlakes(app.db, app.bus, repo, t, { runId: "r2", commit: "bbbbbbb2" }).length).toBe(1);
74 expect(recordFlakes(app.db, app.bus, repo, t, { runId: "r3", commit: "ccccccc3" })).toEqual([]);
75 expect(issues().length).toBe(1);
76 expect(issues()[0]!.body).toContain("Files: t/a.test.ts");
77 const list = flakeList(app.db, Date.now());
78 expect(list[0]).toMatchObject({ repo: "alice/list", test: "grp > flaky", count: 3, lastCommit: "ccccccc3" });
79 // A flake outside the week is not counted.
80 expect(flakeList(app.db, Date.now() + 8 * 24 * 3_600_000)).toEqual([]);
81 });
82
83 const ft = (name: string, file?: string) => ({ failing: [{ name, ...(file === undefined ? {} : { file }) }] });
84
85 test("a queue flake keeps the verdict's one file, the tip and its gate; with no known file it is listed and files no issue", async () => {
86 const made = await createRepo(app, alice, { name: "queued", visibility: "public" });
87 if (!made.ok) throw new Error("repo");
88 const repo = made.value;
89 const issues = () => app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE repo_id = ?1").all(repo.id);
90 const tip = "abcdef0123456789abcdef0123456789abcdef01";
91 const known = [{ tests: ft("grp > known"), learn: ["t/k.test.ts"] }];
92 recordQueueFlakes(app.db, app.bus, repo, known, tip);
93 recordQueueFlakes(app.db, app.bus, repo, known, tip);
94 expect(issues()).toHaveLength(1);
95 expect(issues()[0]!.body).toContain("Files: t/k.test.ts");
96 const row = app.db
97 .query<{ run_id: string; commit_oid: string; file: string }, [string]>("SELECT run_id, commit_oid, file FROM check_flake WHERE repo_id = ?1 AND test = 'grp > known'")
98 .get(repo.id)!;
99 expect(row).toEqual({ run_id: "gate-abcdef012345", commit_oid: tip, file: "t/k.test.ts" });
100 // Two files named, or none: the test is counted, no issue is filed.
101 const vague = [{ tests: ft("grp > vague"), learn: ["t/x.test.ts", "t/y.test.ts"] }];
102 recordQueueFlakes(app.db, app.bus, repo, vague, tip);
103 recordQueueFlakes(app.db, app.bus, repo, vague, tip);
104 recordQueueFlakes(app.db, app.bus, repo, [{ learn: [] }], tip);
105 expect(issues()).toHaveLength(1);
106 // A test that carries its own file files an issue though the verdicts learned several.
107 recordQueueFlakes(app.db, app.bus, repo, [{ tests: ft("grp > own", "t/own.test.ts"), learn: ["t/x.test.ts", "t/y.test.ts"] }], tip);
108 recordQueueFlakes(app.db, app.bus, repo, [{ tests: ft("grp > own", "t/own.test.ts"), learn: [] }], tip);
109 expect(issues()).toHaveLength(2);
110 expect(issues()[1]!.body).toContain("Files: t/own.test.ts");
111 expect(flakeList(app.db, Date.now()).find((r) => r.test === "grp > vague")?.count).toBe(2);
112 });
113
114 test("a test that flaked three times in the week is an attention need; two is only a note", () => {
115 const row = (count: number) => ({ repo: "a/b", test: "t", file: "f", count, lastCommit: "c", last: "" });
116 expect(flakeNeeds([row(2)])).toEqual([]);
117 expect(flakeNeeds([row(3)])).toEqual(["flaky: t failed and passed alone 3 times this week (a/b)"]);
118 });
119
120 /** A check whose full run prints one failing test and exits 1; the re-run is the real test file. */
121 async function mainCheck(name: string, testBody: string) {
122 process.env["FORGE_CELLS"] = "off";
123 const made = await createRepo(app, alice, { name, visibility: "public" });
124 if (!made.ok) throw new Error("repo");
125 const repo = made.value;
126 const fake = `printf 't/a.test.ts:\\n(fail) grp > flaky\\n 0 pass\\n 1 fail\\n'; exit 1`;
127 const checks = startChecks(app, { forgeUrl: "http://forge.test", concurrency: 1 });
128 try {
129 const pushed = await commitToBranch(app, alice, repo, {
130 message: "a flaky main",
131 files: [
132 { path: "forge.json", content: JSON.stringify({ check: { run: fake, install: "" } }) },
133 { path: "t/a.test.ts", content: `import { test, expect, describe } from "bun:test";\ndescribe("grp", () => { test("flaky", () => { ${testBody} }); });\n` },
134 ],
135 });
136 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
137 const oid = pushed.value.commitOid;
138 const deadline = Date.now() + 40_000;
139 const finished = () => listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find((r) => r.commitOid === oid && (r.state === "ok" || r.state === "failed"));
140 while (Date.now() < deadline && finished() === undefined) await Bun.sleep(100);
141 const run = finished()!;
142 return { repo, run, log: readRunLog(app.config.dataDir, run.id).lines.join("\n") };
143 } finally {
144 await checks.stop();
145 }
146 }
147
148 test("a test that fails once and passes alone leaves main green and the flake recorded", async () => {
149 const { repo, run, log } = await mainCheck("flaky", "expect(1).toBe(1);");
150 expect(log).toContain("re-running 1 failed test(s) alone");
151 expect(run.state).toBe("ok");
152 expect(redMain(app.db, repo.id)).toBeUndefined();
153 expect(flakeList(app.db, Date.now()).find((f) => f.repo === "alice/flaky")).toMatchObject({ test: "grp > flaky", file: "t/a.test.ts", count: 1 });
154 }, 60_000);
155
156 test("a test that fails alone too leaves main red", async () => {
157 const { repo, run, log } = await mainCheck("broken", "expect(1).toBe(2);");
158 expect(log).toContain("fails alone too: main is red");
159 expect(run.state).toBe("failed");
160 expect(redMain(app.db, repo.id)?.failing).toEqual(["grp > flaky"]);
161 expect(flakeList(app.db, Date.now()).some((f) => f.repo === "alice/broken")).toBe(false);
162 }, 60_000);
test/merge-queue-search.test.ts +27
@@ -132,4 +132,31 @@ describe("the merge queue searches a red candidate by stacked prefixes", () => {
132132 expect(told).toEqual([14]);
133133 expect(answers).toEqual([true, true, true, false]);
134134 });
135
136 test("prefixes [a,b] red and [a,b,c] green: a, b and c land in one step with no solo gate, the red is reported as a flake, no suspect is gated alone", async () => {
137 told.length = 0;
138 const pulls: Queued[] = [];
139 for (let i = 31; i <= 34; i++) pulls.push(entry(i, await headAdding(`v${i}`, `f${i}.txt`)));
140 const gates: number[][] = [];
141 const flaked: string[] = [];
142 const red = (name: string): LandVerdict => ({ lands: false, reopen: true, comment: "red", learn: [], tests: { passed: 0, failed: 1, failing: [{ name, file: "test/x.test.ts", errors: [] }] } });
143 const gate = async (candidate: Candidate): Promise<LandVerdict> => {
144 const numbers = candidate.steps.map((s) => s.entry.pull.number);
145 gates.push(numbers);
146 if (numbers.length === 4 || numbers.length === 2) return red(numbers.length === 4 ? "whole" : "flaky one");
147 return { lands: true };
148 };
149 let calls = 0;
150 const room = () => (calls++ === 0 ? 0 : 1);
151 const answers = await enqueue("search-flake", pulls, gate, {
152 depth: 1,
153 room,
154 onFlake: (_repo, reds) => flaked.push(...reds.flatMap((r) => ("tests" in r ? (r.tests?.failing.map((t) => t.name) ?? []) : []))),
155 });
156 expect(gates).toEqual([[31, 32, 33, 34], [31, 32], [31, 32, 33]]);
157 // The whole was red and 34 is all that is left of it: the culprit, with no gate of its own.
158 expect(answers).toEqual([true, true, true, false]);
159 expect(told).toEqual([34]);
160 expect(flaked).toEqual(["flaky one"]);
161 });
135162 });
test/tui-screens.test.ts +9
@@ -831,10 +831,19 @@ describe("the forge's health in the terminal", () => {
831831 expect(laned.text()).toContain("Lane: 2 checks queued");
832832 laned.stop();
833833
834 forge.db
835 .query("INSERT INTO check_flake (id, repo_id, run_id, commit_oid, test, file) VALUES ('f1', ?1, 'r1', 'abc1234', 'grp > flaky', 't/a.test.ts')")
836 .run(made.value.id);
837 const flaky = boot(forge, as(a.value));
838 await flaky.settle();
839 expect(flaky.text()).toContain("Flaky this week: grp > flaky ×1");
840 flaky.stop();
841
834842 const other = boot(forge, as(b.value));
835843 await other.settle();
836844 expect(other.text()).not.toContain("quota refused");
837845 expect(other.text()).not.toContain("Lane:");
846 expect(other.text()).not.toContain("Flaky this week");
838847 expect(other.text()).not.toContain("All fine");
839848 other.stop();
840849 } finally {
test/vitals.test.ts +3 −3
@@ -91,7 +91,7 @@ test("a site admin with a read-only token gets the shape; others get the short o
9191 const v = (await res.json()) as Record<string, any>;
9292 expect(Object.keys(v).sort()).toEqual(
9393 [
94 "apps", "at", "checkedAt", "commit", "degraded", "disk", "draining", "errors",
94 "apps", "at", "checkedAt", "commit", "degraded", "disk", "draining", "errors", "flakes",
9595 "incidents", "lane", "memory", "needs", "ok", "queue", "rollout", "stale", "stuck",
9696 "version",
9797 ].sort(),
@@ -168,8 +168,8 @@ test("the number of queries does not grow with runs", () => {
168168 if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
169169 }
170170 expect(count()).toBe(before);
171 // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, (the checks red on a full disk only while the disk is full), and the newest drill of every domain, and the checks no red main holds.
172 expect(before).toBe(9);
171 // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, (the checks red on a full disk only while the disk is full), and the newest drill of every domain, the checks no red main holds, and the week's flake list.
172 expect(before).toBe(10);
173173 });
174174
175175 test("errors are counted, bounded, redacted, and a burst is said", async () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs