Public: anyone can read this; the crew can change it.

A maintenance duty names tests that failed then passed on rerun this week #13950

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13937-a-maintenance-duty-names-tests-that-fail → main 2 commits fork

  • Merged. mason/13937-a-maintenance-duty-names-tests-that-fail was merged into main as e5f3b40.

Review

Sign in to review this change.
6 files changed +114 −3
docs/features/maintenance.md +8
@@ -100,3 +100,11 @@ edit is over 60 days old, that no open pull says it closes (`Closes #n`) and tha
100100 blocker (oldest first; the rest are counted). While that report is open, later nights file none; closing
101101 it lets the next night file a fresh list. The duty never comments on, labels or closes an issue it names.
102102 It shows as the `stale-issues` row on `/maintenance`. Test: `test/maintain-stale-issues.test.ts`.
103
104 ## Flaky tests (#13937)
105
106 The `flaky-tests` duty runs once a day and names each test file whose tests failed and then passed
107 alone on the same commit in the last 7 days, with the count, distinct tests and commits. It reads
108 `check_flake`, which stores a test name, its file and the commit (no finer per-test data), so the report
109 is per file. A quiet week reports nothing; it files no issue. It shows as the `flaky-tests` row on
110 `/maintenance`. Test: `test/maintain-flaky-tests.test.ts`.
src/maintain/duties/flaky_tests.ts added +58
@@ -0,0 +1,58 @@
1 /**
2 * Flaky tests (#13937): each test file whose tests failed and then passed alone on the same commit
3 * in the last 7 days, with counts. It reads `check_flake`, the rows a default-branch check writes
4 * when a failed test passes on its alone re-run (checks/flake.ts); the check rows keep a test name,
5 * its file and the commit, nothing finer, so the report is per file with its tests counted.
6 * Crash flakes (no file) are left out. The duty only names; it files nothing.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { WINDOW_MS } from "../../checks/flake.ts";
11 import type { Duty } from "../duty.ts";
12
13 export interface FlakyFile {
14 readonly repo: string;
15 readonly file: string;
16 /** Rows: one per test that failed then passed on a commit. */
17 readonly count: number;
18 /** Distinct tests in the file that did. */
19 readonly tests: number;
20 /** Distinct commits it happened on. */
21 readonly commits: number;
22 }
23
24 /** Files with flakes in the week before `now`, most first (ties by repo and file). */
25 export function flakyFiles(db: Database, now: Date): FlakyFile[] {
26 return db
27 .query<FlakyFile, [string]>(
28 `SELECT u.username || '/' || r.name AS repo, f.file AS file, COUNT(*) AS count,
29 COUNT(DISTINCT f.test) AS tests, COUNT(DISTINCT f.commit_oid) AS commits
30 FROM check_flake f
31 JOIN repo r ON r.id = f.repo_id
32 JOIN user u ON u.id = r.owner_id
33 WHERE f.at >= ?1 AND f.file <> ''
34 GROUP BY f.repo_id, f.file
35 ORDER BY count DESC, repo, file`,
36 )
37 .all(new Date(now.getTime() - WINDOW_MS).toISOString());
38 }
39
40 const lineOf = (f: FlakyFile): string => `${f.file} ×${f.count} (${f.tests} test${f.tests === 1 ? "" : "s"}, ${f.commits} commit${f.commits === 1 ? "" : "s"}, ${f.repo})`;
41
42 export const duty: Duty = {
43 name: "flaky-tests",
44 intervalMs: 24 * 3_600_000,
45 budget: { runsPerDay: 3, dollarsPerDay: 1, reason: "once a day; the rest is room for a catch-up, and it spends no money" },
46 run({ db, now }) {
47 const found = flakyFiles(db, now);
48 if (found.length === 0) return { findings: [], actions: [] };
49 const shown = found.slice(0, 5).map(lineOf).join("; ");
50 return {
51 findings: [{
52 summary: `${found.length} test file${found.length === 1 ? "" : "s"} failed then passed on rerun this week: ${shown}${found.length > 5 ? `; and ${found.length - 5} more` : ""}`,
53 probe: { kind: "query", what: "test files with check_flake rows in the last 7 days", number: found.length, at: now.toISOString() },
54 }],
55 actions: [],
56 };
57 },
58 };
src/maintain/registry.ts +2 −1
@@ -14,9 +14,10 @@ import { duty as previewDirs } from "./previews.ts";
1414 import { duty as diskFree } from "./duties/disk_free.ts";
1515 import { duty as staleIssues } from "./duties/stale_issues.ts";
1616 import { duty as waitSplit } from "./duties/wait_split.ts";
17 import { duty as flakyTests } from "./duties/flaky_tests.ts";
1718 import { dutyIdle } from "./duties/duty_idle.ts";
1819
19 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit];
20 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests];
2021
2122 /** The duties plus the one that watches them for silence. */
2223 export const duties: readonly Duty[] = [...others, dutyIdle(others)];
src/perf/page_budget.ts +2 −2
@@ -43,8 +43,8 @@ export const PAGE_BUDGETS: readonly PageBudget[] = [
4343 // The admin board of duties (#13388): walked as the admin, stored rows only. Statements: measured
4444 // warm (53, with main's duties) plus eight: each of the seven duties reads its own state and
4545 // runs, so the count moves with duty state and background ticks; an N+1 over the duties would
46 // still show.
47 { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 61 },
46 // still show. One more duty (flaky-tests, #13937) reads its own state: 62 measured, limit 63.
47 { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 63 },
4848 // A stranger's way to a forge (#11410): walked signed out. `/claim` is
4949 // measured on a spawned forge with a live link, as the child's owner sees it.
5050 { page: "pricing", pattern: "/pricing", medianMs: 50, statements: 10, signedOut: true },
test/maintain-flaky-tests.test.ts added +43
@@ -0,0 +1,43 @@
1 /**
2 * Flaky tests (#13937): check_flake rows in, one finding per week's files out; a quiet week, none.
3 */
4 import { expect, test } from "bun:test";
5 import { rulesWorld } from "./helpers/rules-app.ts";
6 import { duty } from "../src/maintain/duties/flaky_tests.ts";
7 import { duties } from "../src/maintain/registry.ts";
8 import { id } from "../src/kernel/id.ts";
9
10 const w = rulesWorld();
11 const NOW = new Date("2026-11-15T12:00:00Z");
12 const ago = (d: number): string => new Date(NOW.getTime() - d * 86_400_000).toISOString();
13 const flake = (file: string, name: string, commit: string, days: number): void => {
14 w.app.db.query("INSERT INTO check_flake (id, repo_id, run_id, commit_oid, test, file, at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)").run(id(), w.repoId, id(), commit, name, file, ago(days));
15 };
16 const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, now: NOW });
17
18 test("a quiet week reports nothing", async () => {
19 expect(await run()).toEqual({ findings: [], actions: [] });
20 flake("test/old.test.ts", "old", "a1", 8); // outside the week
21 flake("", "crash", "a2", 1); // no file: not named
22 expect(await run()).toEqual({ findings: [], actions: [] });
23 });
24
25 test("files that flaked in the week are named with counts", async () => {
26 flake("test/a.test.ts", "one", "c1", 1);
27 flake("test/a.test.ts", "two", "c1", 1);
28 flake("test/a.test.ts", "one", "c2", 2);
29 flake("test/b.test.ts", "x", "c3", 3);
30 const out = await run();
31 expect(out.actions).toEqual([]);
32 expect(out.findings).toHaveLength(1);
33 const f = out.findings[0]!;
34 expect(f.probe!.number).toBe(2);
35 expect(f.summary).toContain("test/a.test.ts ×3 (2 tests, 2 commits, alice/rules)");
36 expect(f.summary).toContain("test/b.test.ts ×1 (1 test, 1 commit");
37 expect(f.summary).not.toContain("old.test.ts");
38 expect(f.summary.indexOf("a.test.ts")).toBeLessThan(f.summary.indexOf("b.test.ts"));
39 });
40
41 test("it is registered", () => {
42 expect(duties.map((d) => d.name)).toContain("flaky-tests");
43 });
test/pages-budget.test.ts +1
@@ -56,6 +56,7 @@ const HEAVIER: Record<string, { statements?: number; git?: number }> = {
5656 "/api/v1/asks": { statements: 61 }, // +1: the stopped apps someone asked for, one read of the viewer's archived repositories (#12277)
5757 "/board": { statements: 72 }, // +1: the forge-copy read for the tell box (#12037); +1: the vitals read the database's growth (#12025); +6: the money line (#510) — one settings read, then the month's and the 30 days' meter, storage and ledger reads; constant in the number of apps
5858 "/settings/capacity": { statements: 70 },
59 "/maintenance": { statements: 63 }, // +1..2: the flaky-tests duty reads its own state and runs (#13937)
5960 };
6061
6162 /** Patterns with no fixture today: lower this as fixtures are added, never raise it silently. */

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs