Public: anyone can read this; the crew can change it.
A maintenance duty names tests that failed then passed on rerun this week #13950
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13937-a-maintenance-duty-names-tests-that-fail → main 2 commits fork
6 files
changed +114
−3
docs/features/maintenance.md +8
| @@ -100,3 +100,11 @@ edit is over 60 days old, that no open pull says it closes (`Closes #n`) and tha | ||
| 100 | 100 | blocker (oldest first; the rest are counted). While that report is open, later nights file none; closing |
| 101 | 101 | it lets the next night file a fresh list. The duty never comments on, labels or closes an issue it names. |
| 102 | 102 | It shows as the `stale-issues` row on `/maintenance`. Test: `test/maintain-stale-issues.test.ts`. |
| 103 | ||
| 104 | ## Flaky tests (#13937) | |
| 105 | ||
| 106 | The `flaky-tests` duty runs once a day and names each test file whose tests failed and then passed | |
| 107 | alone on the same commit in the last 7 days, with the count, distinct tests and commits. It reads | |
| 108 | `check_flake`, which stores a test name, its file and the commit (no finer per-test data), so the report | |
| 109 | is per file. A quiet week reports nothing; it files no issue. It shows as the `flaky-tests` row on | |
| 110 | `/maintenance`. Test: `test/maintain-flaky-tests.test.ts`. |
src/maintain/duties/flaky_tests.ts added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | /** | |
| 2 | * Flaky tests (#13937): each test file whose tests failed and then passed alone on the same commit | |
| 3 | * in the last 7 days, with counts. It reads `check_flake`, the rows a default-branch check writes | |
| 4 | * when a failed test passes on its alone re-run (checks/flake.ts); the check rows keep a test name, | |
| 5 | * its file and the commit, nothing finer, so the report is per file with its tests counted. | |
| 6 | * Crash flakes (no file) are left out. The duty only names; it files nothing. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { WINDOW_MS } from "../../checks/flake.ts"; | |
| 11 | import type { Duty } from "../duty.ts"; | |
| 12 | ||
| 13 | export interface FlakyFile { | |
| 14 | readonly repo: string; | |
| 15 | readonly file: string; | |
| 16 | /** Rows: one per test that failed then passed on a commit. */ | |
| 17 | readonly count: number; | |
| 18 | /** Distinct tests in the file that did. */ | |
| 19 | readonly tests: number; | |
| 20 | /** Distinct commits it happened on. */ | |
| 21 | readonly commits: number; | |
| 22 | } | |
| 23 | ||
| 24 | /** Files with flakes in the week before `now`, most first (ties by repo and file). */ | |
| 25 | export function flakyFiles(db: Database, now: Date): FlakyFile[] { | |
| 26 | return db | |
| 27 | .query<FlakyFile, [string]>( | |
| 28 | `SELECT u.username || '/' || r.name AS repo, f.file AS file, COUNT(*) AS count, | |
| 29 | COUNT(DISTINCT f.test) AS tests, COUNT(DISTINCT f.commit_oid) AS commits | |
| 30 | FROM check_flake f | |
| 31 | JOIN repo r ON r.id = f.repo_id | |
| 32 | JOIN user u ON u.id = r.owner_id | |
| 33 | WHERE f.at >= ?1 AND f.file <> '' | |
| 34 | GROUP BY f.repo_id, f.file | |
| 35 | ORDER BY count DESC, repo, file`, | |
| 36 | ) | |
| 37 | .all(new Date(now.getTime() - WINDOW_MS).toISOString()); | |
| 38 | } | |
| 39 | ||
| 40 | const lineOf = (f: FlakyFile): string => `${f.file} ×${f.count} (${f.tests} test${f.tests === 1 ? "" : "s"}, ${f.commits} commit${f.commits === 1 ? "" : "s"}, ${f.repo})`; | |
| 41 | ||
| 42 | export const duty: Duty = { | |
| 43 | name: "flaky-tests", | |
| 44 | intervalMs: 24 * 3_600_000, | |
| 45 | budget: { runsPerDay: 3, dollarsPerDay: 1, reason: "once a day; the rest is room for a catch-up, and it spends no money" }, | |
| 46 | run({ db, now }) { | |
| 47 | const found = flakyFiles(db, now); | |
| 48 | if (found.length === 0) return { findings: [], actions: [] }; | |
| 49 | const shown = found.slice(0, 5).map(lineOf).join("; "); | |
| 50 | return { | |
| 51 | findings: [{ | |
| 52 | summary: `${found.length} test file${found.length === 1 ? "" : "s"} failed then passed on rerun this week: ${shown}${found.length > 5 ? `; and ${found.length - 5} more` : ""}`, | |
| 53 | probe: { kind: "query", what: "test files with check_flake rows in the last 7 days", number: found.length, at: now.toISOString() }, | |
| 54 | }], | |
| 55 | actions: [], | |
| 56 | }; | |
| 57 | }, | |
| 58 | }; |
src/maintain/registry.ts +2 −1
| @@ -14,9 +14,10 @@ import { duty as previewDirs } from "./previews.ts"; | ||
| 14 | 14 | import { duty as diskFree } from "./duties/disk_free.ts"; |
| 15 | 15 | import { duty as staleIssues } from "./duties/stale_issues.ts"; |
| 16 | 16 | import { duty as waitSplit } from "./duties/wait_split.ts"; |
| 17 | import { duty as flakyTests } from "./duties/flaky_tests.ts"; | |
| 17 | 18 | import { dutyIdle } from "./duties/duty_idle.ts"; |
| 18 | 19 | |
| 19 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit]; | |
| 20 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests]; | |
| 20 | 21 | |
| 21 | 22 | /** The duties plus the one that watches them for silence. */ |
| 22 | 23 | export const duties: readonly Duty[] = [...others, dutyIdle(others)]; |
src/perf/page_budget.ts +2 −2
| @@ -43,8 +43,8 @@ export const PAGE_BUDGETS: readonly PageBudget[] = [ | ||
| 43 | 43 | // The admin board of duties (#13388): walked as the admin, stored rows only. Statements: measured |
| 44 | 44 | // warm (53, with main's duties) plus eight: each of the seven duties reads its own state and |
| 45 | 45 | // runs, so the count moves with duty state and background ticks; an N+1 over the duties would |
| 46 | // still show. | |
| 47 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 61 }, | |
| 46 | // still show. One more duty (flaky-tests, #13937) reads its own state: 62 measured, limit 63. | |
| 47 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 63 }, | |
| 48 | 48 | // A stranger's way to a forge (#11410): walked signed out. `/claim` is |
| 49 | 49 | // measured on a spawned forge with a live link, as the child's owner sees it. |
| 50 | 50 | { page: "pricing", pattern: "/pricing", medianMs: 50, statements: 10, signedOut: true }, |
test/maintain-flaky-tests.test.ts added +43
| @@ -0,0 +1,43 @@ | ||
| 1 | /** | |
| 2 | * Flaky tests (#13937): check_flake rows in, one finding per week's files out; a quiet week, none. | |
| 3 | */ | |
| 4 | import { expect, test } from "bun:test"; | |
| 5 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 6 | import { duty } from "../src/maintain/duties/flaky_tests.ts"; | |
| 7 | import { duties } from "../src/maintain/registry.ts"; | |
| 8 | import { id } from "../src/kernel/id.ts"; | |
| 9 | ||
| 10 | const w = rulesWorld(); | |
| 11 | const NOW = new Date("2026-11-15T12:00:00Z"); | |
| 12 | const ago = (d: number): string => new Date(NOW.getTime() - d * 86_400_000).toISOString(); | |
| 13 | const flake = (file: string, name: string, commit: string, days: number): void => { | |
| 14 | w.app.db.query("INSERT INTO check_flake (id, repo_id, run_id, commit_oid, test, file, at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)").run(id(), w.repoId, id(), commit, name, file, ago(days)); | |
| 15 | }; | |
| 16 | const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, now: NOW }); | |
| 17 | ||
| 18 | test("a quiet week reports nothing", async () => { | |
| 19 | expect(await run()).toEqual({ findings: [], actions: [] }); | |
| 20 | flake("test/old.test.ts", "old", "a1", 8); // outside the week | |
| 21 | flake("", "crash", "a2", 1); // no file: not named | |
| 22 | expect(await run()).toEqual({ findings: [], actions: [] }); | |
| 23 | }); | |
| 24 | ||
| 25 | test("files that flaked in the week are named with counts", async () => { | |
| 26 | flake("test/a.test.ts", "one", "c1", 1); | |
| 27 | flake("test/a.test.ts", "two", "c1", 1); | |
| 28 | flake("test/a.test.ts", "one", "c2", 2); | |
| 29 | flake("test/b.test.ts", "x", "c3", 3); | |
| 30 | const out = await run(); | |
| 31 | expect(out.actions).toEqual([]); | |
| 32 | expect(out.findings).toHaveLength(1); | |
| 33 | const f = out.findings[0]!; | |
| 34 | expect(f.probe!.number).toBe(2); | |
| 35 | expect(f.summary).toContain("test/a.test.ts ×3 (2 tests, 2 commits, alice/rules)"); | |
| 36 | expect(f.summary).toContain("test/b.test.ts ×1 (1 test, 1 commit"); | |
| 37 | expect(f.summary).not.toContain("old.test.ts"); | |
| 38 | expect(f.summary.indexOf("a.test.ts")).toBeLessThan(f.summary.indexOf("b.test.ts")); | |
| 39 | }); | |
| 40 | ||
| 41 | test("it is registered", () => { | |
| 42 | expect(duties.map((d) => d.name)).toContain("flaky-tests"); | |
| 43 | }); |
test/pages-budget.test.ts +1
| @@ -56,6 +56,7 @@ const HEAVIER: Record<string, { statements?: number; git?: number }> = { | ||
| 56 | 56 | "/api/v1/asks": { statements: 61 }, // +1: the stopped apps someone asked for, one read of the viewer's archived repositories (#12277) |
| 57 | 57 | "/board": { statements: 72 }, // +1: the forge-copy read for the tell box (#12037); +1: the vitals read the database's growth (#12025); +6: the money line (#510) — one settings read, then the month's and the 30 days' meter, storage and ledger reads; constant in the number of apps |
| 58 | 58 | "/settings/capacity": { statements: 70 }, |
| 59 | "/maintenance": { statements: 63 }, // +1..2: the flaky-tests duty reads its own state and runs (#13937) | |
| 59 | 60 | }; |
| 60 | 61 | |
| 61 | 62 | /** Patterns with no fixture today: lower this as fixtures are added, never raise it silently. */ |
new above
offline