Public: anyone can read this; the crew can change it.
A maintenance duty files one defect issue per app whose latest build has been red for three days #13951
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13938-a-maintenance-duty-files-one-defect-issu → main 5 commits fork 2 threads · 2 unresolved
6 files
changed +176
−5
docs/features/maintenance.md +11
| @@ -108,3 +108,14 @@ alone on the same commit in the last 7 days, with the count, distinct tests and | ||
| 108 | 108 | `check_flake`, which stores a test name, its file and the commit (no finer per-test data), so the report |
| 109 | 109 | is per file. A quiet week reports nothing; it files no issue. It shows as the `flaky-tests` row on |
| 110 | 110 | `/maintenance`. Test: `test/maintain-flaky-tests.test.ts`. |
| 111 | ||
| 112 | ## Red apps (#13938) | |
| 113 | ||
| 114 | The `red-apps` duty runs once a day. An app is red when the latest finished check of its default branch | |
| 115 | has failed and the first failure of that unbroken streak (no passing check since) finished 3 or more | |
| 116 | days ago; an app whose latest check passed, or that has none, is not. For each red app it files one defect | |
| 117 | issue, labelled `bug` (made if missing), in the app's own repository, titled "The latest build of | |
| 118 | owner/name has been red for 3 days", naming the failing check, the run and the commit. One issue stays | |
| 119 | open at a time: while it is open no second is filed, and closing it while the check is still red lets the | |
| 120 | next day file a fresh one. The duty never comments on or closes an issue. It shows as the `red-apps` row | |
| 121 | on `/maintenance`. Test: `test/maintain-red-apps.test.ts`. |
src/maintain/duties/red_apps.ts added +103
| @@ -0,0 +1,103 @@ | ||
| 1 | /** | |
| 2 | * Red apps (#13938): an app whose default branch's latest finished check has failed, and whose | |
| 3 | * red run began 3 or more days ago, gets one defect issue in its own repository naming the failing | |
| 4 | * check. The issue is keyed on the app (its title) and its open state (`fileIssueOnce`): while one | |
| 5 | * is open no second is filed, and a recovered app (latest check ok) gets none. Reads finished | |
| 6 | * default-branch checks by `run_repo`, per app; it never comments on or closes an issue. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 11 | import { redMain, type RedMain } from "../../checks/red_main.ts"; | |
| 12 | import { getRepo } from "../../core/repos.ts"; | |
| 13 | import { getLabel, createLabel } from "../../core/issues.ts"; | |
| 14 | import { LABEL_COLORS } from "../../hunt/finding.ts"; | |
| 15 | import type { Duty } from "../duty.ts"; | |
| 16 | ||
| 17 | export const RED_DAYS = 3; | |
| 18 | const DAY_MS = 86_400_000; | |
| 19 | ||
| 20 | export interface RedApp { | |
| 21 | readonly owner: string; | |
| 22 | readonly name: string; | |
| 23 | readonly repoId: string; | |
| 24 | readonly red: RedMain; | |
| 25 | /** When the current red streak began: the finish of its first failed check. */ | |
| 26 | readonly redSince: string; | |
| 27 | /** Whole days the streak has run at the time of the read. */ | |
| 28 | readonly daysRed: number; | |
| 29 | } | |
| 30 | ||
| 31 | export const titleOf = (owner: string, name: string): string => `The latest build of ${owner}/${name} has been red for ${RED_DAYS} days`; | |
| 32 | ||
| 33 | /** Apps red for at least `days` at `now`, by repository id. */ | |
| 34 | export function redApps(db: Database, now: Date, days = RED_DAYS): RedApp[] { | |
| 35 | const apps = db | |
| 36 | .query<{ repo_id: string; owner: string; name: string }, []>( | |
| 37 | `SELECT repo.id AS repo_id, u.username AS owner, repo.name FROM app JOIN repo ON repo.id = app.repo_id JOIN user u ON u.id = repo.owner_id ORDER BY repo.id`, | |
| 38 | ) | |
| 39 | .all(); | |
| 40 | const out: RedApp[] = []; | |
| 41 | const cutoff = new Date(now.getTime() - days * DAY_MS).toISOString(); | |
| 42 | for (const a of apps) { | |
| 43 | const red = redMain(db, a.repo_id); | |
| 44 | if (red === undefined) continue; // none yet, or the latest finished check is green | |
| 45 | const since = db | |
| 46 | .query<{ at: string | null }, [string]>( | |
| 47 | `SELECT MIN(r.finished_at) AS at FROM run r JOIN repo ON repo.id = r.repo_id | |
| 48 | WHERE r.repo_id = ?1 AND r.kind = 'check' AND r.issue_id IS NULL AND r.env = repo.default_branch | |
| 49 | AND r.state = 'failed' AND r.finished_at IS NOT NULL | |
| 50 | AND r.id > COALESCE((SELECT MAX(o.id) FROM run o | |
| 51 | WHERE o.repo_id = r.repo_id AND o.kind = 'check' AND o.issue_id IS NULL | |
| 52 | AND o.env = repo.default_branch AND o.state = 'ok'), '')`, | |
| 53 | ) | |
| 54 | .get(a.repo_id)?.at; | |
| 55 | if (!since || since > cutoff) continue; | |
| 56 | out.push({ owner: a.owner, name: a.name, repoId: a.repo_id, red, redSince: since, daysRed: Math.floor((now.getTime() - Date.parse(since)) / DAY_MS) }); | |
| 57 | } | |
| 58 | return out; | |
| 59 | } | |
| 60 | ||
| 61 | const oneLine = (s: string): string => s.replace(/\s+/g, " ").replace(/[<>`]/g, "").trim().slice(0, 200); | |
| 62 | ||
| 63 | export function bodyOf(a: RedApp): string { | |
| 64 | const names = a.red.failing.map(oneLine).filter((n) => n !== ""); | |
| 65 | return ( | |
| 66 | `_Filed by the red apps duty from the run table, not written by a person._\n\n` + | |
| 67 | `The default branch's check of ${a.owner}/${a.name} has failed since ${a.redSince} (${a.daysRed} days) and no check has passed since.\n\n` + | |
| 68 | `Failing check: ${names.length > 0 ? names.join("; ") : oneLine(a.red.summary) || "(no name recorded)"}\n` + | |
| 69 | `Latest run: ${a.red.runId} on ${a.red.commitOid.slice(0, 12) || "(no commit)"}\n\n` + | |
| 70 | `Fix it, or say here why it stays red. One issue stays open at a time; close it and a check still red files a fresh one the next day.` | |
| 71 | ); | |
| 72 | } | |
| 73 | ||
| 74 | /** The repository's `bug` label, made when it has none; undefined when it cannot be made. */ | |
| 75 | function bugLabel(db: Database, repoId: string) { | |
| 76 | const existing = getLabel(db, repoId, "bug"); | |
| 77 | if (existing) return existing; | |
| 78 | const made = createLabel(db, repoId, "bug", LABEL_COLORS.bug ?? "ededed"); | |
| 79 | return made.ok ? made.value : undefined; | |
| 80 | } | |
| 81 | ||
| 82 | export const duty: Duty = { | |
| 83 | name: "red-apps", | |
| 84 | intervalMs: 24 * 3_600_000, | |
| 85 | budget: { runsPerDay: 3, dollarsPerDay: 1, reason: "once a day; the rest is room for a catch-up, and it spends no money" }, | |
| 86 | run({ db, bus, now, app }) { | |
| 87 | if (app === undefined) return { findings: [], actions: [] }; | |
| 88 | const findings = []; | |
outdated
Same days-red calculation as in bodyOf; compute once (e.g. a field on RedApp). |
||
| 89 | const actions: string[] = []; | |
| 90 | for (const a of redApps(db, now)) { | |
| 91 | const repo = getRepo(db, a.owner, a.name); | |
| 92 | if (repo === undefined) continue; | |
| 93 | const label = bugLabel(db, repo.id); | |
| 94 | const filed = fileIssueOnce(db, bus, repo, { title: titleOf(a.owner, a.name), body: bodyOf(a), ...(label ? { labelId: label.id } : {}) }); | |
| 95 | findings.push({ | |
| 96 | summary: `${a.owner}/${a.name} has been red since ${a.redSince}`, | |
| 97 | probe: { kind: "measurement" as const, what: `days the latest check of ${a.owner}/${a.name} has been red`, number: a.daysRed, at: now.toISOString() }, | |
| 98 | }); | |
| 99 | if (filed.opened) actions.push(`filed #${filed.number} on ${a.owner}/${a.name}`); | |
| 100 | } | |
| 101 | return { findings, actions }; | |
| 102 | }, | |
| 103 | }; |
src/maintain/registry.ts +2 −1
| @@ -14,10 +14,11 @@ import { duty as previewDirs } from "./previews.ts"; | ||
| 14 | 14 | import { duty as diskFree } from "./duties/disk_free.ts"; |
| 15 | 15 | import { duty as staleIssues } from "./duties/stale_issues.ts"; |
| 16 | 16 | import { duty as waitSplit } from "./duties/wait_split.ts"; |
| 17 | import { duty as redApps } from "./duties/red_apps.ts"; | |
| 17 | 18 | import { duty as flakyTests } from "./duties/flaky_tests.ts"; |
| 18 | 19 | import { dutyIdle } from "./duties/duty_idle.ts"; |
| 19 | 20 | |
| 20 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests]; | |
| 21 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests, redApps]; | |
| 21 | 22 | |
| 22 | 23 | /** The duties plus the one that watches them for silence. */ |
| 23 | 24 | export const duties: readonly Duty[] = [...others, dutyIdle(others)]; |
src/perf/page_budget.ts +3 −3
| @@ -41,10 +41,10 @@ export const PAGE_BUDGETS: readonly PageBudget[] = [ | ||
| 41 | 41 | { page: "space", pattern: "/:owner/:name/space", medianMs: 50, statements: 17 }, |
| 42 | 42 | { page: "space data", pattern: "/:owner/:name/space.json", medianMs: 50, statements: 8 }, |
| 43 | 43 | // The admin board of duties (#13388): walked as the admin, stored rows only. Statements: measured |
| 44 | // warm (53, with main's duties) plus eight: each of the seven duties reads its own state and | |
| 44 | // warm (53, with main's duties) plus nine: each of the eight duties reads its own state and | |
| 45 | 45 | // runs, so the count moves with duty state and background ticks; an N+1 over the duties would |
| 46 | // still show. One more duty (flaky-tests, #13937) reads its own state: 62 measured, limit 63. | |
| 47 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 63 }, | |
| 46 | // still show. Two more duties (flaky-tests #13937, red apps #13938) each read their own state. | |
| 47 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 65 }, | |
| 48 | 48 | // A stranger's way to a forge (#11410): walked signed out. `/claim` is |
| 49 | 49 | // measured on a spawned forge with a live link, as the child's owner sees it. |
| 50 | 50 | { page: "pricing", pattern: "/pricing", medianMs: 50, statements: 10, signedOut: true }, |
test/maintain-red-apps.test.ts added +56
| @@ -0,0 +1,56 @@ | ||
| 1 | /** | |
| 2 | * Red apps (#13938): red 3 days files one defect issue naming the check; recovered files none; | |
| 3 | * never twice. | |
| 4 | */ | |
| 5 | import { expect, test } from "bun:test"; | |
| 6 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 7 | import { duty, titleOf } from "../src/maintain/duties/red_apps.ts"; | |
| 8 | import { duties } from "../src/maintain/registry.ts"; | |
| 9 | import { id } from "../src/kernel/id.ts"; | |
| 10 | ||
| 11 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 12 | const w = rulesWorld(); | |
| 13 | const NOW = new Date("2026-11-15T12:00:00Z"); | |
| 14 | const ago = (d: number): string => new Date(NOW.getTime() - d * 86_400_000).toISOString(); | |
| 15 | let seq = 0; | |
| 16 | function check(state: "ok" | "failed", days: number, names: string[] = []): void { | |
| 17 | const r = w.app.db.query<{ default_branch: string }, [string]>("SELECT default_branch FROM repo WHERE id = ?1").get(w.repoId)!; | |
| 18 | w.app.db.query("INSERT INTO run (id, repo_id, kind, env, commit_oid, state, summary, check_detail, created_at, finished_at) VALUES (?1, ?2, 'check', ?3, 'abc123abc123abc', ?4, ?5, ?6, ?7, ?7)") | |
| 19 | .run(`${String(++seq).padStart(6, "0")}${id()}`, w.repoId, r.default_branch, state, state === "ok" ? "" : "check failed", JSON.stringify({ names }), ago(days)); | |
| 20 | } | |
| 21 | const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: NOW }); | |
| 22 | const open = () => w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE repo_id = ?1 AND state = 'open' AND title LIKE 'The latest build%'").all(w.repoId); | |
| 23 | ||
| 24 | test("an app that is not an app, or red under 3 days, files nothing", async () => { | |
| 25 | expect((await run()).findings).toEqual([]); | |
| 26 | w.app.db.query("INSERT INTO app (id, repo_id, slug, access) VALUES (?1, ?2, 'rules', 'public')").run(id(), w.repoId); | |
| 27 | check("failed", 2, ["a test"]); | |
| 28 | expect((await run()).actions).toEqual([]); | |
| 29 | expect(open()).toHaveLength(0); | |
| 30 | }); | |
| 31 | ||
| 32 | test("red for 3 days: one defect issue naming the failing check, never a second while open", async () => { | |
| 33 | check("failed", 4, ["parses the thing"]); | |
| 34 | check("failed", 1, ["parses the thing"]); | |
| 35 | const out = await run(); | |
| 36 | expect(out.actions).toHaveLength(1); | |
| 37 | expect(out.findings[0]!.probe!.number).toBe(4); | |
| 38 | expect(open()).toHaveLength(1); | |
| 39 | expect(open()[0]!.body).toContain("parses the thing"); | |
| 40 | const labels = w.app.db.query<{ name: string }, [string]>("SELECT l.name FROM label l JOIN issue_label il ON il.label_id = l.id JOIN issue i ON i.id = il.issue_id WHERE i.repo_id = ?1 AND i.title LIKE 'The latest build%'").all(w.repoId); | |
| 41 | expect(labels.map((l) => l.name)).toContain("bug"); | |
| 42 | expect((await run()).actions).toEqual([]); // already reported | |
| 43 | expect(open()).toHaveLength(1); | |
| 44 | }); | |
| 45 | ||
| 46 | test("recovered: a green latest check files nothing, even after the old issue is closed", async () => { | |
| 47 | w.app.db.query("UPDATE issue SET state = 'closed' WHERE title = ?1").run(titleOf("alice", "rules")); | |
| 48 | check("ok", 0); | |
| 49 | const out = await run(); | |
| 50 | expect(out).toEqual({ findings: [], actions: [] }); | |
| 51 | expect(open()).toHaveLength(0); | |
| 52 | }); | |
| 53 | ||
| 54 | test("the duty is in the registry", () => { | |
| 55 | expect(duties.map((d) => d.name)).toContain("red-apps"); | |
| 56 | }); |
test/pages-budget.test.ts +1 −1
| @@ -56,7 +56,7 @@ const HEAVIER: Record<string, { statements?: number; git?: number }> = { | ||
| 56 | 56 | "/api/v1/asks": { statements: 61 }, // +1: the stopped apps someone asked for, one read of the viewer's archived repositories (#12277) |
| 57 | 57 | "/board": { statements: 72 }, // +1: the forge-copy read for the tell box (#12037); +1: the vitals read the database's growth (#12025); +6: the money line (#510) — one settings read, then the month's and the 30 days' meter, storage and ledger reads; constant in the number of apps |
| 58 | 58 | "/settings/capacity": { statements: 70 }, |
| 59 | "/maintenance": { statements: 63 }, // +1..2: the flaky-tests duty reads its own state and runs (#13937) | |
| 59 | "/maintenance": { statements: 65 }, // +2: the flaky-tests (#13937) and red apps (#13938) duties each read their own state and run | |
| 60 | 60 | }; |
| 61 | 61 | |
| 62 | 62 | /** Patterns with no fixture today: lower this as fixtures are added, never raise it silently. */ |
new above
offline
Extract the bug-label get-or-create into a named helper; this one-liner closure is hard to read.