Public: anyone can read this; the crew can change it.

A maintenance duty files one defect issue per app whose latest build has been red for three days #13951

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13938-a-maintenance-duty-files-one-defect-issu → main 5 commits fork 2 threads · 2 unresolved

  • Merged. scout/13938-a-maintenance-duty-files-one-defect-issu was merged into main as b214c3a.

Review

Sign in to review this change.
6 files changed +176 −5
docs/features/maintenance.md +11
@@ -108,3 +108,14 @@ alone on the same commit in the last 7 days, with the count, distinct tests and
108108 `check_flake`, which stores a test name, its file and the commit (no finer per-test data), so the report
109109 is per file. A quiet week reports nothing; it files no issue. It shows as the `flaky-tests` row on
110110 `/maintenance`. Test: `test/maintain-flaky-tests.test.ts`.
111
112 ## Red apps (#13938)
113
114 The `red-apps` duty runs once a day. An app is red when the latest finished check of its default branch
115 has failed and the first failure of that unbroken streak (no passing check since) finished 3 or more
116 days ago; an app whose latest check passed, or that has none, is not. For each red app it files one defect
117 issue, labelled `bug` (made if missing), in the app's own repository, titled "The latest build of
118 owner/name has been red for 3 days", naming the failing check, the run and the commit. One issue stays
119 open at a time: while it is open no second is filed, and closing it while the check is still red lets the
120 next day file a fresh one. The duty never comments on or closes an issue. It shows as the `red-apps` row
121 on `/maintenance`. Test: `test/maintain-red-apps.test.ts`.
src/maintain/duties/red_apps.ts added +103
@@ -0,0 +1,103 @@
1 /**
2 * Red apps (#13938): an app whose default branch's latest finished check has failed, and whose
3 * red run began 3 or more days ago, gets one defect issue in its own repository naming the failing
4 * check. The issue is keyed on the app (its title) and its open state (`fileIssueOnce`): while one
5 * is open no second is filed, and a recovered app (latest check ok) gets none. Reads finished
6 * default-branch checks by `run_repo`, per app; it never comments on or closes an issue.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { fileIssueOnce } from "../../checks/flake.ts";
11 import { redMain, type RedMain } from "../../checks/red_main.ts";
12 import { getRepo } from "../../core/repos.ts";
13 import { getLabel, createLabel } from "../../core/issues.ts";
14 import { LABEL_COLORS } from "../../hunt/finding.ts";
15 import type { Duty } from "../duty.ts";
16
17 export const RED_DAYS = 3;
18 const DAY_MS = 86_400_000;
19
20 export interface RedApp {
21 readonly owner: string;
22 readonly name: string;
23 readonly repoId: string;
24 readonly red: RedMain;
25 /** When the current red streak began: the finish of its first failed check. */
26 readonly redSince: string;
27 /** Whole days the streak has run at the time of the read. */
28 readonly daysRed: number;
29 }
30
31 export const titleOf = (owner: string, name: string): string => `The latest build of ${owner}/${name} has been red for ${RED_DAYS} days`;
32
33 /** Apps red for at least `days` at `now`, by repository id. */
34 export function redApps(db: Database, now: Date, days = RED_DAYS): RedApp[] {
35 const apps = db
36 .query<{ repo_id: string; owner: string; name: string }, []>(
37 `SELECT repo.id AS repo_id, u.username AS owner, repo.name FROM app JOIN repo ON repo.id = app.repo_id JOIN user u ON u.id = repo.owner_id ORDER BY repo.id`,
38 )
39 .all();
40 const out: RedApp[] = [];
41 const cutoff = new Date(now.getTime() - days * DAY_MS).toISOString();
42 for (const a of apps) {
43 const red = redMain(db, a.repo_id);
44 if (red === undefined) continue; // none yet, or the latest finished check is green
45 const since = db
46 .query<{ at: string | null }, [string]>(
47 `SELECT MIN(r.finished_at) AS at FROM run r JOIN repo ON repo.id = r.repo_id
48 WHERE r.repo_id = ?1 AND r.kind = 'check' AND r.issue_id IS NULL AND r.env = repo.default_branch
49 AND r.state = 'failed' AND r.finished_at IS NOT NULL
50 AND r.id > COALESCE((SELECT MAX(o.id) FROM run o
51 WHERE o.repo_id = r.repo_id AND o.kind = 'check' AND o.issue_id IS NULL
52 AND o.env = repo.default_branch AND o.state = 'ok'), '')`,
53 )
54 .get(a.repo_id)?.at;
55 if (!since || since > cutoff) continue;
56 out.push({ owner: a.owner, name: a.name, repoId: a.repo_id, red, redSince: since, daysRed: Math.floor((now.getTime() - Date.parse(since)) / DAY_MS) });
57 }
58 return out;
59 }
60
61 const oneLine = (s: string): string => s.replace(/\s+/g, " ").replace(/[<>`]/g, "").trim().slice(0, 200);
62
63 export function bodyOf(a: RedApp): string {
64 const names = a.red.failing.map(oneLine).filter((n) => n !== "");
65 return (
66 `_Filed by the red apps duty from the run table, not written by a person._\n\n` +
67 `The default branch's check of ${a.owner}/${a.name} has failed since ${a.redSince} (${a.daysRed} days) and no check has passed since.\n\n` +
68 `Failing check: ${names.length > 0 ? names.join("; ") : oneLine(a.red.summary) || "(no name recorded)"}\n` +
69 `Latest run: ${a.red.runId} on ${a.red.commitOid.slice(0, 12) || "(no commit)"}\n\n` +
70 `Fix it, or say here why it stays red. One issue stays open at a time; close it and a check still red files a fresh one the next day.`
71 );
72 }
73
74 /** The repository's `bug` label, made when it has none; undefined when it cannot be made. */
75 function bugLabel(db: Database, repoId: string) {
76 const existing = getLabel(db, repoId, "bug");
77 if (existing) return existing;
78 const made = createLabel(db, repoId, "bug", LABEL_COLORS.bug ?? "ededed");
79 return made.ok ? made.value : undefined;
80 }
81
82 export const duty: Duty = {
83 name: "red-apps",
84 intervalMs: 24 * 3_600_000,
outdated

Extract the bug-label get-or-create into a named helper; this one-liner closure is hard to read.

85 budget: { runsPerDay: 3, dollarsPerDay: 1, reason: "once a day; the rest is room for a catch-up, and it spends no money" },
86 run({ db, bus, now, app }) {
87 if (app === undefined) return { findings: [], actions: [] };
88 const findings = [];
outdated

Same days-red calculation as in bodyOf; compute once (e.g. a field on RedApp).

89 const actions: string[] = [];
90 for (const a of redApps(db, now)) {
91 const repo = getRepo(db, a.owner, a.name);
92 if (repo === undefined) continue;
93 const label = bugLabel(db, repo.id);
94 const filed = fileIssueOnce(db, bus, repo, { title: titleOf(a.owner, a.name), body: bodyOf(a), ...(label ? { labelId: label.id } : {}) });
95 findings.push({
96 summary: `${a.owner}/${a.name} has been red since ${a.redSince}`,
97 probe: { kind: "measurement" as const, what: `days the latest check of ${a.owner}/${a.name} has been red`, number: a.daysRed, at: now.toISOString() },
98 });
99 if (filed.opened) actions.push(`filed #${filed.number} on ${a.owner}/${a.name}`);
100 }
101 return { findings, actions };
102 },
103 };
src/maintain/registry.ts +2 −1
@@ -14,10 +14,11 @@ import { duty as previewDirs } from "./previews.ts";
1414 import { duty as diskFree } from "./duties/disk_free.ts";
1515 import { duty as staleIssues } from "./duties/stale_issues.ts";
1616 import { duty as waitSplit } from "./duties/wait_split.ts";
17 import { duty as redApps } from "./duties/red_apps.ts";
1718 import { duty as flakyTests } from "./duties/flaky_tests.ts";
1819 import { dutyIdle } from "./duties/duty_idle.ts";
1920
20 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests];
21 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests, redApps];
2122
2223 /** The duties plus the one that watches them for silence. */
2324 export const duties: readonly Duty[] = [...others, dutyIdle(others)];
src/perf/page_budget.ts +3 −3
@@ -41,10 +41,10 @@ export const PAGE_BUDGETS: readonly PageBudget[] = [
4141 { page: "space", pattern: "/:owner/:name/space", medianMs: 50, statements: 17 },
4242 { page: "space data", pattern: "/:owner/:name/space.json", medianMs: 50, statements: 8 },
4343 // The admin board of duties (#13388): walked as the admin, stored rows only. Statements: measured
44 // warm (53, with main's duties) plus eight: each of the seven duties reads its own state and
44 // warm (53, with main's duties) plus nine: each of the eight duties reads its own state and
4545 // runs, so the count moves with duty state and background ticks; an N+1 over the duties would
46 // still show. One more duty (flaky-tests, #13937) reads its own state: 62 measured, limit 63.
47 { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 63 },
46 // still show. Two more duties (flaky-tests #13937, red apps #13938) each read their own state.
47 { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 65 },
4848 // A stranger's way to a forge (#11410): walked signed out. `/claim` is
4949 // measured on a spawned forge with a live link, as the child's owner sees it.
5050 { page: "pricing", pattern: "/pricing", medianMs: 50, statements: 10, signedOut: true },
test/maintain-red-apps.test.ts added +56
@@ -0,0 +1,56 @@
1 /**
2 * Red apps (#13938): red 3 days files one defect issue naming the check; recovered files none;
3 * never twice.
4 */
5 import { expect, test } from "bun:test";
6 import { rulesWorld } from "./helpers/rules-app.ts";
7 import { duty, titleOf } from "../src/maintain/duties/red_apps.ts";
8 import { duties } from "../src/maintain/registry.ts";
9 import { id } from "../src/kernel/id.ts";
10
11 process.env.FORGE_SENTINEL_REPO = "alice/rules";
12 const w = rulesWorld();
13 const NOW = new Date("2026-11-15T12:00:00Z");
14 const ago = (d: number): string => new Date(NOW.getTime() - d * 86_400_000).toISOString();
15 let seq = 0;
16 function check(state: "ok" | "failed", days: number, names: string[] = []): void {
17 const r = w.app.db.query<{ default_branch: string }, [string]>("SELECT default_branch FROM repo WHERE id = ?1").get(w.repoId)!;
18 w.app.db.query("INSERT INTO run (id, repo_id, kind, env, commit_oid, state, summary, check_detail, created_at, finished_at) VALUES (?1, ?2, 'check', ?3, 'abc123abc123abc', ?4, ?5, ?6, ?7, ?7)")
19 .run(`${String(++seq).padStart(6, "0")}${id()}`, w.repoId, r.default_branch, state, state === "ok" ? "" : "check failed", JSON.stringify({ names }), ago(days));
20 }
21 const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: NOW });
22 const open = () => w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE repo_id = ?1 AND state = 'open' AND title LIKE 'The latest build%'").all(w.repoId);
23
24 test("an app that is not an app, or red under 3 days, files nothing", async () => {
25 expect((await run()).findings).toEqual([]);
26 w.app.db.query("INSERT INTO app (id, repo_id, slug, access) VALUES (?1, ?2, 'rules', 'public')").run(id(), w.repoId);
27 check("failed", 2, ["a test"]);
28 expect((await run()).actions).toEqual([]);
29 expect(open()).toHaveLength(0);
30 });
31
32 test("red for 3 days: one defect issue naming the failing check, never a second while open", async () => {
33 check("failed", 4, ["parses the thing"]);
34 check("failed", 1, ["parses the thing"]);
35 const out = await run();
36 expect(out.actions).toHaveLength(1);
37 expect(out.findings[0]!.probe!.number).toBe(4);
38 expect(open()).toHaveLength(1);
39 expect(open()[0]!.body).toContain("parses the thing");
40 const labels = w.app.db.query<{ name: string }, [string]>("SELECT l.name FROM label l JOIN issue_label il ON il.label_id = l.id JOIN issue i ON i.id = il.issue_id WHERE i.repo_id = ?1 AND i.title LIKE 'The latest build%'").all(w.repoId);
41 expect(labels.map((l) => l.name)).toContain("bug");
42 expect((await run()).actions).toEqual([]); // already reported
43 expect(open()).toHaveLength(1);
44 });
45
46 test("recovered: a green latest check files nothing, even after the old issue is closed", async () => {
47 w.app.db.query("UPDATE issue SET state = 'closed' WHERE title = ?1").run(titleOf("alice", "rules"));
48 check("ok", 0);
49 const out = await run();
50 expect(out).toEqual({ findings: [], actions: [] });
51 expect(open()).toHaveLength(0);
52 });
53
54 test("the duty is in the registry", () => {
55 expect(duties.map((d) => d.name)).toContain("red-apps");
56 });
test/pages-budget.test.ts +1 −1
@@ -56,7 +56,7 @@ const HEAVIER: Record<string, { statements?: number; git?: number }> = {
5656 "/api/v1/asks": { statements: 61 }, // +1: the stopped apps someone asked for, one read of the viewer's archived repositories (#12277)
5757 "/board": { statements: 72 }, // +1: the forge-copy read for the tell box (#12037); +1: the vitals read the database's growth (#12025); +6: the money line (#510) — one settings read, then the month's and the 30 days' meter, storage and ledger reads; constant in the number of apps
5858 "/settings/capacity": { statements: 70 },
59 "/maintenance": { statements: 63 }, // +1..2: the flaky-tests duty reads its own state and runs (#13937)
59 "/maintenance": { statements: 65 }, // +2: the flaky-tests (#13937) and red apps (#13938) duties each read their own state and run
6060 };
6161
6262 /** Patterns with no fixture today: lower this as fixtures are added, never raise it silently. */

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs