Public: anyone can read this; the crew can change it.

A maintenance duty that throws three runs in a row is shown as failing on /maintenance and in the maintenance tool #12699

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12679-a-maintenance-duty-that-throws-three-run → main 4 commits fork 3 threads · 3 unresolved

  • Merged. scout/12679-a-maintenance-duty-that-throws-three-run was merged into main as 7d9342b.

Review

Sign in to review this change.
3 files changed +151 −7
src/maintain/pause.ts +28 −6
@@ -23,15 +23,19 @@ const stateRow = (db: Database, name: string): Row | null =>
2323
2424 export const isPaused = (db: Database, name: string): boolean => stateRow(db, name)?.paused === 1;
2525
26 /** A run that did not come out `ok`: it threw, timed out or had a finding refused. */
27 export const errored = (outcome: string): boolean => outcome !== "ok";
26 /**
27 * A run that did not come out `ok`: it threw, timed out or had a finding refused. A `budget:`
28 * stop is not an error: the duty did not run. Pausing and the failing line share this one
29 * predicate, and both skip budget rows in their queries, so a stop never splits a streak.
30 */
31 export const errored = (outcome: string): boolean => outcome !== "ok" && !outcome.startsWith("budget:");
2832
2933 function put(db: Database, name: string, paused: boolean, reason: string, by: string, now: Date): void {
3034 const from = (db.query("SELECT COALESCE(MAX(id), 0) AS id FROM maintenance_run WHERE duty = ?1").get(name) as { id: number }).id;
3135 db.query(
3236 `INSERT INTO maintenance_duty_state (duty, paused, reason, by, since, from_run) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
3337 ON CONFLICT(duty) DO UPDATE SET paused = ?2, reason = ?3, by = ?4, since = ?5, from_run = CASE WHEN ?2 = 0 THEN ?6 ELSE from_run END`,
34 ).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), from);
38 ).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), paused ? 0 : from);
3539 }
3640
3741 export const setPaused = (db: Database, name: string, paused: boolean, by: string, now = new Date()): void =>
@@ -40,20 +44,38 @@ export const setPaused = (db: Database, name: string, paused: boolean, by: strin
4044 /** After a run: three errored runs in a row (since the last resume) pause the duty, with the last error as the reason. Returns whether it paused. */
4145 export function pauseOnErrors(db: Database, name: string, now = new Date()): boolean {
4246 const from = stateRow(db, name)?.from_run ?? 0;
43 const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[];
47 const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 AND outcome NOT LIKE 'budget:%' ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[];
4448 if (rows.length < ERRORS_TO_PAUSE || !rows.every((r) => errored(r.outcome))) return false;
4549 put(db, name, true, `paused after ${ERRORS_TO_PAUSE} errors in a row; last error: ${rows[0]!.outcome}`, "the runner", now);
4650 return true;
4751 }
4852
53 /**
outdated

The query excludes budget: rows but pauseOnErrors does not, so the two disagree on what a streak is. Share one predicate.

54 * Three errored runs in a row and no success since. A success (or a resume) clears it.
55 * `since` is when the streak began; `line` is the latest error's first line.
56 */
outdated

Duplicate: the paused reason already carries the last error. Append only the since-time here, or skip the line when the reason came from pauseOnErrors.

57 export function failing(db: Database, name: string): { since: string; line: string } | undefined {
58 const from = stateRow(db, name)?.from_run ?? 0;
59 const rows = db.query("SELECT started, outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 AND outcome NOT LIKE 'budget:%' ORDER BY id DESC").all(name, from) as { started: string; outcome: string }[];
60 let n = 0;
61 while (n < rows.length && errored(rows[n]!.outcome)) n++;
62 if (n < ERRORS_TO_PAUSE) return undefined;
63 const line = rows[0]!.outcome.replace(/^failed: /, "").split(/\r?\n/, 1)[0]!;
64 return { since: `${rows[n - 1]!.started.slice(0, 16).replace("T", " ")} UTC`, line };
65 }
66
4967 export function dutyState(db: Database, name: string): DutyState {
5068 const s = stateRow(db, name);
5169 if (s?.paused === 1) {
52 const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}`;
70 const f = failing(db, name);
71 // The runner's own pause reason already carries the last error: add only the time.
72 const more = f === undefined ? "" : s.by === "the runner" ? `; failing since ${f.since}` : `; failing since ${f.since}: ${f.line}`;
73 const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}${more}`;
5374 return { name, paused: true, says: `${name}: ${detail}.`, detail };
5475 }
5576 const last = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(name) as { outcome: string } | null;
56 const detail = `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`;
77 const f = failing(db, name);
78 const detail = f ? `on; failing since ${f.since}: ${f.line}` : `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`;
5779 return { name, paused: false, says: `${name}: ${detail}.`, detail };
5880 }
5981
src/maintain/runner.ts +4 −1
@@ -12,6 +12,9 @@ import { duties as registered } from "./registry.ts";
1212 import { isPaused, pauseOnErrors } from "./pause.ts";
1313 import { lastStarted, prune, record } from "./runs.ts";
1414
15 /** The first line of an error's text: what a row can say in one breath. */
16 const firstLine = (text: string): string => text.split(/\r?\n/, 1)[0]!.trim();
17
1518 export const TICK_MS = 60_000;
1619 const FIRST_MS = 15_000;
1720 /** A duty that has not answered in this long is recorded as timed out and the next duty runs. */
@@ -61,7 +64,7 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
6164 dollars = typeof result.dollars === "number" && Number.isFinite(result.dollars) && result.dollars > 0 ? result.dollars : 0;
6265 }
6366 } catch (error) {
64 outcome = clip(`failed: ${error instanceof Error ? error.message : String(error)}`);
67 outcome = clip(`failed: ${firstLine(error instanceof Error ? error.message : String(error))}`);
6568 }
6669 try {
6770 record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome, dollars });
test/maintain-failing.test.ts added +119
@@ -0,0 +1,119 @@
1 /**
2 * A duty that throws three runs in a row is shown as failing, in the page's words, the MCP's and
3 * the terminal's; a success or a resume clears it.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtempSync, rmSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createToken } from "../src/core/tokens.ts";
11 import { createUser, getUserById } from "../src/core/users.ts";
12 import { serve, type Servers } from "../src/http/server.ts";
13 import type { Duty } from "../src/maintain/duty.ts";
14 import { dutyState, failing, isPaused, pauseOnErrors, switchDuty } from "../src/maintain/pause.ts";
15 import { duties } from "../src/maintain/registry.ts";
16 import { startMaintain } from "../src/maintain/runner.ts";
17 import { record } from "../src/maintain/runs.ts";
18 import { dutyLines } from "../src/tui/screens/dutyswitch.ts";
19 import { Session } from "../src/tui/session.ts";
20 import { quietLogs } from "./helpers/globals.ts";
21
22 quietLogs();
23 let root = "";
24 let app: App;
25 let servers: Servers;
26 let admin = "";
27 let adminId = "";
28 const name = duties[0]!.name;
29 beforeAll(async () => {
30 root = mkdtempSync(join(tmpdir(), "forge-failing-"));
31 app = createApp({ dataDir: root, port: 0 });
32 servers = await serve(app);
33 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
34 if (!a.ok) throw new Error("user");
35 adminId = a.value.id;
36 admin = createToken(app.db, a.value.id, "a", "admin").token;
37 });
38 afterAll(() => {
39 servers.stop?.();
40 rmSync(root, { recursive: true, force: true });
41 });
42
43 const mcpStatus = async (): Promise<string> => {
44 const res = await fetch(`${servers.url}/mcp`, {
45 method: "POST",
46 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${admin}` },
47 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "maintenance", arguments: { action: "status" } } }),
48 });
49 return ((await res.json()) as { result: { content: { text: string }[] } }).result.content[0]!.text;
50 };
51 const pageText = async (): Promise<string> =>
52 (await (await fetch(`${servers.url}/maintenance`, { headers: { authorization: `Bearer ${admin}` } })).text());
53 const terminal = (): string => {
outdated

This does not exercise recovery through resume. Pause via the runner, resume, assert cleared; also assert the MCP status and /maintenance row text.

54 const user = getUserById(app.db, adminId);
55 if (user === undefined) throw new Error("user");
56 const session = new Session(app, { sshUsername: "alice", remoteAddress: "127.0.0.1", user });
57 return dutyLines(session, 0).join("\n");
58 };
59
60 const thrower: Duty = {
61 name,
62 intervalMs: 1,
63 budget: { runsPerDay: 100000, dollarsPerDay: 10000, reason: "test" },
64 run: async () => {
65 throw new Error("boom\n at somewhere");
66 },
67 };
68
69 test("three throws pause the duty and say failing since once, on the page, the MCP and the terminal", async () => {
70 const m = startMaintain(app, { duties: [thrower], everyMs: 3_600_000 });
71 const t0 = Date.parse("2030-01-01T00:00:00Z");
72 await m.tick(new Date(t0));
73 await m.tick(new Date(t0 + 1000));
74 expect(dutyState(app.db, name).detail).not.toContain("failing since");
75 await m.tick(new Date(t0 + 2000));
76 m.stop();
77 expect(isPaused(app.db, name)).toBe(true);
78 const s = dutyState(app.db, name);
79 expect(s.detail).toContain("failing since 2030-01-01 00:00 UTC");
80 expect(s.says).not.toContain("somewhere");
81 // The paused reason already carries the error: it is said once.
82 expect(s.says.split("boom").length - 1).toBe(1);
83 expect(await mcpStatus()).toContain("failing since 2030-01-01 00:00 UTC");
84 expect(await pageText()).toContain("failing since 2030-01-01 00:00 UTC");
85 expect(terminal()).toContain("failing since 2030-01-01 00:00 UTC");
86 });
87
88 test("a person's resume clears it everywhere", async () => {
89 expect(switchDuty(app.db, name, false, "alice").ok).toBe(true);
90 expect(failing(app.db, name)).toBeUndefined();
91 expect(dutyState(app.db, name).says).not.toContain("failing");
92 expect(await mcpStatus()).not.toContain("failing since");
93 expect(await pageText()).not.toContain("failing since");
94 expect(terminal()).not.toContain("failing since");
95 });
96
97 test("an unpaused failing duty says the error in full; a success clears it", () => {
98 const d = "other-duty";
99 const at = (n: number) => new Date(Date.parse("2030-02-01T00:00:00Z") + n * 1000);
100 for (let i = 0; i < 3; i++) record(app.db, { duty: d, started: at(i), finished: at(i), findings: [], actions: [], outcome: "failed: nope" });
101 expect(failing(app.db, d)).toEqual({ since: "2030-02-01 00:00 UTC", line: "nope" });
102 record(app.db, { duty: d, started: at(4), finished: at(4), findings: [], actions: [], outcome: "ok" });
103 expect(failing(app.db, d)).toBeUndefined();
104 });
105
106 test("a budget stop is not an error: it neither pauses nor splits a streak", () => {
107 const d = "third-duty";
108 const at = (n: number) => new Date(Date.parse("2030-03-01T00:00:00Z") + n * 1000);
109 for (let i = 0; i < 3; i++) record(app.db, { duty: d, started: at(i), finished: at(i), findings: [], actions: [], outcome: "budget: runs" });
110 expect(failing(app.db, d)).toBeUndefined();
111 expect(pauseOnErrors(app.db, d)).toBe(false);
112 const fail = (n: number) => record(app.db, { duty: d, started: at(n), finished: at(n), findings: [], actions: [], outcome: "failed: x" });
113 fail(10);
114 fail(11);
115 record(app.db, { duty: d, started: at(12), finished: at(12), findings: [], actions: [], outcome: "budget: runs" });
116 fail(13);
117 expect(failing(app.db, d)?.since).toBe("2030-03-01 00:00 UTC");
118 expect(pauseOnErrors(app.db, d)).toBe(true);
119 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs