Public: anyone can read this; the crew can change it.
A maintenance duty that throws three runs in a row is shown as failing on /maintenance and in the maintenance tool #12699
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12679-a-maintenance-duty-that-throws-three-run → main 4 commits fork 3 threads · 3 unresolved
3 files
changed +151
−7
src/maintain/pause.ts +28 −6
| @@ -23,15 +23,19 @@ const stateRow = (db: Database, name: string): Row | null => | ||
| 23 | 23 | |
| 24 | 24 | export const isPaused = (db: Database, name: string): boolean => stateRow(db, name)?.paused === 1; |
| 25 | 25 | |
| 26 | /** A run that did not come out `ok`: it threw, timed out or had a finding refused. */ | |
| 27 | export const errored = (outcome: string): boolean => outcome !== "ok"; | |
| 26 | /** | |
| 27 | * A run that did not come out `ok`: it threw, timed out or had a finding refused. A `budget:` | |
| 28 | * stop is not an error: the duty did not run. Pausing and the failing line share this one | |
| 29 | * predicate, and both skip budget rows in their queries, so a stop never splits a streak. | |
| 30 | */ | |
| 31 | export const errored = (outcome: string): boolean => outcome !== "ok" && !outcome.startsWith("budget:"); | |
| 28 | 32 | |
| 29 | 33 | function put(db: Database, name: string, paused: boolean, reason: string, by: string, now: Date): void { |
| 30 | 34 | const from = (db.query("SELECT COALESCE(MAX(id), 0) AS id FROM maintenance_run WHERE duty = ?1").get(name) as { id: number }).id; |
| 31 | 35 | db.query( |
| 32 | 36 | `INSERT INTO maintenance_duty_state (duty, paused, reason, by, since, from_run) VALUES (?1, ?2, ?3, ?4, ?5, ?6) |
| 33 | 37 | ON CONFLICT(duty) DO UPDATE SET paused = ?2, reason = ?3, by = ?4, since = ?5, from_run = CASE WHEN ?2 = 0 THEN ?6 ELSE from_run END`, |
| 34 | ).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), from); | |
| 38 | ).run(name, paused ? 1 : 0, clip(reason), by, now.toISOString(), paused ? 0 : from); | |
| 35 | 39 | } |
| 36 | 40 | |
| 37 | 41 | export const setPaused = (db: Database, name: string, paused: boolean, by: string, now = new Date()): void => |
| @@ -40,20 +44,38 @@ export const setPaused = (db: Database, name: string, paused: boolean, by: strin | ||
| 40 | 44 | /** After a run: three errored runs in a row (since the last resume) pause the duty, with the last error as the reason. Returns whether it paused. */ |
| 41 | 45 | export function pauseOnErrors(db: Database, name: string, now = new Date()): boolean { |
| 42 | 46 | const from = stateRow(db, name)?.from_run ?? 0; |
| 43 | const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[]; | |
| 47 | const rows = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 AND outcome NOT LIKE 'budget:%' ORDER BY id DESC LIMIT ?3").all(name, from, ERRORS_TO_PAUSE) as { outcome: string }[]; | |
| 44 | 48 | if (rows.length < ERRORS_TO_PAUSE || !rows.every((r) => errored(r.outcome))) return false; |
| 45 | 49 | put(db, name, true, `paused after ${ERRORS_TO_PAUSE} errors in a row; last error: ${rows[0]!.outcome}`, "the runner", now); |
| 46 | 50 | return true; |
| 47 | 51 | } |
| 48 | 52 | |
| 53 | /** | |
| 54 | * Three errored runs in a row and no success since. A success (or a resume) clears it. | |
| 55 | * `since` is when the streak began; `line` is the latest error's first line. | |
| 56 | */ | |
outdated
Duplicate: the paused reason already carries the last error. Append only the since-time here, or skip the line when the reason came from pauseOnErrors. |
||
| 57 | export function failing(db: Database, name: string): { since: string; line: string } | undefined { | |
| 58 | const from = stateRow(db, name)?.from_run ?? 0; | |
| 59 | const rows = db.query("SELECT started, outcome FROM maintenance_run WHERE duty = ?1 AND id > ?2 AND outcome NOT LIKE 'budget:%' ORDER BY id DESC").all(name, from) as { started: string; outcome: string }[]; | |
| 60 | let n = 0; | |
| 61 | while (n < rows.length && errored(rows[n]!.outcome)) n++; | |
| 62 | if (n < ERRORS_TO_PAUSE) return undefined; | |
| 63 | const line = rows[0]!.outcome.replace(/^failed: /, "").split(/\r?\n/, 1)[0]!; | |
| 64 | return { since: `${rows[n - 1]!.started.slice(0, 16).replace("T", " ")} UTC`, line }; | |
| 65 | } | |
| 66 | ||
| 49 | 67 | export function dutyState(db: Database, name: string): DutyState { |
| 50 | 68 | const s = stateRow(db, name); |
| 51 | 69 | if (s?.paused === 1) { |
| 52 | const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}`; | |
| 70 | const f = failing(db, name); | |
| 71 | // The runner's own pause reason already carries the last error: add only the time. | |
| 72 | const more = f === undefined ? "" : s.by === "the runner" ? `; failing since ${f.since}` : `; failing since ${f.since}: ${f.line}`; | |
| 73 | const detail = `paused since ${s.since.slice(0, 16).replace("T", " ")} UTC; ${s.reason}${more}`; | |
| 53 | 74 | return { name, paused: true, says: `${name}: ${detail}.`, detail }; |
| 54 | 75 | } |
| 55 | 76 | const last = db.query("SELECT outcome FROM maintenance_run WHERE duty = ?1 ORDER BY id DESC LIMIT 1").get(name) as { outcome: string } | null; |
| 56 | const detail = `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`; | |
| 77 | const f = failing(db, name); | |
| 78 | const detail = f ? `on; failing since ${f.since}: ${f.line}` : `on${last && errored(last.outcome) ? `; last run errored: ${last.outcome}` : ""}`; | |
| 57 | 79 | return { name, paused: false, says: `${name}: ${detail}.`, detail }; |
| 58 | 80 | } |
| 59 | 81 |
src/maintain/runner.ts +4 −1
| @@ -12,6 +12,9 @@ import { duties as registered } from "./registry.ts"; | ||
| 12 | 12 | import { isPaused, pauseOnErrors } from "./pause.ts"; |
| 13 | 13 | import { lastStarted, prune, record } from "./runs.ts"; |
| 14 | 14 | |
| 15 | /** The first line of an error's text: what a row can say in one breath. */ | |
| 16 | const firstLine = (text: string): string => text.split(/\r?\n/, 1)[0]!.trim(); | |
| 17 | ||
| 15 | 18 | export const TICK_MS = 60_000; |
| 16 | 19 | const FIRST_MS = 15_000; |
| 17 | 20 | /** A duty that has not answered in this long is recorded as timed out and the next duty runs. */ |
| @@ -61,7 +64,7 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve | ||
| 61 | 64 | dollars = typeof result.dollars === "number" && Number.isFinite(result.dollars) && result.dollars > 0 ? result.dollars : 0; |
| 62 | 65 | } |
| 63 | 66 | } catch (error) { |
| 64 | outcome = clip(`failed: ${error instanceof Error ? error.message : String(error)}`); | |
| 67 | outcome = clip(`failed: ${firstLine(error instanceof Error ? error.message : String(error))}`); | |
| 65 | 68 | } |
| 66 | 69 | try { |
| 67 | 70 | record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome, dollars }); |
test/maintain-failing.test.ts added +119
| @@ -0,0 +1,119 @@ | ||
| 1 | /** | |
| 2 | * A duty that throws three runs in a row is shown as failing, in the page's words, the MCP's and | |
| 3 | * the terminal's; a success or a resume clears it. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createToken } from "../src/core/tokens.ts"; | |
| 11 | import { createUser, getUserById } from "../src/core/users.ts"; | |
| 12 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 13 | import type { Duty } from "../src/maintain/duty.ts"; | |
| 14 | import { dutyState, failing, isPaused, pauseOnErrors, switchDuty } from "../src/maintain/pause.ts"; | |
| 15 | import { duties } from "../src/maintain/registry.ts"; | |
| 16 | import { startMaintain } from "../src/maintain/runner.ts"; | |
| 17 | import { record } from "../src/maintain/runs.ts"; | |
| 18 | import { dutyLines } from "../src/tui/screens/dutyswitch.ts"; | |
| 19 | import { Session } from "../src/tui/session.ts"; | |
| 20 | import { quietLogs } from "./helpers/globals.ts"; | |
| 21 | ||
| 22 | quietLogs(); | |
| 23 | let root = ""; | |
| 24 | let app: App; | |
| 25 | let servers: Servers; | |
| 26 | let admin = ""; | |
| 27 | let adminId = ""; | |
| 28 | const name = duties[0]!.name; | |
| 29 | beforeAll(async () => { | |
| 30 | root = mkdtempSync(join(tmpdir(), "forge-failing-")); | |
| 31 | app = createApp({ dataDir: root, port: 0 }); | |
| 32 | servers = await serve(app); | |
| 33 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 34 | if (!a.ok) throw new Error("user"); | |
| 35 | adminId = a.value.id; | |
| 36 | admin = createToken(app.db, a.value.id, "a", "admin").token; | |
| 37 | }); | |
| 38 | afterAll(() => { | |
| 39 | servers.stop?.(); | |
| 40 | rmSync(root, { recursive: true, force: true }); | |
| 41 | }); | |
| 42 | ||
| 43 | const mcpStatus = async (): Promise<string> => { | |
| 44 | const res = await fetch(`${servers.url}/mcp`, { | |
| 45 | method: "POST", | |
| 46 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${admin}` }, | |
| 47 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "maintenance", arguments: { action: "status" } } }), | |
| 48 | }); | |
| 49 | return ((await res.json()) as { result: { content: { text: string }[] } }).result.content[0]!.text; | |
| 50 | }; | |
| 51 | const pageText = async (): Promise<string> => | |
| 52 | (await (await fetch(`${servers.url}/maintenance`, { headers: { authorization: `Bearer ${admin}` } })).text()); | |
| 53 | const terminal = (): string => { | |
outdated
This does not exercise recovery through resume. Pause via the runner, resume, assert cleared; also assert the MCP status and /maintenance row text. |
||
| 54 | const user = getUserById(app.db, adminId); | |
| 55 | if (user === undefined) throw new Error("user"); | |
| 56 | const session = new Session(app, { sshUsername: "alice", remoteAddress: "127.0.0.1", user }); | |
| 57 | return dutyLines(session, 0).join("\n"); | |
| 58 | }; | |
| 59 | ||
| 60 | const thrower: Duty = { | |
| 61 | name, | |
| 62 | intervalMs: 1, | |
| 63 | budget: { runsPerDay: 100000, dollarsPerDay: 10000, reason: "test" }, | |
| 64 | run: async () => { | |
| 65 | throw new Error("boom\n at somewhere"); | |
| 66 | }, | |
| 67 | }; | |
| 68 | ||
| 69 | test("three throws pause the duty and say failing since once, on the page, the MCP and the terminal", async () => { | |
| 70 | const m = startMaintain(app, { duties: [thrower], everyMs: 3_600_000 }); | |
| 71 | const t0 = Date.parse("2030-01-01T00:00:00Z"); | |
| 72 | await m.tick(new Date(t0)); | |
| 73 | await m.tick(new Date(t0 + 1000)); | |
| 74 | expect(dutyState(app.db, name).detail).not.toContain("failing since"); | |
| 75 | await m.tick(new Date(t0 + 2000)); | |
| 76 | m.stop(); | |
| 77 | expect(isPaused(app.db, name)).toBe(true); | |
| 78 | const s = dutyState(app.db, name); | |
| 79 | expect(s.detail).toContain("failing since 2030-01-01 00:00 UTC"); | |
| 80 | expect(s.says).not.toContain("somewhere"); | |
| 81 | // The paused reason already carries the error: it is said once. | |
| 82 | expect(s.says.split("boom").length - 1).toBe(1); | |
| 83 | expect(await mcpStatus()).toContain("failing since 2030-01-01 00:00 UTC"); | |
| 84 | expect(await pageText()).toContain("failing since 2030-01-01 00:00 UTC"); | |
| 85 | expect(terminal()).toContain("failing since 2030-01-01 00:00 UTC"); | |
| 86 | }); | |
| 87 | ||
| 88 | test("a person's resume clears it everywhere", async () => { | |
| 89 | expect(switchDuty(app.db, name, false, "alice").ok).toBe(true); | |
| 90 | expect(failing(app.db, name)).toBeUndefined(); | |
| 91 | expect(dutyState(app.db, name).says).not.toContain("failing"); | |
| 92 | expect(await mcpStatus()).not.toContain("failing since"); | |
| 93 | expect(await pageText()).not.toContain("failing since"); | |
| 94 | expect(terminal()).not.toContain("failing since"); | |
| 95 | }); | |
| 96 | ||
| 97 | test("an unpaused failing duty says the error in full; a success clears it", () => { | |
| 98 | const d = "other-duty"; | |
| 99 | const at = (n: number) => new Date(Date.parse("2030-02-01T00:00:00Z") + n * 1000); | |
| 100 | for (let i = 0; i < 3; i++) record(app.db, { duty: d, started: at(i), finished: at(i), findings: [], actions: [], outcome: "failed: nope" }); | |
| 101 | expect(failing(app.db, d)).toEqual({ since: "2030-02-01 00:00 UTC", line: "nope" }); | |
| 102 | record(app.db, { duty: d, started: at(4), finished: at(4), findings: [], actions: [], outcome: "ok" }); | |
| 103 | expect(failing(app.db, d)).toBeUndefined(); | |
| 104 | }); | |
| 105 | ||
| 106 | test("a budget stop is not an error: it neither pauses nor splits a streak", () => { | |
| 107 | const d = "third-duty"; | |
| 108 | const at = (n: number) => new Date(Date.parse("2030-03-01T00:00:00Z") + n * 1000); | |
| 109 | for (let i = 0; i < 3; i++) record(app.db, { duty: d, started: at(i), finished: at(i), findings: [], actions: [], outcome: "budget: runs" }); | |
| 110 | expect(failing(app.db, d)).toBeUndefined(); | |
| 111 | expect(pauseOnErrors(app.db, d)).toBe(false); | |
| 112 | const fail = (n: number) => record(app.db, { duty: d, started: at(n), finished: at(n), findings: [], actions: [], outcome: "failed: x" }); | |
| 113 | fail(10); | |
| 114 | fail(11); | |
| 115 | record(app.db, { duty: d, started: at(12), finished: at(12), findings: [], actions: [], outcome: "budget: runs" }); | |
| 116 | fail(13); | |
| 117 | expect(failing(app.db, d)?.since).toBe("2030-03-01 00:00 UTC"); | |
| 118 | expect(pauseOnErrors(app.db, d)).toBe(true); | |
| 119 | }); |
new above
offline
The query excludes
budget:rows but pauseOnErrors does not, so the two disagree on what a streak is. Share one predicate.