Public: anyone can read this; the crew can change it.
A maintenance worker that fails three ticks in a row parks itself and files one issue naming its last error #13315
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13301-a-maintenance-worker-that-fails-three-ti → main 3 commits fork 2 threads · 2 unresolved
4 files
changed +168
−4
docs/features/maintenance-duties.md +2 −2
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | # Pause and resume a maintenance duty (#11205) |
| 2 | 2 | |
| 3 | Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once. | |
| 3 | Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once. The pause also files one issue on the forge's own repository (`FORGE_REPO`) naming the duty and its last error (`src/maintain/park.ts`); it carries a `Maintenance key: parked/<duty>` line, so while it is open no second is filed, restarts included. Closed, the next park files a new one. | |
| 4 | 4 | |
| 5 | 5 | All doors are for a forge admin and call one function, `switchDuty` in `src/maintain/pause.ts`, so they say the same words. |
| 6 | 6 | |
| @@ -11,4 +11,4 @@ All doors are for a forge admin and call one function, `switchDuty` in `src/main | ||
| 11 | 11 | |
| 12 | 12 | An unknown duty name is 404 (a flash on the page); a `paused` that is not a boolean is 422. |
| 13 | 13 | |
| 14 | Tests: `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates). | |
| 14 | Tests: `test/maintain-parked-issue.test.ts` (one issue for three failures and a fourth tick), `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates). |
src/maintain/park.ts added +35
| @@ -0,0 +1,35 @@ | ||
| 1 | /** | |
| 2 | * A duty parked after three errors in a row (#13301) files one issue naming the duty and its | |
| 3 | * last error. The issue carries a key line (`Maintenance key: parked/<duty>`), so a restart, a | |
| 4 | * resume that fails again, or a second park while the issue is open files nothing more; only the | |
| 5 | * watcher's own issue counts, so nobody else can silence it by pasting the key; once a person | |
| 6 | * closes it, a new park is a new issue. Plain data in, plain data out. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { Database } from "bun:sqlite"; | |
| 10 | import { createIssue } from "../core/issues.ts"; | |
| 11 | import { watcherId } from "../rules/bot.ts"; | |
| 12 | import { clip } from "./duty.ts"; | |
| 13 | ||
| 14 | const keyOf = (duty: string): string => `Maintenance key: parked/${duty}`; | |
| 15 | ||
| 16 | /** What a park report did: filed an issue, found this duty's still open, or failed and why. */ | |
| 17 | export type Parked = { filed: number } | { open: true } | { failed: string }; | |
| 18 | ||
outdated
instr(body, key) matches parked/flaky inside parked/flaky-worker. Match the full key line and test a prefix pair of duty names. |
||
| 19 | /** Files the issue unless the watcher's issue for this duty is still open. */ | |
| 20 | export function reportParked(db: Database, repoId: string, duty: string, lastError: string, now: Date): Parked { | |
| 21 | const key = keyOf(duty); | |
| 22 | // The whole key line: `parked/flaky` must not match inside `parked/flaky-worker`. | |
| 23 | const open = db | |
| 24 | .query("SELECT 1 FROM issue WHERE repo_id = ?1 AND state = 'open' AND author_id = ?3 AND instr(body || char(10), ?2 || char(10)) > 0 LIMIT 1") | |
| 25 | .get(repoId, key, watcherId(db)); | |
| 26 | if (open !== null) return { open: true }; | |
| 27 | const error = clip(lastError.split(/\r?\n/, 1)[0]!.trim().replace(/^failed: /, "")); | |
| 28 | const made = createIssue(db, { | |
| 29 | repoId, | |
| 30 | authorId: watcherId(db), | |
| 31 | title: `Maintenance worker ${duty} parked after 3 failures: ${error}`.slice(0, 200), | |
| 32 | body: `The maintenance worker \`${duty}\` failed three runs in a row and was paused on ${now.toISOString().slice(0, 10)}.\n\nLast error: ${error}\n\n${key}\n\nFix the cause, then resume it on /maintenance and close this issue. A success clears the count; a resume starts it again.`, | |
| 33 | }); | |
| 34 | return made.ok ? { filed: made.value.number } : { failed: String(made.error) }; | |
| 35 | } |
src/maintain/runner.ts +11 −2
| @@ -5,11 +5,13 @@ | ||
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | 7 | import type { App } from "../app.ts"; |
| 8 | import { getRepo } from "../core/repos.ts"; | |
| 8 | 9 | import { during } from "../perf/jobs.ts"; |
| 9 | 10 | import { overBudget, stoppedToday } from "./budget.ts"; |
| 10 | 11 | import { clip, refusal, validName, type Duty } from "./duty.ts"; |
| 11 | 12 | import { duties as registered } from "./registry.ts"; |
| 12 | 13 | import { isPaused, pauseOnErrors } from "./pause.ts"; |
| 14 | import { reportParked } from "./park.ts"; | |
| 13 | 15 | import { lastStarted, prune, record } from "./runs.ts"; |
| 14 | 16 | |
| 15 | 17 | /** The first line of an error's text: what a row can say in one breath. */ |
| @@ -48,7 +50,8 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve | ||
| 48 | 50 | let outcome = "ok"; |
| 49 | 51 | let dollars = 0; |
| 50 | 52 | try { |
| 51 | // The duty keeps its `running` slot until it really settles, so a timed-out duty never overlaps itself. | |
| 53 | // The duty keeps its `running` slot until it really settles, so a timed-out duty never | |
Unrelated comment reflow; leave it out of this change. |
||
| 54 | // overlaps itself. | |
| 52 | 55 | const work = Promise.resolve().then(() => duty.run({ db: app.db, bus: app.bus, now, app })); |
| 53 | 56 | work.then(() => running.delete(duty.name), () => running.delete(duty.name)); |
| 54 | 57 | let timer: ReturnType<typeof setTimeout> | undefined; |
| @@ -69,7 +72,13 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve | ||
| 69 | 72 | try { |
| 70 | 73 | record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome, dollars }); |
| 71 | 74 | prune(app.db, duty.name, options.keep ?? KEEP_PER_DUTY); |
| 72 | if (pauseOnErrors(app.db, duty.name)) app.log.error("maintenance duty paused after errors", { duty: duty.name }); | |
| 75 | if (pauseOnErrors(app.db, duty.name)) { | |
| 76 | app.log.error("maintenance duty paused after errors", { duty: duty.name }); | |
| 77 | const [owner, repoName] = (app.config.forgeRepo ?? "").split("/"); | |
| 78 | const repo = owner && repoName ? getRepo(app.db, owner, repoName) : undefined; | |
| 79 | const filed = repo ? reportParked(app.db, repo.id, duty.name, outcome, new Date()) : { failed: "no forge repo" }; | |
| 80 | if ("failed" in filed) app.log.error("parked maintenance duty not reported", { duty: duty.name, reason: filed.failed }); | |
| 81 | } | |
| 73 | 82 | } catch (error) { |
| 74 | 83 | app.log.error("maintenance run not recorded", { duty: duty.name, error: String(error) }); |
| 75 | 84 | } |
test/maintain-parked-issue.test.ts added +120
| @@ -0,0 +1,120 @@ | ||
| 1 | /** A duty parked after three errors files one issue naming it and its last error (#13301). */ | |
| 2 | import { afterAll, beforeAll, expect, spyOn, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createIssue, listIssues, setIssueState } from "../src/core/issues.ts"; | |
| 8 | import { createRepo } from "../src/core/repos.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import type { Duty } from "../src/maintain/duty.ts"; | |
| 11 | import { setPaused } from "../src/maintain/pause.ts"; | |
| 12 | import { reportParked } from "../src/maintain/park.ts"; | |
| 13 | import { startMaintain } from "../src/maintain/runner.ts"; | |
| 14 | import { quietLogs } from "./helpers/globals.ts"; | |
| 15 | ||
| 16 | quietLogs(); | |
| 17 | let dir = ""; | |
| 18 | let app: App; | |
| 19 | let repoId = ""; | |
| 20 | beforeAll(async () => { | |
| 21 | dir = await mkdtemp(join(tmpdir(), "forge-parked-")); | |
| 22 | app = createApp({ dataDir: dir, port: 0 }); | |
| 23 | app.config.forgeRepo = "boss/forge"; | |
| 24 | const owner = await createUser(app.db, { username: "boss", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, isAdmin: true }); | |
| 25 | if (!owner.ok) throw new Error("user"); | |
| 26 | const made = await createRepo(app.db, app.reposRoot, { ownerId: owner.value.id, name: "forge", visibility: "public", description: "f" }); | |
| 27 | if (!made.ok) throw new Error("repo"); | |
| 28 | repoId = made.value.id; | |
| 29 | }); | |
| 30 | afterAll(async () => { | |
| 31 | app.close(); | |
| 32 | await rm(dir, { recursive: true, force: true }); | |
| 33 | }); | |
| 34 | ||
| 35 | /** A duty of its own per test, failing until `heal` is called: no test leans on another's state. */ | |
| 36 | function flaky(name: string) { | |
| 37 | let ok = false; | |
| 38 | const duty: Duty = { | |
| 39 | name, | |
| 40 | intervalMs: 1, | |
| 41 | budget: { runsPerDay: 100000, dollarsPerDay: 10000, reason: "test" }, | |
| 42 | run: async () => { | |
| 43 | if (ok) return { findings: [], actions: [] }; | |
| 44 | throw new Error("disk is on fire\n at x"); | |
| 45 | }, | |
| 46 | }; | |
| 47 | return { duty, heal: () => (ok = true) }; | |
| 48 | } | |
| 49 | const issuesOf = (name: string) => | |
| 50 | listIssues(app.db, repoId, { state: "all" }).filter((i) => i.body.split("\n").includes(`Maintenance key: parked/${name}`)); | |
| 51 | let clock = Date.parse("2031-01-01T00:00:00Z"); | |
| 52 | async function ticks(duty: Duty, n: number): Promise<void> { | |
| 53 | const m = startMaintain(app, { duties: [duty], everyMs: 3_600_000 }); | |
| 54 | for (let i = 0; i < n; i++) await m.tick(new Date((clock += 1000))); | |
| 55 | m.stop(); | |
| 56 | } | |
| 57 | ||
| 58 | test("three failures park the worker and file one issue; a fourth tick files nothing more", async () => { | |
| 59 | const { duty } = flaky("first-worker"); | |
| 60 | await ticks(duty, 2); | |
| 61 | expect(issuesOf("first-worker").length).toBe(0); | |
| 62 | await ticks(duty, 1); | |
| 63 | expect(issuesOf("first-worker").length).toBe(1); | |
| 64 | expect(issuesOf("first-worker")[0]!.title).toContain("disk is on fire"); | |
| 65 | expect(issuesOf("first-worker")[0]!.body).not.toContain("at x"); | |
| 66 | await ticks(duty, 1); | |
| 67 | expect(issuesOf("first-worker").length).toBe(1); | |
| 68 | // A resume that fails three more times while the issue is open (a restart forgets nothing) | |
| 69 | // files no second. | |
| 70 | setPaused(app.db, "first-worker", false, "alice"); | |
| 71 | await ticks(duty, 4); | |
| 72 | expect(issuesOf("first-worker").length).toBe(1); | |
| 73 | }); | |
| 74 | ||
| 75 | test("once the issue is closed, a new park files a new one; a success files none", async () => { | |
| 76 | const { duty, heal } = flaky("second-worker"); | |
| 77 | await ticks(duty, 3); | |
| 78 | expect(issuesOf("second-worker").length).toBe(1); | |
| 79 | setIssueState(app.db, issuesOf("second-worker")[0]!.id, "closed"); | |
| 80 | setPaused(app.db, "second-worker", false, "alice"); | |
| 81 | await ticks(duty, 3); | |
| 82 | expect(issuesOf("second-worker").length).toBe(2); | |
| 83 | heal(); | |
| 84 | setPaused(app.db, "second-worker", false, "alice"); | |
| 85 | await ticks(duty, 4); | |
| 86 | expect(issuesOf("second-worker").length).toBe(2); | |
| 87 | }); | |
| 88 | ||
| 89 | test("a duty whose name prefixes another's is not silenced by the other's open issue", async () => { | |
| 90 | const longer = flaky("prefix-worker"); | |
| 91 | const shorter = flaky("prefix"); | |
| 92 | await ticks(longer.duty, 3); | |
| 93 | expect(issuesOf("prefix-worker").length).toBe(1); | |
| 94 | await ticks(shorter.duty, 3); | |
| 95 | expect(issuesOf("prefix").length).toBe(1); | |
| 96 | expect(issuesOf("prefix-worker").length).toBe(1); | |
| 97 | }); | |
| 98 | ||
| 99 | test("a person-authored issue carrying the key does not suppress the filing", () => { | |
| 100 | const person = app.db.query("SELECT owner_id AS id FROM repo WHERE id = ?1").get(repoId) as { id: string }; | |
| 101 | const fake = createIssue(app.db, { repoId, authorId: person.id, title: "hello", body: "Maintenance key: parked/other-worker" }); | |
| 102 | expect(fake.ok).toBe(true); | |
| 103 | expect(reportParked(app.db, repoId, "other-worker", "failed: boom", new Date())).toHaveProperty("filed"); | |
| 104 | expect(reportParked(app.db, repoId, "other-worker", "failed: boom", new Date())).toEqual({ open: true }); | |
| 105 | }); | |
| 106 | ||
| 107 | test("a park that files no issue says so in the log", async () => { | |
| 108 | const { duty } = flaky("unreported-worker"); | |
| 109 | const before = app.config.forgeRepo; | |
| 110 | app.config.forgeRepo = "nobody/missing"; | |
| 111 | const error = spyOn(app.log, "error"); | |
| 112 | try { | |
| 113 | await ticks(duty, 3); | |
| 114 | } finally { | |
| 115 | app.config.forgeRepo = before!; | |
| 116 | } | |
| 117 | expect(error.mock.calls.some(([message]) => message === "parked maintenance duty not reported")).toBe(true); | |
| 118 | error.mockRestore(); | |
| 119 | expect(issuesOf("unreported-worker").length).toBe(0); | |
| 120 | }); |
new above
offline
Add
AND author_id = ?3with watcherId(db). As written, any user can pre-file an issue with the key line and suppress this alert. Test it.