Public: anyone can read this; the crew can change it.

A maintenance worker that fails three ticks in a row parks itself and files one issue naming its last error #13315

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13301-a-maintenance-worker-that-fails-three-ti → main 3 commits fork 2 threads · 2 unresolved

  • Merged. scout/13301-a-maintenance-worker-that-fails-three-ti was merged into main as 31f8a65.

Review

Sign in to review this change.
4 files changed +168 −4
docs/features/maintenance-duties.md +2 −2
@@ -1,6 +1,6 @@
11 # Pause and resume a maintenance duty (#11205)
22
3 Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once.
3 Each duty the forge's maintenance runner runs (`src/maintain/registry.ts`) has an on/off switch, kept in `maintenance_duty_state`. A paused duty is skipped by every tick. After three errored runs in a row (it threw, timed out or had a finding refused) the runner pauses the duty itself and says why: "paused after 3 errors in a row; last error: …". Resuming starts the count again, so one more error does not pause it at once. The pause also files one issue on the forge's own repository (`FORGE_REPO`) naming the duty and its last error (`src/maintain/park.ts`); it carries a `Maintenance key: parked/<duty>` line, so while it is open no second is filed, restarts included. Closed, the next park files a new one.
44
55 All doors are for a forge admin and call one function, `switchDuty` in `src/maintain/pause.ts`, so they say the same words.
66
@@ -11,4 +11,4 @@ All doors are for a forge admin and call one function, `switchDuty` in `src/main
1111
1212 An unknown duty name is 404 (a flash on the page); a `paused` that is not a boolean is 422.
1313
14 Tests: `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates).
14 Tests: `test/maintain-parked-issue.test.ts` (one issue for three failures and a fourth tick), `test/maintain-pause.test.ts` (the three errors, resume from the API, the page and the terminal, the gates).
src/maintain/park.ts added +35
@@ -0,0 +1,35 @@
1 /**
2 * A duty parked after three errors in a row (#13301) files one issue naming the duty and its
3 * last error. The issue carries a key line (`Maintenance key: parked/<duty>`), so a restart, a
4 * resume that fails again, or a second park while the issue is open files nothing more; only the
5 * watcher's own issue counts, so nobody else can silence it by pasting the key; once a person
6 * closes it, a new park is a new issue. Plain data in, plain data out.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { createIssue } from "../core/issues.ts";
11 import { watcherId } from "../rules/bot.ts";
12 import { clip } from "./duty.ts";
13
14 const keyOf = (duty: string): string => `Maintenance key: parked/${duty}`;
15
16 /** What a park report did: filed an issue, found this duty's still open, or failed and why. */
17 export type Parked = { filed: number } | { open: true } | { failed: string };
18
outdated

Add AND author_id = ?3 with watcherId(db). As written, any user can pre-file an issue with the key line and suppress this alert. Test it.

outdated

instr(body, key) matches parked/flaky inside parked/flaky-worker. Match the full key line and test a prefix pair of duty names.

19 /** Files the issue unless the watcher's issue for this duty is still open. */
20 export function reportParked(db: Database, repoId: string, duty: string, lastError: string, now: Date): Parked {
21 const key = keyOf(duty);
22 // The whole key line: `parked/flaky` must not match inside `parked/flaky-worker`.
23 const open = db
24 .query("SELECT 1 FROM issue WHERE repo_id = ?1 AND state = 'open' AND author_id = ?3 AND instr(body || char(10), ?2 || char(10)) > 0 LIMIT 1")
25 .get(repoId, key, watcherId(db));
26 if (open !== null) return { open: true };
27 const error = clip(lastError.split(/\r?\n/, 1)[0]!.trim().replace(/^failed: /, ""));
28 const made = createIssue(db, {
29 repoId,
30 authorId: watcherId(db),
31 title: `Maintenance worker ${duty} parked after 3 failures: ${error}`.slice(0, 200),
32 body: `The maintenance worker \`${duty}\` failed three runs in a row and was paused on ${now.toISOString().slice(0, 10)}.\n\nLast error: ${error}\n\n${key}\n\nFix the cause, then resume it on /maintenance and close this issue. A success clears the count; a resume starts it again.`,
33 });
34 return made.ok ? { filed: made.value.number } : { failed: String(made.error) };
35 }
src/maintain/runner.ts +11 −2
@@ -5,11 +5,13 @@
55 */
66
77 import type { App } from "../app.ts";
8 import { getRepo } from "../core/repos.ts";
89 import { during } from "../perf/jobs.ts";
910 import { overBudget, stoppedToday } from "./budget.ts";
1011 import { clip, refusal, validName, type Duty } from "./duty.ts";
1112 import { duties as registered } from "./registry.ts";
1213 import { isPaused, pauseOnErrors } from "./pause.ts";
14 import { reportParked } from "./park.ts";
1315 import { lastStarted, prune, record } from "./runs.ts";
1416
1517 /** The first line of an error's text: what a row can say in one breath. */
@@ -48,7 +50,8 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
4850 let outcome = "ok";
4951 let dollars = 0;
5052 try {
51 // The duty keeps its `running` slot until it really settles, so a timed-out duty never overlaps itself.
53 // The duty keeps its `running` slot until it really settles, so a timed-out duty never

Unrelated comment reflow; leave it out of this change.

54 // overlaps itself.
5255 const work = Promise.resolve().then(() => duty.run({ db: app.db, bus: app.bus, now, app }));
5356 work.then(() => running.delete(duty.name), () => running.delete(duty.name));
5457 let timer: ReturnType<typeof setTimeout> | undefined;
@@ -69,7 +72,13 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
6972 try {
7073 record(app.db, { duty: duty.name, started, finished: new Date(), findings, actions, outcome, dollars });
7174 prune(app.db, duty.name, options.keep ?? KEEP_PER_DUTY);
72 if (pauseOnErrors(app.db, duty.name)) app.log.error("maintenance duty paused after errors", { duty: duty.name });
75 if (pauseOnErrors(app.db, duty.name)) {
76 app.log.error("maintenance duty paused after errors", { duty: duty.name });
77 const [owner, repoName] = (app.config.forgeRepo ?? "").split("/");
78 const repo = owner && repoName ? getRepo(app.db, owner, repoName) : undefined;
79 const filed = repo ? reportParked(app.db, repo.id, duty.name, outcome, new Date()) : { failed: "no forge repo" };
80 if ("failed" in filed) app.log.error("parked maintenance duty not reported", { duty: duty.name, reason: filed.failed });
81 }
7382 } catch (error) {
7483 app.log.error("maintenance run not recorded", { duty: duty.name, error: String(error) });
7584 }
test/maintain-parked-issue.test.ts added +120
@@ -0,0 +1,120 @@
1 /** A duty parked after three errors files one issue naming it and its last error (#13301). */
2 import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { createIssue, listIssues, setIssueState } from "../src/core/issues.ts";
8 import { createRepo } from "../src/core/repos.ts";
9 import { createUser } from "../src/core/users.ts";
10 import type { Duty } from "../src/maintain/duty.ts";
11 import { setPaused } from "../src/maintain/pause.ts";
12 import { reportParked } from "../src/maintain/park.ts";
13 import { startMaintain } from "../src/maintain/runner.ts";
14 import { quietLogs } from "./helpers/globals.ts";
15
16 quietLogs();
17 let dir = "";
18 let app: App;
19 let repoId = "";
20 beforeAll(async () => {
21 dir = await mkdtemp(join(tmpdir(), "forge-parked-"));
22 app = createApp({ dataDir: dir, port: 0 });
23 app.config.forgeRepo = "boss/forge";
24 const owner = await createUser(app.db, { username: "boss", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, isAdmin: true });
25 if (!owner.ok) throw new Error("user");
26 const made = await createRepo(app.db, app.reposRoot, { ownerId: owner.value.id, name: "forge", visibility: "public", description: "f" });
27 if (!made.ok) throw new Error("repo");
28 repoId = made.value.id;
29 });
30 afterAll(async () => {
31 app.close();
32 await rm(dir, { recursive: true, force: true });
33 });
34
35 /** A duty of its own per test, failing until `heal` is called: no test leans on another's state. */
36 function flaky(name: string) {
37 let ok = false;
38 const duty: Duty = {
39 name,
40 intervalMs: 1,
41 budget: { runsPerDay: 100000, dollarsPerDay: 10000, reason: "test" },
42 run: async () => {
43 if (ok) return { findings: [], actions: [] };
44 throw new Error("disk is on fire\n at x");
45 },
46 };
47 return { duty, heal: () => (ok = true) };
48 }
49 const issuesOf = (name: string) =>
50 listIssues(app.db, repoId, { state: "all" }).filter((i) => i.body.split("\n").includes(`Maintenance key: parked/${name}`));
51 let clock = Date.parse("2031-01-01T00:00:00Z");
52 async function ticks(duty: Duty, n: number): Promise<void> {
53 const m = startMaintain(app, { duties: [duty], everyMs: 3_600_000 });
54 for (let i = 0; i < n; i++) await m.tick(new Date((clock += 1000)));
55 m.stop();
56 }
57
58 test("three failures park the worker and file one issue; a fourth tick files nothing more", async () => {
59 const { duty } = flaky("first-worker");
60 await ticks(duty, 2);
61 expect(issuesOf("first-worker").length).toBe(0);
62 await ticks(duty, 1);
63 expect(issuesOf("first-worker").length).toBe(1);
64 expect(issuesOf("first-worker")[0]!.title).toContain("disk is on fire");
65 expect(issuesOf("first-worker")[0]!.body).not.toContain("at x");
66 await ticks(duty, 1);
67 expect(issuesOf("first-worker").length).toBe(1);
68 // A resume that fails three more times while the issue is open (a restart forgets nothing)
69 // files no second.
70 setPaused(app.db, "first-worker", false, "alice");
71 await ticks(duty, 4);
72 expect(issuesOf("first-worker").length).toBe(1);
73 });
74
75 test("once the issue is closed, a new park files a new one; a success files none", async () => {
76 const { duty, heal } = flaky("second-worker");
77 await ticks(duty, 3);
78 expect(issuesOf("second-worker").length).toBe(1);
79 setIssueState(app.db, issuesOf("second-worker")[0]!.id, "closed");
80 setPaused(app.db, "second-worker", false, "alice");
81 await ticks(duty, 3);
82 expect(issuesOf("second-worker").length).toBe(2);
83 heal();
84 setPaused(app.db, "second-worker", false, "alice");
85 await ticks(duty, 4);
86 expect(issuesOf("second-worker").length).toBe(2);
87 });
88
89 test("a duty whose name prefixes another's is not silenced by the other's open issue", async () => {
90 const longer = flaky("prefix-worker");
91 const shorter = flaky("prefix");
92 await ticks(longer.duty, 3);
93 expect(issuesOf("prefix-worker").length).toBe(1);
94 await ticks(shorter.duty, 3);
95 expect(issuesOf("prefix").length).toBe(1);
96 expect(issuesOf("prefix-worker").length).toBe(1);
97 });
98
99 test("a person-authored issue carrying the key does not suppress the filing", () => {
100 const person = app.db.query("SELECT owner_id AS id FROM repo WHERE id = ?1").get(repoId) as { id: string };
101 const fake = createIssue(app.db, { repoId, authorId: person.id, title: "hello", body: "Maintenance key: parked/other-worker" });
102 expect(fake.ok).toBe(true);
103 expect(reportParked(app.db, repoId, "other-worker", "failed: boom", new Date())).toHaveProperty("filed");
104 expect(reportParked(app.db, repoId, "other-worker", "failed: boom", new Date())).toEqual({ open: true });
105 });
106
107 test("a park that files no issue says so in the log", async () => {
108 const { duty } = flaky("unreported-worker");
109 const before = app.config.forgeRepo;
110 app.config.forgeRepo = "nobody/missing";
111 const error = spyOn(app.log, "error");
112 try {
113 await ticks(duty, 3);
114 } finally {
115 app.config.forgeRepo = before!;
116 }
117 expect(error.mock.calls.some(([message]) => message === "parked maintenance duty not reported")).toBe(true);
118 error.mockRestore();
119 expect(issuesOf("unreported-worker").length).toBe(0);
120 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs