Public: anyone can read this; the crew can change it.
Maintenance framework: a maintainer crew role with scheduled duties, budgets and probe-backed findings #8927
mason/forge:mason/7554-maintenance-framework-a-maintainer-crew → main 4 commits fork
30 files
changed +1126
−2
docs/features/maintainer.md added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | # The maintainer (#7554) | |
| 2 | ||
| 3 | A crew role (`maintainer`) for scheduled duties. A duty is a name, a trigger | |
| 4 | (`{ every: ms }` or `{ on: "event" }`), a budget (runs a day, dollars a day, | |
| 5 | milliseconds and network calls a run), a switch and a handler returning | |
| 6 | findings and actions. | |
| 7 | ||
| 8 | - **Add a duty:** one file in `src/maintain/duties/` using `duty({...})`, one | |
| 9 | import and one entry in `duties/index.ts`. `run_log.ts` is the pattern. | |
| 10 | - **Probes:** every finding and action carries `ctx.probe(query, value)`; the | |
| 11 | runner drops one without it and records the run as `refused`. | |
| 12 | - **Findings** go to the open issue for their stable key as a comment, or open | |
| 13 | one (`maintain/file.ts`, table `maintenance_finding`). | |
| 14 | - **Bulk:** `ctx.bulk(items, { apply, check })` applies one item, checks its | |
| 15 | probe, then continues, capped at 20 a run; a failed check records `canary`. | |
| 16 | - **Budget:** over a day's runs or dollars, or past a run's time or network | |
| 17 | ceiling, the outcome is `budget`. | |
| 18 | - **Errors:** three in a row pause the duty with the reason. | |
| 19 | - **No landing:** `src/maintain/` imports nothing that merges, pushes or | |
| 20 | spawns (`test/maintain-no-land.test.ts`); fixes are pulls through the gate. | |
| 21 | - **Board/API:** `GET /api/v1/maintain/duties`, `GET …/runs?duty=`, | |
| 22 | `POST …/duties/:name/pause|resume` (forge admin). | |
| 23 | ||
| 24 | Runs on the autonomy clock after the sentinel (`crew/autonomy.ts`). |
src/contract/openapi/routes/getApiV1MaintainDuties.ts added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "get", | |
| 5 | path: "/api/v1/maintain/duties", | |
| 6 | summary: | |
| 7 | "The maintainer's duties with trigger, budget, switch, why one is paused and its last run (a person admin's token); answers { duties }", | |
| 8 | auth: true, | |
| 9 | refusals: { forbidden: "the token is not a person who administers the forge" }, | |
| 10 | }; |
src/contract/openapi/routes/getApiV1MaintainRuns.ts added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "get", | |
| 5 | path: "/api/v1/maintain/runs", | |
| 6 | summary: | |
| 7 | "The maintainer's run rows, newest first, each with findings, actions, probes, outcome and cost; ?duty= narrows to one duty (a person admin's token); answers { runs }", | |
| 8 | auth: true, | |
| 9 | refusals: { forbidden: "the token is not a person who administers the forge" }, | |
| 10 | }; |
src/contract/openapi/routes/index.ts +4
| @@ -330,3 +330,7 @@ export { route as putApiV1ReposOwnerNameUpstream } from "./putApiV1ReposOwnerNam | ||
| 330 | 330 | export { route as putApiV1SecretsScopeName } from "./putApiV1SecretsScopeName.ts"; |
| 331 | 331 | export { route as putApiV1UserNotify } from "./putApiV1UserNotify.ts"; |
| 332 | 332 | export { route as putApiV1UserPinsOwnerName } from "./putApiV1UserPinsOwnerName.ts"; |
| 333 | export { route as getApiV1MaintainDuties } from "./getApiV1MaintainDuties.ts"; | |
| 334 | export { route as getApiV1MaintainRuns } from "./getApiV1MaintainRuns.ts"; | |
| 335 | export { route as postApiV1MaintainDutiesNamePause } from "./postApiV1MaintainDutiesNamePause.ts"; | |
| 336 | export { route as postApiV1MaintainDutiesNameResume } from "./postApiV1MaintainDutiesNameResume.ts"; |
src/contract/openapi/routes/postApiV1MaintainDutiesNamePause.ts added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "post", | |
| 5 | path: "/api/v1/maintain/duties/{name}/pause", | |
| 6 | summary: | |
| 7 | "Pause a maintainer duty until it is resumed (admin scope); answers { duties }", | |
| 8 | auth: true, | |
| 9 | refusals: { forbidden: "the token is not a person who administers the forge" }, | |
| 10 | }; |
src/contract/openapi/routes/postApiV1MaintainDutiesNameResume.ts added +10
| @@ -0,0 +1,10 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { | |
| 4 | method: "post", | |
| 5 | path: "/api/v1/maintain/duties/{name}/resume", | |
| 6 | summary: | |
| 7 | "Resume a paused maintainer duty and clear its error count (admin scope); answers { duties }", | |
| 8 | auth: true, | |
| 9 | refusals: { forbidden: "the token is not a person who administers the forge" }, | |
| 10 | }; |
src/core/agents.ts +4 −2
| @@ -303,14 +303,16 @@ export function deleteAgent(db: Database, userId: string): boolean { | ||
| 303 | 303 | /** |
| 304 | 304 | * What a crew member is for on one repository: a builder takes what is |
| 305 | 305 | * ready, a planner turns a direction into a map, a reviewer looks at what |
| 306 | * landed. Per repository, because the same agent may plan one and build | |
| 306 | * landed, a maintainer runs scheduled duties (`src/maintain/`) and proposes | |
| 307 | * only through pulls. Per repository, because the same agent may plan one and build | |
| 307 | 308 | * another. |
| 308 | 309 | */ |
| 309 | export type CrewRole = "builder" | "planner" | "reviewer"; | |
| 310 | export type CrewRole = "builder" | "planner" | "reviewer" | "maintainer"; | |
| 310 | 311 | export const CREW_ROLES: readonly CrewRole[] = [ |
| 311 | 312 | "planner", |
| 312 | 313 | "builder", |
| 313 | 314 | "reviewer", |
| 315 | "maintainer", | |
| 314 | 316 | ]; |
| 315 | 317 | export const isCrewRole = (value: unknown): value is CrewRole => |
| 316 | 318 | (CREW_ROLES as readonly unknown[]).includes(value); |
src/core/migration-pins/20261103150000-maintenance.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 2a3bf27024f4b6762f083124c69e1758bc241e07bd90fe996424a81ca3d85981 |
src/core/migrations/20261103150000-maintenance.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | // The maintainer's duties (#7554): which duties are paused and why, one row | |
| 2 | // per run with the findings, actions and probes it left, and the issue a | |
| 3 | // finding's stable key was filed to. Additive. | |
| 4 | export default ` | |
| 5 | CREATE TABLE maintenance_duty ( | |
| 6 | name TEXT PRIMARY KEY, | |
| 7 | enabled INTEGER NOT NULL DEFAULT 1, | |
| 8 | paused_reason TEXT, | |
| 9 | errors INTEGER NOT NULL DEFAULT 0 | |
| 10 | ) WITHOUT ROWID; | |
| 11 | CREATE TABLE maintenance_run ( | |
| 12 | id TEXT PRIMARY KEY, | |
| 13 | duty TEXT NOT NULL, | |
| 14 | started TEXT NOT NULL, | |
| 15 | finished TEXT NOT NULL, | |
| 16 | findings TEXT NOT NULL, | |
| 17 | actions TEXT NOT NULL, | |
| 18 | probes TEXT NOT NULL, | |
| 19 | outcome TEXT NOT NULL, | |
| 20 | cost REAL NOT NULL DEFAULT 0, | |
| 21 | detail TEXT | |
| 22 | ) WITHOUT ROWID; | |
| 23 | CREATE INDEX maintenance_run_duty ON maintenance_run(duty, started); | |
| 24 | CREATE TABLE maintenance_finding ( | |
| 25 | key TEXT PRIMARY KEY, | |
| 26 | issue_id TEXT NOT NULL, | |
| 27 | first_seen TEXT NOT NULL, | |
| 28 | last_seen TEXT NOT NULL | |
| 29 | ) WITHOUT ROWID; | |
| 30 | `; |
src/crew/autonomy.ts +8
| @@ -64,6 +64,7 @@ import { getHold } from "../core/holds.ts"; | ||
| 64 | 64 | import { certain } from "./decide.ts"; |
| 65 | 65 | import { decideAndRecord } from "./ledger.ts"; |
| 66 | 66 | import { sentinelTick } from "./sentinel.ts"; |
| 67 | import { maintainTick } from "../maintain/service.ts"; | |
| 67 | 68 | import { triageTick } from "./triage.ts"; |
| 68 | 69 | import { liveTriage, none as noTriage, tickFields } from "./livetriage.ts"; |
| 69 | 70 | import { wallOf, wallPlain, wallSentence } from "../contract/policy.ts"; |
| @@ -1463,6 +1464,13 @@ export async function autonomyTick( | ||
| 1463 | 1464 | log.error("sentinel failed", { error: String(error) }); |
| 1464 | 1465 | } |
| 1465 | 1466 | await breathe(); |
| 1467 | // The maintainer's duties that are due (`maintain/runner.ts`). | |
| 1468 | try { | |
| 1469 | await maintainTick(app, now.getTime()); | |
| 1470 | } catch (error) { | |
| 1471 | log.error("maintainer failed", { error: String(error) }); | |
| 1472 | } | |
| 1473 | await breathe(); | |
| 1466 | 1474 | // Review is not autonomy's alone: an agent's green pull is read by the |
| 1467 | 1475 | // crew's reviewer in every repository that has one. |
| 1468 | 1476 | try { |
src/crew/seat_slots.ts +1
| @@ -34,6 +34,7 @@ export const SEAT_UNIT: Record<CrewRole, string> = { | ||
| 34 | 34 | planner: "planning pass at once", |
| 35 | 35 | builder: "build at once", |
| 36 | 36 | reviewer: "review at once", |
| 37 | maintainer: "duty at once", | |
| 37 | 38 | }; |
| 38 | 39 | |
| 39 | 40 | export function slotsOf(repo: Repo, role: CrewRole): number { |
src/http/api.ts +2
| @@ -333,6 +333,7 @@ import { visionRoutes } from "./api/vision.ts"; | ||
| 333 | 333 | import { tokenRoutes } from "./api/tokens.ts"; |
| 334 | 334 | import { accountAdmin, adminRoutes } from "./api/admin.ts"; |
| 335 | 335 | import { flagsRoutes } from "./api/flags.ts"; |
| 336 | import { maintainRoutes } from "./api/maintain.ts"; | |
| 336 | 337 | import { issueRoutes, labelsHandler } from "./api/issues.ts"; |
| 337 | 338 | import { FORGE_NOT_CARRIED } from "../origins/words.ts"; |
| 338 | 339 | import { forgeOriginRoutes } from "./api/forge_origin.ts"; |
| @@ -1166,6 +1167,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1166 | 1167 | forgeOriginRoutes(router); |
| 1167 | 1168 | adminRoutes(router); |
| 1168 | 1169 | flagsRoutes(router); |
| 1170 | maintainRoutes(router); | |
| 1169 | 1171 | |
| 1170 | 1172 | // Open read hooks (#1789): the owner, or an agent holding `telemetry:read`. |
| 1171 | 1173 | // Anyone else is refused; every agent read is logged. |
src/http/api/maintain.ts added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | /** | |
| 2 | * The maintainer's duties over HTTP (#7554): `GET /api/v1/maintain/duties` | |
| 3 | * with each duty's budget, switch and last run; `GET …/runs` for the run | |
| 4 | * rows; `POST …/duties/:name/pause` and `/resume`. A forge admin's door. | |
| 5 | */ | |
| 6 | ||
| 7 | import { dutyBoard, dutyRuns, setDutyPaused } from "../../maintain/service.ts"; | |
| 8 | import type { Ctx, Handler } from "../context.ts"; | |
| 9 | import { json, notFound, requireScope } from "../respond.ts"; | |
| 10 | import type { Router } from "../router.ts"; | |
| 11 | ||
| 12 | const door = (ctx: Ctx, scope: "ro" | "admin"): Response | undefined => | |
| 13 | ctx.auth.user?.isAdmin === true ? requireScope(ctx, scope) : notFound(); | |
| 14 | ||
| 15 | export function maintainRoutes(router: Router<Handler>): void { | |
| 16 | router.add("GET", "/api/v1/maintain/duties", (ctx) => door(ctx, "ro") ?? json({ duties: dutyBoard(ctx.app) })); | |
| 17 | router.add("GET", "/api/v1/maintain/runs", (ctx) => { | |
| 18 | const denied = door(ctx, "ro"); | |
| 19 | if (denied) return denied; | |
| 20 | const duty = ctx.url.searchParams.get("duty") ?? undefined; | |
| 21 | return json({ runs: dutyRuns(ctx.app, duty) }); | |
| 22 | }); | |
| 23 | const switchDuty = (paused: boolean): Handler => (ctx) => { | |
| 24 | const denied = door(ctx, "admin"); | |
| 25 | if (denied) return denied; | |
| 26 | return setDutyPaused(ctx.app, ctx.params.name!, paused, ctx.auth.user!.username) | |
| 27 | ? json({ duties: dutyBoard(ctx.app) }) | |
| 28 | : notFound(); | |
| 29 | }; | |
| 30 | router.add("POST", "/api/v1/maintain/duties/:name/pause", switchDuty(true)); | |
| 31 | router.add("POST", "/api/v1/maintain/duties/:name/resume", switchDuty(false)); | |
| 32 | } |
src/maintain/duties/index.ts added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | /** Every registered duty: a worker adds its file here and imports it on one line. */ | |
| 2 | ||
| 3 | import type { Duty } from "../duty.ts"; | |
| 4 | import { runLog } from "./run_log.ts"; | |
| 5 | ||
| 6 | export const dutyList: readonly Duty[] = [runLog]; |
src/maintain/duties/run_log.ts added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | /** | |
| 2 | * The maintainer's own housekeeping: `maintenance_run` is read by the board | |
| 3 | * and grows with every run, so rows past 90 days are pruned, the oldest | |
| 4 | * first, one first and its probe checked, then the rest. | |
| 5 | */ | |
| 6 | ||
| 7 | import { DAY_MS, duty } from "../duty.ts"; | |
| 8 | ||
| 9 | const KEEP_DAYS = 90; | |
| 10 | ||
| 11 | export const runLog = duty({ | |
| 12 | name: "run-log", | |
| 13 | trigger: { every: DAY_MS }, | |
| 14 | budget: { runsPerDay: 2, usdPerDay: 0, cpuMsPerRun: 10_000, netCallsPerRun: 0 }, | |
| 15 | async handler(ctx) { | |
| 16 | const before = new Date(ctx.now - KEEP_DAYS * DAY_MS).toISOString(); | |
| 17 | const old = ctx.db | |
| 18 | .query<{ id: string }, [string]>("SELECT id FROM maintenance_run WHERE started < ?1 ORDER BY started LIMIT 20") | |
| 19 | .all(before) | |
| 20 | .map((r) => r.id); | |
| 21 | if (old.length === 0) return {}; | |
| 22 | await ctx.bulk(old, { | |
| 23 | what: "prune run", | |
| 24 | target: (id) => id, | |
| 25 | apply: (id) => void ctx.db.query("DELETE FROM maintenance_run WHERE id = ?1").run(id), | |
| 26 | check: (id) => { | |
| 27 | const left = ctx.db.query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM maintenance_run WHERE id = ?1").get(id)?.n ?? 1; | |
| 28 | return { ...ctx.probe("SELECT COUNT(*) FROM maintenance_run WHERE id = :pruned", left), pass: left === 0 }; | |
| 29 | }, | |
| 30 | }); | |
| 31 | return {}; | |
| 32 | }, | |
| 33 | }); |
src/maintain/duty.ts added +97
| @@ -0,0 +1,97 @@ | ||
| 1 | /** | |
| 2 | * What a duty is (#7554): a name, a trigger, a budget, a switch and a handler | |
| 3 | * that returns findings and actions. Plain data in, plain data out; a duty | |
| 4 | * is given only what `DutyContext` hands it, which has no way to merge or to | |
| 5 | * push a branch. | |
| 6 | */ | |
| 7 | ||
| 8 | /** The measurement behind a finding or an action: the query, what it returned, when. */ | |
| 9 | export interface ProbeRecord { | |
| 10 | readonly query: string; | |
| 11 | readonly value: number | string | boolean; | |
| 12 | readonly at: string; | |
| 13 | } | |
| 14 | ||
| 15 | export interface FindingInput { | |
| 16 | /** Stable across runs: the same fault has the same key. */ | |
| 17 | readonly key: string; | |
| 18 | readonly title: string; | |
| 19 | readonly evidence: readonly string[]; | |
| 20 | readonly probe: ProbeRecord | undefined; | |
| 21 | } | |
| 22 | ||
| 23 | export interface ActionInput { | |
| 24 | readonly what: string; | |
| 25 | readonly target: string; | |
| 26 | readonly probe: ProbeRecord | undefined; | |
| 27 | } | |
| 28 | ||
| 29 | export interface Finding extends FindingInput { | |
| 30 | readonly probe: ProbeRecord; | |
| 31 | } | |
| 32 | export interface Action extends ActionInput { | |
| 33 | readonly probe: ProbeRecord; | |
| 34 | } | |
| 35 | ||
| 36 | export type Trigger = | |
| 37 | | { readonly every: number } | |
| 38 | | { readonly on: string }; | |
| 39 | ||
| 40 | export interface Budget { | |
| 41 | readonly runsPerDay: number; | |
| 42 | readonly usdPerDay: number; | |
| 43 | /** Wall time one run may take, a stand-in for CPU. */ | |
| 44 | readonly cpuMsPerRun: number; | |
| 45 | /** Network calls one run may make through `ctx.net()`. */ | |
| 46 | readonly netCallsPerRun: number; | |
| 47 | } | |
| 48 | ||
| 49 | export interface BulkStep<T> { | |
| 50 | readonly what: string; | |
| 51 | readonly target: (item: T) => string; | |
| 52 | readonly apply: (item: T) => void | Promise<void>; | |
| 53 | /** Measured after the first item; the rest run only if `pass`. */ | |
| 54 | readonly check: (item: T) => ProbeRecord & { readonly pass: boolean } | Promise<ProbeRecord & { readonly pass: boolean }>; | |
| 55 | } | |
| 56 | ||
| 57 | export interface DutyContext { | |
| 58 | readonly now: number; | |
| 59 | readonly db: import("bun:sqlite").Database; | |
| 60 | probe(query: string, value: ProbeRecord["value"]): ProbeRecord; | |
| 61 | /** Count a network call; throws once the run's ceiling is passed. */ | |
| 62 | net(): void; | |
| 63 | /** Add to the run's dollar cost. */ | |
| 64 | spend(usd: number): void; | |
| 65 | /** One item first, its probe checked, then the rest, capped. */ | |
| 66 | bulk<T>(items: readonly T[], step: BulkStep<T>): Promise<Action[]>; | |
| 67 | } | |
| 68 | ||
| 69 | export interface DutyResult { | |
| 70 | readonly findings?: readonly FindingInput[]; | |
| 71 | readonly actions?: readonly ActionInput[]; | |
| 72 | } | |
| 73 | ||
| 74 | export interface Duty { | |
| 75 | readonly name: string; | |
| 76 | readonly trigger: Trigger; | |
| 77 | readonly budget: Budget; | |
| 78 | readonly enabled: boolean; | |
| 79 | readonly handler: (ctx: DutyContext) => DutyResult | Promise<DutyResult>; | |
| 80 | } | |
| 81 | ||
| 82 | export const BULK_CAP = 20; | |
| 83 | export const ERRORS_TO_PAUSE = 3; | |
| 84 | export const DAY_MS = 86_400_000; | |
| 85 | ||
| 86 | export type Outcome = "ok" | "error" | "budget" | "canary" | "refused"; | |
| 87 | ||
| 88 | /** A duty with the defaults every duty takes. */ | |
| 89 | export function duty( | |
| 90 | spec: Pick<Duty, "name" | "trigger" | "handler"> & Partial<Pick<Duty, "budget" | "enabled">>, | |
| 91 | ): Duty { | |
| 92 | return { | |
| 93 | enabled: true, | |
| 94 | budget: { runsPerDay: 24, usdPerDay: 0, cpuMsPerRun: 30_000, netCallsPerRun: 0 }, | |
| 95 | ...spec, | |
| 96 | }; | |
| 97 | } |
src/maintain/file.ts added +53
| @@ -0,0 +1,53 @@ | ||
| 1 | /** | |
| 2 | * Where a finding goes (#7554): the open issue for its key gets a comment; | |
| 3 | * a key with none gets an issue. The same de-duplication the sentinel does, | |
| 4 | * by a stable key, kept in `maintenance_finding`. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { App } from "../app.ts"; | |
| 8 | import { addComment, openIssue } from "../actions/issues.ts"; | |
| 9 | import { getAgentByName } from "../core/agents.ts"; | |
| 10 | import { getIssueById } from "../core/issues.ts"; | |
| 11 | import { getRepo } from "../core/repos.ts"; | |
| 12 | import { getUserByName } from "../core/users.ts"; | |
| 13 | import { ownRepoPath } from "../kernel/own_repo.ts"; | |
| 14 | import type { Finding } from "./duty.ts"; | |
| 15 | import { trackedIssue, trackFinding } from "./store.ts"; | |
| 16 | ||
| 17 | const bodyOf = (f: Finding): string => | |
| 18 | [ | |
| 19 | f.title, | |
| 20 | "", | |
| 21 | "**Evidence**", | |
| 22 | ...f.evidence.map((e) => `- ${e}`), | |
| 23 | "", | |
| 24 | `**Probe:** \`${f.probe.query}\` returned ${String(f.probe.value)} at ${f.probe.at}`, | |
| 25 | ].join("\n"); | |
| 26 | ||
| 27 | export function issueFiler(app: App, repoPath = ownRepoPath()): (f: Finding, now: number) => string | undefined { | |
| 28 | return (f, now) => { | |
| 29 | const [owner = "", name = ""] = repoPath.split("/"); | |
| 30 | const repo = getRepo(app.db, owner, name); | |
| 31 | const actor = | |
| 32 | getAgentByName(app.db, "maintainer") !== undefined | |
| 33 | ? getUserByName(app.db, "maintainer") | |
| 34 | : getUserByName( | |
| 35 | app.db, | |
| 36 | app.db.query<{ username: string }, []>("SELECT username FROM user WHERE is_admin = 1 ORDER BY created_at LIMIT 1").get()?.username ?? "", | |
| 37 | ); | |
| 38 | if (repo === undefined || actor === undefined) return undefined; | |
| 39 | const at = new Date(now).toISOString(); | |
| 40 | const known = trackedIssue(app.db, f.key); | |
| 41 | const issue = known === undefined ? undefined : getIssueById(app.db, known); | |
| 42 | if (issue !== undefined && issue.state === "open") { | |
| 43 | const said = addComment(app, actor, repo, issue, `Still there at ${at}:\n${f.evidence.map((e) => `- ${e}`).join("\n")}\n\nProbe \`${f.probe.query}\` returned ${String(f.probe.value)}.`); | |
| 44 | if (!said.ok) throw new Error(`comment: ${String(said.error)}`); | |
| 45 | trackFinding(app.db, f.key, issue.id, at); | |
| 46 | return `#${issue.number} (commented)`; | |
| 47 | } | |
| 48 | const made = openIssue(app, actor, repo, { title: f.title, body: bodyOf(f) }); | |
| 49 | if (!made.ok) throw new Error(`file: ${String(made.error)}`); | |
| 50 | trackFinding(app.db, f.key, made.value.id, at); | |
| 51 | return `#${made.value.number}`; | |
| 52 | }; | |
| 53 | } |
src/maintain/probe.ts added +35
| @@ -0,0 +1,35 @@ | ||
| 1 | /** A finding or an action without the measurement behind it is refused (#7554). */ | |
| 2 | ||
| 3 | import type { Action, ActionInput, Finding, FindingInput, ProbeRecord } from "./duty.ts"; | |
| 4 | ||
| 5 | export function validProbe(p: ProbeRecord | undefined): p is ProbeRecord { | |
| 6 | return ( | |
| 7 | p !== undefined && | |
| 8 | typeof p.query === "string" && | |
| 9 | p.query.trim() !== "" && | |
| 10 | (typeof p.value === "number" ? Number.isFinite(p.value) : typeof p.value === "string" || typeof p.value === "boolean") && | |
| 11 | typeof p.at === "string" && | |
| 12 | !Number.isNaN(Date.parse(p.at)) | |
| 13 | ); | |
| 14 | } | |
| 15 | ||
| 16 | export interface Admitted { | |
| 17 | readonly findings: Finding[]; | |
| 18 | readonly actions: Action[]; | |
| 19 | /** One line per item refused, and why. */ | |
| 20 | readonly refused: string[]; | |
| 21 | } | |
| 22 | ||
| 23 | export function admit(findings: readonly FindingInput[], actions: readonly ActionInput[]): Admitted { | |
| 24 | const out: Admitted = { findings: [], actions: [], refused: [] }; | |
| 25 | for (const f of findings) { | |
| 26 | if (!validProbe(f.probe)) out.refused.push(`finding ${f.key}: no probe`); | |
| 27 | else if (f.key.trim() === "" || f.title.trim() === "") out.refused.push(`finding ${f.key || "(no key)"}: no key or title`); | |
| 28 | else out.findings.push({ ...f, probe: f.probe }); | |
| 29 | } | |
| 30 | for (const a of actions) { | |
| 31 | if (!validProbe(a.probe)) out.refused.push(`action ${a.what} ${a.target}: no probe`); | |
| 32 | else out.actions.push({ ...a, probe: a.probe }); | |
| 33 | } | |
| 34 | return out; | |
| 35 | } |
src/maintain/registry.ts added +21
| @@ -0,0 +1,21 @@ | ||
| 1 | /** The duty registry (#7554): a worker adds one file in `duties/` and one line to `duties/index.ts`. */ | |
| 2 | ||
| 3 | import type { Duty } from "./duty.ts"; | |
| 4 | ||
| 5 | const NAME = /^[a-z][a-z0-9-]{0,63}$/; | |
| 6 | ||
| 7 | export function makeRegistry(initial: readonly Duty[] = []) { | |
| 8 | const byName = new Map<string, Duty>(); | |
| 9 | const add = (d: Duty): void => { | |
| 10 | if (!NAME.test(d.name)) throw new Error(`duty name ${JSON.stringify(d.name)} is not a lowercase word`); | |
| 11 | if (byName.has(d.name)) throw new Error(`duty ${d.name} is registered twice`); | |
| 12 | byName.set(d.name, d); | |
| 13 | }; | |
| 14 | for (const d of initial) add(d); | |
| 15 | return { | |
| 16 | add, | |
| 17 | get: (name: string): Duty | undefined => byName.get(name), | |
| 18 | all: (): Duty[] => [...byName.values()], | |
| 19 | }; | |
| 20 | } | |
| 21 | export type Registry = ReturnType<typeof makeRegistry>; |
src/maintain/runner.ts added +196
| @@ -0,0 +1,196 @@ | ||
| 1 | /** | |
| 2 | * The duty runner (#7554): runs the duties that are due, one at a time per | |
| 3 | * duty, and leaves a `maintenance_run` row for each. It holds no way to | |
| 4 | * merge or push: a duty's fixes are pulls, which the normal gate lands. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { | |
| 9 | BULK_CAP, | |
| 10 | DAY_MS, | |
| 11 | ERRORS_TO_PAUSE, | |
| 12 | type Action, | |
| 13 | type BulkStep, | |
| 14 | type Duty, | |
| 15 | type DutyContext, | |
| 16 | type Finding, | |
| 17 | type Outcome, | |
| 18 | type ProbeRecord, | |
| 19 | } from "./duty.ts"; | |
| 20 | import { admit } from "./probe.ts"; | |
| 21 | import type { Registry } from "./registry.ts"; | |
| 22 | import { | |
| 23 | dutyState, | |
| 24 | lastRun, | |
| 25 | pauseDuty, | |
| 26 | recordRun, | |
| 27 | setErrors, | |
| 28 | usedSince, | |
| 29 | type RunRow, | |
| 30 | } from "./store.ts"; | |
| 31 | ||
| 32 | class OverBudget extends Error {} | |
| 33 | ||
| 34 | export interface RunnerDeps { | |
| 35 | readonly db: Database; | |
| 36 | readonly registry: Registry; | |
| 37 | /** Files or comments a finding on the issue for its key; returns the issue it is on. */ | |
| 38 | readonly file?: (finding: Finding, now: number) => string | undefined; | |
| 39 | } | |
| 40 | ||
| 41 | const iso = (ms: number): string => new Date(ms).toISOString(); | |
| 42 | ||
| 43 | export function makeRunner(deps: RunnerDeps) { | |
| 44 | const { db } = deps; | |
| 45 | const running = new Set<string>(); | |
| 46 | ||
| 47 | function due(duty: Duty, now: number): boolean { | |
| 48 | if (!("every" in duty.trigger)) return false; | |
| 49 | const last = lastRun(db, duty.name); | |
| 50 | return last === undefined || now - Date.parse(last.started) >= duty.trigger.every; | |
| 51 | } | |
| 52 | ||
| 53 | function overBudget(duty: Duty, now: number): string | undefined { | |
| 54 | const used = usedSince(db, duty.name, iso(now - DAY_MS)); | |
| 55 | if (used.runs >= duty.budget.runsPerDay) return `${used.runs} runs in the last day, budget ${duty.budget.runsPerDay}`; | |
| 56 | if (used.usd > duty.budget.usdPerDay) return `$${used.usd.toFixed(2)} in the last day, budget $${duty.budget.usdPerDay}`; | |
| 57 | return undefined; | |
| 58 | } | |
| 59 | ||
| 60 | async function runDuty(duty: Duty, now: number): Promise<RunRow | undefined> { | |
| 61 | const state = dutyState(db, duty.name); | |
| 62 | if (!duty.enabled || !state.enabled) return undefined; | |
| 63 | if (running.has(duty.name)) return undefined; | |
| 64 | running.add(duty.name); | |
| 65 | try { | |
| 66 | return await execute(duty, now); | |
| 67 | } finally { | |
| 68 | running.delete(duty.name); | |
| 69 | } | |
| 70 | } | |
| 71 | ||
| 72 | async function execute(duty: Duty, now: number): Promise<RunRow> { | |
| 73 | const started = iso(now); | |
| 74 | const row = (p: Partial<RunRow>): RunRow => ({ | |
| 75 | id: crypto.randomUUID(), duty: duty.name, started, finished: iso(Date.now()), | |
| 76 | findings: [], actions: [], probes: [], outcome: "ok", cost: 0, detail: null, ...p, | |
| 77 | }); | |
| 78 | const over = overBudget(duty, now); | |
| 79 | if (over !== undefined) { | |
| 80 | // Said once per window, not every tick. | |
| 81 | const last = lastRun(db, duty.name); | |
| 82 | if (last?.outcome === "budget" && now - Date.parse(last.started) < DAY_MS) return last; | |
| 83 | const r = row({ outcome: "budget", detail: over }); | |
| 84 | recordRun(db, r); | |
| 85 | return r; | |
| 86 | } | |
| 87 | ||
| 88 | const used = usedSince(db, duty.name, iso(now - DAY_MS)); | |
| 89 | let cost = 0; | |
| 90 | let calls = 0; | |
| 91 | let halted: string | undefined; | |
| 92 | const done: Action[] = []; | |
| 93 | const probe = (query: string, value: ProbeRecord["value"]): ProbeRecord => ({ query, value, at: iso(Date.now()) }); | |
| 94 | const ctx: DutyContext = { | |
| 95 | now, db, probe, | |
| 96 | net() { | |
| 97 | if (++calls > duty.budget.netCallsPerRun) throw new OverBudget(`more than ${duty.budget.netCallsPerRun} network calls in a run`); | |
| 98 | }, | |
| 99 | spend(usd) { | |
| 100 | cost += usd; | |
| 101 | if (used.usd + cost > duty.budget.usdPerDay) throw new OverBudget(`over $${duty.budget.usdPerDay} a day`); | |
| 102 | }, | |
| 103 | async bulk<T>(items: readonly T[], step: BulkStep<T>): Promise<Action[]> { | |
| 104 | const out: Action[] = []; | |
| 105 | for (const item of items) { | |
| 106 | if (halted !== undefined || done.length >= BULK_CAP) break; | |
| 107 | await step.apply(item); | |
| 108 | const seen = await step.check(item); | |
| 109 | const action: Action = { what: step.what, target: step.target(item), probe: { query: seen.query, value: seen.value, at: seen.at } }; | |
| 110 | out.push(action); | |
| 111 | done.push(action); | |
| 112 | if (!seen.pass) halted = `${step.what} ${action.target}: probe ${seen.query} returned ${String(seen.value)}; nothing else touched`; | |
| 113 | } | |
| 114 | return out; | |
| 115 | }, | |
| 116 | }; | |
| 117 | ||
| 118 | const clock = Date.now(); | |
| 119 | let outcome: Outcome = "ok"; | |
| 120 | let detail: string | null = null; | |
| 121 | let findings: Finding[] = []; | |
| 122 | let actions: Action[] = []; | |
| 123 | try { | |
| 124 | const result = await duty.handler(ctx); | |
| 125 | const admitted = admit(result.findings ?? [], result.actions ?? []); | |
| 126 | findings = admitted.findings; | |
| 127 | actions = [...done, ...admitted.actions]; | |
| 128 | if (admitted.refused.length > 0) { | |
| 129 | outcome = "refused"; | |
| 130 | detail = admitted.refused.join("; "); | |
| 131 | } | |
| 132 | if (halted !== undefined) { | |
| 133 | outcome = "canary"; | |
| 134 | detail = halted; | |
| 135 | } | |
| 136 | if (Date.now() - clock > duty.budget.cpuMsPerRun) { | |
| 137 | outcome = "budget"; | |
| 138 | detail = `took longer than ${duty.budget.cpuMsPerRun} ms`; | |
| 139 | } | |
| 140 | } catch (error) { | |
| 141 | actions = done; | |
| 142 | outcome = error instanceof OverBudget ? "budget" : "error"; | |
| 143 | detail = error instanceof Error ? error.message : String(error); | |
| 144 | } | |
| 145 | ||
| 146 | const filed: string[] = []; | |
| 147 | if (outcome !== "error") { | |
| 148 | for (const f of findings) { | |
| 149 | try { | |
| 150 | const ref = deps.file?.(f, now); | |
| 151 | if (ref !== undefined) filed.push(`${f.key} -> ${ref}`); | |
| 152 | } catch (error) { | |
| 153 | detail = `${detail === null ? "" : `${detail}; `}could not file ${f.key}: ${String(error)}`; | |
| 154 | } | |
| 155 | } | |
| 156 | } | |
| 157 | const r = row({ | |
| 158 | findings, actions, outcome, cost, detail, | |
| 159 | probes: [...findings.map((f) => f.probe), ...actions.map((a) => a.probe)], | |
| 160 | }); | |
| 161 | recordRun(db, r); | |
| 162 | ||
| 163 | if (outcome === "error") { | |
| 164 | const errors = dutyState(db, duty.name).errors + 1; | |
| 165 | setErrors(db, duty.name, errors); | |
| 166 | if (errors >= ERRORS_TO_PAUSE) | |
| 167 | pauseDuty(db, duty.name, `errored ${errors} times in a row; last: ${detail ?? "unknown"}`); | |
| 168 | } else if (dutyState(db, duty.name).errors !== 0) setErrors(db, duty.name, 0); | |
| 169 | return r; | |
| 170 | } | |
| 171 | ||
| 172 | return { | |
| 173 | runDuty, | |
| 174 | /** Every duty whose interval has passed, each once. */ | |
| 175 | async runDue(now: number): Promise<RunRow[]> { | |
| 176 | const out: RunRow[] = []; | |
| 177 | for (const d of deps.registry.all()) { | |
| 178 | if (!due(d, now)) continue; | |
| 179 | const r = await runDuty(d, now); | |
| 180 | if (r !== undefined) out.push(r); | |
| 181 | } | |
| 182 | return out; | |
| 183 | }, | |
| 184 | /** Every duty that waits for this event, e.g. "queue-gate-red". */ | |
| 185 | async fire(event: string, now: number): Promise<RunRow[]> { | |
| 186 | const out: RunRow[] = []; | |
| 187 | for (const d of deps.registry.all()) { | |
| 188 | if (!("on" in d.trigger) || d.trigger.on !== event) continue; | |
| 189 | const r = await runDuty(d, now); | |
| 190 | if (r !== undefined) out.push(r); | |
| 191 | } | |
| 192 | return out; | |
| 193 | }, | |
| 194 | }; | |
| 195 | } | |
| 196 | export type Runner = ReturnType<typeof makeRunner>; |
src/maintain/service.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** The maintainer wired to the app (#7554): one runner, the default registry, the sentinel's clock. */ | |
| 2 | ||
| 3 | import type { App } from "../app.ts"; | |
| 4 | import { dutyList } from "./duties/index.ts"; | |
| 5 | import { issueFiler } from "./file.ts"; | |
| 6 | import { makeRegistry } from "./registry.ts"; | |
| 7 | import { makeRunner, type Runner } from "./runner.ts"; | |
| 8 | import { dutyState, listRuns, pauseDuty, resumeDuty, lastRun } from "./store.ts"; | |
| 9 | ||
| 10 | const runners = new WeakMap<App, Runner>(); | |
| 11 | const registry = makeRegistry(dutyList); | |
| 12 | ||
| 13 | export function runnerOf(app: App): Runner { | |
| 14 | let r = runners.get(app); | |
| 15 | if (r === undefined) { | |
| 16 | r = makeRunner({ db: app.db, registry, file: issueFiler(app) }); | |
| 17 | runners.set(app, r); | |
| 18 | } | |
| 19 | return r; | |
| 20 | } | |
| 21 | ||
| 22 | /** On the autonomy clock: whatever is due runs. */ | |
| 23 | export const maintainTick = (app: App, now = Date.now()) => runnerOf(app).runDue(now); | |
| 24 | ||
| 25 | export function dutyBoard(app: App) { | |
| 26 | return registry.all().map((d) => { | |
| 27 | const s = dutyState(app.db, d.name); | |
| 28 | return { | |
| 29 | name: d.name, | |
| 30 | trigger: d.trigger, | |
| 31 | budget: d.budget, | |
| 32 | enabled: d.enabled && s.enabled, | |
| 33 | pausedReason: s.pausedReason, | |
| 34 | lastRun: lastRun(app.db, d.name) ?? null, | |
| 35 | }; | |
| 36 | }); | |
| 37 | } | |
| 38 | ||
| 39 | export function setDutyPaused(app: App, name: string, paused: boolean, by: string): boolean { | |
| 40 | if (registry.get(name) === undefined) return false; | |
| 41 | if (paused) pauseDuty(app.db, name, `paused by ${by}`); | |
| 42 | else resumeDuty(app.db, name); | |
| 43 | return true; | |
| 44 | } | |
| 45 | ||
| 46 | export const dutyRuns = (app: App, name?: string) => listRuns(app.db, name); |
src/maintain/store.ts added +96
| @@ -0,0 +1,96 @@ | ||
| 1 | /** The maintainer's tables (#7554): duty switches, run rows, and the issue each finding key went to. */ | |
| 2 | ||
| 3 | import type { Database } from "bun:sqlite"; | |
| 4 | import type { Action, Finding, Outcome, ProbeRecord } from "./duty.ts"; | |
| 5 | ||
| 6 | export interface DutyState { | |
| 7 | readonly name: string; | |
| 8 | readonly enabled: boolean; | |
| 9 | readonly pausedReason: string | null; | |
| 10 | readonly errors: number; | |
| 11 | } | |
| 12 | ||
| 13 | export interface RunRow { | |
| 14 | readonly id: string; | |
| 15 | readonly duty: string; | |
| 16 | readonly started: string; | |
| 17 | readonly finished: string; | |
| 18 | readonly findings: Finding[]; | |
| 19 | readonly actions: Action[]; | |
| 20 | readonly probes: ProbeRecord[]; | |
| 21 | readonly outcome: Outcome; | |
| 22 | readonly cost: number; | |
| 23 | readonly detail: string | null; | |
| 24 | } | |
| 25 | ||
| 26 | export function dutyState(db: Database, name: string): DutyState { | |
| 27 | const row = db | |
| 28 | .query<{ enabled: number; paused_reason: string | null; errors: number }, [string]>( | |
| 29 | "SELECT enabled, paused_reason, errors FROM maintenance_duty WHERE name = ?1", | |
| 30 | ) | |
| 31 | .get(name); | |
| 32 | return { name, enabled: row === null || row.enabled === 1, pausedReason: row?.paused_reason ?? null, errors: row?.errors ?? 0 }; | |
| 33 | } | |
| 34 | ||
| 35 | function setState(db: Database, name: string, enabled: boolean, reason: string | null, errors: number): void { | |
| 36 | db.query( | |
| 37 | `INSERT INTO maintenance_duty (name, enabled, paused_reason, errors) VALUES (?1, ?2, ?3, ?4) | |
| 38 | ON CONFLICT(name) DO UPDATE SET enabled = ?2, paused_reason = ?3, errors = ?4`, | |
| 39 | ).run(name, enabled ? 1 : 0, reason, errors); | |
| 40 | } | |
| 41 | ||
| 42 | export const pauseDuty = (db: Database, name: string, reason: string): void => | |
| 43 | setState(db, name, false, reason, dutyState(db, name).errors); | |
| 44 | export const resumeDuty = (db: Database, name: string): void => setState(db, name, true, null, 0); | |
| 45 | export const setErrors = (db: Database, name: string, errors: number): void => { | |
| 46 | const s = dutyState(db, name); | |
| 47 | setState(db, name, s.enabled, s.pausedReason, errors); | |
| 48 | }; | |
| 49 | ||
| 50 | export function recordRun(db: Database, row: RunRow): void { | |
| 51 | db.query( | |
| 52 | `INSERT INTO maintenance_run (id, duty, started, finished, findings, actions, probes, outcome, cost, detail) | |
| 53 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)`, | |
| 54 | ).run( | |
| 55 | row.id, row.duty, row.started, row.finished, JSON.stringify(row.findings), JSON.stringify(row.actions), | |
| 56 | JSON.stringify(row.probes), row.outcome, row.cost, row.detail, | |
| 57 | ); | |
| 58 | } | |
| 59 | ||
| 60 | export function listRuns(db: Database, duty?: string, limit = 50): RunRow[] { | |
| 61 | const rows = db | |
| 62 | .query<Record<string, string | number | null>, [string | null, number]>( | |
| 63 | `SELECT * FROM maintenance_run WHERE (?1 IS NULL OR duty = ?1) ORDER BY started DESC, id DESC LIMIT ?2`, | |
| 64 | ) | |
| 65 | .all(duty ?? null, limit); | |
| 66 | return rows.map((r) => ({ | |
| 67 | id: String(r.id), duty: String(r.duty), started: String(r.started), finished: String(r.finished), | |
| 68 | findings: JSON.parse(String(r.findings)), actions: JSON.parse(String(r.actions)), | |
| 69 | probes: JSON.parse(String(r.probes)), outcome: String(r.outcome) as Outcome, | |
| 70 | cost: Number(r.cost), detail: r.detail === null ? null : String(r.detail), | |
| 71 | })); | |
| 72 | } | |
| 73 | ||
| 74 | /** Runs started and dollars spent by a duty since a moment (the day's budget). */ | |
| 75 | export function usedSince(db: Database, duty: string, sinceIso: string): { runs: number; usd: number } { | |
| 76 | const row = db | |
| 77 | .query<{ runs: number; usd: number | null }, [string, string]>( | |
| 78 | "SELECT COUNT(*) AS runs, SUM(cost) AS usd FROM maintenance_run WHERE duty = ?1 AND started >= ?2 AND outcome != 'budget'", | |
| 79 | ) | |
| 80 | .get(duty, sinceIso); | |
| 81 | return { runs: row?.runs ?? 0, usd: row?.usd ?? 0 }; | |
| 82 | } | |
| 83 | ||
| 84 | export function lastRun(db: Database, duty: string): RunRow | undefined { | |
| 85 | return listRuns(db, duty, 1)[0]; | |
| 86 | } | |
| 87 | ||
| 88 | export function trackedIssue(db: Database, key: string): string | undefined { | |
| 89 | return db.query<{ issue_id: string }, [string]>("SELECT issue_id FROM maintenance_finding WHERE key = ?1").get(key)?.issue_id; | |
| 90 | } | |
| 91 | export function trackFinding(db: Database, key: string, issueId: string, at: string): void { | |
| 92 | db.query( | |
| 93 | `INSERT INTO maintenance_finding (key, issue_id, first_seen, last_seen) VALUES (?1, ?2, ?3, ?3) | |
| 94 | ON CONFLICT(key) DO UPDATE SET issue_id = ?2, last_seen = ?3`, | |
| 95 | ).run(key, issueId, at); | |
| 96 | } |
src/web/studio/crew.ts +2
| @@ -62,6 +62,7 @@ const ROLE_WORD: Record<CrewRole, string> = { | ||
| 62 | 62 | builder: "builds", |
| 63 | 63 | reviewer: "reviews", |
| 64 | 64 | planner: "plans", |
| 65 | maintainer: "keeps up", | |
| 65 | 66 | }; |
| 66 | 67 | |
| 67 | 68 | /** |
| @@ -139,6 +140,7 @@ const SEAT_TITLE: Record<CrewRole, string> = { | ||
| 139 | 140 | planner: "Planners", |
| 140 | 141 | builder: "Builders", |
| 141 | 142 | reviewer: "Reviewers", |
| 143 | maintainer: "Maintainers", | |
| 142 | 144 | }; |
| 143 | 145 | |
| 144 | 146 | /** |
test/helpers/maintain.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | import { Database } from "bun:sqlite"; | |
| 2 | import maintenance from "../../src/core/migrations/20261103150000-maintenance.ts"; | |
| 3 | import { type Duty, type FindingInput, duty } from "../../src/maintain/duty.ts"; | |
| 4 | import { makeRegistry } from "../../src/maintain/registry.ts"; | |
| 5 | import { makeRunner } from "../../src/maintain/runner.ts"; | |
| 6 | import type { Finding } from "../../src/maintain/duty.ts"; | |
| 7 | ||
| 8 | export const NOW = Date.parse("2026-11-02T12:00:00Z"); | |
| 9 | export const HOUR = 3_600_000; | |
| 10 | ||
| 11 | export function maintainer(duties: Duty[], filed: Finding[] = []) { | |
| 12 | const db = new Database(":memory:"); | |
| 13 | db.exec(maintenance); | |
| 14 | const runner = makeRunner({ | |
| 15 | db, | |
| 16 | registry: makeRegistry(duties), | |
| 17 | file: (f) => { | |
| 18 | filed.push(f); | |
| 19 | return `#${filed.length}`; | |
| 20 | }, | |
| 21 | }); | |
| 22 | return { db, runner }; | |
| 23 | } | |
| 24 | ||
| 25 | export const probed = (ctx: { probe: (q: string, v: number) => unknown }, key = "k"): FindingInput => | |
| 26 | ({ key, title: `finding ${key}`, evidence: ["e"], probe: ctx.probe("SELECT 1", 1) }) as FindingInput; | |
| 27 | ||
| 28 | export { duty }; |
test/maintain-api.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** The board's switch for a duty, over the API, and the issue a finding is filed to (#7554). */ | |
| 2 | ||
| 3 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 10 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 11 | import { runnerOf } from "../src/maintain/service.ts"; | |
| 12 | import { dutyList } from "../src/maintain/duties/index.ts"; | |
| 13 | import { quietLogs } from "./helpers/globals.ts"; | |
| 14 | ||
| 15 | quietLogs(); | |
| 16 | const PASS = "a-long-maintain-passphrase"; | |
| 17 | ||
| 18 | describe("maintainer over HTTP", () => { | |
| 19 | let dataDir = ""; | |
| 20 | let app: App; | |
| 21 | let servers: Servers; | |
| 22 | const tokenOf = (name: string, scope: "ro" | "admin" = "admin") => createToken(app.db, getUserByName(app.db, name)!.id, "t", scope).token; | |
| 23 | const call = (method: string, path: string, token: string | null) => | |
| 24 | fetch(`${servers.url}/api/v1/maintain/${path}`, { method, headers: token ? { authorization: `Bearer ${token}` } : {} }); | |
| 25 | ||
| 26 | beforeAll(async () => { | |
| 27 | dataDir = await mkdtemp(join(tmpdir(), "forge-maintain-")); | |
| 28 | app = createApp({ dataDir, port: 0 }); | |
| 29 | servers = await serve(app); | |
| 30 | for (const [username, isAdmin] of [["alice", true], ["bob", false]] as const) | |
| 31 | await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@x.test`, password: PASS, isAdmin }); | |
| 32 | }, 30_000); | |
| 33 | ||
| 34 | afterAll(async () => { | |
| 35 | servers.stop(); | |
| 36 | app.close(); | |
| 37 | await rm(dataDir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | test("an admin lists duties, pauses one with a reason, and resumes it", async () => { | |
| 41 | const name = dutyList[0]!.name; | |
| 42 | const listed = (await (await call("GET", "duties", tokenOf("alice", "ro"))).json()) as any; | |
| 43 | expect(listed.duties.map((d: any) => d.name)).toContain(name); | |
| 44 | const paused = (await (await call("POST", `duties/${name}/pause`, tokenOf("alice"))).json()) as any; | |
| 45 | expect(paused.duties.find((d: any) => d.name === name)).toMatchObject({ enabled: false, pausedReason: "paused by alice" }); | |
| 46 | expect(await runnerOf(app).runDue(Date.now() + 10 * 86_400_000)).toEqual([]); | |
| 47 | const resumed = (await (await call("POST", `duties/${name}/resume`, tokenOf("alice"))).json()) as any; | |
| 48 | expect(resumed.duties.find((d: any) => d.name === name).enabled).toBe(true); | |
| 49 | }); | |
| 50 | ||
| 51 | test("a read-only token cannot pause; a plain member and a stranger are told nothing is there", async () => { | |
| 52 | const name = dutyList[0]!.name; | |
| 53 | expect((await call("POST", `duties/${name}/pause`, tokenOf("alice", "ro"))).status).toBe(403); | |
| 54 | expect((await call("GET", "duties", tokenOf("bob"))).status).toBe(404); | |
| 55 | expect((await call("GET", "runs", null)).status).toBe(404); | |
| 56 | expect((await call("POST", "duties/nope/pause", tokenOf("alice"))).status).toBe(404); | |
| 57 | }); | |
| 58 | ||
| 59 | test("the scheduled run leaves a row the API reads", async () => { | |
| 60 | await runnerOf(app).runDue(Date.now() + 100 * 86_400_000); | |
| 61 | const runs = (await (await call("GET", `runs?duty=${dutyList[0]!.name}`, tokenOf("alice", "ro"))).json()) as any; | |
| 62 | expect(runs.runs.length).toBeGreaterThan(0); | |
| 63 | }); | |
| 64 | ||
| 65 | test("the routes are in the OpenAPI document", async () => { | |
| 66 | const doc = (await (await fetch(`${servers.url}/api/v1/openapi.json`)).json()) as any; | |
| 67 | expect(doc.paths["/api/v1/maintain/duties"].get).toBeDefined(); | |
| 68 | expect(doc.paths["/api/v1/maintain/duties/{name}/pause"].post).toBeDefined(); | |
| 69 | }); | |
| 70 | }); |
test/maintain-budget.test.ts added +69
| @@ -0,0 +1,69 @@ | ||
| 1 | /** A duty over budget stops and records "budget" as the outcome (#7554). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { HOUR, NOW, duty, maintainer } from "./helpers/maintain.ts"; | |
| 5 | import { listRuns } from "../src/maintain/store.ts"; | |
| 6 | import type { Budget } from "../src/maintain/duty.ts"; | |
| 7 | ||
| 8 | const budget = (b: Partial<Budget>): Budget => ({ runsPerDay: 100, usdPerDay: 1, cpuMsPerRun: 10_000, netCallsPerRun: 2, ...b }); | |
| 9 | ||
| 10 | describe("budget", () => { | |
| 11 | test("runs per day: the run past the budget does not run and says budget", async () => { | |
| 12 | let ran = 0; | |
| 13 | const d = duty({ name: "few", trigger: { every: 1 }, budget: budget({ runsPerDay: 2 }), handler: () => { ran++; return {}; } }); | |
| 14 | const { db, runner } = maintainer([d]); | |
| 15 | for (let i = 0; i < 4; i++) await runner.runDue(NOW + i * 10); | |
| 16 | expect(ran).toBe(2); | |
| 17 | const rows = listRuns(db, "few"); | |
| 18 | expect(rows.filter((r) => r.outcome === "budget")).toHaveLength(1); // said once, not each tick | |
| 19 | expect(rows[0]?.outcome).toBe("budget"); | |
| 20 | }); | |
| 21 | ||
| 22 | test("the day rolls over", async () => { | |
| 23 | const d = duty({ name: "daily", trigger: { every: 1 }, budget: budget({ runsPerDay: 1 }), handler: () => ({}) }); | |
| 24 | const { runner } = maintainer([d]); | |
| 25 | await runner.runDue(NOW); | |
| 26 | expect((await runner.runDue(NOW + HOUR))[0]?.outcome).toBe("budget"); | |
| 27 | expect((await runner.runDue(NOW + 25 * HOUR))[0]?.outcome).toBe("ok"); | |
| 28 | }); | |
| 29 | ||
| 30 | test("dollars: spending past the day's ceiling stops the run", async () => { | |
| 31 | const d = duty({ | |
| 32 | name: "spender", | |
| 33 | trigger: { every: 1 }, | |
| 34 | budget: budget({ usdPerDay: 1 }), | |
| 35 | handler: (ctx) => { | |
| 36 | ctx.spend(0.6); | |
| 37 | ctx.spend(0.6); | |
| 38 | return {}; | |
| 39 | }, | |
| 40 | }); | |
| 41 | const { db, runner } = maintainer([d]); | |
| 42 | await runner.runDue(NOW); | |
| 43 | const [row] = listRuns(db); | |
| 44 | expect(row?.outcome).toBe("budget"); | |
| 45 | expect(row?.detail).toContain("$1"); | |
| 46 | expect(row?.cost).toBeCloseTo(1.2); | |
| 47 | }); | |
| 48 | ||
| 49 | test("network: calls past the ceiling stop the run", async () => { | |
| 50 | const d = duty({ name: "chatty", trigger: { every: 1 }, budget: budget({ netCallsPerRun: 2 }), handler: (ctx) => { ctx.net(); ctx.net(); ctx.net(); return {}; } }); | |
| 51 | const { db, runner } = maintainer([d]); | |
| 52 | await runner.runDue(NOW); | |
| 53 | expect(listRuns(db)[0]?.outcome).toBe("budget"); | |
| 54 | }); | |
| 55 | ||
| 56 | test("time: a run past its ceiling is recorded as budget", async () => { | |
| 57 | const d = duty({ name: "slow", trigger: { every: 1 }, budget: budget({ cpuMsPerRun: 5 }), handler: async () => { await Bun.sleep(30); return {}; } }); | |
| 58 | const { db, runner } = maintainer([d]); | |
| 59 | await runner.runDue(NOW); | |
| 60 | expect(listRuns(db)[0]?.outcome).toBe("budget"); | |
| 61 | }); | |
| 62 | ||
| 63 | test("a budget stop is not an error: it never pauses the duty", async () => { | |
| 64 | const d = duty({ name: "chat", trigger: { every: 1 }, budget: budget({ netCallsPerRun: 0 }), handler: (ctx) => { ctx.net(); return {}; } }); | |
| 65 | const { db, runner } = maintainer([d]); | |
| 66 | for (let i = 0; i < 4; i++) await runner.runDue(NOW + i * 10); | |
| 67 | expect(db.query("SELECT enabled FROM maintenance_duty").all()).toEqual([]); | |
| 68 | }); | |
| 69 | }); |
test/maintain-canary.test.ts added +53
| @@ -0,0 +1,53 @@ | ||
| 1 | /** A bulk action touches one item, waits for its probe, then continues (#7554). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { NOW, duty, maintainer } from "./helpers/maintain.ts"; | |
| 5 | import { listRuns } from "../src/maintain/store.ts"; | |
| 6 | import { BULK_CAP, type DutyContext } from "../src/maintain/duty.ts"; | |
| 7 | ||
| 8 | const bulky = (items: number[], touched: number[], passes: (n: number) => boolean) => | |
| 9 | duty({ | |
| 10 | name: "bulk", | |
| 11 | trigger: { every: 1 }, | |
| 12 | handler: async (ctx: DutyContext) => { | |
| 13 | await ctx.bulk(items, { | |
| 14 | what: "touch", | |
| 15 | target: String, | |
| 16 | apply: (n) => void touched.push(n), | |
| 17 | check: (n) => ({ ...ctx.probe(`check ${n}`, passes(n)), pass: passes(n) }), | |
| 18 | }); | |
| 19 | return {}; | |
| 20 | }, | |
| 21 | }); | |
| 22 | ||
| 23 | describe("canary", () => { | |
| 24 | test("a passing canary lets the rest through, each with its probe", async () => { | |
| 25 | const touched: number[] = []; | |
| 26 | const { db, runner } = maintainer([bulky([1, 2, 3], touched, () => true)]); | |
| 27 | await runner.runDue(NOW); | |
| 28 | expect(touched).toEqual([1, 2, 3]); | |
| 29 | const [row] = listRuns(db); | |
| 30 | expect(row?.outcome).toBe("ok"); | |
| 31 | expect(row?.actions.map((a) => a.target)).toEqual(["1", "2", "3"]); | |
| 32 | }); | |
| 33 | ||
| 34 | test("a failing canary touches nothing else", async () => { | |
| 35 | const touched: number[] = []; | |
| 36 | const { db, runner } = maintainer([bulky([1, 2, 3], touched, () => false)]); | |
| 37 | await runner.runDue(NOW); | |
| 38 | expect(touched).toEqual([1]); | |
| 39 | const [row] = listRuns(db); | |
| 40 | expect(row?.outcome).toBe("canary"); | |
| 41 | expect(row?.detail).toContain("nothing else touched"); | |
| 42 | expect(row?.actions).toHaveLength(1); | |
| 43 | expect(row?.actions[0]?.probe.value).toBe(false); | |
| 44 | }); | |
| 45 | ||
| 46 | test("a bulk is capped per run", async () => { | |
| 47 | const touched: number[] = []; | |
| 48 | const many = Array.from({ length: BULK_CAP + 15 }, (_, i) => i); | |
| 49 | const { runner } = maintainer([bulky(many, touched, () => true)]); | |
| 50 | await runner.runDue(NOW); | |
| 51 | expect(touched).toHaveLength(BULK_CAP); | |
| 52 | }); | |
| 53 | }); |
test/maintain-no-land.test.ts added +34
| @@ -0,0 +1,34 @@ | ||
| 1 | /** The maintainer role has no path to merge or to push main (#7554). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { readdirSync, readFileSync } from "node:fs"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { isCrewRole } from "../src/core/agents.ts"; | |
| 7 | ||
| 8 | const root = join(import.meta.dir, "../src/maintain"); | |
| 9 | const sources = (dir: string): string[] => | |
| 10 | readdirSync(dir, { withFileTypes: true }).flatMap((e) => | |
| 11 | e.isDirectory() ? sources(join(dir, e.name)) : e.name.endsWith(".ts") ? [join(dir, e.name)] : [], | |
| 12 | ); | |
| 13 | ||
| 14 | describe("the maintainer proposes, a person lands", () => { | |
| 15 | test("maintainer is a crew role, the same kind of agent as a builder", () => { | |
| 16 | expect(isCrewRole("maintainer")).toBe(true); | |
| 17 | }); | |
| 18 | ||
| 19 | test("nothing under src/maintain imports a way to merge, push or run git", () => { | |
| 20 | const banned = [/mergePull/, /core\/merges/, /git\/(receive|push|hooks)/, /Bun\.spawn/, /node:child_process/, /\bpush\b.*\bmain\b/, /\beval\(/]; | |
| 21 | for (const file of sources(root)) { | |
| 22 | const text = readFileSync(file, "utf8"); | |
| 23 | for (const rule of banned) expect([file, rule.test(text)]).toEqual([file, false]); | |
| 24 | } | |
| 25 | }); | |
| 26 | ||
| 27 | test("a duty's context offers no merge or push", async () => { | |
| 28 | const { maintainer, NOW, duty } = await import("./helpers/maintain.ts"); | |
| 29 | let keys: string[] = []; | |
| 30 | const { runner } = maintainer([duty({ name: "peek", trigger: { every: 1 }, handler: (ctx) => { keys = Object.keys(ctx); return {}; } })]); | |
| 31 | await runner.runDue(NOW); | |
| 32 | expect(keys.sort()).toEqual(["bulk", "db", "net", "now", "probe", "spend"]); | |
| 33 | }); | |
| 34 | }); |
test/maintain-probe-required.test.ts added +39
| @@ -0,0 +1,39 @@ | ||
| 1 | /** A finding or an action without the probe behind it is refused (#7554). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { NOW, duty, maintainer, probed } from "./helpers/maintain.ts"; | |
| 5 | import { listRuns } from "../src/maintain/store.ts"; | |
| 6 | import { validProbe } from "../src/maintain/probe.ts"; | |
| 7 | import type { Finding } from "../src/maintain/duty.ts"; | |
| 8 | ||
| 9 | describe("probe required", () => { | |
| 10 | test("a finding with no probe is refused, one with a probe is kept and filed", async () => { | |
| 11 | const filed: Finding[] = []; | |
| 12 | const d = duty({ | |
| 13 | name: "mixed", | |
| 14 | trigger: { every: 1 }, | |
| 15 | handler: (ctx) => ({ | |
| 16 | findings: [{ key: "bare", title: "no probe", evidence: [], probe: undefined }, probed(ctx, "backed")], | |
| 17 | actions: [{ what: "fix", target: "t", probe: undefined }], | |
| 18 | }), | |
| 19 | }); | |
| 20 | const { db, runner } = maintainer([d], filed); | |
| 21 | await runner.runDue(NOW); | |
| 22 | const [row] = listRuns(db); | |
| 23 | expect(row?.outcome).toBe("refused"); | |
| 24 | expect(row?.findings.map((f) => f.key)).toEqual(["backed"]); | |
| 25 | expect(row?.actions).toEqual([]); | |
| 26 | expect(row?.detail).toContain("finding bare: no probe"); | |
| 27 | expect(row?.detail).toContain("action fix t: no probe"); | |
| 28 | expect(filed.map((f) => f.key)).toEqual(["backed"]); | |
| 29 | }); | |
| 30 | ||
| 31 | test("a probe needs a query, a finite value and a time", () => { | |
| 32 | const ok = { query: "SELECT 1", value: 1, at: "2026-11-02T12:00:00Z" }; | |
| 33 | expect(validProbe(ok)).toBe(true); | |
| 34 | expect(validProbe({ ...ok, query: " " })).toBe(false); | |
| 35 | expect(validProbe({ ...ok, value: Number.NaN })).toBe(false); | |
| 36 | expect(validProbe({ ...ok, at: "yesterday-ish" })).toBe(false); | |
| 37 | expect(validProbe(undefined)).toBe(false); | |
| 38 | }); | |
| 39 | }); |
test/maintain-runner.test.ts added +102
| @@ -0,0 +1,102 @@ | ||
| 1 | /** A duty registered in one place runs on schedule, serialized, and leaves a run row (#7554). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { HOUR, NOW, duty, maintainer, probed } from "./helpers/maintain.ts"; | |
| 5 | import { listRuns, dutyState, resumeDuty } from "../src/maintain/store.ts"; | |
| 6 | import { makeRegistry } from "../src/maintain/registry.ts"; | |
| 7 | import { dutyList } from "../src/maintain/duties/index.ts"; | |
| 8 | import type { Finding } from "../src/maintain/duty.ts"; | |
| 9 | ||
| 10 | describe("the duty runner", () => { | |
| 11 | test("runs a due duty and leaves a row with findings, actions and probes", async () => { | |
| 12 | const filed: Finding[] = []; | |
| 13 | const d = duty({ | |
| 14 | name: "look", | |
| 15 | trigger: { every: HOUR }, | |
| 16 | handler: (ctx) => ({ | |
| 17 | findings: [probed(ctx, "a")], | |
| 18 | actions: [{ what: "note", target: "x", probe: ctx.probe("SELECT 2", 2) }], | |
| 19 | }), | |
| 20 | }); | |
| 21 | const { db, runner } = maintainer([d], filed); | |
| 22 | const ran = await runner.runDue(NOW); | |
| 23 | expect(ran).toHaveLength(1); | |
| 24 | const [row] = listRuns(db, "look"); | |
| 25 | expect(row?.outcome).toBe("ok"); | |
| 26 | expect(row?.findings.map((f) => f.key)).toEqual(["a"]); | |
| 27 | expect(row?.actions[0]?.target).toBe("x"); | |
| 28 | expect(row?.probes.map((p) => p.value)).toEqual([1, 2]); | |
| 29 | expect(filed.map((f) => f.key)).toEqual(["a"]); | |
| 30 | }); | |
| 31 | ||
| 32 | test("waits for its interval, then runs again", async () => { | |
| 33 | const { db, runner } = maintainer([duty({ name: "tick", trigger: { every: HOUR }, handler: () => ({}) })]); | |
| 34 | await runner.runDue(NOW); | |
| 35 | expect(await runner.runDue(NOW + HOUR / 2)).toHaveLength(0); | |
| 36 | expect(await runner.runDue(NOW + HOUR)).toHaveLength(1); | |
| 37 | expect(listRuns(db)).toHaveLength(2); | |
| 38 | }); | |
| 39 | ||
| 40 | test("an event duty runs on its event, not on the clock", async () => { | |
| 41 | const { db, runner } = maintainer([duty({ name: "red", trigger: { on: "queue-gate-red" }, handler: () => ({}) })]); | |
| 42 | expect(await runner.runDue(NOW)).toHaveLength(0); | |
| 43 | expect(await runner.fire("something-else", NOW)).toHaveLength(0); | |
| 44 | expect(await runner.fire("queue-gate-red", NOW)).toHaveLength(1); | |
| 45 | expect(listRuns(db)).toHaveLength(1); | |
| 46 | }); | |
| 47 | ||
| 48 | test("a duty never overlaps itself", async () => { | |
| 49 | let release = () => {}; | |
| 50 | let starts = 0; | |
| 51 | const d = duty({ | |
| 52 | name: "slow", | |
| 53 | trigger: { every: 1 }, | |
| 54 | handler: () => { | |
| 55 | starts++; | |
| 56 | return new Promise((r) => (release = () => r({}))); | |
| 57 | }, | |
| 58 | }); | |
| 59 | const { runner } = maintainer([d]); | |
| 60 | const first = runner.runDuty(d, NOW); | |
| 61 | expect(await runner.runDuty(d, NOW + 5)).toBeUndefined(); | |
| 62 | release(); | |
| 63 | await first; | |
| 64 | expect(starts).toBe(1); | |
| 65 | }); | |
| 66 | ||
| 67 | test("a switched-off duty does not run", async () => { | |
| 68 | const { runner } = maintainer([duty({ name: "off", trigger: { every: 1 }, enabled: false, handler: () => ({}) })]); | |
| 69 | expect(await runner.runDue(NOW)).toHaveLength(0); | |
| 70 | }); | |
| 71 | ||
| 72 | test("three errors in a row pause the duty and say why; resume clears it", async () => { | |
| 73 | const d = duty({ name: "bad", trigger: { every: 1 }, handler: () => { throw new Error("boom"); } }); | |
| 74 | const { db, runner } = maintainer([d]); | |
| 75 | for (let i = 0; i < 3; i++) await runner.runDue(NOW + i * 10); | |
| 76 | const state = dutyState(db, "bad"); | |
| 77 | expect(state.enabled).toBe(false); | |
| 78 | expect(state.pausedReason).toContain("3 times"); | |
| 79 | expect(state.pausedReason).toContain("boom"); | |
| 80 | expect(await runner.runDue(NOW + 100)).toHaveLength(0); | |
| 81 | resumeDuty(db, "bad"); | |
| 82 | expect(await runner.runDue(NOW + 200)).toHaveLength(1); | |
| 83 | }); | |
| 84 | ||
| 85 | test("a success resets the error count", async () => { | |
| 86 | let fail = true; | |
| 87 | const d = duty({ name: "flaky", trigger: { every: 1 }, handler: () => { if (fail) throw new Error("x"); return {}; } }); | |
| 88 | const { db, runner } = maintainer([d]); | |
| 89 | await runner.runDue(NOW); | |
| 90 | await runner.runDue(NOW + 10); | |
| 91 | fail = false; | |
| 92 | await runner.runDue(NOW + 20); | |
| 93 | expect(dutyState(db, "flaky").errors).toBe(0); | |
| 94 | }); | |
| 95 | ||
| 96 | test("the registry refuses a twin and a bad name; the shipped duties register", () => { | |
| 97 | const d = duty({ name: "one", trigger: { every: 1 }, handler: () => ({}) }); | |
| 98 | expect(() => makeRegistry([d, d])).toThrow("twice"); | |
| 99 | expect(() => makeRegistry([duty({ name: "Bad Name", trigger: { every: 1 }, handler: () => ({}) })])).toThrow(); | |
| 100 | expect(makeRegistry(dutyList).all().length).toBeGreaterThan(0); | |
| 101 | }); | |
| 102 | }); |
new above
offline