Public: anyone can read this; the crew can change it.
scripts/drills/run.ts <domain> walks the operator through a standby drill, posts its result, and says so when the forge never comes back #2179
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2174-scripts-drills-run-ts-domain-walks-the-o → main 2 commits fork 3 threads · 3 unresolved
4 files
changed +316
−1
README.md +1 −1
Diff not shown — a line 4,279 characters wide (limit 2,000). +1 −1
View filedocs/DOMAINS.md +15
| @@ -18,3 +18,18 @@ keeps a drill's verdict for any domain here (see the README). | ||
| 18 | 18 | | Identity | Sign-in fails; sessions and tokens already issued may still work. | nothing yet | 15 | Restore the sign-in provider's settings; sign in with the admin password. | none: Breaking sign-in locks the owner out of the forge that would fix it. | |
| 19 | 19 | | The model credential | Agents cannot start work; everything else runs. | nothing yet | 5 | Set a fresh credential in the settings. | expire-token | |
| 20 | 20 | | The runner | Checks and runs queue behind a lane that is not moving. | "wait behind one lane; the oldest since"; "is stuck: no heartbeat for" | 10 | Cancel the stuck run; restart the runner if the lane stays empty. | lose-runner | |
| 21 | ||
| 22 | ## Running a drill | |
| 23 | ||
| 24 | `scripts/drills/run.ts` walks you through one. It refuses a domain whose drill | |
| 25 | is none, prints the fault to cause, and waits for Enter. It then polls | |
| 26 | `/api/v1/health` and `/` once a second, keeping each distinct line the forge | |
| 27 | says while unhealthy; when the forge is healthy again it posts what it saw and | |
| 28 | the seconds it took to `POST /api/v1/drills` and prints the verdict. If the | |
| 29 | forge is still down after an hour it posts nothing, says so, and exits 0. A | |
| 30 | refused post exits non-zero with the forge's message. `FORGE_TOKEN` must be a | |
| 31 | site admin's token. | |
| 32 | ||
| 33 | ||
| 34 | FORGE_TOKEN=… bun scripts/drills/run.ts runner --forge https://forge.open-platform.sh | |
| 35 |
scripts/drills/run.ts added +182
| @@ -0,0 +1,182 @@ | ||
| 1 | /** | |
| 2 | * `bun scripts/drills/run.ts <domain> --forge <url>`: walks an operator | |
| 3 | * through a standby drill. It prints the fault to cause, waits for Enter, | |
| 4 | * polls `/api/v1/health` and `/` once a second keeping each distinct line the | |
| 5 | * forge says while unhealthy, and when the forge is healthy again posts what | |
| 6 | * it saw and how long it took to `POST /api/v1/drills` (`FORGE_TOKEN` from | |
| 7 | * the environment). If the forge never comes back within the ceiling it posts | |
| 8 | * nothing. Zero dependencies; it never shells out. | |
| 9 | */ | |
| 10 | ||
| 11 | import { DOMAINS } from "../../src/kernel/domains.ts"; | |
| 12 | ||
| 13 | export interface DrillRun { | |
| 14 | domain: string; | |
| 15 | forge: string; | |
| 16 | token: string; | |
| 17 | /** Wait for the operator's Enter. */ | |
| 18 | enter: () => Promise<void>; | |
| 19 | /** One output line. */ | |
| 20 | say: (line: string) => void; | |
| 21 | now?: () => number; | |
| 22 | sleep?: (ms: number) => Promise<void>; | |
| 23 | intervalMs?: number; | |
| 24 | ceilingMs?: number; | |
| 25 | } | |
| 26 | ||
| 27 | const SEEN_MAX = 8_000; | |
| 28 | const DEFAULT_CEILING_MS = 3_600_000; | |
| 29 | ||
| 30 | /** Every string in a JSON value, or the text's own lines when it is not JSON. */ | |
| 31 | function linesOf(text: string, contentType: string): string[] { | |
| 32 | if (contentType.includes("json")) { | |
| 33 | try { | |
| 34 | const out: string[] = []; | |
| 35 | const walk = (v: unknown): void => { | |
| 36 | if (typeof v === "string") out.push(v); | |
| 37 | else if (Array.isArray(v)) v.forEach(walk); | |
| 38 | else if (v && typeof v === "object") Object.values(v).forEach(walk); | |
| 39 | }; | |
| 40 | walk(JSON.parse(text)); | |
| 41 | return out; | |
| 42 | } catch { | |
| 43 | /* fall through to plain lines */ | |
| 44 | } | |
| 45 | } | |
| 46 | return text | |
| 47 | .replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ") | |
| 48 | .replace(/<[^>]*>/g, "\n") | |
| 49 | .replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">") | |
| 50 | .replace(/"/g, '"').replace(/�?39;/g, "'") | |
| 51 | .split("\n") | |
| 52 | .map((l) => l.trim()) | |
| 53 | .filter((l) => l !== ""); | |
| 54 | } | |
| 55 | ||
| 56 | interface Look { | |
| 57 | healthy: boolean; | |
| 58 | health: string[]; | |
| 59 | board: string[]; | |
| 60 | } | |
| 61 | ||
| 62 | async function look(forge: string, token: string): Promise<Look> { | |
| 63 | const get = async (path: string): Promise<{ ok: boolean; lines: string[]; json: unknown }> => { | |
| 64 | try { | |
| 65 | const res = await fetch(`${forge}${path}`, { | |
| 66 | headers: { authorization: `Bearer ${token}` }, | |
| 67 | redirect: "manual", | |
| 68 | signal: AbortSignal.timeout(5_000), | |
| 69 | }); | |
| 70 | const type = res.headers.get("content-type") ?? ""; | |
| 71 | const text = await res.text(); | |
| 72 | let json: unknown = null; | |
| 73 | if (type.includes("json")) { | |
| 74 | try { json = JSON.parse(text); } catch { /* not JSON */ } | |
| 75 | } | |
| 76 | return { ok: res.status < 400, lines: linesOf(text, type), json }; | |
| 77 | } catch { | |
| 78 | return { ok: false, lines: [], json: null }; | |
| 79 | } | |
| 80 | }; | |
| 81 | const [h, b] = await Promise.all([get("/api/v1/health"), get("/")]); | |
| 82 | const reported = (h.json as { ok?: unknown } | null)?.ok === true; | |
| 83 | return { healthy: h.ok && b.ok && reported, health: h.lines, board: b.lines }; | |
| 84 | } | |
| 85 | ||
| 86 | /** A line with its numbers blanked, so a clock or a counter is not a new line. */ | |
| 87 | const shape = (l: string): string => l.replace(/\d+/g, "#"); | |
| 88 | ||
| 89 | const cap = (lines: Iterable<string>): string => [...lines].join("\n").slice(0, SEEN_MAX); | |
| 90 | ||
| 91 | /** Runs the drill; resolves to the process exit code. */ | |
| 92 | export async function runDrill(run: DrillRun): Promise<number> { | |
| 93 | const domain = DOMAINS.find((d) => d.key === run.domain); | |
| 94 | if (!domain) { | |
| 95 | run.say(`unknown domain "${run.domain}"; one of: ${DOMAINS.map((d) => d.key).join(", ")}`); | |
| 96 | return 1; | |
| 97 | } | |
| 98 | if (domain.drill.name === null) { | |
| 99 | run.say(`${domain.key} has no drill: ${domain.drill.why}`); | |
| 100 | return 1; | |
| 101 | } | |
| 102 | const now = run.now ?? Date.now; | |
| 103 | const sleep = run.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms))); | |
| 104 | const intervalMs = run.intervalMs ?? 1_000; | |
| 105 | const ceilingMs = run.ceilingMs ?? DEFAULT_CEILING_MS; | |
| 106 | ||
| 107 | run.say(`Drill ${domain.drill.name} for ${domain.name}.`); | |
| 108 | run.say(`Cause this fault: ${domain.breaks}`); | |
| 109 | run.say(`The forge should be back within ${domain.backMinutes} minutes. Press Enter when you have caused it.`); | |
| 110 | await run.enter(); | |
| 111 | ||
| 112 | const started = now(); | |
| 113 | const health = new Map<string, string>(); | |
| 114 | const board = new Map<string, string>(); | |
| 115 | let sawDown = false; | |
| 116 | for (;;) { | |
| 117 | const seen = await look(run.forge, run.token); | |
| 118 | if (seen.healthy && sawDown) break; | |
| 119 | if (!seen.healthy) { | |
| 120 | sawDown = true; | |
| 121 | for (const l of seen.health) if (!health.has(shape(l))) { health.set(shape(l), l); run.say(`health: ${l}`); } | |
| 122 | for (const l of seen.board) if (!board.has(shape(l))) { board.set(shape(l), l); run.say(`board: ${l}`); } | |
| 123 | } | |
| 124 | if (now() - started >= ceilingMs) { | |
| 125 | const ceilingS = Math.round(ceilingMs / 1000); | |
| 126 | run.say(`The forge did not come back within the ceiling of ${ceilingS}s; nothing was posted.`); | |
| 127 | run.say(JSON.stringify({ domain: domain.key, backSeconds: null, posted: false, ceilingSeconds: ceilingS })); | |
| 128 | return 0; | |
| 129 | } | |
| 130 | await sleep(intervalMs); | |
| 131 | } | |
| 132 | ||
| 133 | const backSeconds = Math.max(0, Math.round((now() - started) / 1000)); | |
Distinguish sawDown=false at the ceiling: the message should say the forge was never seen unhealthy, not that it did not come back. |
||
| 134 | const body = { domain: domain.key, seen: { health: cap(health.values()), board: cap(board.values()) }, backSeconds }; | |
| 135 | let res: Response; | |
| 136 | try { | |
| 137 | res = await fetch(`${run.forge}/api/v1/drills`, { | |
| 138 | method: "POST", | |
| 139 | headers: { authorization: `Bearer ${run.token}`, "content-type": "application/json" }, | |
| 140 | body: JSON.stringify(body), | |
| 141 | redirect: "manual", | |
| 142 | signal: AbortSignal.timeout(10_000), | |
| 143 | }); | |
| 144 | } catch (e) { | |
| 145 | run.say(`could not post the result: ${e instanceof Error ? e.message : String(e)}`); | |
| 146 | return 1; | |
| 147 | } | |
| 148 | const text = await res.text(); | |
| 149 | let reply: Record<string, unknown> = {}; | |
| 150 | try { reply = JSON.parse(text) as Record<string, unknown>; } catch { /* not JSON */ } | |
| 151 | if (!res.ok) { | |
| 152 | const msg = typeof reply.message === "string" ? reply.message : typeof reply.error === "string" ? reply.error : text.slice(0, 200); | |
| 153 | run.say(`the forge refused the result (${res.status}): ${msg}`); | |
| 154 | return 1; | |
| 155 | } | |
| 156 | const verdict = (reply.verdict ?? reply) as { green?: unknown; why?: unknown }; | |
| 157 | run.say(`${verdict.green === true ? "GREEN" : "RED"}: ${String(verdict.why ?? "")}`); | |
| 158 | run.say(JSON.stringify({ domain: domain.key, backSeconds, posted: true, result: reply })); | |
| 159 | return 0; | |
| 160 | } | |
| 161 | ||
| 162 | if (import.meta.main) { | |
| 163 | const args = process.argv.slice(2); | |
| 164 | const at = args.indexOf("--forge"); | |
| 165 | const forge = at >= 0 ? args[at + 1] : undefined; | |
| 166 | const domain = args.find((a, i) => !a.startsWith("--") && i !== at + 1); | |
| 167 | const token = process.env.FORGE_TOKEN ?? ""; | |
| 168 | if (!domain || !forge || !/^https?:\/\//.test(forge) || token === "") { | |
| 169 | console.error("usage: FORGE_TOKEN=… bun scripts/drills/run.ts <domain> --forge <url>"); | |
| 170 | process.exit(2); | |
| 171 | } | |
| 172 | const code = await runDrill({ | |
| 173 | domain, | |
| 174 | forge: forge.replace(/\/+$/, ""), | |
| 175 | token, | |
| 176 | say: (l) => console.log(l), | |
| 177 | enter: async () => { | |
| 178 | for await (const _ of console) break; | |
| 179 | }, | |
| 180 | }); | |
| 181 | process.exit(code); | |
| 182 | } |
test/drill-run.test.ts added +118
| @@ -0,0 +1,118 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { runDrill } from "../scripts/drills/run.ts"; | |
| 3 | ||
| 4 | let server: ReturnType<typeof Bun.serve> | undefined; | |
| 5 | afterEach(() => server?.stop(true)); | |
| 6 | ||
| 7 | interface Stub { | |
| 8 | down: boolean; | |
| 9 | posts: { auth: string | null; body: unknown }[]; | |
| 10 | refuse: boolean; | |
| 11 | /** `/` answers a sign-in redirect while up. */ | |
| 12 | redirect?: boolean; | |
| 13 | hits?: number; | |
| 14 | } | |
| 15 | ||
| 16 | function stub(state: Stub): string { | |
| 17 | server = Bun.serve({ | |
| 18 | port: 0, | |
| 19 | hostname: "127.0.0.1", | |
| 20 | async fetch(req) { | |
| 21 | const path = new URL(req.url).pathname; | |
| 22 | if (path === "/api/v1/health") | |
| 23 | return state.down | |
| 24 | ? Response.json({ ok: false, note: "Disk is nearly full: 97%", tick: `up ${(state.hits = (state.hits ?? 0) + 1)}s` }, { status: 503 }) | |
| 25 | : Response.json({ ok: true }); | |
| 26 | if (path === "/") | |
| 27 | return state.down | |
| 28 | ? new Response("<p>The runner is stuck</p>", { status: 503, headers: { "content-type": "text/html" } }) | |
| 29 | : state.redirect | |
| 30 | ? new Response(null, { status: 302, headers: { location: "/sign-in" } }) | |
| 31 | : new Response("<p>fine</p>", { headers: { "content-type": "text/html" } }); | |
| 32 | if (path === "/api/v1/drills") { | |
| 33 | state.posts.push({ auth: req.headers.get("authorization"), body: await req.json() }); | |
| 34 | return state.refuse | |
| 35 | ? Response.json({ error: "forbidden", message: "admins only" }, { status: 403 }) | |
| 36 | : Response.json({ id: 1, green: true, why: "every phrase seen" }, { status: 201 }); | |
| 37 | } | |
| 38 | return new Response("no", { status: 404 }); | |
| 39 | }, | |
| 40 | }); | |
| 41 | return `http://127.0.0.1:${server.port}`; | |
| 42 | } | |
outdated
|
||
| 43 | ||
| 44 | function harness(state: Stub, extra: { ceilingMs?: number } = {}) { | |
| 45 | const lines: string[] = []; | |
| 46 | let t = 0; | |
| 47 | let sleeps = 0; | |
| 48 | const run = (domain: string, forge: string) => | |
| 49 | runDrill({ | |
| 50 | domain, | |
| 51 | forge, | |
| 52 | token: "tok", | |
| 53 | say: (l) => lines.push(l), | |
| 54 | enter: async () => { state.down = true; }, | |
| 55 | now: () => t, | |
| 56 | sleep: async (ms) => { | |
| 57 | t += ms; | |
| 58 | sleeps++; | |
| 59 | if (sleeps === 3 && !extra.ceilingMs) state.down = false; | |
| 60 | }, | |
| 61 | intervalMs: 1000, | |
| 62 | ceilingMs: extra.ceilingMs ?? 60_000, | |
| 63 | }); | |
| 64 | return { lines, run }; | |
| 65 | } | |
| 66 | ||
| 67 | test("goes down and comes back: posts what it saw and prints the verdict", async () => { | |
| 68 | const state: Stub = { down: false, posts: [], refuse: false }; | |
| 69 | const h = harness(state); | |
| 70 | const code = await h.run("runner", stub(state)); | |
| 71 | expect(code).toBe(0); | |
| 72 | expect(state.posts).toHaveLength(1); | |
| 73 | expect(state.posts[0]!.auth).toBe("Bearer tok"); | |
| 74 | const body = state.posts[0]!.body as { domain: string; seen: { health: string; board: string }; backSeconds: number }; | |
| 75 | expect(body.domain).toBe("runner"); | |
| 76 | expect(body.seen.health).toContain("Disk is nearly full: 97%"); | |
| 77 | expect(body.seen.board).toContain("The runner is stuck"); | |
| 78 | expect(body.backSeconds).toBe(3); | |
| 79 | expect(h.lines.filter((l) => l.includes("The runner is stuck"))).toHaveLength(1); | |
| 80 | expect(h.lines).toContain("GREEN: every phrase seen"); | |
| 81 | }); | |
| 82 | ||
| 83 | test("stays down past the ceiling: posts nothing, exits 0, says so", async () => { | |
| 84 | const state: Stub = { down: false, posts: [], refuse: false }; | |
| 85 | const h = harness(state, { ceilingMs: 5_000 }); | |
| 86 | const code = await h.run("runner", stub(state)); | |
| 87 | expect(code).toBe(0); | |
| 88 | expect(state.posts).toHaveLength(0); | |
| 89 | expect(h.lines.some((l) => l.includes("did not come back within the ceiling of 5s"))).toBe(true); | |
| 90 | const last = JSON.parse(h.lines.at(-1)!); | |
| 91 | expect(last).toEqual({ domain: "runner", backSeconds: null, posted: false, ceilingSeconds: 5 }); | |
| 92 | }); | |
| 93 | ||
| 94 | test("a refused post prints the forge's message and exits non-zero", async () => { | |
| 95 | const state: Stub = { down: false, posts: [], refuse: true }; | |
| 96 | const h = harness(state); | |
| 97 | const code = await h.run("runner", stub(state)); | |
| 98 | expect(code).toBe(1); | |
| 99 | expect(h.lines.some((l) => l.includes("403") && l.includes("admins only"))).toBe(true); | |
| 100 | }); | |
| 101 | ||
| 102 | test("an unknown domain and a domain with no drill are refused before anything is asked", async () => { | |
| 103 | const state: Stub = { down: false, posts: [], refuse: false }; | |
| 104 | const h = harness(state); | |
| 105 | expect(await h.run("nope", stub(state))).toBe(1); | |
| 106 | expect(await h.run("state", stub(state))).toBe(1); | |
| 107 | expect(h.lines.some((l) => l.includes("no drill"))).toBe(true); | |
| 108 | expect(state.down).toBe(false); | |
| 109 | expect(state.posts).toHaveLength(0); | |
| 110 | }); | |
| 111 | ||
| 112 | test("a sign-in redirect from / is up, and a changing counter is one line", async () => { | |
| 113 | const state: Stub = { down: false, posts: [], refuse: false, redirect: true }; | |
| 114 | const h = harness(state); | |
| 115 | expect(await h.run("runner", stub(state))).toBe(0); | |
| 116 | expect(state.posts).toHaveLength(1); | |
| 117 | expect(h.lines.filter((l) => l.startsWith("health: up "))).toHaveLength(1); | |
| 118 | }); |
new above
offline
With redirect: manual a 3xx on
/makesokfalse, so the drill would wait the full ceiling. Treat a 3xx as up, or follow redirects.