Public: anyone can read this; the crew can change it.
The forge's admin hears once of a domain drill that is red, stale or never run, with the exact command to run it #5369
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/5368-the-forge-s-admin-hears-once-of-a-domain → main 2 commits fork 2 threads · 2 unresolved
2 files
changed +135
−2
src/crew/notify.ts +32 −1
| @@ -34,6 +34,9 @@ import { err, ok, type Result } from "../kernel/result.ts"; | ||
| 34 | 34 | import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts"; |
| 35 | 35 | import { markRollbackPaged, rollbackNeed, rolledBack } from "../release/launch.ts"; |
| 36 | 36 | import { drillNeed, firstSeen, lastVerdict } from "../snapshot/verdict.ts"; |
| 37 | import { domainDrillNeeds } from "../kernel/domain_drill.ts"; | |
| 38 | import { DOMAINS } from "../kernel/domains.ts"; | |
| 39 | import { newestDrills } from "../core/domain_drills.ts"; | |
| 37 | 40 | import { health } from "../http/health.ts"; |
| 38 | 41 | import { forgeVitals } from "../actions/health.ts"; |
| 39 | 42 | import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch } from "./incidents.ts"; |
| @@ -177,6 +180,31 @@ export function drillLines( | ||
| 177 | 180 | return [{ id, line: need.sentence, items: [id] }]; |
| 178 | 181 | } |
| 179 | 182 | |
| 183 | /** | |
| 184 | * The forge's admin hears of a domain drill that is red, stale or never run, | |
| 185 | * with the command that runs it. The needs are the vitals' own | |
| 186 | * (`domainDrillNeeds`); the id is the cause and its `since`, so a repeat tick | |
| 187 | * is not news and a later red is. | |
| 188 | */ | |
| 189 | export function domainDrillLines( | |
| 190 | app: App, | |
| 191 | user: User, | |
| 192 | now = Date.now(), | |
| 193 | forgeUrl = "http://localhost", | |
| 194 | ): { id: string; line: string; items: readonly string[] }[] { | |
| 195 | if (!user.isAdmin || user.kind !== "user") return []; | |
| 196 | return domainDrillNeeds( | |
| 197 | newestDrills(app.db), | |
| 198 | now, | |
| 199 | firstSeen(app.config.dataDir), | |
| 200 | forgeUrl, | |
| 201 | DOMAINS, | |
| 202 | ).map((need) => { | |
| 203 | const id = `${need.cause}@${need.since}`; | |
| 204 | return { id, line: need.sentence, items: [id] }; | |
| 205 | }); | |
| 206 | } | |
| 207 | ||
| 180 | 208 | /** The admin hears of a snapshot that did not land off the box, or a stale mirror, once per cause. */ |
| 181 | 209 | export function mirrorLines( |
| 182 | 210 | app: App, |
| @@ -237,9 +265,11 @@ export interface NotifyOptions { | ||
| 237 | 265 | causes?: () => readonly { cause: string; sentence: string }[]; |
| 238 | 266 | } |
| 239 | 267 | |
| 268 | /** Long enough for a domain drill line to keep every command it names. */ | |
| 269 | const LINE_MAX = 600; | |
| 240 | 270 | const oneLine = (s: string): string => |
| 241 | 271 | // eslint-disable-next-line no-control-regex |
| 242 | s.replace(/[\u0000-\u001f\u007f]+/g, " ").slice(0, 160); | |
| 272 | s.replace(/[\u0000-\u001f\u007f]+/g, " ").slice(0, LINE_MAX); | |
| 243 | 273 | |
| 244 | 274 | export function startNotifier(app: App, options: NotifyOptions): Notifier { |
| 245 | 275 | const send = options.send ?? post; |
| @@ -391,6 +421,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier { | ||
| 391 | 421 | ...landedLines(app.db, user, now()), |
| 392 | 422 | ...mailCeilingLines(app, user, now()), |
outdated
No test exercises this wiring: delete this line and the suite stays green. Add a |
||
| 393 | 423 | ...drillLines(app, user, now()), |
| 424 | ...domainDrillLines(app, user, now(), options.forgeUrl), | |
| 394 | 425 | ...mirrorLines(app, user, now()), |
| 395 | 426 | ...redLines(app, user, now()), |
| 396 | 427 | ]; |
test/notify-causes.test.ts +103 −1
| @@ -8,9 +8,13 @@ import { mkdtemp, rm } from "node:fs/promises"; | ||
| 8 | 8 | import { tmpdir } from "node:os"; |
| 9 | 9 | import { join } from "node:path"; |
| 10 | 10 | import { createApp, type App } from "../src/app.ts"; |
| 11 | import { seedFirstSeen } from "../src/snapshot/verdict.ts"; | |
| 12 | import { DOMAINS } from "../src/kernel/domains.ts"; | |
| 13 | import { drillCommand } from "../src/kernel/domain_drill.ts"; | |
| 14 | import { recordDrill } from "../src/core/domain_drills.ts"; | |
| 11 | 15 | import { createUser } from "../src/core/users.ts"; |
| 12 | 16 | import { owned } from "../src/crew/incidents.ts"; |
| 13 | import { setNotifyUrl, startNotifier, type Notifier } from "../src/crew/notify.ts"; | |
| 17 | import { domainDrillLines, setNotifyUrl, startNotifier, type Notifier } from "../src/crew/notify.ts"; | |
| 14 | 18 | |
| 15 | 19 | const MIN = 60_000; |
| 16 | 20 | |
| @@ -115,3 +119,101 @@ describe("the cause pager", () => { | ||
| 115 | 119 | expect(posts).toHaveLength(0); |
| 116 | 120 | }); |
| 117 | 121 | }); |
| 122 | ||
| 123 | describe("domain drill lines", () => { | |
| 124 | const URL_ = "https://forge.example.test"; | |
| 125 | const DAY = 86_400_000; | |
| 126 | const t0 = Date.parse("2026-01-01T00:00:00Z"); | |
| 127 | let dataDir: string; | |
| 128 | let app: App; | |
| 129 | beforeEach(async () => { | |
| 130 | dataDir = await mkdtemp(join(tmpdir(), "forge-domain-drill-")); | |
| 131 | app = createApp({ dataDir, port: 0 }); | |
| 132 | seedFirstSeen(dataDir, t0); | |
| 133 | }); | |
| 134 | afterEach(async () => { | |
| 135 | await rm(dataDir, { recursive: true, force: true }); | |
| 136 | }); | |
| 137 | const person = async (name: string, adminIfFirst: boolean) => { | |
| 138 | const made = await createUser(app.db, { | |
| 139 | origin: "seed", | |
| 140 | trusted: true, | |
| 141 | username: name, | |
| 142 | email: `${name}@forge.local`, | |
| 143 | password: "a-good-passphrase", | |
| 144 | adminIfFirst, | |
| 145 | }); | |
| 146 | if (!made.ok) throw new Error(made.error); | |
| 147 | return made.value; | |
| 148 | }; | |
| 149 | ||
| 150 | test("an admin hears drill:never naming every command, once per need", async () => { | |
| 151 | const admin = await person("alice", true); | |
| 152 | const now = t0 + 9 * DAY; | |
| 153 | const lines = domainDrillLines(app, admin, now, URL_); | |
| 154 | expect(lines).toHaveLength(1); | |
| 155 | expect(lines[0]!.id.startsWith("drill:never")).toBe(true); | |
| 156 | for (const d of DOMAINS) | |
| 157 | if (d.drill.name !== null) expect(lines[0]!.line).toContain(drillCommand(d.key, URL_)); | |
| 158 | // A repeat tick carries the same id: nothing new to send. | |
| 159 | expect(domainDrillLines(app, admin, now + 60_000, URL_)[0]!.id).toBe(lines[0]!.id); | |
| 160 | }); | |
| 161 | ||
| 162 | test("a later red is a new id", async () => { | |
| 163 | const admin = await person("alice", true); | |
| 164 | recordDrill(app.db, admin.id, "compute", { health: "", board: "" }, 1, { green: false, why: 'the forge never said "x"' }); | |
| 165 | const first = domainDrillLines(app, admin, t0 + DAY, URL_).find((l) => l.id.startsWith("drill:compute")); | |
| 166 | expect(first).toBeDefined(); | |
| 167 | app.db.query("UPDATE domain_drill SET at = '2099-01-01T00:00:00.000Z'").run(); | |
| 168 | const next = domainDrillLines(app, admin, t0 + DAY, URL_).find((l) => l.id.startsWith("drill:compute")); | |
| 169 | expect(next!.id).not.toBe(first!.id); | |
| 170 | }); | |
| 171 | ||
| 172 | test("through the notifier: one post, none on a repeat tick, one on a later red", async () => { | |
| 173 | const admin = await person("alice", true); | |
| 174 | setNotifyUrl(app.db, admin.id, "https://ntfy.example.test/root-topic"); | |
| 175 | let clock = t0 + 7 * DAY; | |
| 176 | const posts: string[] = []; | |
| 177 | const notifier = startNotifier(app, { | |
| 178 | forgeUrl: URL_, | |
| 179 | now: () => clock, | |
| 180 | drainingSince: () => null, | |
| 181 | causes: () => [], | |
| 182 | send: async (_u, body) => { | |
| 183 | posts.push(body); | |
| 184 | return true; | |
| 185 | }, | |
| 186 | }); | |
| 187 | try { | |
| 188 | await notifier.settled(); | |
| 189 | clock = t0 + 9 * DAY; | |
| 190 | app.bus.append({ kind: "drill.finished", payload: {} }); | |
| 191 | await new Promise((r) => setTimeout(r, 20)); | |
| 192 | await notifier.settled(); | |
| 193 | const drills = () => posts.filter((p) => p.includes("scripts/drills/run.ts")); | |
| 194 | expect(drills()).toHaveLength(1); | |
| 195 | for (const d of DOMAINS) | |
| 196 | if (d.drill.name !== null) expect(drills()[0]).toContain(drillCommand(d.key, URL_)); | |
| 197 | clock += MIN; | |
| 198 | app.bus.append({ kind: "drill.finished", payload: {} }); | |
| 199 | await new Promise((r) => setTimeout(r, 20)); | |
| 200 | await notifier.settled(); | |
| 201 | expect(drills()).toHaveLength(1); | |
| 202 | recordDrill(app.db, admin.id, "compute", { health: "", board: "" }, 1, { green: false, why: 'the forge never said "x"' }); | |
| 203 | clock += MIN; | |
| 204 | app.bus.append({ kind: "drill.finished", payload: {} }); | |
| 205 | await new Promise((r) => setTimeout(r, 20)); | |
| 206 | await notifier.settled(); | |
| 207 | expect(drills()).toHaveLength(2); | |
| 208 | expect(drills()[1]).toContain(drillCommand("compute", URL_)); | |
| 209 | } finally { | |
| 210 | await notifier.close(); | |
| 211 | } | |
| 212 | }); | |
| 213 | ||
| 214 | test("a non-admin hears nothing", async () => { | |
| 215 | await person("alice", true); | |
| 216 | const bob = await person("bob", false); | |
| 217 | expect(domainDrillLines(app, bob, t0 + 9 * DAY, URL_)).toEqual([]); | |
| 218 | }); | |
| 219 | }); |
new above
offline
This cap now applies to every pushed line. That is fine for ntfy, but pin it with a test, or only lengthen the drill lines.