Public: anyone can read this; the crew can change it.
The fault hunt files one issue for each cause degraded more than ten minutes in its day, quoting the cause's sentence #1709
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1297-the-fault-hunt-files-one-issue-for-each → main 2 commits fork
2 files
changed +122
−0
src/crew/sentinel.ts +44
| @@ -30,6 +30,7 @@ import { migrations } from "../core/schema.ts"; | ||
| 30 | 30 | import { getUserByName, type User } from "../core/users.ts"; |
| 31 | 31 | import { migrate } from "../kernel/migrate.ts"; |
| 32 | 32 | import { createLog, recentErrors, redactSecrets } from "../kernel/log.ts"; |
| 33 | import { spans } from "../kernel/degraded_history.ts"; | |
| 33 | 34 | import { QUEUE_WAIT_MAX_MS } from "../kernel/vitals.ts"; |
| 34 | 35 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 35 | 36 | |
| @@ -42,6 +43,8 @@ export const HUNT_EVERY_MS = HOUR_MS; | ||
| 42 | 43 | export const SIGNAL_GAP_MS = 10 * 60_000; |
| 43 | 44 | /** A setting away from its default this long is a finding. */ |
| 44 | 45 | export const SETTING_AWAY_MS = DAY_MS; |
| 46 | /** A cause degraded longer than this in its day is a finding. */ | |
| 47 | export const DEGRADED_DAY_MS = 10 * 60_000; | |
| 45 | 48 | /** How many alike make a streak. */ |
| 46 | 49 | export const STREAK = 3; |
| 47 | 50 | /** The run summaries read per repository. */ |
| @@ -342,6 +345,46 @@ export function repeatedErrors(now: number): Finding[] { | ||
| 342 | 345 | return out; |
| 343 | 346 | } |
| 344 | 347 | |
| 348 | /** | |
| 349 | * Each cause whose spans total more than ten minutes in the day ending at | |
| 350 | * `now`, quoting its sentence. Shorter totals stay in the history. | |
| 351 | */ | |
| 352 | export function degradedCauses(db: Database, now: number): Finding[] { | |
| 353 | const from = now - DAY_MS; | |
| 354 | const byCause = new Map<string, { sentence: string; href: string; first: number; ms: number }>(); | |
| 355 | for (const s of spans(db, from, now)) { | |
| 356 | const start = Math.max(s.openedAt, from); | |
| 357 | const held = Math.min(s.closedAt ?? now, now) - start; | |
| 358 | const seen = byCause.get(s.cause); | |
| 359 | // Spans run oldest first: the first sets the open time, the last the sentence. | |
| 360 | byCause.set(s.cause, { | |
| 361 | sentence: s.sentence, | |
| 362 | href: s.href, | |
| 363 | first: seen?.first ?? start, | |
| 364 | ms: (seen?.ms ?? 0) + Math.max(0, held), | |
| 365 | }); | |
| 366 | } | |
| 367 | const out: Finding[] = []; | |
| 368 | for (const [cause, d] of byCause) { | |
| 369 | if (d.ms <= DEGRADED_DAY_MS) continue; | |
| 370 | const minutes = Math.round(d.ms / 60_000); | |
| 371 | out.push({ | |
| 372 | key: `degraded:${keyPart(cause)}`, | |
| 373 | title: `${cause} was degraded ${minutes} minutes today: ${clip(oneLine(d.sentence), 120)}`, | |
| 374 | evidence: [ | |
| 375 | `cause: ${cause}`, | |
| 376 | `sentence: ${oneLine(d.sentence)}`, | |
| 377 | `first open: ${new Date(d.first).toISOString()}`, | |
| 378 | `total: ${minutes} minutes in the day`, | |
| 379 | `href: ${d.href}`, | |
| 380 | ], | |
| 381 | pillar: "P0 Trust: the forge says when it is not well, and is well again", | |
| 382 | lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`, | |
| 383 | }); | |
| 384 | } | |
| 385 | return out; | |
| 386 | } | |
| 387 | ||
| 345 | 388 | /** Everything the sentinel reads, in one pass. */ |
| 346 | 389 | export function findFaults(db: Database, now = Date.now()): Finding[] { |
| 347 | 390 | const found = [ |
| @@ -351,6 +394,7 @@ export function findFaults(db: Database, now = Date.now()): Finding[] { | ||
| 351 | 394 | ...ledgerStreaks(db, now), |
| 352 | 395 | ...runStreaks(db), |
| 353 | 396 | ...repeatedErrors(now), |
| 397 | ...degradedCauses(db, now), | |
| 354 | 398 | ]; |
| 355 | 399 | return found.map((f) => ({ |
| 356 | 400 | ...f, |
test/sentinel.test.ts +78
| @@ -18,6 +18,7 @@ import { getRepoById, type Repo } from "../src/core/repos.ts"; | ||
| 18 | 18 | import { createRun } from "../src/core/runs.ts"; |
| 19 | 19 | import { createUser, type User } from "../src/core/users.ts"; |
| 20 | 20 | import { |
| 21 | degradedCauses, | |
| 21 | 22 | findFaults, |
| 22 | 23 | hunt, |
| 23 | 24 | sentinelLine, |
| @@ -346,3 +347,80 @@ describe("the signals inside the hour", () => { | ||
| 346 | 347 | expect(withRepo(() => sentinelTick(app, lastHunt() + 20 * 60_000))).toBeNull(); |
| 347 | 348 | }); |
| 348 | 349 | }); |
| 350 | ||
| 351 | // ---- degraded causes (#1297) ---- | |
| 352 | ||
| 353 | let dapp: App; | |
| 354 | let drepo: Repo; | |
| 355 | const D_MIN = 60_000; | |
| 356 | const D_NIGHT = Date.parse("2026-10-13T03:00:00.000Z"); | |
| 357 | const D_DAY = 24 * 60 * D_MIN; | |
| 358 | ||
| 359 | function plantSpan(cause: string, openedAt: number, minutes: number): void { | |
| 360 | dapp.db | |
| 361 | .query( | |
| 362 | "INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES (?1, ?2, '/admin/queue', ?3, ?4)", | |
| 363 | ) | |
| 364 | .run(cause, `The ${cause} is backed up.`, openedAt, openedAt + minutes * D_MIN); | |
| 365 | } | |
| 366 | const openIssues = () => listIssues(dapp.db, drepo.id, { state: "open" } as never); | |
| 367 | ||
| 368 | ||
| 369 | describe("degraded causes", () => { | |
| 370 | let ddir: string; | |
| 371 | beforeAll(async () => { | |
| 372 | ddir = await mkdtemp(join(tmpdir(), "forge-sentinel-degraded-")); | |
| 373 | dapp = createApp({ dataDir: ddir, port: 0 }); | |
| 374 | const a = await createUser(dapp.db, { | |
| 375 | trusted: true, | |
| 376 | username: "alice", | |
| 377 | email: "[email protected]", | |
| 378 | password: "a-good-passphrase", | |
| 379 | adminIfFirst: true, | |
| 380 | }); | |
| 381 | if (!a.ok) throw new Error(a.error); | |
| 382 | const made = await createRepo(dapp, a.value, { name: "forge", visibility: "private" }); | |
| 383 | if (!made.ok) throw new Error(String(made.error)); | |
| 384 | drepo = made.value; | |
| 385 | }); | |
| 386 | afterAll(async () => { | |
| 387 | dapp.db.close(); | |
| 388 | await rm(ddir, { recursive: true, force: true }); | |
| 389 | }); | |
| 390 | ||
| 391 | test("a 4-minute span files nothing", () => { | |
| 392 | plantSpan("short", D_NIGHT - 2 * 3_600_000, 4); | |
| 393 | expect(degradedCauses(dapp.db, D_NIGHT)).toEqual([]); | |
| 394 | expect(hunt(dapp, "alice/forge", D_NIGHT).filed).toBe(0); | |
| 395 | }); | |
| 396 | ||
| 397 | test("a 25-minute span files one issue; the next night comments on it", () => { | |
| 398 | plantSpan("queue", D_NIGHT - 2 * 3_600_000, 25); | |
| 399 | const found = degradedCauses(dapp.db, D_NIGHT); | |
| 400 | expect(found.map((f) => f.key)).toEqual(["degraded:queue"]); | |
| 401 | const text = found[0]!.evidence.join("\n"); | |
| 402 | expect(text).toContain("The queue is backed up."); | |
| 403 | expect(text).toContain("25 minutes"); | |
| 404 | expect(text).toContain("/admin/queue"); | |
| 405 | expect(found[0]!.pillar).toStartWith("P0 Trust"); | |
| 406 | ||
| 407 | expect(hunt(dapp, "alice/forge", D_NIGHT).filed).toBe(1); | |
| 408 | expect(hunt(dapp, "alice/forge", D_NIGHT + 30 * D_MIN).filed).toBe(0); | |
| 409 | expect(openIssues().length).toBe(1); | |
| 410 | ||
| 411 | plantSpan("queue", D_NIGHT + D_DAY - 2 * 3_600_000, 25); | |
| 412 | const next = hunt(dapp, "alice/forge", D_NIGHT + D_DAY); | |
| 413 | expect(next.filed).toBe(0); | |
| 414 | expect(next.commented).toBe(1); | |
| 415 | const open = openIssues(); | |
| 416 | expect(open.length).toBe(1); | |
| 417 | expect(listComments(dapp.db, open[0]!.id).length).toBe(1); | |
| 418 | }); | |
| 419 | ||
| 420 | test("spans that add up past ten minutes are one finding", () => { | |
| 421 | const day = D_NIGHT + 3 * D_DAY; | |
| 422 | plantSpan("disk", day - 5 * 3_600_000, 6); | |
| 423 | plantSpan("disk", day - 3 * 3_600_000, 6); | |
| 424 | expect(degradedCauses(dapp.db, day).map((f) => f.key)).toEqual(["degraded:disk"]); | |
| 425 | }); | |
| 426 | }); |
new above
offline