Public: anyone can read this; the crew can change it.
The sentinel's memory-growth finding is one open issue that gets a comment each day, not a new issue and commit per day #14525
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14491-the-sentinel-s-memory-growth-finding-is → main 1 commit fork
3 files
changed +112
−11
src/crew/sentinel.ts +28 −8
| @@ -69,6 +69,12 @@ export const SIGNAL_GAP_MS = 10 * 60_000; | ||
| 69 | 69 | export const SENTINEL_PREFIX = "Sentinel: "; |
| 70 | 70 | /** A cause degraded longer than this in its day is a finding. */ |
| 71 | 71 | export const DEGRADED_DAY_MS = 10 * 60_000; |
| 72 | /** A degraded cause must stay out of the history this long (two days) before its issue closes. */ | |
| 73 | export const HEALTHY_DAYS = 2; | |
| 74 | /** | |
| 75 | * A tracked finding is told new figures at most this often: one comment a day, not one per hunt. | |
| 76 | */ | |
| 77 | export const COMMENT_EVERY_MS = DAY_MS; | |
| 72 | 78 | /** How many alike make a streak. */ |
| 73 | 79 | export const STREAK = 3; |
| 74 | 80 | /** The run summaries read per repository. */ |
| @@ -1003,9 +1009,15 @@ function settledBefore(db: Database, actor: User, f: Finding, now: number): stri | ||
| 1003 | 1009 | ].join("\n"); |
| 1004 | 1010 | } |
| 1005 | 1011 | |
| 1012 | /** The issue's `cause: …` evidence line quoted for the closing comment, or nothing. */ | |
| 1013 | function causeLine(body: string): string { | |
| 1014 | const line = body.split("\n").find((l) => l.startsWith("- cause: ")); | |
| 1015 | return line === undefined ? "" : `> ${line.slice(2)}\n\n`; | |
| 1016 | } | |
| 1017 | ||
| 1006 | 1018 | /** |
| 1007 | 1019 | * A finding backed by an incident whose cause has stayed out of the degraded |
| 1008 | * history for a day is done: the sentinel says since when and closes it. Not a | |
| 1020 | * history for two days is done: the sentinel says since when and closes it. Not a | |
| 1009 | 1021 | * setting (its own rule), and not one a person has spoken on — they decide. |
| 1010 | 1022 | * `active` are the keys found now; a finding that is there is not held. |
| 1011 | 1023 | */ |
| @@ -1028,7 +1040,7 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string | ||
| 1028 | 1040 | if (s.closedAt === null) live = true; |
| 1029 | 1041 | else since = Math.max(since, s.closedAt); |
| 1030 | 1042 | } |
| 1031 | if (live || since === -Infinity || now - since < DAY_MS) continue; | |
| 1043 | if (live || since === -Infinity || now - since < HEALTHY_DAYS * DAY_MS) continue; | |
| 1032 | 1044 | if (personSpoke(app.db, t.issue_id)) continue; |
| 1033 | 1045 | const issue = getIssueById(app.db, t.issue_id); |
| 1034 | 1046 | if (issue === undefined || issue.repoId !== repo.id) continue; |
| @@ -1037,7 +1049,7 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string | ||
| 1037 | 1049 | actor, |
| 1038 | 1050 | repo, |
| 1039 | 1051 | issue, |
| 1040 | `Held since ${new Date(since).toISOString()}: no new incident on this cause for over a day. Closing as done; if it comes back it is filed again.`, | |
| 1052 | `Held since ${new Date(since).toISOString()}: no new incident on this cause for ${HEALTHY_DAYS} days running. ${causeLine(issue.body)}Closing as done; if it comes back it is filed again.`, | |
| 1041 | 1053 | ); |
| 1042 | 1054 | if (!said.ok) { |
| 1043 | 1055 | log.warn("sentinel could not comment", { key: t.key, error: String(said.error) }); |
| @@ -1081,15 +1093,19 @@ export function hunt( | ||
| 1081 | 1093 | const evidence = f.evidence.join("\n"); |
| 1082 | 1094 | const tracked = app.db |
| 1083 | 1095 | .query< |
| 1084 | { issue_id: string; evidence: string }, | |
| 1096 | { issue_id: string; evidence: string; commented_at: string | null; first_seen: string }, | |
| 1085 | 1097 | [string] |
| 1086 | >(`SELECT t.issue_id, t.evidence FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 1098 | >(`SELECT t.issue_id, t.evidence, t.commented_at, t.first_seen FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 1087 | 1099 | WHERE t.key = ?1 AND i.state = 'open' LIMIT 1`) |
| 1088 | 1100 | .get(f.key); |
| 1089 | 1101 | if (tracked !== null) { |
| 1090 | 1102 | // The stored evidence moves only once the comment is made: a |
| 1091 | 1103 | // comment that failed is tried again by the next hunt. |
| 1092 | if (tracked.evidence !== evidence) { | |
| 1104 | // Matched on the key (duty and subject), never the title: the title carries changing | |
| 1105 | // numbers. Told at most once a day, so a cause degraded all day is one issue with a | |
| 1106 | // comment a day. | |
| 1107 | const toldAt = Date.parse(tracked.commented_at ?? tracked.first_seen); | |
| 1108 | if (tracked.evidence !== evidence && !(f.key.startsWith("degraded:") && now - toldAt < COMMENT_EVERY_MS)) { | |
| 1093 | 1109 | const issue = getIssueById(app.db, tracked.issue_id); |
| 1094 | 1110 | if (issue === undefined) continue; |
| 1095 | 1111 | const said = addComment(app, actor, repo, issue, `Still there at ${at}, evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`); |
| @@ -1098,10 +1114,14 @@ export function hunt( | ||
| 1098 | 1114 | continue; |
| 1099 | 1115 | } |
| 1100 | 1116 | commented++; |
| 1117 | app.db | |
| 1118 | .query("UPDATE sentinel_finding SET last_seen = ?1, evidence = ?2, commented_at = ?1 WHERE key = ?3 AND issue_id = ?4") | |
| 1119 | .run(at, evidence, f.key, tracked.issue_id); | |
| 1120 | continue; | |
| 1101 | 1121 | } |
| 1102 | 1122 | app.db |
| 1103 | .query("UPDATE sentinel_finding SET last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4") | |
| 1104 | .run(at, evidence, f.key, tracked.issue_id); | |
| 1123 | .query("UPDATE sentinel_finding SET last_seen = ?1 WHERE key = ?2 AND issue_id = ?3") | |
| 1124 | .run(at, f.key, tracked.issue_id); | |
| 1105 | 1125 | continue; |
| 1106 | 1126 | } |
| 1107 | 1127 | // A shared red that passed again on its own is no work: the day's total does not file it again. |
test/maintain-dedupe.test.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * A degraded finding is one issue however many days it lasts (#14491): the same cause while its | |
| 3 | * issue is open is a comment with the new figures, and it closes after two healthy days, quoting | |
| 4 | * its cause line. Matched on the cause's key, never the title, which carries changing numbers. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import "./helpers/globals.ts"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createAgentRow } from "../src/core/agents.ts"; | |
| 13 | import { createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { listComments } from "../src/core/issues.ts"; | |
| 16 | import { hunt } from "../src/crew/sentinel.ts"; | |
| 17 | ||
| 18 | const HOUR = 3_600_000; | |
| 19 | const DAY = 24 * HOUR; | |
| 20 | const MIN = 60_000; | |
| 21 | const T0 = Date.parse("2026-11-01T12:00:00.000Z"); | |
| 22 | let dir: string; | |
| 23 | let app: App; | |
| 24 | ||
| 25 | beforeAll(async () => { | |
| 26 | dir = await mkdtemp(join(tmpdir(), "forge-dedupe-")); | |
| 27 | app = createApp({ dataDir: dir, port: 0 }); | |
| 28 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 29 | if (!a.ok) throw new Error(a.error); | |
| 30 | // The sentinel files as its own agent: a person's word, not its, would hold the issue open. | |
| 31 | const s = await createUser(app.db, { username: "sentinel", email: "[email protected]", agent: true, origin: "agent" }); | |
| 32 | if (!s.ok) throw new Error(String(s.error)); | |
| 33 | createAgentRow(app.db, { userId: s.value.id, roles: ["builder"] }); | |
| 34 | app.db.query("UPDATE user SET is_admin = 1 WHERE id = ?1").run(s.value.id); | |
| 35 | const made = await createRepo(app, a.value, { name: "forge", visibility: "private" }); | |
| 36 | if (!made.ok) throw new Error(String(made.error)); | |
| 37 | }); | |
| 38 | afterAll(async () => { | |
| 39 | app.db.close(); | |
| 40 | await rm(dir, { recursive: true, force: true }); | |
| 41 | }); | |
| 42 | ||
| 43 | const span = (openedAt: number, minutes: number, closed: boolean): void => { | |
| 44 | app.db | |
| 45 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('memory-growth', ?1, '/system', ?2, ?3)") | |
| 46 | .run(`The forge's own memory is ${minutes} MiB.`, openedAt, closed ? openedAt + minutes * MIN : null); | |
| 47 | }; | |
| 48 | const issues = () => app.db.query<{ id: string; state: string }, []>("SELECT id, state FROM issue WHERE title LIKE '%memory-growth%'").all(); | |
| 49 | ||
| 50 | test("three degraded days are one issue and three comments; two healthy days close it", () => { | |
| 51 | // Day 1: filed. | |
| 52 | span(T0 - HOUR, 30, true); | |
| 53 | expect(hunt(app, "alice/forge", T0).filed).toBe(1); | |
| 54 | const id = issues()[0]!.id; | |
| 55 | // Hourly hunts within the day add nothing, though the figures move. | |
| 56 | span(T0 + HOUR, 11, true); | |
| 57 | expect(hunt(app, "alice/forge", T0 + 2 * HOUR).commented).toBe(0); | |
| 58 | // Days 2, 3 and 4 (one comment each); the title's number changes every time. | |
| 59 | for (const [i, minutes] of [40, 50, 60].entries()) { | |
| 60 | const t = T0 + (i + 1) * DAY; | |
| 61 | span(t - HOUR, minutes, true); | |
| 62 | const r = hunt(app, "alice/forge", t); | |
| 63 | expect(r.filed).toBe(0); | |
| 64 | expect(r.commented).toBe(1); | |
| 65 | } | |
| 66 | expect(issues().length).toBe(1); | |
| 67 | expect(listComments(app.db, id).filter((c) => c.body.startsWith("Still there")).length).toBe(3); | |
| 68 | // Healthy: the last span closed an hour before day 4's hunt; a day on, still open. | |
| 69 | const last = T0 + 3 * DAY; | |
| 70 | hunt(app, "alice/forge", last + DAY); | |
| 71 | expect(issues()[0]!.state).toBe("open"); | |
| 72 | // Two healthy days: closed once, quoting the cause. | |
| 73 | hunt(app, "alice/forge", last + 2 * DAY); | |
| 74 | hunt(app, "alice/forge", last + 3 * DAY); | |
| 75 | expect(issues().length).toBe(1); | |
| 76 | expect(issues()[0]!.state).toBe("closed"); | |
| 77 | const held = listComments(app.db, id).filter((c) => c.body.startsWith("Held since ")); | |
| 78 | expect(held.length).toBe(1); | |
| 79 | expect(held[0]!.body).toContain("> cause: memory-growth"); | |
| 80 | }); |
test/sentinel.test.ts +4 −3
| @@ -1005,20 +1005,21 @@ describe("a degraded finding whose cause has stayed closed", () => { | ||
| 1005 | 1005 | await rm(hdir, { recursive: true, force: true }); |
| 1006 | 1006 | }); |
| 1007 | 1007 | |
| 1008 | test("23 hours closed stays open; a person's comment keeps theirs at 25", () => { | |
| 1008 | test("47 hours closed stays open; a person's comment keeps theirs at 49", () => { | |
| 1009 | 1009 | const disk = happ.db.query<{ id: string }, []>("SELECT id FROM issue WHERE title LIKE '%disk%'").get()!; |
| 1010 | 1010 | addComment(happ.db, { issueId: disk.id, authorId: halice.id, body: "looking" }); |
| 1011 | 1011 | |
| 1012 | hunt(happ, "alice/forge", T0 + 23 * HOUR); | |
| 1012 | hunt(happ, "alice/forge", T0 + 47 * HOUR); | |
| 1013 | 1013 | expect(state("drill")).toBe("open"); |
| 1014 | 1014 | |
| 1015 | hunt(happ, "alice/forge", T0 + 25 * HOUR); | |
| 1015 | hunt(happ, "alice/forge", T0 + 49 * HOUR); | |
| 1016 | 1016 | expect(state("drill")).toBe("closed"); |
| 1017 | 1017 | const drill = happ.db.query<{ id: string; close_reason: string }, []>("SELECT id, close_reason FROM issue WHERE title LIKE '%drill%'").get()!; |
| 1018 | 1018 | expect(drill.close_reason).toBe("done"); |
| 1019 | 1019 | const said = listComments(happ.db, drill.id).filter((c) => c.body.startsWith("Held since ")); |
| 1020 | 1020 | expect(said.length).toBe(1); |
| 1021 | 1021 | expect(said[0]!.body).toContain(new Date(T0).toISOString()); |
| 1022 | expect(said[0]!.body).toContain("> cause: drill"); | |
| 1022 | 1023 | expect(state("disk")).toBe("open"); |
| 1023 | 1024 | }); |
| 1024 | 1025 |
new above
offline