Public: anyone can read this; the crew can change it.

The sentinel's memory-growth finding is one open issue that gets a comment each day, not a new issue and commit per day #14525

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14491-the-sentinel-s-memory-growth-finding-is → main 1 commit fork

  • Merged. mason/14491-the-sentinel-s-memory-growth-finding-is was merged into main as a46be6f.

Review

Sign in to review this change.
3 files changed +112 −11
src/crew/sentinel.ts +28 −8
@@ -69,6 +69,12 @@ export const SIGNAL_GAP_MS = 10 * 60_000;
6969 export const SENTINEL_PREFIX = "Sentinel: ";
7070 /** A cause degraded longer than this in its day is a finding. */
7171 export const DEGRADED_DAY_MS = 10 * 60_000;
72 /** A degraded cause must stay out of the history this long (two days) before its issue closes. */
73 export const HEALTHY_DAYS = 2;
74 /**
75 * A tracked finding is told new figures at most this often: one comment a day, not one per hunt.
76 */
77 export const COMMENT_EVERY_MS = DAY_MS;
7278 /** How many alike make a streak. */
7379 export const STREAK = 3;
7480 /** The run summaries read per repository. */
@@ -1003,9 +1009,15 @@ function settledBefore(db: Database, actor: User, f: Finding, now: number): stri
10031009 ].join("\n");
10041010 }
10051011
1012 /** The issue's `cause: …` evidence line quoted for the closing comment, or nothing. */
1013 function causeLine(body: string): string {
1014 const line = body.split("\n").find((l) => l.startsWith("- cause: "));
1015 return line === undefined ? "" : `> ${line.slice(2)}\n\n`;
1016 }
1017
10061018 /**
10071019 * A finding backed by an incident whose cause has stayed out of the degraded
1008 * history for a day is done: the sentinel says since when and closes it. Not a
1020 * history for two days is done: the sentinel says since when and closes it. Not a
10091021 * setting (its own rule), and not one a person has spoken on — they decide.
10101022 * `active` are the keys found now; a finding that is there is not held.
10111023 */
@@ -1028,7 +1040,7 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string
10281040 if (s.closedAt === null) live = true;
10291041 else since = Math.max(since, s.closedAt);
10301042 }
1031 if (live || since === -Infinity || now - since < DAY_MS) continue;
1043 if (live || since === -Infinity || now - since < HEALTHY_DAYS * DAY_MS) continue;
10321044 if (personSpoke(app.db, t.issue_id)) continue;
10331045 const issue = getIssueById(app.db, t.issue_id);
10341046 if (issue === undefined || issue.repoId !== repo.id) continue;
@@ -1037,7 +1049,7 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string
10371049 actor,
10381050 repo,
10391051 issue,
1040 `Held since ${new Date(since).toISOString()}: no new incident on this cause for over a day. Closing as done; if it comes back it is filed again.`,
1052 `Held since ${new Date(since).toISOString()}: no new incident on this cause for ${HEALTHY_DAYS} days running. ${causeLine(issue.body)}Closing as done; if it comes back it is filed again.`,
10411053 );
10421054 if (!said.ok) {
10431055 log.warn("sentinel could not comment", { key: t.key, error: String(said.error) });
@@ -1081,15 +1093,19 @@ export function hunt(
10811093 const evidence = f.evidence.join("\n");
10821094 const tracked = app.db
10831095 .query<
1084 { issue_id: string; evidence: string },
1096 { issue_id: string; evidence: string; commented_at: string | null; first_seen: string },
10851097 [string]
1086 >(`SELECT t.issue_id, t.evidence FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
1098 >(`SELECT t.issue_id, t.evidence, t.commented_at, t.first_seen FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
10871099 WHERE t.key = ?1 AND i.state = 'open' LIMIT 1`)
10881100 .get(f.key);
10891101 if (tracked !== null) {
10901102 // The stored evidence moves only once the comment is made: a
10911103 // comment that failed is tried again by the next hunt.
1092 if (tracked.evidence !== evidence) {
1104 // Matched on the key (duty and subject), never the title: the title carries changing
1105 // numbers. Told at most once a day, so a cause degraded all day is one issue with a
1106 // comment a day.
1107 const toldAt = Date.parse(tracked.commented_at ?? tracked.first_seen);
1108 if (tracked.evidence !== evidence && !(f.key.startsWith("degraded:") && now - toldAt < COMMENT_EVERY_MS)) {
10931109 const issue = getIssueById(app.db, tracked.issue_id);
10941110 if (issue === undefined) continue;
10951111 const said = addComment(app, actor, repo, issue, `Still there at ${at}, evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`);
@@ -1098,10 +1114,14 @@ export function hunt(
10981114 continue;
10991115 }
11001116 commented++;
1117 app.db
1118 .query("UPDATE sentinel_finding SET last_seen = ?1, evidence = ?2, commented_at = ?1 WHERE key = ?3 AND issue_id = ?4")
1119 .run(at, evidence, f.key, tracked.issue_id);
1120 continue;
11011121 }
11021122 app.db
1103 .query("UPDATE sentinel_finding SET last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4")
1104 .run(at, evidence, f.key, tracked.issue_id);
1123 .query("UPDATE sentinel_finding SET last_seen = ?1 WHERE key = ?2 AND issue_id = ?3")
1124 .run(at, f.key, tracked.issue_id);
11051125 continue;
11061126 }
11071127 // A shared red that passed again on its own is no work: the day's total does not file it again.
test/maintain-dedupe.test.ts added +80
@@ -0,0 +1,80 @@
1 /**
2 * A degraded finding is one issue however many days it lasts (#14491): the same cause while its
3 * issue is open is a comment with the new figures, and it closes after two healthy days, quoting
4 * its cause line. Matched on the cause's key, never the title, which carries changing numbers.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import "./helpers/globals.ts";
11 import { createApp, type App } from "../src/app.ts";
12 import { createAgentRow } from "../src/core/agents.ts";
13 import { createRepo } from "../src/actions/repos.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { listComments } from "../src/core/issues.ts";
16 import { hunt } from "../src/crew/sentinel.ts";
17
18 const HOUR = 3_600_000;
19 const DAY = 24 * HOUR;
20 const MIN = 60_000;
21 const T0 = Date.parse("2026-11-01T12:00:00.000Z");
22 let dir: string;
23 let app: App;
24
25 beforeAll(async () => {
26 dir = await mkdtemp(join(tmpdir(), "forge-dedupe-"));
27 app = createApp({ dataDir: dir, port: 0 });
28 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
29 if (!a.ok) throw new Error(a.error);
30 // The sentinel files as its own agent: a person's word, not its, would hold the issue open.
31 const s = await createUser(app.db, { username: "sentinel", email: "[email protected]", agent: true, origin: "agent" });
32 if (!s.ok) throw new Error(String(s.error));
33 createAgentRow(app.db, { userId: s.value.id, roles: ["builder"] });
34 app.db.query("UPDATE user SET is_admin = 1 WHERE id = ?1").run(s.value.id);
35 const made = await createRepo(app, a.value, { name: "forge", visibility: "private" });
36 if (!made.ok) throw new Error(String(made.error));
37 });
38 afterAll(async () => {
39 app.db.close();
40 await rm(dir, { recursive: true, force: true });
41 });
42
43 const span = (openedAt: number, minutes: number, closed: boolean): void => {
44 app.db
45 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES ('memory-growth', ?1, '/system', ?2, ?3)")
46 .run(`The forge's own memory is ${minutes} MiB.`, openedAt, closed ? openedAt + minutes * MIN : null);
47 };
48 const issues = () => app.db.query<{ id: string; state: string }, []>("SELECT id, state FROM issue WHERE title LIKE '%memory-growth%'").all();
49
50 test("three degraded days are one issue and three comments; two healthy days close it", () => {
51 // Day 1: filed.
52 span(T0 - HOUR, 30, true);
53 expect(hunt(app, "alice/forge", T0).filed).toBe(1);
54 const id = issues()[0]!.id;
55 // Hourly hunts within the day add nothing, though the figures move.
56 span(T0 + HOUR, 11, true);
57 expect(hunt(app, "alice/forge", T0 + 2 * HOUR).commented).toBe(0);
58 // Days 2, 3 and 4 (one comment each); the title's number changes every time.
59 for (const [i, minutes] of [40, 50, 60].entries()) {
60 const t = T0 + (i + 1) * DAY;
61 span(t - HOUR, minutes, true);
62 const r = hunt(app, "alice/forge", t);
63 expect(r.filed).toBe(0);
64 expect(r.commented).toBe(1);
65 }
66 expect(issues().length).toBe(1);
67 expect(listComments(app.db, id).filter((c) => c.body.startsWith("Still there")).length).toBe(3);
68 // Healthy: the last span closed an hour before day 4's hunt; a day on, still open.
69 const last = T0 + 3 * DAY;
70 hunt(app, "alice/forge", last + DAY);
71 expect(issues()[0]!.state).toBe("open");
72 // Two healthy days: closed once, quoting the cause.
73 hunt(app, "alice/forge", last + 2 * DAY);
74 hunt(app, "alice/forge", last + 3 * DAY);
75 expect(issues().length).toBe(1);
76 expect(issues()[0]!.state).toBe("closed");
77 const held = listComments(app.db, id).filter((c) => c.body.startsWith("Held since "));
78 expect(held.length).toBe(1);
79 expect(held[0]!.body).toContain("> cause: memory-growth");
80 });
test/sentinel.test.ts +4 −3
@@ -1005,20 +1005,21 @@ describe("a degraded finding whose cause has stayed closed", () => {
10051005 await rm(hdir, { recursive: true, force: true });
10061006 });
10071007
1008 test("23 hours closed stays open; a person's comment keeps theirs at 25", () => {
1008 test("47 hours closed stays open; a person's comment keeps theirs at 49", () => {
10091009 const disk = happ.db.query<{ id: string }, []>("SELECT id FROM issue WHERE title LIKE '%disk%'").get()!;
10101010 addComment(happ.db, { issueId: disk.id, authorId: halice.id, body: "looking" });
10111011
1012 hunt(happ, "alice/forge", T0 + 23 * HOUR);
1012 hunt(happ, "alice/forge", T0 + 47 * HOUR);
10131013 expect(state("drill")).toBe("open");
10141014
1015 hunt(happ, "alice/forge", T0 + 25 * HOUR);
1015 hunt(happ, "alice/forge", T0 + 49 * HOUR);
10161016 expect(state("drill")).toBe("closed");
10171017 const drill = happ.db.query<{ id: string; close_reason: string }, []>("SELECT id, close_reason FROM issue WHERE title LIKE '%drill%'").get()!;
10181018 expect(drill.close_reason).toBe("done");
10191019 const said = listComments(happ.db, drill.id).filter((c) => c.body.startsWith("Held since "));
10201020 expect(said.length).toBe(1);
10211021 expect(said[0]!.body).toContain(new Date(T0).toISOString());
1022 expect(said[0]!.body).toContain("> cause: drill");
10221023 expect(state("disk")).toBe("open");
10231024 });
10241025

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs