4c27e64 · 595ad39
@@ -0,0 +1,7 @@
// A degraded finding remembers the incident it was filed for (#2553): the
// time the cause first opened, and when its closed issue was last told the
// incident goes on. Both are null for every other finding.
export default `
  ALTER TABLE sentinel_finding ADD COLUMN incident TEXT;
  ALTER TABLE sentinel_finding ADD COLUMN commented_at TEXT;
  `;
@@ -64,6 +64,8 @@
export interface Finding {
  readonly fix?: string;
  /** A setting finding: the value and `updated_at` a person may have kept (#2001). */
  readonly kept?: { readonly value: string; readonly updatedAt: string };
  /** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */
  readonly incident?: string;
}
const clip = (text: string, n = 200): string =>
@@ -386,7 +388,7 @@
export function repeatedErrors(now: number): Finding[] {
 */
export function degradedCauses(db: Database, now: number): Finding[] {
  const from = now - DAY_MS;
  const byCause = new Map<string, { sentence: string; href: string; first: number; ms: number }>();
  const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>();
  for (const s of spans(db, from, now)) {
    const start = Math.max(s.openedAt, from);
    const held = Math.min(s.closedAt ?? now, now) - start;
@@ -396,6 +398,7 @@
export function degradedCauses(db: Database, now: number): Finding[] {
      sentence: s.sentence,
      href: s.href,
      first: seen?.first ?? start,
      opened: seen?.opened ?? s.openedAt,
      ms: (seen?.ms ?? 0) + Math.max(0, held),
    });
  }
@@ -415,6 +418,7 @@
export function degradedCauses(db: Database, now: number): Finding[] {
      ],
      pillar: "P0 Trust: the forge says when it is not well, and is well again",
      lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
      incident: new Date(d.opened).toISOString(),
    });
  }
  return out;
@@ -626,6 +630,42 @@
export function recordKept(db: Database, key: string, value: string, updatedAt:
  ).run(key, value, updatedAt, at);
}
/**
 * A degraded incident whose issue was closed as done and which goes on (same
 * cause, same first-open time) is told to that issue, not filed again: the
 * fix is likely merged but not live. At most one comment a day. True when
 * the incident is that one: the comments made (0 or 1), else undefined.
 */
function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 | 1 | undefined {
  const prior = app.db
    .query<{ issue_id: string; commented_at: string | null }, [string, string]>(
      `SELECT t.issue_id, t.commented_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
        WHERE t.key = ?1 AND t.incident = ?2 AND i.state = 'closed'
        ORDER BY t.first_seen DESC LIMIT 1`,
    )
    .get(f.key, f.incident ?? "");
  if (prior === null) return undefined;
  const issue = getIssueById(app.db, prior.issue_id);
  if (issue === undefined || issue.closeReason !== "done") return undefined;
  if (prior.commented_at !== null && now - Date.parse(prior.commented_at) < DAY_MS) return 0;
  const minutes = f.evidence.find((e) => e.startsWith("total: "));
  const said = addComment(
    app,
    actor,
    repo,
    issue,
    `Still degraded after the fix; the fix may not be live yet. At ${at}${minutes === undefined ? "" : `, ${minutes.slice(7)}`}; evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`,
  );
  if (!said.ok) {
    log.warn("sentinel could not comment", { key: f.key, error: String(said.error) });
    return 0;
  }
  app.db
    .query("UPDATE sentinel_finding SET commented_at = ?1, last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4")
    .run(at, evidence, f.key, prior.issue_id);
  return 1;
}
/**
 * Read, file, record. One open issue per finding: a tracked finding whose
 * issue is open is commented on only when its evidence changed; one whose
@@ -674,6 +714,11 @@
export function hunt(
            .run(at, evidence, f.key, tracked.issue_id);
          continue;
        }
        const told = f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now);
        if (told !== undefined) {
          commented += told;
          continue;
        }
        const title = `Sentinel: ${clip(f.title, 200)}`;
        // An issue already filed under this title whose record was lost
        // (the insert failed last hunt) is adopted, not filed twice.
@@ -694,8 +739,8 @@
export function hunt(
          if (f.urgent && f.kept === undefined) handOver(app, repo, actor, made.value);
        }
        app.db
          .query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen) VALUES (?1, ?2, ?3, ?4, ?4)")
          .run(f.key, issueId, evidence, at);
          .query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen, incident) VALUES (?1, ?2, ?3, ?4, ?4, ?5)")
          .run(f.key, issueId, evidence, at, f.incident ?? null);
      } catch (error) {
        log.error("sentinel filing failed", { key: f.key, error: String(error) });
      }
@@ -739,3 +739,77 @@
describe("a budget breach is crew work at once", () => {
    }
  });
});
// ---- a degraded incident closed as done (#2553) ----
describe("a degraded incident whose issue closed as done", () => {
  let idir: string;
  let iapp: App;
  let irepo: Repo;
  const T0 = Date.parse("2026-10-18T12:00:00.000Z");
  const OPEN = T0 - 3 * HOUR;
  const span = (cause: string, openedAt: number, minutes: number): void => {
    iapp.db
      .query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES (?1, 'It is slow.', '/x', ?2, ?3)")
      .run(cause, openedAt, openedAt + minutes * D_MIN);
  };
  const issues = (state: string) => listIssues(iapp.db, irepo.id, { state } as never);
  const close = (reason: string): void => {
    iapp.db.query("UPDATE issue SET state = 'closed', close_reason = ?1, closed_at = ?2").run(reason, new Date(T0).toISOString());
  };
  beforeAll(async () => {
    idir = await mkdtemp(join(tmpdir(), "forge-sentinel-incident-"));
    iapp = createApp({ dataDir: idir, port: 0 });
    const a = await createUser(iapp.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
    if (!a.ok) throw new Error(a.error);
    const made = await createRepo(iapp, a.value, { name: "forge", visibility: "private" });
    if (!made.ok) throw new Error(String(made.error));
    irepo = made.value;
  });
  afterAll(async () => {
    iapp.db.close();
    await rm(idir, { recursive: true, force: true });
  });
  test("is commented once a day, not refiled", () => {
    span("drill", OPEN, 40 * 60);
    expect(hunt(iapp, "alice/forge", T0).filed).toBe(1);
    close("done");
    const again = hunt(iapp, "alice/forge", T0 + 30 * D_MIN);
    expect(again.filed).toBe(0);
    expect(again.commented).toBe(1);
    expect(issues("open").length).toBe(0);
    const closed = issues("closed");
    expect(closed.length).toBe(1);
    const comments = listComments(iapp.db, closed[0]!.id);
    expect(comments.length).toBe(1);
    expect(comments[0]!.body).toContain("Still degraded after the fix; the fix may not be live yet");
    // Same day: nothing more, and still no new issue.
    const third = hunt(iapp, "alice/forge", T0 + 2 * HOUR);
    expect(third.filed).toBe(0);
    expect(third.commented).toBe(0);
    expect(listComments(iapp.db, closed[0]!.id).length).toBe(1);
    // The next day it is told again.
    expect(hunt(iapp, "alice/forge", T0 + 25 * HOUR).commented).toBe(1);
    expect(listComments(iapp.db, closed[0]!.id).length).toBe(2);
    expect(issues("open").length).toBe(0);
  });
  test("a new incident of the same cause is refiled", () => {
    const later = T0 + 3 * DAY;
    span("drill", later - HOUR, 30);
    const r = hunt(iapp, "alice/forge", later);
    expect(r.filed).toBe(1);
    expect(issues("open").length).toBe(1);
  });
  test("a finding closed as dropped that returns is refiled", () => {
    const t = T0 + 10 * DAY;
    span("disk", t - 2 * HOUR, 30);
    expect(hunt(iapp, "alice/forge", t).filed).toBe(1);
    iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'dropped', closed_at = ?1 WHERE state = 'open'").run(new Date(t).toISOString());
    const r = hunt(iapp, "alice/forge", t + 30 * D_MIN);
    expect(r.filed).toBe(1);
    expect(r.commented).toBe(0);
  });
});
		@@ -0,0 +1,7 @@
	1	// A degraded finding remembers the incident it was filed for (#2553): the
	2	// time the cause first opened, and when its closed issue was last told the
	3	// incident goes on. Both are null for every other finding.
	4	export default `
	5	ALTER TABLE sentinel_finding ADD COLUMN incident TEXT;
	6	ALTER TABLE sentinel_finding ADD COLUMN commented_at TEXT;
	7	`;
		@@ -64,6 +64,8 @@ export interface Finding {
64	64	readonly fix?: string;
65	65	/** A setting finding: the value and `updated_at` a person may have kept (#2001). */
66	66	readonly kept?: { readonly value: string; readonly updatedAt: string };
	67	/** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */
	68	readonly incident?: string;
67	69	}
68	70
69	71	const clip = (text: string, n = 200): string =>
		@@ -386,7 +388,7 @@ export function repeatedErrors(now: number): Finding[] {
386	388	*/
387	389	export function degradedCauses(db: Database, now: number): Finding[] {
388	390	const from = now - DAY_MS;
389		const byCause = new Map<string, { sentence: string; href: string; first: number; ms: number }>();
	391	const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>();
390	392	for (const s of spans(db, from, now)) {
391	393	const start = Math.max(s.openedAt, from);
392	394	const held = Math.min(s.closedAt ?? now, now) - start;
		@@ -396,6 +398,7 @@ export function degradedCauses(db: Database, now: number): Finding[] {
396	398	sentence: s.sentence,
397	399	href: s.href,
398	400	first: seen?.first ?? start,
	401	opened: seen?.opened ?? s.openedAt,
399	402	ms: (seen?.ms ?? 0) + Math.max(0, held),
400	403	});
401	404	}
		@@ -415,6 +418,7 @@ export function degradedCauses(db: Database, now: number): Finding[] {
415	418	],
416	419	pillar: "P0 Trust: the forge says when it is not well, and is well again",
417	420	lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
	421	incident: new Date(d.opened).toISOString(),
418	422	});
419	423	}
420	424	return out;
		@@ -626,6 +630,42 @@ export function recordKept(db: Database, key: string, value: string, updatedAt:
626	630	).run(key, value, updatedAt, at);
627	631	}
628	632
	633	/**
	634	* A degraded incident whose issue was closed as done and which goes on (same
	635	* cause, same first-open time) is told to that issue, not filed again: the
	636	* fix is likely merged but not live. At most one comment a day. True when
	637	* the incident is that one: the comments made (0 or 1), else undefined.
	638	*/
	639	function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 \| 1 \| undefined {
	640	const prior = app.db
	641	.query<{ issue_id: string; commented_at: string \| null }, [string, string]>(
	642	`SELECT t.issue_id, t.commented_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
	643	WHERE t.key = ?1 AND t.incident = ?2 AND i.state = 'closed'
	644	ORDER BY t.first_seen DESC LIMIT 1`,
	645	)
	646	.get(f.key, f.incident ?? "");
	647	if (prior === null) return undefined;
	648	const issue = getIssueById(app.db, prior.issue_id);
	649	if (issue === undefined \|\| issue.closeReason !== "done") return undefined;
	650	if (prior.commented_at !== null && now - Date.parse(prior.commented_at) < DAY_MS) return 0;
	651	const minutes = f.evidence.find((e) => e.startsWith("total: "));
	652	const said = addComment(
	653	app,
	654	actor,
	655	repo,
	656	issue,
	657	`Still degraded after the fix; the fix may not be live yet. At ${at}${minutes === undefined ? "" : `, ${minutes.slice(7)}`}; evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`,
	658	);
	659	if (!said.ok) {
	660	log.warn("sentinel could not comment", { key: f.key, error: String(said.error) });
	661	return 0;
	662	}
	663	app.db
	664	.query("UPDATE sentinel_finding SET commented_at = ?1, last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4")
	665	.run(at, evidence, f.key, prior.issue_id);
	666	return 1;
	667	}
	668
629	669	/**
630	670	* Read, file, record. One open issue per finding: a tracked finding whose
631	671	* issue is open is commented on only when its evidence changed; one whose
		@@ -674,6 +714,11 @@ export function hunt(
674	714	.run(at, evidence, f.key, tracked.issue_id);
675	715	continue;
676	716	}
	717	const told = f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now);
	718	if (told !== undefined) {
	719	commented += told;
	720	continue;
	721	}
677	722	const title = `Sentinel: ${clip(f.title, 200)}`;
678	723	// An issue already filed under this title whose record was lost
679	724	// (the insert failed last hunt) is adopted, not filed twice.
		@@ -694,8 +739,8 @@ export function hunt(
694	739	if (f.urgent && f.kept === undefined) handOver(app, repo, actor, made.value);
695	740	}
696	741	app.db
697		.query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen) VALUES (?1, ?2, ?3, ?4, ?4)")
698		.run(f.key, issueId, evidence, at);
	742	.query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen, incident) VALUES (?1, ?2, ?3, ?4, ?4, ?5)")
	743	.run(f.key, issueId, evidence, at, f.incident ?? null);
699	744	} catch (error) {
700	745	log.error("sentinel filing failed", { key: f.key, error: String(error) });
701	746	}
		@@ -739,3 +739,77 @@ describe("a budget breach is crew work at once", () => {
739	739	}
740	740	});
741	741	});
	742
	743	// ---- a degraded incident closed as done (#2553) ----
	744
	745	describe("a degraded incident whose issue closed as done", () => {
	746	let idir: string;
	747	let iapp: App;
	748	let irepo: Repo;
	749	const T0 = Date.parse("2026-10-18T12:00:00.000Z");
	750	const OPEN = T0 - 3 * HOUR;
	751	const span = (cause: string, openedAt: number, minutes: number): void => {
	752	iapp.db
	753	.query("INSERT INTO degraded_span (cause, sentence, href, opened_at, closed_at) VALUES (?1, 'It is slow.', '/x', ?2, ?3)")
	754	.run(cause, openedAt, openedAt + minutes * D_MIN);
	755	};
	756	const issues = (state: string) => listIssues(iapp.db, irepo.id, { state } as never);
	757	const close = (reason: string): void => {
	758	iapp.db.query("UPDATE issue SET state = 'closed', close_reason = ?1, closed_at = ?2").run(reason, new Date(T0).toISOString());
	759	};
	760	beforeAll(async () => {
	761	idir = await mkdtemp(join(tmpdir(), "forge-sentinel-incident-"));
	762	iapp = createApp({ dataDir: idir, port: 0 });
	763	const a = await createUser(iapp.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
	764	if (!a.ok) throw new Error(a.error);
	765	const made = await createRepo(iapp, a.value, { name: "forge", visibility: "private" });
	766	if (!made.ok) throw new Error(String(made.error));
	767	irepo = made.value;
	768	});
	769	afterAll(async () => {
	770	iapp.db.close();
	771	await rm(idir, { recursive: true, force: true });
	772	});
	773
	774	test("is commented once a day, not refiled", () => {
	775	span("drill", OPEN, 40 * 60);
	776	expect(hunt(iapp, "alice/forge", T0).filed).toBe(1);
	777	close("done");
	778	const again = hunt(iapp, "alice/forge", T0 + 30 * D_MIN);
	779	expect(again.filed).toBe(0);
	780	expect(again.commented).toBe(1);
	781	expect(issues("open").length).toBe(0);
	782	const closed = issues("closed");
	783	expect(closed.length).toBe(1);
	784	const comments = listComments(iapp.db, closed[0]!.id);
	785	expect(comments.length).toBe(1);
	786	expect(comments[0]!.body).toContain("Still degraded after the fix; the fix may not be live yet");
	787	// Same day: nothing more, and still no new issue.
	788	const third = hunt(iapp, "alice/forge", T0 + 2 * HOUR);
	789	expect(third.filed).toBe(0);
	790	expect(third.commented).toBe(0);
	791	expect(listComments(iapp.db, closed[0]!.id).length).toBe(1);
	792	// The next day it is told again.
	793	expect(hunt(iapp, "alice/forge", T0 + 25 * HOUR).commented).toBe(1);
	794	expect(listComments(iapp.db, closed[0]!.id).length).toBe(2);
	795	expect(issues("open").length).toBe(0);
	796	});
	797
	798	test("a new incident of the same cause is refiled", () => {
	799	const later = T0 + 3 * DAY;
	800	span("drill", later - HOUR, 30);
	801	const r = hunt(iapp, "alice/forge", later);
	802	expect(r.filed).toBe(1);
	803	expect(issues("open").length).toBe(1);
	804	});
	805
	806	test("a finding closed as dropped that returns is refiled", () => {
	807	const t = T0 + 10 * DAY;
	808	span("disk", t - 2 * HOUR, 30);
	809	expect(hunt(iapp, "alice/forge", t).filed).toBe(1);
	810	iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'dropped', closed_at = ?1 WHERE state = 'open'").run(new Date(t).toISOString());
	811	const r = hunt(iapp, "alice/forge", t + 30 * D_MIN);
	812	expect(r.filed).toBe(1);
	813	expect(r.commented).toBe(0);
	814	});
	815	});