Public: anyone can read this; the crew can change it.

The sentinel does not refile a test's budget breach in the hour after the pull that fixed it landed, unless it fails again on checks started after the fix #8190

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/8185-the-sentinel-does-not-refile-a-test-s-bu → main 1 commit fork

  • Merged. mason/8185-the-sentinel-does-not-refile-a-test-s-bu was merged into main as 0c4d3a4.

Review

Sign in to review this change.
3 files changed +180 −2
src/checks/shared_red.ts +21
@@ -141,6 +141,27 @@ export function sharedCause(t: Pick<SharedTest, "repoId" | "test">): string {
141141 return `shared:${t.repoId}:${Bun.hash(t.test).toString(36).slice(0, 10)}`;
142142 }
143143
144 /**
145 * Distinct pulls whose check, created at or after `sinceMs`, failed the test
146 * a shared cause names (#8185): the reds a fix that landed then did not end,
147 * where the hour's window still holds the ones from before it.
148 */
149 const FAILED_SINCE = `
150 SELECT DISTINCT run.issue_id, n.value AS test
151 FROM run,
152 json_each(CASE WHEN json_valid(run.check_detail) THEN run.check_detail ELSE '{}' END, '$.names') n
153 WHERE run.repo_id = ?1 AND run.id >= ?2 AND run.created_at >= ?3
154 AND run.kind = 'check' AND run.issue_id IS NOT NULL AND run.state = 'failed'
155 AND n.type = 'text'`;
156
157 export function pullsFailedSince(db: Database, cause: string, sinceMs: number): number {
158 const repoId = cause.split(":")[1] ?? "";
159 const rows = db
160 .query<{ issue_id: string; test: string }, [string, string, string]>(FAILED_SINCE)
161 .all(repoId, idFloor(sinceMs), new Date(sinceMs).toISOString());
162 return new Set(rows.filter((r) => sharedCause({ repoId, test: r.test }) === cause).map((r) => r.issue_id)).size;
163 }
164
144165 /**
145166 * The sentinel's finding key for a cause, `degraded:${keyPart(cause)}`
146167 * (`crew/sentinel.ts`, which reads the vitals and so is not imported here):
src/crew/sentinel.ts +75 −2
@@ -32,6 +32,7 @@ import { spans } from "../kernel/degraded_history.ts";
3232 import { QUEUE_WAIT_MAX_MS } from "../kernel/vitals.ts";
3333 import { STALE_AFTER_MS } from "../runs/lease.ts";
3434 import { isOutageSummary } from "./network.ts";
35 import { pullsFailedSince, SHARED_PULLS, SHARED_WINDOW_MS } from "../checks/shared_red.ts";
3536 import { clip } from "../text/clip.ts";
3637
3738 const log = createLog("sentinel");
@@ -62,6 +63,8 @@ export interface Finding {
6263 readonly fix?: string;
6364 /** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */
6465 readonly incident?: string;
66 /** The vitals cause a budget-breach or degraded finding reads. */
67 readonly cause?: string;
6568 }
6669
6770 const QUOTE_MAX = 200;
@@ -398,6 +401,7 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string):
398401 pillar: "P0 Trust: the forge says when it is not well, and is well again",
399402 lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
400403 incident: new Date(d.opened).toISOString(),
404 cause,
401405 });
402406 }
403407 return out;
@@ -454,6 +458,7 @@ export function budgetBreaches(db: Database, now: number, forgeRepoId?: string):
454458 ? "Make the named test's verdict depend only on the code under test, not on the check machine's load, disk or clock, in one pull. Pulls red on it alone wait, unparked, for main's next green check."
455459 : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
456460 urgent: true,
461 cause: s.cause,
457462 });
458463 }
459464 return out;
@@ -610,6 +615,67 @@ function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence:
610615 return 1;
611616 }
612617
618 interface RecentFix {
619 readonly issueId: string;
620 readonly closedAt: number;
621 /** Its issue was already told the test is still red since that close. */
622 readonly told: boolean;
623 }
624
625 /**
626 * A shared test's issue closed as done or split within the hour (#8185): its
627 * close is the landing on main, since a merged pull closes what it fixes only
628 * once it lands. The hour's window still holds the reds from before it.
629 */
630 function recentFix(db: Database, f: Finding, now: number): RecentFix | undefined {
631 if (f.cause?.startsWith("shared:") !== true) return undefined;
632 const prior = db
633 .query<{ issue_id: string; commented_at: string | null; closed_at: string | null }, [string]>(
634 `SELECT t.issue_id, t.commented_at, i.closed_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
635 WHERE t.key = ?1 AND i.state = 'closed' AND i.parked = 0 AND i.close_reason IN ('done', 'split')
636 ORDER BY i.closed_at DESC LIMIT 1`,
637 )
638 .get(f.key);
639 const closedAt = prior?.closed_at == null ? NaN : Date.parse(prior.closed_at);
640 if (prior === null || !(now - closedAt < SHARED_WINDOW_MS)) return undefined;
641 const told = prior.commented_at !== null && Date.parse(prior.commented_at) >= closedAt;
642 return { issueId: prior.issue_id, closedAt, told };
643 }
644
645 /** Whether a recent fix holds the finding back: told already, or too few reds since it landed. */
646 const heldByFix = (db: Database, f: Finding, fix: RecentFix): boolean =>
647 fix.told || pullsFailedSince(db, f.cause ?? "", fix.closedAt) < SHARED_PULLS;
648
649 /**
650 * A shared test fixed within the hour files nothing new (#8185). With fewer
651 * than SHARED_PULLS pulls red on it on checks created after the fix landed,
652 * it says nothing; with that many, it tells the closed issue once, as
653 * `continuesDone` does. Undefined when no fix is that recent.
654 */
655 function sinceFix(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 | 1 | undefined {
656 const fix = recentFix(app.db, f, now);
657 if (fix === undefined) return undefined;
658 if (heldByFix(app.db, f, fix)) return 0;
659 const issue = getIssueById(app.db, fix.issueId);
660 if (issue === undefined) return 0;
661 const pulls = pullsFailedSince(app.db, f.cause ?? "", fix.closedAt);
662 const said = addComment(
663 app,
664 actor,
665 repo,
666 issue,
667 `Still red after the fix landed at ${new Date(fix.closedAt).toISOString()}: ${pulls} pulls failed it on checks created since; the fix may not be live yet. At ${at}; evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`,
668 );
669 if (!said.ok) {
670 log.warn("sentinel could not comment", { key: f.key, error: String(said.error) });
671 return 0;
672 }
673 app.db
674 .query("UPDATE sentinel_finding SET commented_at = ?1, last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4")
675 .run(at, evidence, f.key, fix.issueId);
676 return 1;
677 }
678
613679 /**
614680 * A finding backed by an incident whose cause has stayed out of the degraded
615681 * history for a day is done: the sentinel says since when and closes it. Not a
@@ -711,7 +777,9 @@ export function hunt(
711777 .run(at, evidence, f.key, tracked.issue_id);
712778 continue;
713779 }
714 const told = f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now);
780 const told =
781 sinceFix(app, repo, actor, f, evidence, at, now) ??
782 (f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now));
715783 if (told !== undefined) {
716784 commented += told;
717785 continue;
@@ -825,7 +893,12 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null {
825893 .all()
826894 .map((r) => r.key),
827895 );
828 if (!signals.some((s) => !tracked.has(s.key))) return null;
896 // A shared test held back by its recent fix is no signal: no hunt a tick for its hour.
897 const held = (s: Finding): boolean => {
898 const fix = recentFix(app.db, s, now);
899 return fix !== undefined && heldByFix(app.db, s, fix);
900 };
901 if (!signals.some((s) => !tracked.has(s.key) && !held(s))) return null;
829902 }
830903 return hunt(app, repoPath, now);
831904 }
test/sentinel.test.ts +84
@@ -20,6 +20,7 @@ import { getRepoById, type Repo } from "../src/core/repos.ts";
2020 import { createRun } from "../src/core/runs.ts";
2121 import { createUser, type User } from "../src/core/users.ts";
2222 import { recordCap } from "../src/crew/guard.ts";
23 import { sharedCause } from "../src/checks/shared_red.ts";
2324 import {
2425 budgetBreaches,
2526 degradedCauses,
@@ -838,6 +839,89 @@ describe("a degraded incident whose issue closed as done", () => {
838839 });
839840 });
840841
842 // ---- a shared test fixed within the hour (#8185) ----
843
844 describe("a shared test whose fix landed within the hour", () => {
845 let fdir: string;
846 let fapp: App;
847 let frepo: Repo;
848 let author: User;
849 const TEST = "MCP backups > a listing that fails";
850 const T = Date.parse("2026-10-20T12:00:00.000Z");
851 const cause = (): string => sharedCause({ repoId: frepo.id, test: TEST });
852 const filedFor = () =>
853 listIssues(fapp.db, frepo.id, { state: "all" }).filter((i) => i.title.startsWith(`Sentinel: ${cause()}`));
854 /** One pull whose check failed on the test, created at `at`. */
855 const red = (n: number, at: number): void => {
856 const pull = createIssue(fapp.db, { repoId: frepo.id, authorId: author.id, title: `pull ${n}`, kind: "pull" });
857 if (!pull.ok) throw new Error(String(pull.error));
858 const run = createRun(fapp.db, { repoId: frepo.id, kind: "check", issueId: pull.value.id });
859 fapp.db
860 .query("UPDATE run SET state = 'failed', created_at = ?2, finished_at = ?2, check_detail = ?3 WHERE id = ?1")
861 .run(run.id, new Date(at).toISOString(), JSON.stringify({ names: [TEST] }));
862 };
863 beforeAll(async () => {
864 fdir = await mkdtemp(join(tmpdir(), "forge-sentinel-fix-"));
865 fapp = createApp({ dataDir: fdir, port: 0 });
866 const a = await createUser(fapp.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
867 if (!a.ok) throw new Error(a.error);
868 author = a.value;
869 const made = await createRepo(fapp, a.value, { name: "forge", visibility: "private" });
870 if (!made.ok) throw new Error(String(made.error));
871 frepo = made.value;
872 fapp.db
873 .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'It is red on 12 pulls.', '/x', ?2)")
874 .run(cause(), T - 50 * D_MIN);
875 for (let n = 0; n < 12; n++) red(n, T - 40 * D_MIN + n * D_MIN);
876 expect(filedFor().length).toBe(0);
877 hunt(fapp, "alice/forge", T - 2 * D_MIN);
878 expect(filedFor().length).toBe(1);
879 // The fix lands a minute before the hunt: its pull closed the issue as done.
880 fapp.db
881 .query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE id = ?2")
882 .run(new Date(T - D_MIN).toISOString(), filedFor()[0]!.id);
883 });
884 afterAll(async () => {
885 fapp.db.close();
886 await rm(fdir, { recursive: true, force: true });
887 });
888
889 test("with 12 earlier reds and none since, files no issue and says nothing", () => {
890 // Still a breach, as the terminal and the MCP count it.
891 expect(budgetBreaches(fapp.db, T, frepo.id).map((f) => f.cause)).toEqual([cause()]);
892 const r = hunt(fapp, "alice/forge", T);
893 expect(r.commented).toBe(0);
894 expect(filedFor().length).toBe(1);
895 expect(listComments(fapp.db, filedFor()[0]!.id).length).toBe(0);
896 const prior = process.env.FORGE_SENTINEL_REPO;
897 process.env.FORGE_SENTINEL_REPO = "alice/forge";
898 try {
899 // No hunt a tick for the hour it is held.
900 expect(sentinelTick(fapp, T + D_MIN)).toBeNull();
901 } finally {
902 if (prior === undefined) delete process.env.FORGE_SENTINEL_REPO;
903 else process.env.FORGE_SENTINEL_REPO = prior;
904 }
905 });
906
907 test("with 3 reds on checks created after the fix, comments once on the closed issue", () => {
908 for (let n = 0; n < 3; n++) red(100 + n, T + n * D_MIN);
909 const r = hunt(fapp, "alice/forge", T + 5 * D_MIN);
910 expect(r.commented).toBe(1);
911 const issues = filedFor();
912 expect(issues.length).toBe(1);
913 expect(issues[0]!.state).toBe("closed");
914 const comments = listComments(fapp.db, issues[0]!.id);
915 expect(comments.length).toBe(1);
916 expect(comments[0]!.body).toContain("Still red after the fix landed at");
917 expect(comments[0]!.body).toContain("3 pulls failed it on checks created since");
918 // Told once: the next hunt inside the hour neither comments nor files.
919 expect(hunt(fapp, "alice/forge", T + 10 * D_MIN).commented).toBe(0);
920 expect(filedFor().length).toBe(1);
921 expect(listComments(fapp.db, issues[0]!.id).length).toBe(1);
922 });
923 });
924
841925 // ---- a finding held for a day is closed as done (#2635) ----
842926
843927 describe("a degraded finding whose cause has stayed closed", () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs