Public: anyone can read this; the crew can change it.
The sentinel does not refile a test's budget breach in the hour after the pull that fixed it landed, unless it fails again on checks started after the fix #8190
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8185-the-sentinel-does-not-refile-a-test-s-bu → main 1 commit fork
3 files
changed +180
−2
src/crew/sentinel.ts +75 −2
| @@ -32,6 +32,7 @@ import { spans } from "../kernel/degraded_history.ts"; | ||
| 32 | 32 | import { QUEUE_WAIT_MAX_MS } from "../kernel/vitals.ts"; |
| 33 | 33 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 34 | 34 | import { isOutageSummary } from "./network.ts"; |
| 35 | import { pullsFailedSince, SHARED_PULLS, SHARED_WINDOW_MS } from "../checks/shared_red.ts"; | |
| 35 | 36 | import { clip } from "../text/clip.ts"; |
| 36 | 37 | |
| 37 | 38 | const log = createLog("sentinel"); |
| @@ -62,6 +63,8 @@ export interface Finding { | ||
| 62 | 63 | readonly fix?: string; |
| 63 | 64 | /** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */ |
| 64 | 65 | readonly incident?: string; |
| 66 | /** The vitals cause a budget-breach or degraded finding reads. */ | |
| 67 | readonly cause?: string; | |
| 65 | 68 | } |
| 66 | 69 | |
| 67 | 70 | const QUOTE_MAX = 200; |
| @@ -398,6 +401,7 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string): | ||
| 398 | 401 | pillar: "P0 Trust: the forge says when it is not well, and is well again", |
| 399 | 402 | lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`, |
| 400 | 403 | incident: new Date(d.opened).toISOString(), |
| 404 | cause, | |
| 401 | 405 | }); |
| 402 | 406 | } |
| 403 | 407 | return out; |
| @@ -454,6 +458,7 @@ export function budgetBreaches(db: Database, now: number, forgeRepoId?: string): | ||
| 454 | 458 | ? "Make the named test's verdict depend only on the code under test, not on the check machine's load, disk or clock, in one pull. Pulls red on it alone wait, unparked, for main's next green check." |
| 455 | 459 | : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.", |
| 456 | 460 | urgent: true, |
| 461 | cause: s.cause, | |
| 457 | 462 | }); |
| 458 | 463 | } |
| 459 | 464 | return out; |
| @@ -610,6 +615,67 @@ function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence: | ||
| 610 | 615 | return 1; |
| 611 | 616 | } |
| 612 | 617 | |
| 618 | interface RecentFix { | |
| 619 | readonly issueId: string; | |
| 620 | readonly closedAt: number; | |
| 621 | /** Its issue was already told the test is still red since that close. */ | |
| 622 | readonly told: boolean; | |
| 623 | } | |
| 624 | ||
| 625 | /** | |
| 626 | * A shared test's issue closed as done or split within the hour (#8185): its | |
| 627 | * close is the landing on main, since a merged pull closes what it fixes only | |
| 628 | * once it lands. The hour's window still holds the reds from before it. | |
| 629 | */ | |
| 630 | function recentFix(db: Database, f: Finding, now: number): RecentFix | undefined { | |
| 631 | if (f.cause?.startsWith("shared:") !== true) return undefined; | |
| 632 | const prior = db | |
| 633 | .query<{ issue_id: string; commented_at: string | null; closed_at: string | null }, [string]>( | |
| 634 | `SELECT t.issue_id, t.commented_at, i.closed_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 635 | WHERE t.key = ?1 AND i.state = 'closed' AND i.parked = 0 AND i.close_reason IN ('done', 'split') | |
| 636 | ORDER BY i.closed_at DESC LIMIT 1`, | |
| 637 | ) | |
| 638 | .get(f.key); | |
| 639 | const closedAt = prior?.closed_at == null ? NaN : Date.parse(prior.closed_at); | |
| 640 | if (prior === null || !(now - closedAt < SHARED_WINDOW_MS)) return undefined; | |
| 641 | const told = prior.commented_at !== null && Date.parse(prior.commented_at) >= closedAt; | |
| 642 | return { issueId: prior.issue_id, closedAt, told }; | |
| 643 | } | |
| 644 | ||
| 645 | /** Whether a recent fix holds the finding back: told already, or too few reds since it landed. */ | |
| 646 | const heldByFix = (db: Database, f: Finding, fix: RecentFix): boolean => | |
| 647 | fix.told || pullsFailedSince(db, f.cause ?? "", fix.closedAt) < SHARED_PULLS; | |
| 648 | ||
| 649 | /** | |
| 650 | * A shared test fixed within the hour files nothing new (#8185). With fewer | |
| 651 | * than SHARED_PULLS pulls red on it on checks created after the fix landed, | |
| 652 | * it says nothing; with that many, it tells the closed issue once, as | |
| 653 | * `continuesDone` does. Undefined when no fix is that recent. | |
| 654 | */ | |
| 655 | function sinceFix(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 | 1 | undefined { | |
| 656 | const fix = recentFix(app.db, f, now); | |
| 657 | if (fix === undefined) return undefined; | |
| 658 | if (heldByFix(app.db, f, fix)) return 0; | |
| 659 | const issue = getIssueById(app.db, fix.issueId); | |
| 660 | if (issue === undefined) return 0; | |
| 661 | const pulls = pullsFailedSince(app.db, f.cause ?? "", fix.closedAt); | |
| 662 | const said = addComment( | |
| 663 | app, | |
| 664 | actor, | |
| 665 | repo, | |
| 666 | issue, | |
| 667 | `Still red after the fix landed at ${new Date(fix.closedAt).toISOString()}: ${pulls} pulls failed it on checks created since; the fix may not be live yet. At ${at}; evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`, | |
| 668 | ); | |
| 669 | if (!said.ok) { | |
| 670 | log.warn("sentinel could not comment", { key: f.key, error: String(said.error) }); | |
| 671 | return 0; | |
| 672 | } | |
| 673 | app.db | |
| 674 | .query("UPDATE sentinel_finding SET commented_at = ?1, last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4") | |
| 675 | .run(at, evidence, f.key, fix.issueId); | |
| 676 | return 1; | |
| 677 | } | |
| 678 | ||
| 613 | 679 | /** |
| 614 | 680 | * A finding backed by an incident whose cause has stayed out of the degraded |
| 615 | 681 | * history for a day is done: the sentinel says since when and closes it. Not a |
| @@ -711,7 +777,9 @@ export function hunt( | ||
| 711 | 777 | .run(at, evidence, f.key, tracked.issue_id); |
| 712 | 778 | continue; |
| 713 | 779 | } |
| 714 | const told = f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now); | |
| 780 | const told = | |
| 781 | sinceFix(app, repo, actor, f, evidence, at, now) ?? | |
| 782 | (f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now)); | |
| 715 | 783 | if (told !== undefined) { |
| 716 | 784 | commented += told; |
| 717 | 785 | continue; |
| @@ -825,7 +893,12 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null { | ||
| 825 | 893 | .all() |
| 826 | 894 | .map((r) => r.key), |
| 827 | 895 | ); |
| 828 | if (!signals.some((s) => !tracked.has(s.key))) return null; | |
| 896 | // A shared test held back by its recent fix is no signal: no hunt a tick for its hour. | |
| 897 | const held = (s: Finding): boolean => { | |
| 898 | const fix = recentFix(app.db, s, now); | |
| 899 | return fix !== undefined && heldByFix(app.db, s, fix); | |
| 900 | }; | |
| 901 | if (!signals.some((s) => !tracked.has(s.key) && !held(s))) return null; | |
| 829 | 902 | } |
| 830 | 903 | return hunt(app, repoPath, now); |
| 831 | 904 | } |
test/sentinel.test.ts +84
| @@ -20,6 +20,7 @@ import { getRepoById, type Repo } from "../src/core/repos.ts"; | ||
| 20 | 20 | import { createRun } from "../src/core/runs.ts"; |
| 21 | 21 | import { createUser, type User } from "../src/core/users.ts"; |
| 22 | 22 | import { recordCap } from "../src/crew/guard.ts"; |
| 23 | import { sharedCause } from "../src/checks/shared_red.ts"; | |
| 23 | 24 | import { |
| 24 | 25 | budgetBreaches, |
| 25 | 26 | degradedCauses, |
| @@ -838,6 +839,89 @@ describe("a degraded incident whose issue closed as done", () => { | ||
| 838 | 839 | }); |
| 839 | 840 | }); |
| 840 | 841 | |
| 842 | // ---- a shared test fixed within the hour (#8185) ---- | |
| 843 | ||
| 844 | describe("a shared test whose fix landed within the hour", () => { | |
| 845 | let fdir: string; | |
| 846 | let fapp: App; | |
| 847 | let frepo: Repo; | |
| 848 | let author: User; | |
| 849 | const TEST = "MCP backups > a listing that fails"; | |
| 850 | const T = Date.parse("2026-10-20T12:00:00.000Z"); | |
| 851 | const cause = (): string => sharedCause({ repoId: frepo.id, test: TEST }); | |
| 852 | const filedFor = () => | |
| 853 | listIssues(fapp.db, frepo.id, { state: "all" }).filter((i) => i.title.startsWith(`Sentinel: ${cause()}`)); | |
| 854 | /** One pull whose check failed on the test, created at `at`. */ | |
| 855 | const red = (n: number, at: number): void => { | |
| 856 | const pull = createIssue(fapp.db, { repoId: frepo.id, authorId: author.id, title: `pull ${n}`, kind: "pull" }); | |
| 857 | if (!pull.ok) throw new Error(String(pull.error)); | |
| 858 | const run = createRun(fapp.db, { repoId: frepo.id, kind: "check", issueId: pull.value.id }); | |
| 859 | fapp.db | |
| 860 | .query("UPDATE run SET state = 'failed', created_at = ?2, finished_at = ?2, check_detail = ?3 WHERE id = ?1") | |
| 861 | .run(run.id, new Date(at).toISOString(), JSON.stringify({ names: [TEST] })); | |
| 862 | }; | |
| 863 | beforeAll(async () => { | |
| 864 | fdir = await mkdtemp(join(tmpdir(), "forge-sentinel-fix-")); | |
| 865 | fapp = createApp({ dataDir: fdir, port: 0 }); | |
| 866 | const a = await createUser(fapp.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 867 | if (!a.ok) throw new Error(a.error); | |
| 868 | author = a.value; | |
| 869 | const made = await createRepo(fapp, a.value, { name: "forge", visibility: "private" }); | |
| 870 | if (!made.ok) throw new Error(String(made.error)); | |
| 871 | frepo = made.value; | |
| 872 | fapp.db | |
| 873 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'It is red on 12 pulls.', '/x', ?2)") | |
| 874 | .run(cause(), T - 50 * D_MIN); | |
| 875 | for (let n = 0; n < 12; n++) red(n, T - 40 * D_MIN + n * D_MIN); | |
| 876 | expect(filedFor().length).toBe(0); | |
| 877 | hunt(fapp, "alice/forge", T - 2 * D_MIN); | |
| 878 | expect(filedFor().length).toBe(1); | |
| 879 | // The fix lands a minute before the hunt: its pull closed the issue as done. | |
| 880 | fapp.db | |
| 881 | .query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE id = ?2") | |
| 882 | .run(new Date(T - D_MIN).toISOString(), filedFor()[0]!.id); | |
| 883 | }); | |
| 884 | afterAll(async () => { | |
| 885 | fapp.db.close(); | |
| 886 | await rm(fdir, { recursive: true, force: true }); | |
| 887 | }); | |
| 888 | ||
| 889 | test("with 12 earlier reds and none since, files no issue and says nothing", () => { | |
| 890 | // Still a breach, as the terminal and the MCP count it. | |
| 891 | expect(budgetBreaches(fapp.db, T, frepo.id).map((f) => f.cause)).toEqual([cause()]); | |
| 892 | const r = hunt(fapp, "alice/forge", T); | |
| 893 | expect(r.commented).toBe(0); | |
| 894 | expect(filedFor().length).toBe(1); | |
| 895 | expect(listComments(fapp.db, filedFor()[0]!.id).length).toBe(0); | |
| 896 | const prior = process.env.FORGE_SENTINEL_REPO; | |
| 897 | process.env.FORGE_SENTINEL_REPO = "alice/forge"; | |
| 898 | try { | |
| 899 | // No hunt a tick for the hour it is held. | |
| 900 | expect(sentinelTick(fapp, T + D_MIN)).toBeNull(); | |
| 901 | } finally { | |
| 902 | if (prior === undefined) delete process.env.FORGE_SENTINEL_REPO; | |
| 903 | else process.env.FORGE_SENTINEL_REPO = prior; | |
| 904 | } | |
| 905 | }); | |
| 906 | ||
| 907 | test("with 3 reds on checks created after the fix, comments once on the closed issue", () => { | |
| 908 | for (let n = 0; n < 3; n++) red(100 + n, T + n * D_MIN); | |
| 909 | const r = hunt(fapp, "alice/forge", T + 5 * D_MIN); | |
| 910 | expect(r.commented).toBe(1); | |
| 911 | const issues = filedFor(); | |
| 912 | expect(issues.length).toBe(1); | |
| 913 | expect(issues[0]!.state).toBe("closed"); | |
| 914 | const comments = listComments(fapp.db, issues[0]!.id); | |
| 915 | expect(comments.length).toBe(1); | |
| 916 | expect(comments[0]!.body).toContain("Still red after the fix landed at"); | |
| 917 | expect(comments[0]!.body).toContain("3 pulls failed it on checks created since"); | |
| 918 | // Told once: the next hunt inside the hour neither comments nor files. | |
| 919 | expect(hunt(fapp, "alice/forge", T + 10 * D_MIN).commented).toBe(0); | |
| 920 | expect(filedFor().length).toBe(1); | |
| 921 | expect(listComments(fapp.db, issues[0]!.id).length).toBe(1); | |
| 922 | }); | |
| 923 | }); | |
| 924 | ||
| 841 | 925 | // ---- a finding held for a day is closed as done (#2635) ---- |
| 842 | 926 | |
| 843 | 927 | describe("a degraded finding whose cause has stayed closed", () => { |
new above
offline