Public: anyone can read this; the crew can change it.
The sentinel closes its own shared-red issue when the test passes again with no fix, instead of handing it to a builder #8346
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/8287-the-sentinel-closes-its-own-shared-red-i → main 2 commits fork
2 files
changed +350
−11
src/crew/sentinel.ts +176 −11
| @@ -20,8 +20,13 @@ import type { App } from "../app.ts"; | ||
| 20 | 20 | import { addComment, openIssue, setIssueState } from "../actions/issues.ts"; |
| 21 | 21 | import { decisionsSince } from "../core/decisions.ts"; |
| 22 | 22 | import { getAgentByName } from "../core/agents.ts"; |
| 23 | import { getIssueById, getLabel, setIssueLabels as writeLabels, type Issue } from "../core/issues.ts"; | |
| 24 | import { getRepo, type Repo } from "../core/repos.ts"; | |
| 23 | import { | |
| 24 | getIssueById, | |
| 25 | getLabel, | |
| 26 | setIssueLabels as writeLabels, | |
| 27 | type Issue, | |
| 28 | } from "../core/issues.ts"; | |
| 29 | import { canWrite, getRepo, type Repo } from "../core/repos.ts"; | |
| 25 | 30 | import { ownRepoPath } from "../kernel/own_repo.ts"; |
| 26 | 31 | import { leaseLapsed, listRunning } from "../core/runs.ts"; |
| 27 | 32 | import { migrations } from "../core/schema.ts"; |
| @@ -733,6 +738,164 @@ function sinceFix(app: App, repo: Repo, actor: User, f: Finding, evidence: strin | ||
| 733 | 738 | return 1; |
| 734 | 739 | } |
| 735 | 740 | |
| 741 | /** Whether a person, not an agent, has commented on the issue: then they decide, not the sentinel. */ | |
| 742 | function personSpoke(db: Database, issueId: string): boolean { | |
| 743 | const spoke = db | |
| 744 | .query<{ n: number }, [string]>( | |
| 745 | `SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id | |
| 746 | WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`, | |
| 747 | ) | |
| 748 | .get(issueId); | |
| 749 | return spoke !== null && spoke.n > 0; | |
| 750 | } | |
| 751 | ||
| 752 | /** The test a shared finding names, read back from its evidence. */ | |
| 753 | const testOf = (evidence: readonly string[]): string | undefined => | |
| 754 | evidence.find((e) => e.startsWith("test: "))?.slice(6); | |
| 755 | ||
| 756 | /** A finding's test, or, once its runs left the hour's window, the one its last record named. */ | |
| 757 | function testFor(db: Database, f: Finding): string | undefined { | |
| 758 | const named = testOf(f.evidence); | |
| 759 | if (named !== undefined || f.cause?.startsWith("shared:") !== true) return named; | |
| 760 | const last = db | |
| 761 | .query<{ evidence: string }, [string]>( | |
| 762 | "SELECT evidence FROM sentinel_finding WHERE key = ?1 ORDER BY last_seen DESC LIMIT 1", | |
| 763 | ) | |
| 764 | .get(f.key); | |
| 765 | return last === null ? undefined : testOf(last.evidence.split("\n")); | |
| 766 | } | |
| 767 | ||
| 768 | interface Settled { | |
| 769 | readonly at: number; | |
| 770 | readonly evidence: readonly string[]; | |
| 771 | } | |
| 772 | ||
| 773 | /** | |
| 774 | * A shared red that ended by itself (#8287): its span has closed, no pull | |
| 775 | * failed the test on a check created since, and main's newest check, created | |
| 776 | * after the span closed, passes it: a main check from before the red ended | |
| 777 | * says nothing about it. Undefined while any of those does not hold, or the | |
| 778 | * test is unknown. | |
| 779 | */ | |
| 780 | function settled(db: Database, cause: string, test: string | undefined, now: number): Settled | undefined { | |
| 781 | if (!cause.startsWith("shared:") || test === undefined) return undefined; | |
| 782 | let at = -Infinity; | |
| 783 | for (const s of spans(db, now - DAY_MS, now)) { | |
| 784 | if (s.cause !== cause) continue; | |
| 785 | if (s.closedAt === null) return undefined; | |
| 786 | at = Math.max(at, s.closedAt); | |
| 787 | } | |
| 788 | if (at === -Infinity) return undefined; | |
| 789 | const pulls = pullsFailedSince(db, cause, at); | |
| 790 | if (pulls > 0) return undefined; | |
| 791 | const main = mainVerdict(db, cause.split(":")[1] ?? "", test); | |
| 792 | if (main === undefined || !main.passes) return undefined; | |
| 793 | const created = db.query<{ created_at: string }, [string]>("SELECT created_at FROM run WHERE id = ?1").get(main.runId); | |
| 794 | if (created === null || !(Date.parse(created.created_at) > at)) return undefined; | |
| 795 | return { | |
| 796 | at, | |
| 797 | evidence: [ | |
| 798 | `cause: ${cause}`, | |
| 799 | `test: ${test}`, | |
| 800 | `span closed: ${new Date(at).toISOString()}`, | |
| 801 | `pulls failed it on checks created since: ${pulls}`, | |
| 802 | `main: passes on main's newest check, run ${main.runId} at ${main.commitOid}, created ${created.created_at}`, | |
| 803 | ], | |
| 804 | }; | |
| 805 | } | |
| 806 | ||
| 807 | /** | |
| 808 | * A shared red that passed again on its own closes its own open issue as | |
| 809 | * dropped, before a builder is handed it and finds nothing to fix (#8287). | |
| 810 | * Not one a person has spoken on: they decide. If the cause opens again, | |
| 811 | * the next hunt files it fresh and quotes this one. A sentinel that may not | |
| 812 | * close issues here closes nothing and says so: no access check is skipped. | |
| 813 | * | |
| 814 | * It runs on every `sentinelTick`, not only in the hourly `hunt`, because a | |
| 815 | * builder may take an open issue on any tick; `hunt` runs it too, first, so | |
| 816 | * a hunt never comments on an issue it is about to close. | |
| 817 | */ | |
| 818 | function settleShared(app: App, repo: Repo, actor: User, now: number): void { | |
| 819 | const open = app.db | |
| 820 | .query<{ key: string; issue_id: string; evidence: string }, []>( | |
| 821 | `SELECT t.key, t.issue_id, t.evidence FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 822 | WHERE i.state = 'open' AND t.key LIKE 'degraded:shared-%'`, | |
| 823 | ) | |
| 824 | .all(); | |
| 825 | if (open.length === 0) return; | |
| 826 | if (!canWrite(app.db, repo, actor)) { | |
| 827 | log.warn("sentinel may not close issues here; a settled shared red stays open", { repo: repo.id }); | |
| 828 | return; | |
| 829 | } | |
| 830 | const causes = new Map(spans(app.db, now - DAY_MS, now).map((s) => [`degraded:${keyPart(s.cause)}`, s.cause])); | |
| 831 | for (const t of open) { | |
| 832 | try { | |
| 833 | const cause = causes.get(t.key); | |
| 834 | if (cause === undefined) continue; | |
| 835 | const done = settled(app.db, cause, testOf(t.evidence.split("\n")), now); | |
| 836 | if (done === undefined || personSpoke(app.db, t.issue_id)) continue; | |
| 837 | const issue = getIssueById(app.db, t.issue_id); | |
| 838 | if (issue === undefined || issue.repoId !== repo.id) continue; | |
| 839 | const said = addComment( | |
| 840 | app, | |
| 841 | actor, | |
| 842 | repo, | |
| 843 | issue, | |
| 844 | [ | |
| 845 | `Passed again on its own at ${new Date(done.at).toISOString()}; evidence:`, | |
| 846 | ...done.evidence.map((e) => `- ${e}`), | |
| 847 | "", | |
| 848 | "Closing as dropped: there is nothing to fix. If it opens again it is filed fresh, quoting this.", | |
| 849 | ].join("\n"), | |
| 850 | ); | |
| 851 | if (!said.ok) { | |
| 852 | log.warn("sentinel could not comment", { key: t.key, error: String(said.error) }); | |
| 853 | continue; | |
| 854 | } | |
| 855 | const closed = setIssueState(app, actor, repo, issue, "closed", { reason: "dropped" }); | |
| 856 | if (!closed.ok) { | |
| 857 | log.warn("sentinel could not close", { key: t.key, error: String(closed.error) }); | |
| 858 | continue; | |
| 859 | } | |
| 860 | // When it settled, on the forge's clock: the repeat rule reads this, never the comment's words. | |
| 861 | app.db | |
| 862 | .query("UPDATE sentinel_finding SET commented_at = ?1 WHERE key = ?2 AND issue_id = ?3") | |
| 863 | .run(new Date(now).toISOString(), t.key, t.issue_id); | |
| 864 | } catch (error) { | |
| 865 | log.error("sentinel settle failed", { key: t.key, error: String(error) }); | |
| 866 | } | |
| 867 | } | |
| 868 | } | |
| 869 | ||
| 870 | /** | |
| 871 | * The issue a shared cause settled on within the day, and what the sentinel | |
| 872 | * said closing it: a repeat is filed fresh and quotes it (#8287). | |
| 873 | */ | |
| 874 | function settledBefore(db: Database, actor: User, f: Finding, now: number): string | undefined { | |
| 875 | if (f.cause?.startsWith("shared:") !== true) return undefined; | |
| 876 | // A dropped finding's `commented_at` is when it settled (`settleShared`); done and split ones keep theirs. | |
| 877 | const prior = db | |
| 878 | .query<{ issue_id: string; number: number; commented_at: string }, [string]>( | |
| 879 | `SELECT t.issue_id, i.number, t.commented_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 880 | WHERE t.key = ?1 AND i.state = 'closed' AND i.close_reason = 'dropped' AND t.commented_at IS NOT NULL | |
| 881 | ORDER BY t.commented_at DESC LIMIT 1`, | |
| 882 | ) | |
| 883 | .get(f.key); | |
| 884 | if (prior === null || !(now - Date.parse(prior.commented_at) < DAY_MS)) return undefined; | |
| 885 | // What it said closing it: its last comment there. | |
| 886 | const said = db | |
| 887 | .query<{ body: string }, [string, string]>( | |
| 888 | "SELECT body FROM comment WHERE issue_id = ?1 AND author_id = ?2 ORDER BY created_at DESC, id DESC LIMIT 1", | |
| 889 | ) | |
| 890 | .get(prior.issue_id, actor.id); | |
| 891 | if (said === null) return undefined; | |
| 892 | return [ | |
| 893 | "", | |
| 894 | `**Opened again within a day of #${prior.number}**, which the sentinel closed as passed on its own:`, | |
| 895 | ...said.body.split("\n").map((l) => `> ${l}`), | |
| 896 | ].join("\n"); | |
| 897 | } | |
| 898 | ||
| 736 | 899 | /** |
| 737 | 900 | * A finding backed by an incident whose cause has stayed out of the degraded |
| 738 | 901 | * history for a day is done: the sentinel says since when and closes it. Not a |
| @@ -759,13 +922,7 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string | ||
| 759 | 922 | else since = Math.max(since, s.closedAt); |
| 760 | 923 | } |
| 761 | 924 | if (live || since === -Infinity || now - since < DAY_MS) continue; |
| 762 | const spoke = app.db | |
| 763 | .query<{ n: number }, [string]>( | |
| 764 | `SELECT COUNT(*) AS n FROM comment c JOIN user u ON u.id = c.author_id | |
| 765 | WHERE c.issue_id = ?1 AND u.is_agent = 0 AND u.kind = 'user'`, | |
| 766 | ) | |
| 767 | .get(t.issue_id); | |
| 768 | if (spoke !== null && spoke.n > 0) continue; | |
| 925 | if (personSpoke(app.db, t.issue_id)) continue; | |
| 769 | 926 | const issue = getIssueById(app.db, t.issue_id); |
| 770 | 927 | if (issue === undefined || issue.repoId !== repo.id) continue; |
| 771 | 928 | const said = addComment( |
| @@ -806,6 +963,7 @@ export function hunt( | ||
| 806 | 963 | const findings = findFaults(app.db, now, repo?.id, app.config.dataDir); |
| 807 | 964 | const actor = sentinelActor(app.db); |
| 808 | 965 | if (repo !== undefined && actor !== undefined) { |
| 966 | settleShared(app, repo, actor, now); | |
| 809 | 967 | for (const f of findings) { |
| 810 | 968 | try { |
| 811 | 969 | const evidence = f.evidence.join("\n"); |
| @@ -834,6 +992,8 @@ export function hunt( | ||
| 834 | 992 | .run(at, evidence, f.key, tracked.issue_id); |
| 835 | 993 | continue; |
| 836 | 994 | } |
| 995 | // A shared red that passed again on its own is no work: the day's total does not file it again. | |
| 996 | if (settled(app.db, f.cause ?? "", testFor(app.db, f), now) !== undefined) continue; | |
| 837 | 997 | const told = |
| 838 | 998 | sinceFix(app, repo, actor, f, evidence, at, now) ?? |
| 839 | 999 | (f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now)); |
| @@ -851,7 +1011,8 @@ export function hunt( | ||
| 851 | 1011 | .get(repo.id, title, actor.id); |
| 852 | 1012 | let issueId = adopted?.id; |
| 853 | 1013 | if (issueId === undefined) { |
| 854 | const made = openIssue(app, actor, repo, { title, body: bodyOf(f, at) }); | |
| 1014 | const body = bodyOf(f, at) + (settledBefore(app.db, actor, f, now) ?? ""); | |
| 1015 | const made = openIssue(app, actor, repo, { title, body }); | |
| 855 | 1016 | if (!made.ok) { |
| 856 | 1017 | log.warn("sentinel could not file", { key: f.key, error: String(made.error) }); |
| 857 | 1018 | continue; |
| @@ -934,7 +1095,11 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null { | ||
| 934 | 1095 | const last = sentinelStanding(app.db).lastHuntAt; |
| 935 | 1096 | const since = last === null ? Infinity : now - new Date(last).getTime(); |
| 936 | 1097 | const [own = "", ownName = ""] = repoPath.split("/"); |
| 937 | const breaches = budgetBreaches(app.db, now, getRepo(app.db, own, ownName)?.id); | |
| 1098 | const ownRepo = getRepo(app.db, own, ownName); | |
| 1099 | // Every tick, before a builder takes it: see `settleShared`. | |
| 1100 | const actor = sentinelActor(app.db); | |
| 1101 | if (ownRepo !== undefined && actor !== undefined) settleShared(app, ownRepo, actor, now); | |
| 1102 | const breaches = budgetBreaches(app.db, now, ownRepo?.id); | |
| 938 | 1103 | if (since < SIGNAL_GAP_MS && breaches.length === 0) return null; |
| 939 | 1104 | if (since < HUNT_EVERY_MS) { |
| 940 | 1105 | // A breach is not held back by the gap: the crew hears within a tick. |
new above
offline