Public: anyone can read this; the crew can change it.

The sentinel files no forge issue for another repository's red main; that repository's owner is told instead, in words that name the fix #3996

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/3992-the-sentinel-files-no-forge-issue-for-an → main 1 commit fork 2 threads · 2 unresolved

  • Merged. scout/3992-the-sentinel-files-no-forge-issue-for-an was merged into main as 9f58ae4.

Review

Sign in to review this change.
5 files changed +118 −13
src/checks/red_main.ts +25 −1
@@ -29,6 +29,8 @@ export interface RedMain {
2929 /** The failing tests' names the check recorded, else its summary. */
3030 readonly failing: readonly string[];
3131 readonly summary: string;
32 /** The repository's default branch, for the link to its forge.json. */
33 readonly branch: string;
3234 /** The check was stopped by its cell's limit or its budget, not by a test: no pull shares it. */
3335 readonly limit: boolean;
3436 readonly since: string;
@@ -44,6 +46,7 @@ interface Row {
4446 id: string;
4547 repo_id: string;
4648 repo: string;
49 branch: string;
4750 commit_oid: string;
4851 state: string;
4952 summary: string;
@@ -69,7 +72,7 @@ export function redMain(db: Database, repoId: string): RedMain | undefined {
6972 const lastMain = (from: string): string => `
7073 SELECT run.id, run.repo_id, run.commit_oid, run.state, run.summary,
7174 run.check_detail, run.finished_at, run.created_at,
72 u.username || '/' || repo.name AS repo
75 u.username || '/' || repo.name AS repo, repo.default_branch AS branch
7376 FROM (${from}) l
7477 JOIN repo ON repo.id = l.repo_id
7578 JOIN user u ON u.id = repo.owner_id
@@ -100,6 +103,7 @@ function toRed(row: Row): RedMain | undefined {
100103 commitOid: row.commit_oid,
101104 failing,
102105 summary: row.summary,
106 branch: row.branch,
103107 limit: LIMIT.test(row.summary),
104108 since: row.finished_at ?? row.created_at,
105109 };
@@ -191,8 +195,28 @@ export function freeChecks(
191195 return row ?? { n: 0, oldest: null };
192196 }
193197
198 /**
199 * The manifest error a check failed on before any test ran
200 * (`apps/manifest.ts`), else undefined: such a check is the repository's own
201 * to fix, not a failing test.
202 */
203 export function manifestError(red: RedMain): string | undefined {
204 const text = red.summary.replace(/^check failed: /, "");
205 return red.failing.length === 0 && text.startsWith("forge.json: ") ? text : undefined;
206 }
207
208 /** Where a red main's card points: its forge.json when that is invalid, else the run. */
209 export function redHref(red: RedMain): string {
210 return manifestError(red) === undefined
211 ? `/${red.repo}/runs/${red.runId}`
212 : `/${red.repo}/blob/${encodeURIComponent(red.branch)}/forge.json`;
213 }
214
194215 /** The card's sentence: what is red, by which test, and what waits. */
195216 export function redSentence(red: RedMain, held: number): string {
217 const bad = manifestError(red);
218 if (bad !== undefined)
219 return `${red.repo}: the forge.json on main (${red.commitOid.slice(0, 7)}) is invalid, so no check can run — "${bad.slice(0, 200)}"; fix forge.json (${redHref(red)}); ${held} pull check${held === 1 ? "" : "s"} wait for it.`;
196220 const first = red.failing[0];
197221 const what =
198222 first === undefined
src/crew/notify.ts +25
@@ -28,6 +28,7 @@ import type { App } from "../app.ts";
2828 import { isPrivateHost } from "../actions/migrate.ts";
2929 import { getUserSetting, setUserSetting } from "../core/settings.ts";
3030 import { getUserById, type User } from "../core/users.ts";
31 import { redHref, redMain, redSentence } from "../checks/red_main.ts";
3132 import { landedLines } from "../core/watches.ts";
3233 import { err, ok, type Result } from "../kernel/result.ts";
3334 import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
@@ -189,6 +190,29 @@ export function mirrorLines(
189190 return [{ id, line: need.sentence, items: [id] }];
190191 }
191192
193 /**
194 * A person hears once when main is red in a repository they own, however
195 * the check failed: the crew files nothing for another repository's red
196 * main (#3992), so the owner is the one told. The id is the failing run's.
197 */
198 export function redLines(
199 app: App,
200 user: User,
201 _now = Date.now(),
202 ): { id: string; line: string; items: readonly string[]; link: string }[] {
203 if (user.kind !== "user") return [];
204 const out: { id: string; line: string; items: readonly string[]; link: string }[] = [];
205 for (const { id } of app.db

_now is unused; drop it (and the argument at the call site).

206 .query<{ id: string }, [string]>("SELECT id FROM repo WHERE owner_id = ?1 AND archived_at IS NULL")
207 .all(user.id)) {
208 const red = redMain(app.db, id);
209 if (red === undefined || red.limit) continue;
210 const key = `red:${red.runId}`;
211 out.push({ id: key, line: redSentence(red, 0), items: [key], link: redHref(red) });

redSentence(red, 0) tells the owner "0 pull checks wait for it", which is false when pulls are held. Pass heldChecks count or omit the clause for notifications.

212 }
213 return out;
214 }
215
192216 export interface Notifier {
193217 /** Resolves when the work queued so far is done. For tests and shutdown. */
194218 settled(): Promise<void>;
@@ -368,6 +392,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
368392 ...mailCeilingLines(app, user, now()),
369393 ...drillLines(app, user, now()),
370394 ...mirrorLines(app, user, now()),
395 ...redLines(app, user, now()),
371396 ];
372397 const before = seen.get(userId);
373398 // What is gone may ask again later, and then it is new again. Seen by
src/crew/sentinel.ts +20 −8
@@ -350,10 +350,11 @@ export function repeatedErrors(now: number): Finding[] {
350350 * Each cause whose spans total more than ten minutes in the day ending at
351351 * `now`, quoting its sentence. Shorter totals stay in the history.
352352 */
353 export function degradedCauses(db: Database, now: number): Finding[] {
353 export function degradedCauses(db: Database, now: number, forgeRepoId?: string): Finding[] {
354354 const from = now - DAY_MS;
355355 const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>();
356356 for (const s of spans(db, from, now)) {
357 if (!ownCause(s.cause, forgeRepoId)) continue;
357358 const start = Math.max(s.openedAt, from);
358359 const held = Math.min(s.closedAt ?? now, now) - start;
359360 const seen = byCause.get(s.cause);
@@ -397,15 +398,25 @@ export function degradedCauses(db: Database, now: number): Finding[] {
397398 */
398399 const BREACH_CAUSE = /^(queue:check|stuck:|budget:|red:|shared:)/;
399400
401 /**
402 * A red main in another repository is its owner's to fix, not the forge's
403 * (#3992): the crew cannot see that repository, so no issue is filed here.
404 * Its card stays in the vitals and the owner is notified (`crew/notify.ts`).
405 * `forgeRepoId` unset means no repository is known and nothing is dropped.
406 */
407 export function ownCause(cause: string, forgeRepoId: string | undefined): boolean {
408 return forgeRepoId === undefined || !cause.startsWith("red:") || cause === `red:${forgeRepoId}`;
409 }
410
400411 /**
401412 * Each budget breach that holds right now, however briefly: the crew hears
402413 * at once, not when the day has added up. The finding shares its key with
403414 * `degradedCauses`, so one cause is one issue.
404415 */
405 export function budgetBreaches(db: Database, now: number): Finding[] {
416 export function budgetBreaches(db: Database, now: number, forgeRepoId?: string): Finding[] {
406417 const out: Finding[] = [];
407418 for (const s of spans(db, now - DAY_MS, now)) {
408 if (s.closedAt !== null || !BREACH_CAUSE.test(s.cause)) continue;
419 if (s.closedAt !== null || !BREACH_CAUSE.test(s.cause) || !ownCause(s.cause, forgeRepoId)) continue;
409420 out.push({
410421 key: `degraded:${keyPart(s.cause)}`,
411422 title: `${s.cause} is past its budget: ${clip(oneLine(s.sentence), 120)}`,
@@ -442,15 +453,15 @@ function runLogLines(db: Database, cause: string): string[] {
442453 }
443454
444455 /** Everything the sentinel reads, in one pass. */
445 export function findFaults(db: Database, now = Date.now()): Finding[] {
456 export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string): Finding[] {
446457 const found = [
447 ...budgetBreaches(db, now),
458 ...budgetBreaches(db, now, forgeRepoId),
448459 ...heldSlots(db, now),
449460 ...strayTriggers(db),
450461 ...ledgerStreaks(db, now),
451462 ...runStreaks(db),
452463 ...repeatedErrors(now),
453 ...degradedCauses(db, now),
464 ...degradedCauses(db, now, forgeRepoId),
454465 ];
455466 const keys = new Set<string>();
456467 return found
@@ -644,11 +655,11 @@ export function hunt(
644655 now = Date.now(),
645656 ): HuntResult {
646657 const at = new Date(now).toISOString();
647 const findings = findFaults(app.db, now);
648658 let filed = 0;
649659 let commented = 0;
650660 const [owner = "", name = ""] = repoPath.split("/");
651661 const repo = getRepo(app.db, owner, name);
662 const findings = findFaults(app.db, now, repo?.id);
652663 const actor = sentinelActor(app.db);
653664 if (repo !== undefined && actor !== undefined) {
654665 for (const f of findings) {
@@ -776,7 +787,8 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null {
776787 const repoPath = sentinelRepoPath();
777788 const last = sentinelStanding(app.db).lastHuntAt;
778789 const since = last === null ? Infinity : now - new Date(last).getTime();
779 const breaches = budgetBreaches(app.db, now);
790 const [own = "", ownName = ""] = repoPath.split("/");
791 const breaches = budgetBreaches(app.db, now, getRepo(app.db, own, ownName)?.id);
780792 if (since < SIGNAL_GAP_MS && breaches.length === 0) return null;
781793 if (since < HUNT_EVERY_MS) {
782794 // A breach is not held back by the gap: the crew hears within a tick.
src/kernel/vitals.ts +2 −2
@@ -32,7 +32,7 @@ import { behindStanding } from "../release/behind.ts";
3232 import { newestDrills } from "../core/domain_drills.ts";
3333 import { DOMAINS } from "./domains.ts";
3434 import { domainDrillNeeds } from "./domain_drill.ts";
35 import { freeChecks, heldChecks, redSentence } from "../checks/red_main.ts";
35 import { freeChecks, heldChecks, redHref, redSentence } from "../checks/red_main.ts";
3636 import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
3737
3838 /** Free disk under this share of the volume is something to say. */
@@ -379,7 +379,7 @@ export function vitals(input: VitalsInput): Vitals {
379379 `red:${red.repoId}`,
380380 redSentence(red, held),
381381 red.since,
382 `/${red.repo}/runs/${red.runId}`,
382 redHref(red),
383383 );
384384
385385 // A test failing on many pulls in the hour is one card, not a park per pull.
test/red-main.test.ts +46 −2
@@ -9,9 +9,10 @@ import { tmpdir } from "node:os";
99 import { migrations } from "../src/core/schema.ts";
1010 import { openDb } from "../src/kernel/db.ts";
1111 import { claimRun, createRun, finishRun, listActive, type Run } from "../src/core/runs.ts";
12 import { RED_HOLD_MS, heldChecks, redMain, redSentence, sharedWithMain, unheld } from "../src/checks/red_main.ts";
12 import { RED_HOLD_MS, heldChecks, redHref, redMain, redSentence, sharedWithMain, unheld } from "../src/checks/red_main.ts";
1313 import { coveredByRedFix } from "../src/crew/red_fix.ts";
14 import { budgetBreaches } from "../src/crew/sentinel.ts";
14 import { budgetBreaches, findFaults } from "../src/crew/sentinel.ts";
15 import { redLines } from "../src/crew/notify.ts";
1516 import { vitals } from "../src/kernel/vitals.ts";
1617 import { ErrorLedger } from "../src/kernel/log.ts";
1718 import { RefusedLedger } from "../src/kernel/internal_refused.ts";
@@ -238,3 +239,46 @@ describe("the lane sentence and a red main", () => {
238239 expect(causes).not.toContain("queue:check");
239240 });
240241 });
242
243 describe("a red main in another repository (#3992)", () => {
244 const span = (cause: string) =>
245 db.run(
246 "INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'x: main is red', '/x', 1000)",
247 [cause],
248 );
249
250 test("files no sentinel issue for it; the forge's own red main still files one", () => {
251 span("red:r2");
252 span("red:r1");
253 expect(budgetBreaches(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]);
254 expect(findFaults(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]);
255 expect(budgetBreaches(db, 2000, "r2").map((f) => f.key)).toEqual(["degraded:red-r2"]);
256 });
257
258 test("a manifest error reads as an invalid forge.json, quotes it and links to the file", () => {
259 const run = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID("d"), reason: "push" });
260 claimRun(db, run.id);
261 finishRun(db, run.id, { state: "failed", summary: "check failed: forge.json: check.run is required" });
262 const r = redMain(db, "r1")!;
263 const sentence = redSentence(r, 2);
264 expect(sentence).toContain("forge.json on main");
265 expect(sentence).toContain("invalid");
266 expect(sentence).toContain("forge.json: check.run is required");
267 expect(redHref(r)).toBe("/trevato/forge/blob/main/forge.json");
268 expect(redSentence(redMain(db, "r1")!, 0)).not.toContain("tests");
269 });
270
271 test("a failing test keeps its own sentence and run link", () => {
272 const r = redMain(db, main("e", "failed", ["a test"]).repoId)!;
273 expect(redSentence(r, 1)).toContain('"a test" fails');
274 expect(redHref(r)).toBe(`/trevato/forge/runs/${r.runId}`);
275 });
276
277 test("the owner of the red repository is told, once per failing run", () => {
278 main("f", "failed", ["a test"]);
279 const lines = redLines({ db } as never, { id: "u1", kind: "user" } as never);
280 expect(lines).toHaveLength(1);
281 expect(lines[0]!.line).toContain("trevato/forge: main is red");
282 expect(redLines({ db } as never, { id: "u2", kind: "user" } as never)).toEqual([]);
283 });
284 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs