Public: anyone can read this; the crew can change it.
The sentinel files no forge issue for another repository's red main; that repository's owner is told instead, in words that name the fix #3996
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3992-the-sentinel-files-no-forge-issue-for-an → main 1 commit fork 2 threads · 2 unresolved
5 files
changed +118
−13
src/checks/red_main.ts +25 −1
| @@ -29,6 +29,8 @@ export interface RedMain { | ||
| 29 | 29 | /** The failing tests' names the check recorded, else its summary. */ |
| 30 | 30 | readonly failing: readonly string[]; |
| 31 | 31 | readonly summary: string; |
| 32 | /** The repository's default branch, for the link to its forge.json. */ | |
| 33 | readonly branch: string; | |
| 32 | 34 | /** The check was stopped by its cell's limit or its budget, not by a test: no pull shares it. */ |
| 33 | 35 | readonly limit: boolean; |
| 34 | 36 | readonly since: string; |
| @@ -44,6 +46,7 @@ interface Row { | ||
| 44 | 46 | id: string; |
| 45 | 47 | repo_id: string; |
| 46 | 48 | repo: string; |
| 49 | branch: string; | |
| 47 | 50 | commit_oid: string; |
| 48 | 51 | state: string; |
| 49 | 52 | summary: string; |
| @@ -69,7 +72,7 @@ export function redMain(db: Database, repoId: string): RedMain | undefined { | ||
| 69 | 72 | const lastMain = (from: string): string => ` |
| 70 | 73 | SELECT run.id, run.repo_id, run.commit_oid, run.state, run.summary, |
| 71 | 74 | run.check_detail, run.finished_at, run.created_at, |
| 72 | u.username || '/' || repo.name AS repo | |
| 75 | u.username || '/' || repo.name AS repo, repo.default_branch AS branch | |
| 73 | 76 | FROM (${from}) l |
| 74 | 77 | JOIN repo ON repo.id = l.repo_id |
| 75 | 78 | JOIN user u ON u.id = repo.owner_id |
| @@ -100,6 +103,7 @@ function toRed(row: Row): RedMain | undefined { | ||
| 100 | 103 | commitOid: row.commit_oid, |
| 101 | 104 | failing, |
| 102 | 105 | summary: row.summary, |
| 106 | branch: row.branch, | |
| 103 | 107 | limit: LIMIT.test(row.summary), |
| 104 | 108 | since: row.finished_at ?? row.created_at, |
| 105 | 109 | }; |
| @@ -191,8 +195,28 @@ export function freeChecks( | ||
| 191 | 195 | return row ?? { n: 0, oldest: null }; |
| 192 | 196 | } |
| 193 | 197 | |
| 198 | /** | |
| 199 | * The manifest error a check failed on before any test ran | |
| 200 | * (`apps/manifest.ts`), else undefined: such a check is the repository's own | |
| 201 | * to fix, not a failing test. | |
| 202 | */ | |
| 203 | export function manifestError(red: RedMain): string | undefined { | |
| 204 | const text = red.summary.replace(/^check failed: /, ""); | |
| 205 | return red.failing.length === 0 && text.startsWith("forge.json: ") ? text : undefined; | |
| 206 | } | |
| 207 | ||
| 208 | /** Where a red main's card points: its forge.json when that is invalid, else the run. */ | |
| 209 | export function redHref(red: RedMain): string { | |
| 210 | return manifestError(red) === undefined | |
| 211 | ? `/${red.repo}/runs/${red.runId}` | |
| 212 | : `/${red.repo}/blob/${encodeURIComponent(red.branch)}/forge.json`; | |
| 213 | } | |
| 214 | ||
| 194 | 215 | /** The card's sentence: what is red, by which test, and what waits. */ |
| 195 | 216 | export function redSentence(red: RedMain, held: number): string { |
| 217 | const bad = manifestError(red); | |
| 218 | if (bad !== undefined) | |
| 219 | return `${red.repo}: the forge.json on main (${red.commitOid.slice(0, 7)}) is invalid, so no check can run — "${bad.slice(0, 200)}"; fix forge.json (${redHref(red)}); ${held} pull check${held === 1 ? "" : "s"} wait for it.`; | |
| 196 | 220 | const first = red.failing[0]; |
| 197 | 221 | const what = |
| 198 | 222 | first === undefined |
src/crew/notify.ts +25
| @@ -28,6 +28,7 @@ import type { App } from "../app.ts"; | ||
| 28 | 28 | import { isPrivateHost } from "../actions/migrate.ts"; |
| 29 | 29 | import { getUserSetting, setUserSetting } from "../core/settings.ts"; |
| 30 | 30 | import { getUserById, type User } from "../core/users.ts"; |
| 31 | import { redHref, redMain, redSentence } from "../checks/red_main.ts"; | |
| 31 | 32 | import { landedLines } from "../core/watches.ts"; |
| 32 | 33 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 33 | 34 | import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts"; |
| @@ -189,6 +190,29 @@ export function mirrorLines( | ||
| 189 | 190 | return [{ id, line: need.sentence, items: [id] }]; |
| 190 | 191 | } |
| 191 | 192 | |
| 193 | /** | |
| 194 | * A person hears once when main is red in a repository they own, however | |
| 195 | * the check failed: the crew files nothing for another repository's red | |
| 196 | * main (#3992), so the owner is the one told. The id is the failing run's. | |
| 197 | */ | |
| 198 | export function redLines( | |
| 199 | app: App, | |
| 200 | user: User, | |
| 201 | _now = Date.now(), | |
| 202 | ): { id: string; line: string; items: readonly string[]; link: string }[] { | |
| 203 | if (user.kind !== "user") return []; | |
| 204 | const out: { id: string; line: string; items: readonly string[]; link: string }[] = []; | |
| 205 | for (const { id } of app.db | |
| 206 | .query<{ id: string }, [string]>("SELECT id FROM repo WHERE owner_id = ?1 AND archived_at IS NULL") | |
| 207 | .all(user.id)) { | |
| 208 | const red = redMain(app.db, id); | |
| 209 | if (red === undefined || red.limit) continue; | |
| 210 | const key = `red:${red.runId}`; | |
| 211 | out.push({ id: key, line: redSentence(red, 0), items: [key], link: redHref(red) }); | |
redSentence(red, 0) tells the owner "0 pull checks wait for it", which is false when pulls are held. Pass heldChecks count or omit the clause for notifications. |
||
| 212 | } | |
| 213 | return out; | |
| 214 | } | |
| 215 | ||
| 192 | 216 | export interface Notifier { |
| 193 | 217 | /** Resolves when the work queued so far is done. For tests and shutdown. */ |
| 194 | 218 | settled(): Promise<void>; |
| @@ -368,6 +392,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier { | ||
| 368 | 392 | ...mailCeilingLines(app, user, now()), |
| 369 | 393 | ...drillLines(app, user, now()), |
| 370 | 394 | ...mirrorLines(app, user, now()), |
| 395 | ...redLines(app, user, now()), | |
| 371 | 396 | ]; |
| 372 | 397 | const before = seen.get(userId); |
| 373 | 398 | // What is gone may ask again later, and then it is new again. Seen by |
src/crew/sentinel.ts +20 −8
| @@ -350,10 +350,11 @@ export function repeatedErrors(now: number): Finding[] { | ||
| 350 | 350 | * Each cause whose spans total more than ten minutes in the day ending at |
| 351 | 351 | * `now`, quoting its sentence. Shorter totals stay in the history. |
| 352 | 352 | */ |
| 353 | export function degradedCauses(db: Database, now: number): Finding[] { | |
| 353 | export function degradedCauses(db: Database, now: number, forgeRepoId?: string): Finding[] { | |
| 354 | 354 | const from = now - DAY_MS; |
| 355 | 355 | const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>(); |
| 356 | 356 | for (const s of spans(db, from, now)) { |
| 357 | if (!ownCause(s.cause, forgeRepoId)) continue; | |
| 357 | 358 | const start = Math.max(s.openedAt, from); |
| 358 | 359 | const held = Math.min(s.closedAt ?? now, now) - start; |
| 359 | 360 | const seen = byCause.get(s.cause); |
| @@ -397,15 +398,25 @@ export function degradedCauses(db: Database, now: number): Finding[] { | ||
| 397 | 398 | */ |
| 398 | 399 | const BREACH_CAUSE = /^(queue:check|stuck:|budget:|red:|shared:)/; |
| 399 | 400 | |
| 401 | /** | |
| 402 | * A red main in another repository is its owner's to fix, not the forge's | |
| 403 | * (#3992): the crew cannot see that repository, so no issue is filed here. | |
| 404 | * Its card stays in the vitals and the owner is notified (`crew/notify.ts`). | |
| 405 | * `forgeRepoId` unset means no repository is known and nothing is dropped. | |
| 406 | */ | |
| 407 | export function ownCause(cause: string, forgeRepoId: string | undefined): boolean { | |
| 408 | return forgeRepoId === undefined || !cause.startsWith("red:") || cause === `red:${forgeRepoId}`; | |
| 409 | } | |
| 410 | ||
| 400 | 411 | /** |
| 401 | 412 | * Each budget breach that holds right now, however briefly: the crew hears |
| 402 | 413 | * at once, not when the day has added up. The finding shares its key with |
| 403 | 414 | * `degradedCauses`, so one cause is one issue. |
| 404 | 415 | */ |
| 405 | export function budgetBreaches(db: Database, now: number): Finding[] { | |
| 416 | export function budgetBreaches(db: Database, now: number, forgeRepoId?: string): Finding[] { | |
| 406 | 417 | const out: Finding[] = []; |
| 407 | 418 | for (const s of spans(db, now - DAY_MS, now)) { |
| 408 | if (s.closedAt !== null || !BREACH_CAUSE.test(s.cause)) continue; | |
| 419 | if (s.closedAt !== null || !BREACH_CAUSE.test(s.cause) || !ownCause(s.cause, forgeRepoId)) continue; | |
| 409 | 420 | out.push({ |
| 410 | 421 | key: `degraded:${keyPart(s.cause)}`, |
| 411 | 422 | title: `${s.cause} is past its budget: ${clip(oneLine(s.sentence), 120)}`, |
| @@ -442,15 +453,15 @@ function runLogLines(db: Database, cause: string): string[] { | ||
| 442 | 453 | } |
| 443 | 454 | |
| 444 | 455 | /** Everything the sentinel reads, in one pass. */ |
| 445 | export function findFaults(db: Database, now = Date.now()): Finding[] { | |
| 456 | export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string): Finding[] { | |
| 446 | 457 | const found = [ |
| 447 | ...budgetBreaches(db, now), | |
| 458 | ...budgetBreaches(db, now, forgeRepoId), | |
| 448 | 459 | ...heldSlots(db, now), |
| 449 | 460 | ...strayTriggers(db), |
| 450 | 461 | ...ledgerStreaks(db, now), |
| 451 | 462 | ...runStreaks(db), |
| 452 | 463 | ...repeatedErrors(now), |
| 453 | ...degradedCauses(db, now), | |
| 464 | ...degradedCauses(db, now, forgeRepoId), | |
| 454 | 465 | ]; |
| 455 | 466 | const keys = new Set<string>(); |
| 456 | 467 | return found |
| @@ -644,11 +655,11 @@ export function hunt( | ||
| 644 | 655 | now = Date.now(), |
| 645 | 656 | ): HuntResult { |
| 646 | 657 | const at = new Date(now).toISOString(); |
| 647 | const findings = findFaults(app.db, now); | |
| 648 | 658 | let filed = 0; |
| 649 | 659 | let commented = 0; |
| 650 | 660 | const [owner = "", name = ""] = repoPath.split("/"); |
| 651 | 661 | const repo = getRepo(app.db, owner, name); |
| 662 | const findings = findFaults(app.db, now, repo?.id); | |
| 652 | 663 | const actor = sentinelActor(app.db); |
| 653 | 664 | if (repo !== undefined && actor !== undefined) { |
| 654 | 665 | for (const f of findings) { |
| @@ -776,7 +787,8 @@ export function sentinelTick(app: App, now = Date.now()): HuntResult | null { | ||
| 776 | 787 | const repoPath = sentinelRepoPath(); |
| 777 | 788 | const last = sentinelStanding(app.db).lastHuntAt; |
| 778 | 789 | const since = last === null ? Infinity : now - new Date(last).getTime(); |
| 779 | const breaches = budgetBreaches(app.db, now); | |
| 790 | const [own = "", ownName = ""] = repoPath.split("/"); | |
| 791 | const breaches = budgetBreaches(app.db, now, getRepo(app.db, own, ownName)?.id); | |
| 780 | 792 | if (since < SIGNAL_GAP_MS && breaches.length === 0) return null; |
| 781 | 793 | if (since < HUNT_EVERY_MS) { |
| 782 | 794 | // A breach is not held back by the gap: the crew hears within a tick. |
src/kernel/vitals.ts +2 −2
| @@ -32,7 +32,7 @@ import { behindStanding } from "../release/behind.ts"; | ||
| 32 | 32 | import { newestDrills } from "../core/domain_drills.ts"; |
| 33 | 33 | import { DOMAINS } from "./domains.ts"; |
| 34 | 34 | import { domainDrillNeeds } from "./domain_drill.ts"; |
| 35 | import { freeChecks, heldChecks, redSentence } from "../checks/red_main.ts"; | |
| 35 | import { freeChecks, heldChecks, redHref, redSentence } from "../checks/red_main.ts"; | |
| 36 | 36 | import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts"; |
| 37 | 37 | |
| 38 | 38 | /** Free disk under this share of the volume is something to say. */ |
| @@ -379,7 +379,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 379 | 379 | `red:${red.repoId}`, |
| 380 | 380 | redSentence(red, held), |
| 381 | 381 | red.since, |
| 382 | `/${red.repo}/runs/${red.runId}`, | |
| 382 | redHref(red), | |
| 383 | 383 | ); |
| 384 | 384 | |
| 385 | 385 | // A test failing on many pulls in the hour is one card, not a park per pull. |
test/red-main.test.ts +46 −2
| @@ -9,9 +9,10 @@ import { tmpdir } from "node:os"; | ||
| 9 | 9 | import { migrations } from "../src/core/schema.ts"; |
| 10 | 10 | import { openDb } from "../src/kernel/db.ts"; |
| 11 | 11 | import { claimRun, createRun, finishRun, listActive, type Run } from "../src/core/runs.ts"; |
| 12 | import { RED_HOLD_MS, heldChecks, redMain, redSentence, sharedWithMain, unheld } from "../src/checks/red_main.ts"; | |
| 12 | import { RED_HOLD_MS, heldChecks, redHref, redMain, redSentence, sharedWithMain, unheld } from "../src/checks/red_main.ts"; | |
| 13 | 13 | import { coveredByRedFix } from "../src/crew/red_fix.ts"; |
| 14 | import { budgetBreaches } from "../src/crew/sentinel.ts"; | |
| 14 | import { budgetBreaches, findFaults } from "../src/crew/sentinel.ts"; | |
| 15 | import { redLines } from "../src/crew/notify.ts"; | |
| 15 | 16 | import { vitals } from "../src/kernel/vitals.ts"; |
| 16 | 17 | import { ErrorLedger } from "../src/kernel/log.ts"; |
| 17 | 18 | import { RefusedLedger } from "../src/kernel/internal_refused.ts"; |
| @@ -238,3 +239,46 @@ describe("the lane sentence and a red main", () => { | ||
| 238 | 239 | expect(causes).not.toContain("queue:check"); |
| 239 | 240 | }); |
| 240 | 241 | }); |
| 242 | ||
| 243 | describe("a red main in another repository (#3992)", () => { | |
| 244 | const span = (cause: string) => | |
| 245 | db.run( | |
| 246 | "INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES (?1, 'x: main is red', '/x', 1000)", | |
| 247 | [cause], | |
| 248 | ); | |
| 249 | ||
| 250 | test("files no sentinel issue for it; the forge's own red main still files one", () => { | |
| 251 | span("red:r2"); | |
| 252 | span("red:r1"); | |
| 253 | expect(budgetBreaches(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]); | |
| 254 | expect(findFaults(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]); | |
| 255 | expect(budgetBreaches(db, 2000, "r2").map((f) => f.key)).toEqual(["degraded:red-r2"]); | |
| 256 | }); | |
| 257 | ||
| 258 | test("a manifest error reads as an invalid forge.json, quotes it and links to the file", () => { | |
| 259 | const run = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID("d"), reason: "push" }); | |
| 260 | claimRun(db, run.id); | |
| 261 | finishRun(db, run.id, { state: "failed", summary: "check failed: forge.json: check.run is required" }); | |
| 262 | const r = redMain(db, "r1")!; | |
| 263 | const sentence = redSentence(r, 2); | |
| 264 | expect(sentence).toContain("forge.json on main"); | |
| 265 | expect(sentence).toContain("invalid"); | |
| 266 | expect(sentence).toContain("forge.json: check.run is required"); | |
| 267 | expect(redHref(r)).toBe("/trevato/forge/blob/main/forge.json"); | |
| 268 | expect(redSentence(redMain(db, "r1")!, 0)).not.toContain("tests"); | |
| 269 | }); | |
| 270 | ||
| 271 | test("a failing test keeps its own sentence and run link", () => { | |
| 272 | const r = redMain(db, main("e", "failed", ["a test"]).repoId)!; | |
| 273 | expect(redSentence(r, 1)).toContain('"a test" fails'); | |
| 274 | expect(redHref(r)).toBe(`/trevato/forge/runs/${r.runId}`); | |
| 275 | }); | |
| 276 | ||
| 277 | test("the owner of the red repository is told, once per failing run", () => { | |
| 278 | main("f", "failed", ["a test"]); | |
| 279 | const lines = redLines({ db } as never, { id: "u1", kind: "user" } as never); | |
| 280 | expect(lines).toHaveLength(1); | |
| 281 | expect(lines[0]!.line).toContain("trevato/forge: main is red"); | |
| 282 | expect(redLines({ db } as never, { id: "u2", kind: "user" } as never)).toEqual([]); | |
| 283 | }); | |
| 284 | }); |
new above
offline
_now is unused; drop it (and the argument at the call site).