Public: anyone can read this; the crew can change it.
Sentinel: backup was degraded 182 minutes today: Primary (s3.us-east-005.backblazeb2.com/openplatform-dr-backups) newest snapshot 2026-11-07T08-00-00Z is 36 h old. The … #10315
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10291-sentinel-backup-was-degraded-182-minutes → main 1 commit fork
2 files
changed +58
−6
src/crew/sentinel.ts +24 −6
| @@ -80,6 +80,10 @@ export interface Finding { | ||
| 80 | 80 | readonly fix?: string; |
| 81 | 81 | /** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */ |
| 82 | 82 | readonly incident?: string; |
| 83 | /** A degraded finding: when its newest span opened (#10291). */ | |
| 84 | readonly lastOpenedAt?: string; | |
| 85 | /** A degraded finding whose spans have all closed: when the last one did. Absent while one is open (#10291). */ | |
| 86 | readonly endedAt?: string; | |
| 83 | 87 | /** The vitals cause a budget-breach or degraded finding reads. */ |
| 84 | 88 | readonly cause?: string; |
| 85 | 89 | } |
| @@ -398,7 +402,10 @@ export function repeatedErrors(now: number): Finding[] { | ||
| 398 | 402 | */ |
| 399 | 403 | export function degradedCauses(db: Database, now: number, forgeRepoId?: string, dataDir?: string): Finding[] { |
| 400 | 404 | const from = now - DAY_MS; |
| 401 | const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>(); | |
| 405 | const byCause = new Map< | |
| 406 | string, | |
| 407 | { sentence: string; href: string; first: number; opened: number; lastOpened: number; ms: number; ended: number } | |
| 408 | >(); | |
| 402 | 409 | for (const s of spans(db, from, now)) { |
| 403 | 410 | if (!ownCause(s.cause, forgeRepoId)) continue; |
| 404 | 411 | const start = Math.max(s.openedAt, from); |
| @@ -410,7 +417,10 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string, | ||
| 410 | 417 | href: s.href, |
| 411 | 418 | first: seen?.first ?? start, |
| 412 | 419 | opened: seen?.opened ?? s.openedAt, |
| 420 | lastOpened: s.openedAt, | |
| 413 | 421 | ms: (seen?.ms ?? 0) + Math.max(0, held), |
| 422 | // Infinity while any span is still open: the incident has not ended. | |
| 423 | ended: Math.max(seen?.ended ?? -Infinity, s.closedAt ?? Infinity), | |
| 414 | 424 | }); |
| 415 | 425 | } |
| 416 | 426 | const out: Finding[] = []; |
| @@ -432,6 +442,8 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string, | ||
| 432 | 442 | kind: "defect", |
| 433 | 443 | lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`, |
| 434 | 444 | incident: new Date(d.opened).toISOString(), |
| 445 | lastOpenedAt: new Date(d.lastOpened).toISOString(), | |
| 446 | ...(Number.isFinite(d.ended) ? { endedAt: new Date(d.ended).toISOString() } : {}), | |
| 435 | 447 | cause, |
| 436 | 448 | }); |
| 437 | 449 | } |
| @@ -658,20 +670,26 @@ function labelFinding(app: App, repo: Repo, actor: User, issue: Issue, f: Findin | ||
| 658 | 670 | * again: the fix is likely merged but not live. At most one comment a day. |
| 659 | 671 | * One closed as dropped stays silent. True when |
| 660 | 672 | * the incident is that one: the comments made (0 or 1), else undefined. |
| 673 | * | |
| 674 | * The day's window clips an incident's early spans as it slides, which moves | |
| 675 | * its first-open time; a finding whose newest span had opened by the time the | |
| 676 | * sentinel last saw the closed issue's is still that incident. Ended, it is | |
| 677 | * said nothing: it was filed, fixed and over (#10167, #10237, #10291 were one). | |
| 661 | 678 | */ |
| 662 | 679 | function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 | 1 | undefined { |
| 663 | 680 | const prior = app.db |
| 664 | .query<{ issue_id: string; commented_at: string | null }, [string, string]>( | |
| 665 | `SELECT t.issue_id, t.commented_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 666 | WHERE t.key = ?1 AND t.incident = ?2 AND i.state = 'closed' | |
| 667 | ORDER BY t.first_seen DESC LIMIT 1`, | |
| 681 | .query<{ issue_id: string; commented_at: string | null; same: number }, [string, string, string]>( | |
| 682 | `SELECT t.issue_id, t.commented_at, t.incident IS ?2 AS same FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id | |
| 683 | WHERE t.key = ?1 AND i.state = 'closed' AND (t.incident = ?2 OR t.last_seen >= ?3) | |
| 684 | ORDER BY same DESC, t.first_seen DESC LIMIT 1`, | |
| 668 | 685 | ) |
| 669 | .get(f.key, f.incident ?? ""); | |
| 686 | .get(f.key, f.incident ?? "", f.lastOpenedAt ?? "\uffff"); | |
| 670 | 687 | if (prior === null) return undefined; |
| 671 | 688 | const issue = getIssueById(app.db, prior.issue_id); |
| 672 | 689 | if (issue === undefined) return undefined; |
| 673 | 690 | if (issue.closeReason === "dropped") return 0; |
| 674 | 691 | if (issue.closeReason !== "done" && issue.closeReason !== "split") return undefined; |
| 692 | if (prior.same === 0 && f.endedAt !== undefined) return 0; | |
| 675 | 693 | if (prior.commented_at !== null && now - Date.parse(prior.commented_at) < DAY_MS) return 0; |
| 676 | 694 | const minutes = f.evidence.find((e) => e.startsWith("total: ")); |
| 677 | 695 | const said = addComment( |
test/sentinel.test.ts +34
| @@ -840,6 +840,40 @@ describe("a degraded incident whose issue closed as done", () => { | ||
| 840 | 840 | expect(r.filed).toBe(1); |
| 841 | 841 | expect(issues("open").length).toBe(1); |
| 842 | 842 | }); |
| 843 | ||
| 844 | test("an incident that ended before its issue closed is not refiled as the day slides past its first span (#10291)", () => { | |
| 845 | const t = T0 + 40 * DAY; | |
| 846 | iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE state = 'open'").run(new Date(t).toISOString()); | |
| 847 | // Two spans of one cause: the morning's, and the evening's; both ended. | |
| 848 | span("backup", t - 12 * HOUR, 60); | |
| 849 | span("backup", t - 4 * HOUR, 180); | |
| 850 | expect(hunt(iapp, "alice/forge", t).filed).toBe(1); | |
| 851 | // The fix lands after the evening span ended. | |
| 852 | iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE state = 'open'").run(new Date(t + HOUR).toISOString()); | |
| 853 | // The morning span slides out of the day: the first-open time moves, and the evening's total still counts. | |
| 854 | const r = hunt(iapp, "alice/forge", t + 13 * HOUR); | |
| 855 | expect(r.filed).toBe(0); | |
| 856 | expect(r.commented).toBe(0); | |
| 857 | expect(issues("open").length).toBe(0); | |
| 858 | // A span after the close is a new incident, and is filed. | |
| 859 | span("backup", t + 14 * HOUR, 30); | |
| 860 | expect(hunt(iapp, "alice/forge", t + 15 * HOUR).filed).toBe(1); | |
| 861 | }); | |
| 862 | ||
| 863 | test("one still going on as the day slides past its first span is told to its closed issue, not refiled", () => { | |
| 864 | const t = T0 + 50 * DAY; | |
| 865 | iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE state = 'open'").run(new Date(t).toISOString()); | |
| 866 | span("mirror", t - 12 * HOUR, 60); | |
| 867 | iapp.db | |
| 868 | .query("INSERT INTO degraded_span (cause, sentence, href, opened_at) VALUES ('mirror', 'It is slow.', '/x', ?1)") | |
| 869 | .run(t - 4 * HOUR); | |
| 870 | expect(hunt(iapp, "alice/forge", t).filed).toBe(1); | |
| 871 | iapp.db.query("UPDATE issue SET state = 'closed', close_reason = 'done', closed_at = ?1 WHERE state = 'open'").run(new Date(t + HOUR).toISOString()); | |
| 872 | const r = hunt(iapp, "alice/forge", t + 13 * HOUR); | |
| 873 | expect(r.filed).toBe(0); | |
| 874 | expect(r.commented).toBe(1); | |
| 875 | expect(issues("open").length).toBe(0); | |
| 876 | }); | |
| 843 | 877 | }); |
| 844 | 878 | |
| 845 | 879 | // ---- a shared test fixed within the hour (#8185) ---- |