Public: anyone can read this; the crew can change it.
Sentinel: backup-refs:mason/forge was degraded 20 minutes today: 1 branch of mason/forge cannot be backed up: their commits are missing ([redacted]… #12590
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12581-sentinel-backup-refs-mason-forge-was-deg → main 10 commits fork 2 threads · 2 unresolved
6 files
changed +52
−7
docs/SNAPSHOT.md +2 −1
| @@ -69,7 +69,8 @@ and `sentMb` put, `heldBefore` already there. | ||
| 69 | 69 | fails git's whole bundle, so the refs are sorted and every one that resolves |
| 70 | 70 | is bundled; the others are in `brokenRefs` by name and are one line a |
| 71 | 71 | repository for the forge's admin ("2 branches of alice/notes cannot be |
| 72 | backed up: their commits are missing (…)"). A repository with no bundle at | |
| 72 | backed up: their commits are missing (…)"), linked to its Branches page; | |
| 73 | how to clear it is `docs/features/backup-refs.md`. A repository with no bundle at | |
| 73 | 74 | all turns the backups verdict red, naming it: a snapshot that lacks a |
| 74 | 75 | repository is not green however fresh. A bundle's deadline follows the |
| 75 | 76 | repository's size (2 min, and 2 s for each MB of its objects and those it |
docs/features/README.md +1
| @@ -350,6 +350,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md) | ||
| 350 | 350 | |
| 351 | 351 | Read with the guide page [](../guide-backups-moving.md). |
| 352 | 352 | |
| 353 | - [](backup-refs.md) | |
| 353 | 354 | - [](forge-origin.md) |
| 354 | 355 | - [](origin-pull-state.md) |
| 355 | 356 | - [](app-backup.md) |
docs/features/backup-refs.md added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | # A branch that cannot be backed up | |
| 2 | ||
| 3 | The nightly snapshot bundles every repository (`src/snapshot/bundle.ts`). A | |
| 4 | branch whose history is not all there (a commit, tree or blob it reaches is | |
| 5 | missing) cannot go in a bundle. The snapshot keeps the rest of the repository | |
| 6 | and names that branch. The forge's admin then sees one card for each such | |
| 7 | repository, on the vitals and the home: | |
| 8 | ||
| 9 | > 1 branch of owner/name cannot be backed up: their commits are missing (name). | |
| 10 | > The rest of the repository is in the snapshot. Delete it on its Branches page. | |
| 11 | ||
| 12 | **To clear it:** open the card. It links to the repository's Branches page, | |
| 13 | `/owner/name/branches`. A writer there presses *Delete* on the branch the card | |
| 14 | names. Pushing the branch again from a clone that has every object also works. | |
| 15 | The card goes away when the next snapshot lands without the branch. | |
| 16 | ||
| 17 | - **The commits stay.** Deleting the branch removes only the ref. The forge | |
| 18 | never runs `gc`, so its tip commit stays in the object store, where someone | |
| 19 | on the box can read it with `git cat-file -p <oid>`. | |
| 20 | - **Detaching a fork does not clear it.** A detach repacks every object the | |
| 21 | fork borrows, and a missing object makes that repack fail. | |
| 22 | - **Finding the branch by hand.** `git fetch` of every ref from the repository | |
| 23 | fails on the server with `bad tree object <oid>` (or a missing commit or | |
| 24 | blob). Fetching the refs in halves finds the one that fails. | |
| 25 | ||
| 26 | The first case was `mason/forge` on 2026-11-14 (#12581). The branch was | |
| 27 | `mason/7731-palette-entries-can-live-beside-their-sc`, tip `443cf57e`. Its tree | |
| 28 | `6d6a9f68` was missing. Its run had ended with nothing to merge, so deleting it | |
| 29 | lost no work. |
src/kernel/vitals.ts +1 −1
| @@ -647,7 +647,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 647 | 647 | if (failing !== undefined) need("todo", "snapshot-failed", failing.sentence, failing.since, "/settings/backups"); |
| 648 | 648 | // What the snapshot that landed could not hold is one card a repository until one holds it. |
| 649 | 649 | for (const left of leftOutNeeds(lastSnapshotOutcome(input.dataDir))) |
| 650 | need("todo", left.cause, left.sentence, left.since, `/${left.repo}`); | |
| 650 | need("todo", left.cause, left.sentence, left.since, left.href); | |
| 651 | 651 | |
| 652 | 652 | // The bucket itself, read by this process: late, unreadable, a failed drill, or a check that stopped is one card. |
| 653 | 653 | if (isWatched(input.dataDir)) { |
src/snapshot/outcome.ts +7 −4
| @@ -127,11 +127,12 @@ const NAMED = 5; | ||
| 127 | 127 | /** |
| 128 | 128 | * What the snapshot that landed could not hold, one line a repository: branches whose commits |
| 129 | 129 | * are missing (the rest of the repository is in it), and repositories not in it at all. Each |
| 130 | * stands until a snapshot lands without it. | |
| 130 | * stands until a snapshot lands without it. `href` is where a person clears it: a broken branch | |
| 131 | * is deleted on the repository's Branches page (docs/features/backup-refs.md). | |
| 131 | 132 | */ |
| 132 | export function leftOutNeeds(s: SnapshotOutcome | undefined): { cause: string; repo: string; sentence: string; since: string }[] { | |
| 133 | export function leftOutNeeds(s: SnapshotOutcome | undefined): { cause: string; repo: string; sentence: string; since: string; href: string }[] { | |
| 133 | 134 | if (s?.landedAt === undefined) return []; |
| 134 | const out: { cause: string; repo: string; sentence: string; since: string }[] = []; | |
| 135 | const out: { cause: string; repo: string; sentence: string; since: string; href: string }[] = []; | |
| 135 | 136 | for (const b of s.brokenRefs ?? []) { |
| 136 | 137 | const names = b.refs.map((r) => r.replace(/^refs\/heads\//, "")); |
| 137 | 138 | const shown = names.slice(0, NAMED).join(", "); |
| @@ -139,8 +140,9 @@ export function leftOutNeeds(s: SnapshotOutcome | undefined): { cause: string; r | ||
| 139 | 140 | out.push({ |
| 140 | 141 | cause: `backup-refs:${b.repo}`, |
| 141 | 142 | repo: b.repo, |
| 142 | sentence: `${names.length} ${names.length === 1 ? "branch" : "branches"} of ${b.repo} cannot be backed up: their commits are missing (${shown}${more}). The rest of the repository is in the snapshot.`, | |
| 143 | sentence: `${names.length} ${names.length === 1 ? "branch" : "branches"} of ${b.repo} cannot be backed up: their commits are missing (${shown}${more}). The rest of the repository is in the snapshot. Delete ${names.length === 1 ? "it" : "them"} on its Branches page.`, | |
| 143 | 144 | since: s.landedAt, |
| 145 | href: `/${b.repo}/branches`, | |
| 144 | 146 | }); |
| 145 | 147 | } |
| 146 | 148 | for (const a of s.absent ?? []) |
| @@ -149,6 +151,7 @@ export function leftOutNeeds(s: SnapshotOutcome | undefined): { cause: string; r | ||
| 149 | 151 | repo: a.repo, |
| 150 | 152 | sentence: `${a.repo} is not in the snapshot: ${a.why.replace(/\.$/, "")}. A restore would not bring it back.`, |
| 151 | 153 | since: s.landedAt, |
| 154 | href: `/${a.repo}`, | |
| 152 | 155 | }); |
| 153 | 156 | return out; |
| 154 | 157 | } |
test/snapshot-repositories.test.ts +12 −1
| @@ -241,12 +241,14 @@ describe("a repository with refs whose commits are missing", () => { | ||
| 241 | 241 | ]); |
| 242 | 242 | const lines = leftOutNeeds(kept); |
| 243 | 243 | expect(lines.map((l) => l.sentence)).toEqual([ |
| 244 | "2 branches of alice/notes cannot be backed up: their commits are missing (lost, torn). The rest of the repository is in the snapshot.", | |
| 244 | "2 branches of alice/notes cannot be backed up: their commits are missing (lost, torn). The rest of the repository is in the snapshot. Delete them on its Branches page.", | |
| 245 | 245 | ]); |
| 246 | 246 | const card = forgeVitals(app).degraded.filter((d) => |
| 247 | 247 | d.cause.startsWith("backup-refs:"), |
| 248 | 248 | ); |
| 249 | 249 | expect(card.map((d) => d.cause)).toEqual(["backup-refs:alice/notes"]); |
| 250 | // The card opens the page where a writer deletes the branch. | |
| 251 | expect(card.map((d) => d.href)).toEqual(["/alice/notes/branches"]); | |
| 250 | 252 | const said = |
| 251 | 253 | forgeNeeds(app, alice)?.filter((l) => |
| 252 | 254 | l.includes("cannot be backed up"), |
| @@ -270,6 +272,15 @@ describe("a repository with refs whose commits are missing", () => { | ||
| 270 | 272 | ).toContain( |
| 271 | 273 | "7 branches of a/b cannot be backed up: their commits are missing (b0, b1, b2, b3, b4, and 2 more)", |
| 272 | 274 | ); |
| 275 | // One branch is "it". | |
| 276 | expect( | |
| 277 | leftOutNeeds({ | |
| 278 | at: "t", | |
| 279 | landed: "x", | |
| 280 | landedAt: "2026-11-14T08:05:00Z", | |
| 281 | brokenRefs: [{ repo: "a/b", refs: ["refs/heads/lost"] }], | |
| 282 | })[0]!.sentence, | |
| 283 | ).toEndWith("Delete it on its Branches page."); | |
| 273 | 284 | expect( |
| 274 | 285 | leftOutNeeds({ at: "t", landed: "x", landedAt: "2026-11-14T08:05:00Z" }), |
| 275 | 286 | ).toEqual([]); |
new above
offline
Say where to delete or detach: name the page or action. Shorten it too, because three clauses on a vitals card is a lot.