Public: anyone can read this; the crew can change it.
Sentinel: backup-refs:scout/forge was degraded 20 minutes today: 2 branches of scout/forge cannot be backed up: their commits are missing ([redacted]… #12589
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12582-sentinel-backup-refs-scout-forge-was-deg → main 5 commits fork 2 threads · 2 unresolved
5 files
changed +151
−9
src/git/repos.ts +30 −1
| @@ -4,7 +4,7 @@ | ||
| 4 | 4 | * `tar`. Sovereignty starts with the filesystem being legible. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { join } from "node:path"; | |
| 7 | import { dirname, isAbsolute, join, resolve } from "node:path"; | |
| 8 | 8 | import { rm, stat } from "node:fs/promises"; |
| 9 | 9 | import { defect } from "../kernel/error.ts"; |
| 10 | 10 | import { err, ok, type Result } from "../kernel/result.ts"; |
| @@ -92,6 +92,35 @@ export async function forkBareRepo( | ||
| 92 | 92 | { cwd: forkDir, timeoutMs: 120_000 }, |
| 93 | 93 | ); |
| 94 | 94 | if (!pins.ok) return await undo(forkDir, pins); |
| 95 | const keep = await keepLentObjects(forkDir); | |
| 96 | if (!keep.ok) return await undo(forkDir, keep); | |
| 97 | return ok(); | |
| 98 | } | |
| 99 | ||
| 100 | /** | |
| 101 | * Make every repository this one borrows objects from keep its unreachable objects. | |
| 102 | * | |
| 103 | * The fork's refs reach objects the lender's own refs may later drop (a force-push, a | |
| 104 | * deleted branch). The lender's gc cannot see the fork's refs, so once its grace period | |
| 105 | * ended it would prune them and leave the fork's branches with missing commits, which the | |
| 106 | * snapshot then cannot back up (2026-11-14, scout/forge). Idempotent: run when a fork is | |
| 107 | * made and again before each snapshot, so lenders forked before this existed are covered. | |
| 108 | */ | |
| 109 | export async function keepLentObjects(forkDir: string): Promise<Result<void, GitFailure>> { | |
| 110 | const own = join(forkDir, "objects"); | |
| 111 | let lines: string[]; | |
| 112 | try { | |
| 113 | lines = (await Bun.file(join(own, "info", "alternates")).text()).split("\n"); | |
| 114 | } catch { | |
| 115 | return ok(); | |
| 116 | } | |
| 117 | for (const line of lines) { | |
| 118 | const path = line.trim(); | |
| 119 | if (path === "" || path.startsWith("#")) continue; | |
| 120 | const lender = dirname(isAbsolute(path) ? path : resolve(own, path)); | |
| 121 | const set = await runGit(["config", "gc.pruneExpire", "never"], { cwd: lender }); | |
| 122 | if (!set.ok) return set; | |
| 123 | } | |
| 95 | 124 | return ok(); |
| 96 | 125 | } |
| 97 | 126 |
src/snapshot/snapshot.ts +5 −1
| @@ -43,7 +43,7 @@ import { join, relative } from "node:path"; | ||
| 43 | 43 | import { envDataDir } from "../apps/data.ts"; |
| 44 | 44 | import { listApps } from "../core/apps.ts"; |
| 45 | 45 | import { listEveryRepo } from "../core/repos.ts"; |
| 46 | import { repoDir } from "../git/repos.ts"; | |
| 46 | import { keepLentObjects, repoDir } from "../git/repos.ts"; | |
| 47 | 47 | import { createLog } from "../kernel/log.ts"; |
| 48 | 48 | import { bundleRepo } from "./bundle.ts"; |
| 49 | 49 | import { countRows, type DrillCounts } from "./counts.ts"; |
| @@ -492,6 +492,10 @@ export async function takeSnapshot( | ||
| 492 | 492 | repositories.push(full); |
| 493 | 493 | const dir = repoDir(forge.reposRoot, repo.ownerName, repo.name); |
| 494 | 494 | const bundle = join(scratch, `${repo.ownerName}--${repo.name}.bundle`); |
| 495 | // A lender prunes what only its forks reach, unless told not to: cover those forked | |
| 496 | // before it was. | |
| 497 | const kept = await keepLentObjects(dir); | |
| 498 | if (!kept.ok) log.warn("lender not set to keep its objects", { repo: full }); | |
| 495 | 499 | const made = await bundleRepo(dir, bundle, { deadlineMs: forge.bundleDeadlineMs, sizes }); |
| 496 | 500 | if (!made.ok) { |
| 497 | 501 | rmSync(bundle, { force: true }); |
test/features-tui.test.ts +4 −2
| @@ -21,7 +21,7 @@ import { createForgeApp } from "../src/tui/forge.ts"; | ||
| 21 | 21 | import type { TuiIdentity } from "../src/tui/session.ts"; |
| 22 | 22 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 23 | 23 | import { quietLogs } from "./helpers/globals.ts"; |
| 24 | import { settleApp } from "./helpers/tui.ts"; | |
| 24 | import { settleApp, settleUntil } from "./helpers/tui.ts"; | |
| 25 | 25 | |
| 26 | 26 | quietLogs(); |
| 27 | 27 | |
| @@ -59,7 +59,7 @@ describe("features in the terminal", () => { | ||
| 59 | 59 | await settleApp(app); |
| 60 | 60 | await type(query); |
| 61 | 61 | }; |
| 62 | return { text: () => app.screen.toText(), press, type, palette, stop: () => app.stop() }; | |
| 62 | return { text: () => app.screen.toText(), press, type, palette, until: (what: string) => settleUntil(app, what), stop: () => app.stop() }; | |
| 63 | 63 | } |
| 64 | 64 | |
| 65 | 65 | beforeAll(async () => { |
| @@ -175,6 +175,8 @@ describe("features in the terminal", () => { | ||
| 175 | 175 | await t.palette("Feature feeds"); |
| 176 | 176 | await t.press("enter"); |
| 177 | 177 | await t.press("enter"); |
| 178 | // The feed screen reads the feature after it opens, out of the settle's sight: wait for it. | |
| 179 | await t.until("Scheduled: "); | |
| 178 | 180 | expect(t.text()).toMatch(/Scheduled: scan at 07:15 America\/Chicago · last /); |
| 179 | 181 | expect(t.text()).not.toContain("every 1h"); |
| 180 | 182 | await t.press("h"); |
test/fork-lender-keeps-objects.test.ts added +102
| @@ -0,0 +1,102 @@ | ||
| 1 | /** A repository that lends objects to a fork never prunes what only the fork reaches. */ | |
| 2 | import { expect, test } from "bun:test"; | |
| 3 | import { readdirSync, utimesSync } from "node:fs"; | |
| 4 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { forkBareRepo, initBareRepo, keepLentObjects } from "../src/git/repos.ts"; | |
| 8 | ||
| 9 | const git = async (cwd: string, ...args: string[]): Promise<{ code: number; out: string }> => { | |
| 10 | const p = Bun.spawn(["git", ...args], { | |
| 11 | cwd, | |
| 12 | stdout: "pipe", | |
| 13 | stderr: "pipe", | |
| 14 | env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" }, | |
outdated
This checks the config value, not the behaviour. Drop a parent ref after forking, run gc with prune expiry now in the parent, and assert the fork's commit still resolves. Also cover a parent that was forked before this change. |
||
| 15 | }); | |
| 16 | const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited]); | |
| 17 | return { code, out: out.trim() }; | |
| 18 | }; | |
| 19 | ||
| 20 | /** A parent with one commit on `main`, and a fork with one more commit that only it has. */ | |
| 21 | async function setup(root: string) { | |
| 22 | const parent = join(root, "p.git"); | |
| 23 | expect((await initBareRepo(parent)).ok).toBe(true); | |
| 24 | const work = join(root, "w"); | |
| 25 | await git(root, "clone", parent, work); | |
| 26 | await git(work, "commit", "--allow-empty", "-m", "one"); | |
| 27 | await git(work, "branch", "-M", "main"); | |
| 28 | await git(work, "push", "origin", "main"); | |
| 29 | await git(work, "checkout", "-b", "topic"); | |
| 30 | await git(work, "commit", "--allow-empty", "-m", "only-topic"); | |
| 31 | await git(work, "push", "origin", "topic"); | |
| 32 | const sha = (await git(work, "rev-parse", "topic")).out; | |
| 33 | const fork = join(root, "f.git"); | |
| 34 | expect((await forkBareRepo(parent, fork)).ok).toBe(true); | |
| 35 | return { parent, fork, sha }; | |
| 36 | } | |
| 37 | ||
| 38 | /** Every file under a directory, aged past the grace period git gives unreachable objects. */ | |
| 39 | function age(dir: string): void { | |
| 40 | const then = new Date(Date.now() - 30 * 86_400_000); | |
| 41 | for (const e of readdirSync(dir, { withFileTypes: true })) { | |
| 42 | const path = join(dir, e.name); | |
| 43 | if (e.isDirectory()) age(path); | |
| 44 | else utimesSync(path, then, then); | |
| 45 | } | |
| 46 | } | |
| 47 | ||
| 48 | /** The parent drops the branch the fork also has; a month later its ordinary gc runs. */ | |
| 49 | async function dropAndPrune(parent: string): Promise<void> { | |
| 50 | await git(parent, "update-ref", "-d", "refs/heads/topic"); | |
| 51 | await git(parent, "reflog", "expire", "--expire=now", "--all"); | |
| 52 | age(join(parent, "objects")); | |
| 53 | expect((await git(parent, "gc")).code).toBe(0); | |
| 54 | } | |
| 55 | ||
| 56 | test("what only the fork reaches survives the parent's gc", async () => { | |
| 57 | const root = await mkdtemp(join(tmpdir(), "lender-")); | |
| 58 | try { | |
| 59 | const { parent, fork, sha } = await setup(root); | |
| 60 | await dropAndPrune(parent); | |
| 61 | expect((await git(fork, "cat-file", "-e", sha)).code).toBe(0); | |
| 62 | expect((await git(fork, "bundle", "create", join(root, "b"), "--all")).code).toBe(0); | |
| 63 | } finally { | |
| 64 | await rm(root, { recursive: true, force: true }); | |
| 65 | } | |
| 66 | }); | |
| 67 | ||
| 68 | test("a parent forked before the setting existed is covered by keepLentObjects, and is lost without it", async () => { | |
| 69 | const root = await mkdtemp(join(tmpdir(), "lender-")); | |
| 70 | try { | |
| 71 | const old = await setup(root); | |
| 72 | await git(old.parent, "config", "--unset", "gc.pruneExpire"); | |
| 73 | await dropAndPrune(old.parent); | |
| 74 | // Without the setting git prunes the commit and the fork's branch is broken. | |
| 75 | expect((await git(old.fork, "cat-file", "-e", old.sha)).code).not.toBe(0); | |
| 76 | } finally { | |
| 77 | await rm(root, { recursive: true, force: true }); | |
| 78 | } | |
| 79 | const root2 = await mkdtemp(join(tmpdir(), "lender-")); | |
| 80 | try { | |
| 81 | const { parent, fork, sha } = await setup(root2); | |
| 82 | await git(parent, "config", "--unset", "gc.pruneExpire"); | |
| 83 | expect((await keepLentObjects(fork)).ok).toBe(true); | |
| 84 | expect((await keepLentObjects(fork)).ok).toBe(true); // idempotent | |
| 85 | await dropAndPrune(parent); | |
| 86 | expect((await git(fork, "cat-file", "-e", sha)).code).toBe(0); | |
| 87 | } finally { | |
| 88 | await rm(root2, { recursive: true, force: true }); | |
| 89 | } | |
| 90 | }); | |
| 91 | ||
| 92 | test("a repository that borrows nothing is left alone", async () => { | |
| 93 | const root = await mkdtemp(join(tmpdir(), "lender-")); | |
| 94 | try { | |
| 95 | const p = join(root, "p.git"); | |
| 96 | await initBareRepo(p); | |
| 97 | expect((await keepLentObjects(p)).ok).toBe(true); | |
| 98 | expect((await git(p, "config", "--get", "gc.pruneExpire")).code).not.toBe(0); | |
| 99 | } finally { | |
| 100 | await rm(root, { recursive: true, force: true }); | |
| 101 | } | |
| 102 | }); |
test/mcp-parent-unreachable.test.ts +10 −5
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 9 | 9 | import { mkdtemp, rm } from "node:fs/promises"; |
| 10 | 10 | import { tmpdir } from "node:os"; |
| 11 | import { createServer, type Server } from "node:net"; | |
| 11 | 12 | import { join } from "node:path"; |
| 12 | 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 13 | 14 | import { mergePull, openPull } from "../src/actions/pulls.ts"; |
| @@ -38,6 +39,7 @@ describe("MCP proposes to a parent forge that does not answer", () => { | ||
| 38 | 39 | let lib: Repo; |
| 39 | 40 | let forge: Repo; |
| 40 | 41 | let writer = ""; |
| 42 | let holder: Server | undefined; | |
| 41 | 43 | |
| 42 | 44 | const person = async (app: App, username: string) => { |
| 43 | 45 | const made = await createUser(app.db, { |
| @@ -76,11 +78,11 @@ describe("MCP proposes to a parent forge that does not answer", () => { | ||
| 76 | 78 | |
| 77 | 79 | beforeAll(async () => { |
| 78 | 80 | dir = await mkdtemp(join(tmpdir(), "forge-unreachable-")); |
| 79 | // A port that was free a moment ago and refuses connections until the | |
| 80 | // parent is started on it. | |
| 81 | const probe = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("") }); | |
| 82 | const port = probe.port as number; | |
| 83 | await probe.stop(true); | |
| 81 | // The parent's port is held by a listener that drops every connection unanswered, so | |
| 82 | // no other test on a loaded shard can take it before the parent is started on it. | |
| 83 | holder = createServer((socket) => socket.destroy()); | |
| 84 | await new Promise<void>((done) => holder!.listen(0, "127.0.0.1", done)); | |
| 85 | const port = (holder.address() as { port: number }).port; | |
| 84 | 86 | |
| 85 | 87 | parent = createApp({ dataDir: join(dir, "parent"), port, host: "127.0.0.1" }); |
| 86 | 88 | const alice = await person(parent, "alice"); |
| @@ -146,6 +148,7 @@ describe("MCP proposes to a parent forge that does not answer", () => { | ||
| 146 | 148 | afterAll(async () => { |
| 147 | 149 | await childServer.stop(); |
| 148 | 150 | await parentServer?.stop(); |
| 151 | holder?.close(); | |
| 149 | 152 | await rm(dir, { recursive: true, force: true }); |
| 150 | 153 | }); |
| 151 | 154 | |
| @@ -167,6 +170,8 @@ describe("MCP proposes to a parent forge that does not answer", () => { | ||
| 167 | 170 | }); |
| 168 | 171 | |
| 169 | 172 | test("once the parent answers, a retry proposes exactly once", async () => { |
| 173 | await new Promise<void>((done) => holder!.close(() => done())); | |
| 174 | holder = undefined; | |
| 170 | 175 | parentServer = await serve(parent); |
| 171 | 176 | const first = await call(writer); |
| 172 | 177 | expect(first.error).toBe(false); |
new above
offline
Only forks created from now on set this. Existing lenders (scout/forge) stay unprotected. Move the setting into an idempotent helper and also run it over repos that already have forks.