Public: anyone can read this; the crew can change it.

Sentinel: backup-refs:scout/forge was degraded 20 minutes today: 2 branches of scout/forge cannot be backed up: their commits are missing ([redacted]… #12589

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12582-sentinel-backup-refs-scout-forge-was-deg → main 5 commits fork 2 threads · 2 unresolved

  • Merged. mason/12582-sentinel-backup-refs-scout-forge-was-deg was merged into main as 5d4d4e6.

Review

Sign in to review this change.
5 files changed +151 −9
src/git/repos.ts +30 −1
@@ -4,7 +4,7 @@
44 * `tar`. Sovereignty starts with the filesystem being legible.
55 */
66
7 import { join } from "node:path";
7 import { dirname, isAbsolute, join, resolve } from "node:path";
88 import { rm, stat } from "node:fs/promises";
99 import { defect } from "../kernel/error.ts";
1010 import { err, ok, type Result } from "../kernel/result.ts";
@@ -92,6 +92,35 @@ export async function forkBareRepo(
9292 { cwd: forkDir, timeoutMs: 120_000 },
9393 );
9494 if (!pins.ok) return await undo(forkDir, pins);
95 const keep = await keepLentObjects(forkDir);
96 if (!keep.ok) return await undo(forkDir, keep);
97 return ok();
98 }
99
100 /**
101 * Make every repository this one borrows objects from keep its unreachable objects.
outdated

Only forks created from now on set this. Existing lenders (scout/forge) stay unprotected. Move the setting into an idempotent helper and also run it over repos that already have forks.

102 *
103 * The fork's refs reach objects the lender's own refs may later drop (a force-push, a
104 * deleted branch). The lender's gc cannot see the fork's refs, so once its grace period
105 * ended it would prune them and leave the fork's branches with missing commits, which the
106 * snapshot then cannot back up (2026-11-14, scout/forge). Idempotent: run when a fork is
107 * made and again before each snapshot, so lenders forked before this existed are covered.
108 */
109 export async function keepLentObjects(forkDir: string): Promise<Result<void, GitFailure>> {
110 const own = join(forkDir, "objects");
111 let lines: string[];
112 try {
113 lines = (await Bun.file(join(own, "info", "alternates")).text()).split("\n");
114 } catch {
115 return ok();
116 }
117 for (const line of lines) {
118 const path = line.trim();
119 if (path === "" || path.startsWith("#")) continue;
120 const lender = dirname(isAbsolute(path) ? path : resolve(own, path));
121 const set = await runGit(["config", "gc.pruneExpire", "never"], { cwd: lender });
122 if (!set.ok) return set;
123 }
95124 return ok();
96125 }
97126
src/snapshot/snapshot.ts +5 −1
@@ -43,7 +43,7 @@ import { join, relative } from "node:path";
4343 import { envDataDir } from "../apps/data.ts";
4444 import { listApps } from "../core/apps.ts";
4545 import { listEveryRepo } from "../core/repos.ts";
46 import { repoDir } from "../git/repos.ts";
46 import { keepLentObjects, repoDir } from "../git/repos.ts";
4747 import { createLog } from "../kernel/log.ts";
4848 import { bundleRepo } from "./bundle.ts";
4949 import { countRows, type DrillCounts } from "./counts.ts";
@@ -492,6 +492,10 @@ export async function takeSnapshot(
492492 repositories.push(full);
493493 const dir = repoDir(forge.reposRoot, repo.ownerName, repo.name);
494494 const bundle = join(scratch, `${repo.ownerName}--${repo.name}.bundle`);
495 // A lender prunes what only its forks reach, unless told not to: cover those forked
496 // before it was.
497 const kept = await keepLentObjects(dir);
498 if (!kept.ok) log.warn("lender not set to keep its objects", { repo: full });
495499 const made = await bundleRepo(dir, bundle, { deadlineMs: forge.bundleDeadlineMs, sizes });
496500 if (!made.ok) {
497501 rmSync(bundle, { force: true });
test/features-tui.test.ts +4 −2
@@ -21,7 +21,7 @@ import { createForgeApp } from "../src/tui/forge.ts";
2121 import type { TuiIdentity } from "../src/tui/session.ts";
2222 import { headlessTerminal } from "../src/tui/term/terminal.ts";
2323 import { quietLogs } from "./helpers/globals.ts";
24 import { settleApp } from "./helpers/tui.ts";
24 import { settleApp, settleUntil } from "./helpers/tui.ts";
2525
2626 quietLogs();
2727
@@ -59,7 +59,7 @@ describe("features in the terminal", () => {
5959 await settleApp(app);
6060 await type(query);
6161 };
62 return { text: () => app.screen.toText(), press, type, palette, stop: () => app.stop() };
62 return { text: () => app.screen.toText(), press, type, palette, until: (what: string) => settleUntil(app, what), stop: () => app.stop() };
6363 }
6464
6565 beforeAll(async () => {
@@ -175,6 +175,8 @@ describe("features in the terminal", () => {
175175 await t.palette("Feature feeds");
176176 await t.press("enter");
177177 await t.press("enter");
178 // The feed screen reads the feature after it opens, out of the settle's sight: wait for it.
179 await t.until("Scheduled: ");
178180 expect(t.text()).toMatch(/Scheduled: scan at 07:15 America\/Chicago · last /);
179181 expect(t.text()).not.toContain("every 1h");
180182 await t.press("h");
test/fork-lender-keeps-objects.test.ts added +102
@@ -0,0 +1,102 @@
1 /** A repository that lends objects to a fork never prunes what only the fork reaches. */
2 import { expect, test } from "bun:test";
3 import { readdirSync, utimesSync } from "node:fs";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { forkBareRepo, initBareRepo, keepLentObjects } from "../src/git/repos.ts";
8
9 const git = async (cwd: string, ...args: string[]): Promise<{ code: number; out: string }> => {
10 const p = Bun.spawn(["git", ...args], {
11 cwd,
12 stdout: "pipe",
13 stderr: "pipe",
14 env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" },
outdated

This checks the config value, not the behaviour. Drop a parent ref after forking, run gc with prune expiry now in the parent, and assert the fork's commit still resolves. Also cover a parent that was forked before this change.

15 });
16 const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited]);
17 return { code, out: out.trim() };
18 };
19
20 /** A parent with one commit on `main`, and a fork with one more commit that only it has. */
21 async function setup(root: string) {
22 const parent = join(root, "p.git");
23 expect((await initBareRepo(parent)).ok).toBe(true);
24 const work = join(root, "w");
25 await git(root, "clone", parent, work);
26 await git(work, "commit", "--allow-empty", "-m", "one");
27 await git(work, "branch", "-M", "main");
28 await git(work, "push", "origin", "main");
29 await git(work, "checkout", "-b", "topic");
30 await git(work, "commit", "--allow-empty", "-m", "only-topic");
31 await git(work, "push", "origin", "topic");
32 const sha = (await git(work, "rev-parse", "topic")).out;
33 const fork = join(root, "f.git");
34 expect((await forkBareRepo(parent, fork)).ok).toBe(true);
35 return { parent, fork, sha };
36 }
37
38 /** Every file under a directory, aged past the grace period git gives unreachable objects. */
39 function age(dir: string): void {
40 const then = new Date(Date.now() - 30 * 86_400_000);
41 for (const e of readdirSync(dir, { withFileTypes: true })) {
42 const path = join(dir, e.name);
43 if (e.isDirectory()) age(path);
44 else utimesSync(path, then, then);
45 }
46 }
47
48 /** The parent drops the branch the fork also has; a month later its ordinary gc runs. */
49 async function dropAndPrune(parent: string): Promise<void> {
50 await git(parent, "update-ref", "-d", "refs/heads/topic");
51 await git(parent, "reflog", "expire", "--expire=now", "--all");
52 age(join(parent, "objects"));
53 expect((await git(parent, "gc")).code).toBe(0);
54 }
55
56 test("what only the fork reaches survives the parent's gc", async () => {
57 const root = await mkdtemp(join(tmpdir(), "lender-"));
58 try {
59 const { parent, fork, sha } = await setup(root);
60 await dropAndPrune(parent);
61 expect((await git(fork, "cat-file", "-e", sha)).code).toBe(0);
62 expect((await git(fork, "bundle", "create", join(root, "b"), "--all")).code).toBe(0);
63 } finally {
64 await rm(root, { recursive: true, force: true });
65 }
66 });
67
68 test("a parent forked before the setting existed is covered by keepLentObjects, and is lost without it", async () => {
69 const root = await mkdtemp(join(tmpdir(), "lender-"));
70 try {
71 const old = await setup(root);
72 await git(old.parent, "config", "--unset", "gc.pruneExpire");
73 await dropAndPrune(old.parent);
74 // Without the setting git prunes the commit and the fork's branch is broken.
75 expect((await git(old.fork, "cat-file", "-e", old.sha)).code).not.toBe(0);
76 } finally {
77 await rm(root, { recursive: true, force: true });
78 }
79 const root2 = await mkdtemp(join(tmpdir(), "lender-"));
80 try {
81 const { parent, fork, sha } = await setup(root2);
82 await git(parent, "config", "--unset", "gc.pruneExpire");
83 expect((await keepLentObjects(fork)).ok).toBe(true);
84 expect((await keepLentObjects(fork)).ok).toBe(true); // idempotent
85 await dropAndPrune(parent);
86 expect((await git(fork, "cat-file", "-e", sha)).code).toBe(0);
87 } finally {
88 await rm(root2, { recursive: true, force: true });
89 }
90 });
91
92 test("a repository that borrows nothing is left alone", async () => {
93 const root = await mkdtemp(join(tmpdir(), "lender-"));
94 try {
95 const p = join(root, "p.git");
96 await initBareRepo(p);
97 expect((await keepLentObjects(p)).ok).toBe(true);
98 expect((await git(p, "config", "--get", "gc.pruneExpire")).code).not.toBe(0);
99 } finally {
100 await rm(root, { recursive: true, force: true });
101 }
102 });
test/mcp-parent-unreachable.test.ts +10 −5
@@ -8,6 +8,7 @@
88 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
99 import { mkdtemp, rm } from "node:fs/promises";
1010 import { tmpdir } from "node:os";
11 import { createServer, type Server } from "node:net";
1112 import { join } from "node:path";
1213 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
1314 import { mergePull, openPull } from "../src/actions/pulls.ts";
@@ -38,6 +39,7 @@ describe("MCP proposes to a parent forge that does not answer", () => {
3839 let lib: Repo;
3940 let forge: Repo;
4041 let writer = "";
42 let holder: Server | undefined;
4143
4244 const person = async (app: App, username: string) => {
4345 const made = await createUser(app.db, {
@@ -76,11 +78,11 @@ describe("MCP proposes to a parent forge that does not answer", () => {
7678
7779 beforeAll(async () => {
7880 dir = await mkdtemp(join(tmpdir(), "forge-unreachable-"));
79 // A port that was free a moment ago and refuses connections until the
80 // parent is started on it.
81 const probe = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("") });
82 const port = probe.port as number;
83 await probe.stop(true);
81 // The parent's port is held by a listener that drops every connection unanswered, so
82 // no other test on a loaded shard can take it before the parent is started on it.
83 holder = createServer((socket) => socket.destroy());
84 await new Promise<void>((done) => holder!.listen(0, "127.0.0.1", done));
85 const port = (holder.address() as { port: number }).port;
8486
8587 parent = createApp({ dataDir: join(dir, "parent"), port, host: "127.0.0.1" });
8688 const alice = await person(parent, "alice");
@@ -146,6 +148,7 @@ describe("MCP proposes to a parent forge that does not answer", () => {
146148 afterAll(async () => {
147149 await childServer.stop();
148150 await parentServer?.stop();
151 holder?.close();
149152 await rm(dir, { recursive: true, force: true });
150153 });
151154
@@ -167,6 +170,8 @@ describe("MCP proposes to a parent forge that does not answer", () => {
167170 });
168171
169172 test("once the parent answers, a retry proposes exactly once", async () => {
173 await new Promise<void>((done) => holder!.close(() => done()));
174 holder = undefined;
170175 parentServer = await serve(parent);
171176 const first = await call(writer);
172177 expect(first.error).toBe(false);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs