Public: anyone can read this; the crew can change it.

Sentinel: release was degraded 743 minutes today: Release 0.7.578 is unproven: a stranger's journey failed: the forge could not be named (TimeoutError). It is proven aga… #12223

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12171-sentinel-release-was-degraded-743-minute → main 4 commits fork

  • Merged. torque/12171-sentinel-release-was-degraded-743-minute was merged into main as 325ce45.

Review

Sign in to review this change.
5 files changed +256 −0
src/actions/spawn.ts +4
@@ -39,6 +39,7 @@ import {
3939 import { createUser, getUserByName, type User } from "../core/users.ts";
4040 import { runGit } from "../git/env.ts";
4141 import { repoDir } from "../git/repos.ts";
42 import { seedHistory } from "../git/seed_history.ts";
4243 import { commitFiles } from "../git/write.ts";
4344 import { adoptForgeCode, forkForgeCode } from "../spawn/forge_code.ts";
4445 import { lendIncludedCredit } from "../spawn/included_credit.ts";
@@ -473,6 +474,9 @@ async function copyInto(
473474 repo = made.value;
474475 }
475476 // Branches and tags with every commit behind them; re-running only tops up.
477 // The history packed straight into the child first,
478 // so the fetch has nothing to send (#12171).
479 await seedHistory(repoDir(app.reposRoot, o, n), repoDir(child.reposRoot, org.username, n));
476480 const fetched = await runGit(
477481 [
478482 "fetch",
src/git/seed_history.ts added +102
@@ -0,0 +1,102 @@
1 /**
2 * A repository's branches and tags, with every commit behind them, written
3 * into an empty one by `git pack-objects` before `git fetch` is asked
4 * (#12171).
5 *
6 * A spawn copies the forge's own code, and each app it sends, with
7 * `git fetch <parent's repository>`. From a local path that fetch packs the
8 * whole history on one side and indexes it again on the other, inflating
9 * and hashing every object a second time: about 7 s for this repository
10 * (34 MiB, 81 000 objects, 548 refs) on a quiet machine, more on a busy
11 * host. The release proof's stranger waits on that answer with a 10 s
12 * deadline, and "the forge could not be named (TimeoutError)" held release
13 * 0.7.578 unproven for 743 minutes on 2026-11-13.
14 *
15 * `pack-objects <base>` writes the pack and its index itself (it already
16 * knows every object's name), straight into the child's `objects/pack`; the
17 * refs are then set to the very tips that were packed. The same fetch that
18 * follows finds every ref where it wants it and sends nothing: about 2.1 s
19 * in all for the same history.
20 *
21 * What is copied is what a fetch would copy: the objects reachable from the
22 * parent's `refs/heads/*` and `refs/tags/*` at the moment they were read,
23 * never the rest of its object store (a commit force-pushed away stays with
24 * the parent). Git still decides: when this does nothing (a repository that
25 * is not empty, a pack that could not be written, a refused ref) the fetch
26 * does the whole copy as before. A ref is set only after its pack is whole.
27 *
28 * The destination may be a hosted forge's directory (`spawn/forge_code.ts`
29 * backfills a running child): each directory on the way down from `to` is
30 * lstat'ed and must be a directory itself, never a link. Paths go to git as
31 * arguments, never through a shell. Plain data in (two bare repositories'
32 * directories), plain data out (how many refs were set). Never throws.
33 */
34
35 import { lstat } from "node:fs/promises";
36 import { join } from "node:path";
37 import { runGit } from "./env.ts";
38
39 /** The longest the pack may take; the fetch after it has its own. */
40 export const SEED_TIMEOUT_MS = 300_000;
41
42 const SHA = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
43 const REF = /^refs\/(?:heads|tags)\/[^\s\0]+$/;
44
45 async function isDir(path: string): Promise<boolean> {
46 try {
47 return (await lstat(path)).isDirectory();
48 } catch {
49 return false;
50 }
51 }
52
53 /** The branch and tag tips of `from`, read once: `sha refname` each. */
54 export async function tipsOf(dir: string): Promise<{ sha: string; ref: string }[]> {
55 const listed = await runGit(["for-each-ref", "--format=%(objectname) %(refname)", "refs/heads", "refs/tags"], { cwd: dir });
56 if (!listed.ok) return [];
57 const tips: { sha: string; ref: string }[] = [];
58 for (const line of listed.value.split("\n")) {
59 const at = line.indexOf(" ");
60 const sha = line.slice(0, at);
61 const ref = line.slice(at + 1);
62 if (at > 0 && SHA.test(sha) && REF.test(ref)) tips.push({ sha, ref });
63 }
64 return tips;
65 }
66
67 /**
68 * Packs `from`'s branches and tags into `to` and sets the same refs there,
69 * when `to` has none yet. Answers how many refs were set: 0 when nothing was
70 * done, and the caller's fetch then copies everything itself.
71 */
72 export async function seedHistory(from: string, to: string): Promise<number> {
73 try {
74 return await seed(from, to);
75 } catch {
76 return 0; // git could not be started there: the fetch decides
77 }
78 }
79
80 async function seed(from: string, to: string): Promise<number> {
81 if (!(await isDir(from))) return 0;
82 const pack = join(to, "objects", "pack");
83 if (!(await isDir(to)) || !(await isDir(join(to, "objects"))) || !(await isDir(pack))) return 0;
84 // Only an empty repository: a retry already holds most of it, and the fetch tops it up.
85 const held = await runGit(["for-each-ref", "--count=1", "--format=%(refname)"], { cwd: to });
86 if (!held.ok || held.value !== "") return 0;
87 const tips = await tipsOf(from);
88 if (tips.length === 0) return 0;
89 const packed = await runGit(["pack-objects", "--revs", "--delta-base-offset", "-q", join(pack, "pack")], {
90 cwd: from,
91 stdin: [...new Set(tips.map((t) => t.sha))].join("\n") + "\n",
92 timeoutMs: SEED_TIMEOUT_MS,
93 });
94 if (!packed.ok) return 0;
95 // `create`: each ref only where it is still absent, all or none,
96 // never over one something else set meanwhile.
97 const set = await runGit(["update-ref", "--stdin"], {
98 cwd: to,
99 stdin: tips.map((t) => `create ${t.ref} ${t.sha}\n`).join(""),
100 });
101 return set.ok ? tips.length : 0;
102 }
src/spawn/forge_code.ts +4
@@ -23,6 +23,7 @@ import { getOrg, listTeamMembers, listTeams, OWNERS_TEAM } from "../core/orgs.ts
2323 import { createRepo, getRepo, listReposOwnedBy, type Repo } from "../core/repos.ts";
2424 import { runGit } from "../git/env.ts";
2525 import { repoDir } from "../git/repos.ts";
26 import { seedHistory } from "../git/seed_history.ts";
2627 import { readPrivate, writePrivate } from "../kernel/file_modes.ts";
2728 import { err, ok, type Result } from "../kernel/result.ts";
2829 import { forgeSource, setForgeSource, storedForgeSource } from "../settings/forge_source.ts";
@@ -75,6 +76,9 @@ export async function forkForgeCode(
7576 if (!made.ok) return err(`could not make the child's forge repo: ${made.error}`);
7677 repo = made.value;
7778 }
79 // The history packed straight into the child first:
80 // the fetch below then has nothing to send (#12171).
81 await seedHistory(repoDir(app.reposRoot, o, n), repoDir(child.reposRoot, orgName, FORGE_CODE_NAME));
7882 const fetched = await runGit(
7983 [
8084 "fetch",
test/forge-code-own-source.test.ts +39
@@ -171,3 +171,42 @@ describe("a forge's own source", () => {
171171 expect(await backfillChildren(parent)).toEqual({});
172172 }, 240_000);
173173 });
174
175 describe("a forge's own source is packed into the child, not fetched (#12171)", () => {
176 test("the child's forge repository holds one pack of the parent's history and no loose object", async () => {
177 const dir = await mkdtemp(join(tmpdir(), "forge-seed-"));
178 const root = createApp({ dataDir: dir, url: "https://root.test", port: 0, forgeRepo: "alice/forge-code" });
179 try {
180 const u = await createUser(root.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true });
181 if (!u.ok) throw new Error("user");
182 const made = await createRepo(root, u.value, { name: "forge-code", visibility: "public" });
183 if (!made.ok) throw new Error("repo");
184 const c = await commitFiles(repoDir(root.reposRoot, "alice", "forge-code"), {
185 branch: "main",
186 message: "forge v1",
187 files: [{ path: "README.md", content: "# forge\n" }],
188 author: { name: "a", email: "[email protected]" },
189 });
190 if (!c.ok) throw new Error("commit");
191 const r = await spawn(root, u.value, "packed");
192 if (!r.ok) throw new Error(JSON.stringify(r.error));
193 const child = createApp({ dataDir: childDirOf(root, r.value.bodyId), port: 0 });
194 try {
195 const forge = repoDir(child.reposRoot, "packed", "forge");
196 // A fetch of so small a history leaves its objects loose (below git's unpack limit);
197 // the seed writes one pack.
198 const count = await runGit(["count-objects", "-v"], { cwd: forge });
199 if (!count.ok) throw new Error("count");
200 expect(count.value).toMatch(/^count: 0$/m);
201 expect(count.value).toMatch(/^packs: 1$/m);
202 const tip = await runGit(["rev-parse", "refs/heads/main"], { cwd: forge });
203 expect(tip.ok && tip.value).toBe(c.value.commitOid);
204 } finally {
205 child.close();
206 }
207 } finally {
208 root.close();
209 await rm(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
210 }
211 }, 240_000);
212 });
test/seed-history.test.ts added +107
@@ -0,0 +1,107 @@
1 /**
2 * `seedHistory` (#12171): a repository's branches and tags packed straight
3 * into an empty one, so the spawn's fetch after it has nothing to send; what
4 * a fetch would not copy is not copied; a link in the way stops it.
5 */
6
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdir, mkdtemp, readdir, rm, symlink } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { runGit } from "../src/git/env.ts";
12 import { seedHistory } from "../src/git/seed_history.ts";
13
14 const git = async (cwd: string, ...args: string[]): Promise<string> => {
15 const r = await runGit(args, { cwd });
16 if (!r.ok) throw new Error(`git ${args.join(" ")}: ${r.error.stderr}`);
17 return r.value;
18 };
19
20 describe("seedHistory", () => {
21 let dir = "";
22 let parent = "";
23 let main = "";
24 let side = "";
25 let gone = "";
26
27 beforeAll(async () => {
28 dir = await mkdtemp(join(tmpdir(), "seed-history-"));
29 const work = join(dir, "work");
30 await mkdir(work);
31 await git(work, "init", "-q", "-b", "main");
32 const commit = async (file: string, text: string): Promise<string> => {
33 await Bun.write(join(work, file), text);
34 await git(work, "add", file);
35 await git(work, "-c", "user.name=t", "-c", "[email protected]", "commit", "-q", "-m", file);
36 return git(work, "rev-parse", "HEAD");
37 };
38 main = await commit("a.txt", "one\n");
39 await git(work, "tag", "v1");
40 await git(work, "checkout", "-q", "-b", "side");
41 side = await commit("b.txt", "two\n");
42 // Pushed, then the branch deleted: an object the parent still holds that no ref reaches.
43 await git(work, "checkout", "-q", "-b", "secret");
44 gone = await commit("key.txt", "a secret force-pushed away\n");
45 await git(work, "checkout", "-q", "main");
46 parent = join(dir, "parent.git");
47 await git(dir, "init", "-q", "--bare", parent);
48 await git(work, "push", "-q", parent, "main", "side", "secret", "v1");
49 await git(parent, "branch", "-D", "secret");
50 await git(parent, "repack", "-a", "-d", "-q");
51 });
52
53 afterAll(async () => {
54 await rm(dir, { recursive: true, force: true });
55 });
56
57 const fresh = async (name: string): Promise<string> => {
58 const to = join(dir, `${name}.git`);
59 await git(dir, "init", "-q", "--bare", to);
60 return to;
61 };
62
63 test("sets every branch and tag to the parent's tip, with a whole pack behind them", async () => {
64 const to = await fresh("child");
65 expect(await seedHistory(parent, to)).toBe(3);
66 expect(await git(to, "rev-parse", "refs/heads/main")).toBe(main);
67 expect(await git(to, "rev-parse", "refs/heads/side")).toBe(side);
68 expect(await git(to, "rev-parse", "refs/tags/v1")).toBe(main);
69 await git(to, "fsck", "--connectivity-only", "--no-dangling");
70 // The fetch the spawn runs after it sends nothing: no loose object, no second pack.
71 await git(to, "fetch", "-q", "--no-write-fetch-head", parent, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*");
72 const count = await git(to, "count-objects", "-v");
73 expect(count).toMatch(/^count: 0$/m);
74 expect(count).toMatch(/^packs: 1$/m);
75 });
76
77 test("copies only what a fetch would: a commit no branch reaches stays with the parent", async () => {
78 expect((await runGit(["cat-file", "-e", gone], { cwd: parent })).ok).toBe(true);
79 const to = await fresh("private");
80 await seedHistory(parent, to);
81 expect((await runGit(["cat-file", "-e", gone], { cwd: to })).ok).toBe(false);
82 });
83
84 test("leaves a repository that already has a ref to the fetch", async () => {
85 const to = await fresh("held");
86 await git(dir, "--git-dir", to, "fetch", "-q", parent, "main:main");
87 const before = await readdir(join(to, "objects", "pack"));
88 expect(await seedHistory(parent, to)).toBe(0);
89 expect(await readdir(join(to, "objects", "pack"))).toEqual(before);
90 });
91
92 test("a link where the pack would go stops it, and nothing is written where the link points", async () => {
93 const to = await fresh("linked");
94 const elsewhere = join(dir, "elsewhere");
95 await mkdir(elsewhere);
96 await rm(join(to, "objects", "pack"), { recursive: true });
97 await symlink(elsewhere, join(to, "objects", "pack"));
98 expect(await seedHistory(parent, to)).toBe(0);
99 expect(await readdir(elsewhere)).toEqual([]);
100 expect((await runGit(["for-each-ref"], { cwd: to })).ok && (await git(to, "for-each-ref"))).toBe("");
101 });
102
103 test("an address that is not a repository does nothing", async () => {
104 expect(await seedHistory(join(dir, "nowhere"), await fresh("orphan"))).toBe(0);
105 expect(await seedHistory(parent, join(dir, "nowhere"))).toBe(0);
106 });
107 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs