Public: anyone can read this; the crew can change it.

A forge that fails to start from a template leaves nothing half-made #13173

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13135-a-forge-that-fails-to-start-from-a-templ → main 9 commits fork 2 threads · 2 unresolved

  • Merged. scout/13135-a-forge-that-fails-to-start-from-a-templ was merged into main as 0736f4b.

Review

Sign in to review this change.
5 files changed +188 −19
docs/SPAWN.md +1 −1
@@ -20,7 +20,7 @@ and user accounts. The child mints its own. A test in #420 and #422 proves it.
2020 | #6023 | a cell child boots inside a forge that is itself in a cell (nothing under the cell's own `/run` is remounted read-only); its cell starts behind the first request, which is answered 503 with `Retry-After` until it is up, and a cell that fails turns the child hosted; the terminal's family lines say where each child runs | #2926 |
2121 | #421 | spawn action: org, instance, apps as repos, briefs | #420, #427 |
2222
23 `src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token).
23 `src/actions/spawn.ts` (#421): the child lives in `<data>/children/<bodyId>`; the parent keeps a ledger line in `<data>/spawn/<name>.json` so a retry resumes or answers. Subscription names are in the plan but not yet created in the child (a subscription is a token). A forge whose first attempt fails is removed whole (directory, ledger, placement) and the error names the step (#13135); only a retry of an older unfinished ledger resumes.
2424
2525 | #422 | first boot: direction, own tokens, domain, claim link | #421 |
2626 | #423 | cross-forge following over HTTPS, read-only token | #422 |
src/actions/spawn.ts +50 −13
@@ -14,7 +14,7 @@
1414
1515 import { nameShapeVerdict, nameVerdict, suggestNames, takenWords, type NameVerdict } from "../spawn/name_verdict.ts";
1616 import { readdirSync, readFileSync } from "node:fs";
17 import { mkdir, readdir } from "node:fs/promises";
17 import { mkdir, readdir, rm } from "node:fs/promises";
1818 import { join } from "node:path";
1919 import { createApp, type App } from "../app.ts";
2020 import { tellPlan } from "../billing/notice.ts";
@@ -332,6 +332,8 @@ async function make(
332332 if (deps.modelKey !== undefined) deps = { ...deps, modelKey: normalizeToken(deps.modelKey) };
333333 const planHash = sha256(JSON.stringify(plan));
334334 let ledger = await readLedger(app, plan.name);
335 // Only a child this call began is removed on failure; a resumed one is left for the retry.
336 const began = ledger === undefined;
335337 if (ledger) {
336338 if (ledger.planHash !== planHash)
337339 return err({ kind: "invalid", field: "name", reason: await takenReason(app, plan.name) });
@@ -402,7 +404,10 @@ async function make(
402404 }
403405
404406 const copied = await copyInto(app, owner, plan, ledger, deps.modelKey, deps.modelEndpoint);
405 if (!copied.ok) return failed(app, deps, "copy", copied.error);
407 if (!copied.ok) {
408 if (began) await unmake(app, deps, plan.name, ledger.bodyId);
409 return failed(app, deps, "copy", copied.error);
410 }
406411 const claimUrl = claimLink(liveChildUrl(app, plan.name), copied.value);
407412 // A plan carries its own credit (grantPlanCredit); the included credit is for a spawn on no plan.
408413 if (plan.plan === undefined) lendIncludedCredit(app, plan.name, childDirOf(app, ledger.bodyId), deps.modelKey !== undefined);
@@ -427,6 +432,22 @@ async function make(
427432 });
428433 }
429434
435 /**
436 * A forge that failed to start is not left half-made: its data directory (the child's database,
437 * repositories and keys), its ledger and its placement go, so a retry begins clean.
438 */
439 async function unmake(app: App, deps: SpawnDeps, name: string, bodyId: string): Promise<void> {
440 if (!SAFE_ID.test(bodyId)) return;
441 try {
442 await rm(childDirOf(app, bodyId), { recursive: true, force: true });
443 await rm(ledgerPath(app, name), { force: true });
444 const retired = await deps.placement.retire(bodyId);
445 if (!retired.ok) app.log.warn(`the placement of the half-made ${name} was not retired: ${retired.error.message}`);
446 } catch (e) {
447 app.log.warn(`could not remove the half-made ${name}: ${e instanceof Error ? e.message : String(e)}`);
448 }
449 }
450
430451 /**
431452 * The crew credit the child's plan carries, read now (so an edit on the plans
432453 * page is what the next spawn gets): lent as a monthly cap, the spend counted
@@ -449,6 +470,9 @@ export function grantPlanCredit(app: App, plan: SpawnPlan, bodyId: string): void
449470 }
450471 }
451472
473 /** An error that names the step of making a forge that broke. */
474 const inStep = (step: string, message: string): string => `step "${step}" failed: ${message}`;
475
452476 /**
453477 * Every step skips what is already there, so a resumed child is finished,
454478 * not doubled. Answers the child's claim token, minted afresh each time.
@@ -464,6 +488,8 @@ async function copyInto(
464488 const dir = childDirOf(app, ledger.bodyId);
465489 await mkdir(dir, { recursive: true });
466490 const child = createApp({ dataDir: dir });
491 // The step under way, named in the error when it breaks.
492 let step = "founder";
467493 try {
468494 // The founder is a name only: no password, no email worth mailing. The
469495 // child's first boot hands the account to a person (#422). A stranger's
@@ -483,16 +509,17 @@ async function copyInto(
483509 });
484510 return made.ok ? made.value : undefined;
485511 })());
486 if (!founder) return err("could not make the child's founder");
512 if (!founder) return err(inStep(step, "could not make the child's founder"));
487513 // A member, not a guest: the child's first boot acts as the founder to
488514 // open the direction and hand it to the planner, which starts a run.
489515 setMember(child.db, founder.id, true, founder.id, new Date());
490516
517 step = "organisation";
491518 let org = getOrg(child.db, plan.org);
492519 if (!org) {
493520 // Named by the spawner, who was held to the rule on the parent.
494521 const made = createOrg(child.db, plan.org, founder.id, { allowReserved: true });
495 if (!made.ok) return err(`could not make org ${plan.org}: ${made.error}`);
522 if (!made.ok) return err(inStep(step, `could not make org ${plan.org}: ${made.error}`));
496523 org = made.value;
497524 }
498525
@@ -501,10 +528,11 @@ async function copyInto(
501528 // Each app's link to the parent's repository: the whole history is copied below, and the
502529 // child follows it from the first minute (`spawn/app_links.ts`).
503530 const appEntries: AppEntry[] = [];
531 step = "apps";
504532 for (const a of plan.apps) {
505533 const [o, n] = a.ref.split("/") as [string, string];
506534 const parent = getRepo(app.db, o, n);
507 if (!parent) return err(`${a.ref} is gone`);
535 if (!parent) return err(inStep(step, `${a.ref} is gone`));
508536 let repo = getRepo(child.db, org.username, n);
509537 if (!repo) {
510538 const made = await createRepo(child.db, child.reposRoot, {
@@ -514,7 +542,7 @@ async function copyInto(
514542 description: parent.description,
515543 defaultBranch: parent.defaultBranch,
516544 });
517 if (!made.ok) return err(`could not make ${a.ref}: ${made.error}`);
545 if (!made.ok) return err(inStep(step, `could not make ${a.ref}: ${made.error}`));
518546 repo = made.value;
519547 }
520548 // Branches and tags with every commit behind them; re-running only tops up.
@@ -531,7 +559,7 @@ async function copyInto(
531559 ],
532560 { cwd: repoDir(child.reposRoot, org.username, n), timeoutMs: 600_000 },
533561 );
534 if (!fetched.ok) return err(`could not copy ${a.ref}: ${fetched.error.stderr}`);
562 if (!fetched.ok) return err(inStep(step, `could not copy ${a.ref}: ${fetched.error.stderr}`));
535563 // The spawner's words are each app's direction; the child's first
536564 // boot opens the direction issue from them (`spawn/first-boot.ts`).
537565 if (plan.direction !== "" && repo.autonomyDirection !== plan.direction)
@@ -558,6 +586,7 @@ async function copyInto(
558586 // one repository, not two (`spawn/template_birth.ts`).
559587 // A template repository supplies the code below instead; the sentence, if any, still
560588 // begins a fresh app.
589 step = "template";
561590 const fromRepo = plan.template === undefined ? undefined : repoTemplateRef(plan.template);
562591 if (fromRepo !== undefined) templateMark = { id: fromRepo, repo: fromRepo };
563592 if (plan.template !== undefined && fromRepo === undefined) {
@@ -567,11 +596,11 @@ async function copyInto(
567596 direction: plan.direction,
568597 taken: new Set(plan.apps.map((a) => a.ref.split("/")[1]!)),
569598 });
570 if (!born.ok) return err(born.error);
599 if (!born.ok) return err(inStep(step, born.error));
571600 templateMark = born.value;
572601 } else if (plan.fresh !== undefined) {
573602 const made = await startFresh(child, org, plan.fresh, plan.direction);
574 if (!made.ok) return err(made.error);
603 if (!made.ok) return err(inStep(step, made.error));
575604 }
576605
577606 // The forge's own code, forked into the child with its full history,
@@ -579,12 +608,14 @@ async function copyInto(
579608 // A cross-forge fork: the history is copied by git, and the upstream
580609 // is recorded in parent.json for the child to pull from. Skipped when
581610 // the forge does not name its code repository (`forgeRepo`).
611 step = "forge code";
582612 const forgeCode = await forkForgeCode(app, child, org.username, plan.followMode ?? DEFAULT_MODE, fromRepo);
583 if (!forgeCode.ok) return err(forgeCode.error);
613 if (!forgeCode.ok) return err(inStep(step, forgeCode.error));
584614 // The commit a template repository was read at, kept with the template (#13038).
585615 if (fromRepo !== undefined && forgeCode.value !== undefined && forgeCode.value.tip !== "")
586616 templateMark = { id: fromRepo, repo: fromRepo, tip: forgeCode.value.tip };
587617
618 step = "crew";
588619 for (const b of plan.briefs) {
589620 // The jobs it declares travel with it (#8907).
590621 const roles = getAgentByName(app.db, b.agent)?.roles;
@@ -599,7 +630,7 @@ async function copyInto(
599630 agent: true,
600631 origin: "spawn",
601632 });
602 if (!made.ok) return err(`could not make crew member ${b.agent}: ${made.error}`);
633 if (!made.ok) return err(inStep(step, `could not make crew member ${b.agent}: ${made.error}`));
603634 createAgentRow(child.db, { userId: made.value.id, brief: b.text, roles, createdBy: founder.id });
604635 }
605636
@@ -625,6 +656,7 @@ async function copyInto(
625656 // The child's own keys, made in its own directory now, so they exist
626657 // before its first boot and are provably not the parent's: the key that
627658 // seals its secrets, and its SSH host key.
659 step = "keys";
628660 loadKey(dir);
629661 loadOrCreateHostKey(join(dir, "ssh"));
630662
@@ -632,6 +664,7 @@ async function copyInto(
632664 // offered to its organisation's crew, and bound to a whole-life budget
633665 // the child cannot raise — enforced where every run picks its account,
634666 // whoever started the run and on whichever repository.
667 step = "model key";
635668 if (modelKey !== undefined && plan.budgetUsd !== null) {
636669 const sealed =
637670 getSubscriptionByName(child.db, STARTER_KEY) ??
@@ -646,12 +679,13 @@ async function copyInto(
646679 });
647680 return added.ok ? added.value : undefined;
648681 })();
649 if (!sealed) return err("could not seal the child's model key");
682 if (!sealed) return err(inStep(step, "could not seal the child's model key"));
650683 setGrants(child.db, org.id, "", [sealed.id]);
651684 if (!lockBudget(child.db, sealed.id, plan.budgetUsd))
652 return err("could not set the key's budget");
685 return err(inStep(step, "could not set the key's budget"));
653686 }
654687
688 step = "record";
655689 const upstream = app.config.url ?? `http://${app.config.host ?? "localhost"}`;
656690 writePrivate(
657691 join(dir, "parent.json"),
@@ -681,6 +715,9 @@ async function copyInto(
681715 }),
682716 );
683717 return ok(mintClaim(child.db, org.id));
718 } catch (e) {
719 // A throw is a failed step too: the caller removes what this spawn made.
720 return err(inStep(step, e instanceof Error ? e.message : String(e)));
684721 } finally {
685722 child.close();
686723 }
src/kernel/destroy_reach.ts +11 −3
@@ -83,6 +83,7 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
8383 { unit: "src/git/env.ts:trustedCaBundle", why: "removes the process's own temporary CA bundle at exit" },
8484 { unit: "src/apps/contain.ts:openCell", why: "clears the cell's socket directory before the cell opens" },
8585 { unit: "src/apps/cgroup.ts:removeDir", why: "removes an empty cgroup directory" },
86 { unit: "src/actions/spawn.ts:unmake", why: "removes the directory and ledger of a child whose first attempt failed before it was ever finished, and only that call's own (never a finished or resumed one): nothing of anyone's work is in it (#13135)" },
8687 { unit: "src/spawn/own_code.ts:checkout", why: "removes a half-made export of the forge's own code, keyed by commit" },
8788 { unit: "src/spawn/own_code.ts:recordGood", why: "removes exports of the forge's own code for other commits than the good one" },
8889 { unit: "src/web/asset.ts:prune", why: "removes cached page scripts older than their keep time" },
@@ -103,9 +104,13 @@ interface Module {
103104 imports: Map<string, { from: string; name: string }>;
104105 spaces: Map<string, string>;
105106 reexports: string[];
106 /** Local names that mean a file removal in this module (`import { rmSync as r } from "node:fs"`). */
107 /**
108 * Local names that mean a file removal in this module (`import { rmSync as r } from "node:fs"`).
109 */
107110 removers: Set<string>;
108 /** Every identifier in the file: a class is a candidate for a method call only where it is named. */
111 /**
112 * Every identifier in the file: a class is a candidate for a method call only where it is named.
113 */
109114 words: Set<string>;
110115 }
111116
@@ -248,7 +253,10 @@ export function destroyReach(root: string, entry: string, allowed: readonly { un
248253 for (let c: Unit | undefined = u; c; c = via.get(c)) names.unshift(`${c.file}:${c.name}`);
249254 return names.join(" -> ");
250255 };
251 /** A method is called from a unit only where its file is the unit's or imported by it, and its class is named there. */
256 /**
257 * A method is called from a unit only where its file is the unit's or imported by it, and its
258 * class is named there.
259 */
252260 const candidate = (caller: Unit, method: Unit): boolean =>
253261 (method.file === caller.file || touchesFile(caller.file, method.file)) && mod(caller.file).words.has(method.name.slice(0, method.name.indexOf(".")));
254262 const stopped = new Set<string>();
test/spawn-unmake.test.ts added +122
@@ -0,0 +1,122 @@
1 /** A forge that fails to start leaves nothing half-made; the name can be retried (#13135). */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { existsSync, mkdirSync } from "node:fs";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { childDirOf, spawnForge } from "../src/actions/spawn.ts";
8 import { createRepo } from "../src/actions/repos.ts";
9 import { createApp, type App } from "../src/app.ts";
10 import { deleteRepo, getRepo } from "../src/core/repos.ts";
11 import { createUser, type User } from "../src/core/users.ts";
12 import { hostedPlacement } from "../src/spawn/placement.ts";
13 import { quietLogs } from "./helpers/globals.ts";
14
15 quietLogs();
16
17 describe("a forge that fails to start is removed whole", () => {
18 let dir = "";
19 let parent: App;
20 let alice: User;
21
22 const spawn = (name: string, placement: ReturnType<typeof hostedPlacement>, apps: string[] = []) =>
23 spawnForge(
24 parent,
25 alice,
26 { name, org: name, apps, carryBriefs: false, carrySubscriptions: false, template: "starter" },
27 { catalog: apps.map((ref) => ({ ref, readable: true, bytes: 1, forgeJsonBytes: 0 })), briefs: [], subscriptions: [] },
28 {
29 placement,
30 request: new Request("https://root.test/x", { method: "POST", headers: { origin: "https://root.test" } }),
31 url: new URL("https://root.test/x"),
32 },
33 );
34
35 beforeAll(async () => {
36 dir = await mkdtemp(join(tmpdir(), "forge-unmake-"));
37 parent = createApp({ dataDir: join(dir, "parent"), port: 0, host: "root.test" });
38 const made = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true });
39 if (!made.ok) throw new Error("no alice");
40 alice = made.value;
41 });
42 afterAll(async () => {
43 parent.close();
44 await rm(dir, { recursive: true, force: true });
45 });
46
47 test("a late step fails: no directory, ledger or placement stays, the error names the step, a retry works", async () => {
48 const real = hostedPlacement({ host: "root.test" });
49 let bodyId = "";
50 const retired: string[] = [];
51 const placement: typeof real = {
52 ...real,
53 async retire(id) {
54 retired.push(id);
55 return real.retire(id);
56 },
57 async place(plan) {
outdated

Also assert placement.retire was called with bodyId, and add an early-step failure case.

58 const p = await real.place(plan);
59 if (p.ok) {
60 bodyId = p.value.bodyId;
61 // The last step writes parent.json: a directory in its place breaks it late.
62 mkdirSync(join(childDirOf(parent, bodyId), "parent.json"), { recursive: true });
63 }
64 return p;
65 },
66 };
67 const r = await spawn("shaky", placement);
68 expect(r.ok).toBe(false);
69 if (!r.ok) expect(JSON.stringify(r.error)).toContain('step \\"record\\" failed');
70 expect(existsSync(childDirOf(parent, bodyId))).toBe(false);
71 expect(existsSync(join(parent.config.dataDir, "spawn", "shaky.json"))).toBe(false);
72 expect(retired).toEqual([bodyId]);
73
74 const again = await spawn("shaky", real);
75 expect(again.ok).toBe(true);
76 if (again.ok) expect(existsSync(join(childDirOf(parent, again.value.bodyId), "parent.json"))).toBe(true);
77 });
78
79 /** A parent app that is readable when the plan is made and gone by the "apps" step. */
80 const vanishing = async (name: string, refuse: boolean) => {
81 const made = await createRepo(parent, alice, { name, visibility: "private" });
82 if (!made.ok) throw new Error("no repo");
83 const real = hostedPlacement({ host: "root.test" });
84 const out = { bodyId: "", retired: [] as string[] };
85 const placement: typeof real = {
86 ...real,
87 async place(plan) {
88 const p = await real.place(plan);
89 if (p.ok) out.bodyId = p.value.bodyId;
90 const repo = getRepo(parent.db, "alice", name);
91 if (repo) await deleteRepo(parent.db, parent.reposRoot, repo);
92 return p;
93 },
94 async retire(id) {
95 out.retired.push(id);
96 return refuse ? { ok: false as const, error: { reason: "children" as const, message: "nope" } } : real.retire(id);
97 },
98 };
99 return { out, placement, ref: `alice/${name}` };
100 };
101
102 test("an early step fails: the error names it and nothing stays", async () => {
103 const { out, placement, ref } = await vanishing("gone", false);
104 const r = await spawn("early", placement, [ref]);
105 expect(r.ok).toBe(false);
106 if (!r.ok) expect(JSON.stringify(r.error)).toContain('step \\"apps\\" failed');
107 expect(existsSync(childDirOf(parent, out.bodyId))).toBe(false);
108 expect(existsSync(join(parent.config.dataDir, "spawn", "early.json"))).toBe(false);
109 expect(out.retired).toEqual([out.bodyId]);
110 });
111
112 test("a refused retire is logged, not silent", async () => {
113 const { placement, ref } = await vanishing("gone2", true);
114 const warned: string[] = [];
115 const warn = parent.log.warn;
116 parent.log.warn = (m: string) => void warned.push(m);
117 const r = await spawn("stuck", placement, [ref]);
118 parent.log.warn = warn;
119 expect(r.ok).toBe(false);
120 expect(warned.some((m) => m.includes("not retired") && m.includes("nope"))).toBe(true);
121 });
122 });
test/spawn.test.ts +4 −2
@@ -1,4 +1,5 @@
11 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
2 import { existsSync } from "node:fs";
23 import { mkdtemp, readdir, rename, rm } from "node:fs/promises";
34 import { tmpdir } from "node:os";
45 import { join } from "node:path";
@@ -132,13 +133,14 @@ describe("spawnForge", () => {
132133 expect(events).toEqual({ n: 1 });
133134 });
134135
135 test("a half-made child is resumed, not duplicated", async () => {
136 test("a child that fails to start is removed whole, and a retry makes it fresh (#13135)", async () => {
136137 // First attempt dies after placing: the parent's repo vanishes mid-copy.
137138 const d = repoDir(app.reposRoot, "alice", "notes");
138139 await rename(d, `${d}.away`);
139140 const first = await spawn();
140141 expect(first.ok).toBe(false);
141 expect(await readdir(join(dir, "children"))).toHaveLength(1);
142 expect(await readdir(join(dir, "children"))).toEqual([]);
143 expect(existsSync(join(dir, "spawn", "kid.json"))).toBe(false);
142144 await rename(`${d}.away`, d);
143145 const second = await spawn();
144146 if (!second.ok) throw new Error(JSON.stringify(second.error));

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs