Public: anyone can read this; the crew can change it.

Real-cell proof, build half: a Small child builds a first app from the template, passes its tests and lands a second change #12000

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/11999-real-cell-proof-build-half-a-small-child → main 3 commits fork

  • Merged. torque/11999-real-cell-proof-build-half-a-small-child was merged into main as b5fc815.

Review

Sign in to review this change.
4 files changed +265 −0
docs/features/real-cell-proof.md added +6
@@ -0,0 +1,6 @@
1 # Real-cell proof: the build half, in process (#11999)
2
3 - `test/helpers/build-standin.ts` (its work in `build-standin-builder.ts`) is a deterministic stand-in for the model: it adds a `/reading` route and its test to the template's app (`READING-V1` in the prompt), or changes what it says (`READING-V2`), runs the app's real `bun test`, and pushes only when that passes.
4 - `test/build-journey.test.ts` walks a first ask to production in one process: `newApp` copies the template, the crew's stand-in builds, the pull's preview run says its tests passed, a person lands it, production serves `/reading`, and a second change lands the same way. About 8 s.
5
6 Not proved here, and tracked in #12001: the same walk inside a 2 GiB, 1-core cell, and a real memory breach restarting a child.
test/build-journey.test.ts added +144
@@ -0,0 +1,144 @@
1 /**
2 * The build half of a child's first hour (#11999), in this process: a first
3 * ask builds an app from the template, the app's own tests pass in the
4 * check, a person lands the pull, production serves the change, and a
5 * second change lands the same way. The model is `build-standin.ts`, which
6 * is deterministic and runs the app's real `bun test` before it pushes; the
7 * template, the crew, the check, the merge and the runtime are the forge's
8 * own. It runs in this process; the same walk in a Small cell is its own issue.
9 */
10 import { afterAll, beforeAll, expect, setDefaultTimeout, test } from "bun:test";
11 import { mkdtemp, rm } from "node:fs/promises";
12 import { tmpdir } from "node:os";
13 import { join } from "node:path";
14 import { newApp } from "../src/actions/apps.ts";
15 import { createAgent } from "../src/actions/crew.ts";
16 import { assignIssue, openIssue } from "../src/actions/issues.ts";
17 import { mergePull } from "../src/actions/pulls.ts";
18 import { createApp, type App } from "../src/app.ts";
19 import { AppsRuntime } from "../src/apps/runtime.ts";
20 import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
21 import { getRepoById, type Repo } from "../src/core/repos.ts";
22 import { listOpenPulls, type Pull } from "../src/core/pulls.ts";
23 import { listRuns } from "../src/core/runs.ts";
24 import { createUser, type User } from "../src/core/users.ts";
25 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
26 import { serve, type Servers } from "../src/http/server.ts";
27 import { READING_V1, READING_V2, writeBuildStandIn } from "./helpers/build-standin.ts";
28 import { quietLogs } from "./helpers/globals.ts";
29 import "./helpers/roomy-disk.ts";
30
31 quietLogs();
32 // The walk takes about 8 s; the rest is room for a loaded check machine.
33 setDefaultTimeout(60_000);
34
35 const DOMAIN = "apps.test";
36 let dir = "";
37 let app: App;
38 let servers: Servers;
39 let runtime: AppsRuntime;
40 let checks: ChecksHandle;
41 let crew: CrewHandle;
42 let alice: User;
43 let bot: User;
44 let repo: Repo;
45 let slug = "";
46
47 const viaHost = (path: string) => fetch(`${servers.url}${path}`, { headers: { host: `${slug}.${DOMAIN}` }, redirect: "manual" });
48
49 async function until<T>(what: string, read: () => T | undefined | Promise<T | undefined>, ms = 30_000): Promise<T> {
50 const deadline = Date.now() + ms;
51 while (Date.now() < deadline) {
52 const value = await read();
53 if (value !== undefined) return value;
54 await Bun.sleep(100); // real-timer: polling a real forge, crew and runtime that run on their own clocks
55 }
56 throw new Error(`${what} did not happen in ${ms / 1000}s`);
57 }
58
59 beforeAll(async () => {
60 dir = await mkdtemp(join(tmpdir(), "forge-build-journey-"));
61 app = createApp({ dataDir: dir, port: 0 });
62 servers = await serve(app);
63 runtime = new AppsRuntime(app, { domain: DOMAIN, scheme: "http", publicPort: servers.port, forgeUrl: servers.url });
64 await runtime.start();
65 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
66 crew = await startCrew(app, {
67 forgeUrl: servers.url,
68 localUrl: servers.url,
69 binary: writeBuildStandIn(dir).fake,
70 concurrency: 1,
71 budgetUsd: 1,
72 runTimeoutMs: 30_000,
73 networkRetryScale: 0,
74 tickMs: 100,
75 });
76 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
77 if (!made.ok) throw new Error(made.error);
78 alice = made.value;
79 const agent = await createAgent(app, alice, { username: "bot", brief: "Build what is asked." });
80 if (!agent.ok) throw new Error(String(agent.error));
81 bot = agent.value.user;
82 });
83
84 afterAll(async () => {
85 await crew?.stop();
86 await checks?.stop();
87 await runtime?.stop();
88 servers?.stop();
89 app?.close();
90 await rm(dir, { recursive: true, force: true });
91 });
92
93 const fresh = () => getRepoById(app.db, repo.id)!;
94
95 /** The pull the crew's run opened for `issueId`'s work, once there is one that is not `known`. */
96 const newPull = (known: ReadonlySet<number>) =>
97 until<Pull>("the crew's pull", () => listOpenPulls(app.db, repo.id).find((p) => !known.has(p.number)));
98
99 /**
100 * Land `pull` as the person, once its preview has been built and its tests
101 * have passed: the pull's own deploy run says "its tests passed".
102 */
103 async function land(pull: Pull): Promise<void> {
104 const built = await until("the pull's preview and its tests", () =>
105 listRuns(app.db, repo.id, { kind: "deploy", env: `pull-${pull.number}` }).find((r) => r.state === "ok" || r.state === "failed"),
106 );
107 expect(built.state, built.summary ?? "").toBe("ok");
108 expect(built.summary ?? "").toContain("its tests passed");
109 const merged = await mergePull(app, alice, fresh(), pull, "squash");
110 if (!merged.ok) throw new Error(JSON.stringify(merged.error));
111 }
112
113 test("a first ask builds an app from the template; its tests pass; production serves it", async () => {
114 const made = await newApp(app, alice, { name: "reading", description: "A page that lists my reading. READING-V1", agent: "bot", visibility: "public" });
115 if (!made.ok) throw new Error(JSON.stringify(made.error));
116 repo = made.value.repo;
117 slug = made.value.app.slug;
118 // The template's files are the app's, whole: its own tests came with it.
119 expect(made.value.issue).toBeDefined();
120 const pull = await newPull(new Set());
121 await land(pull);
122 const served = await until("production to serve the change", async () => {
123 const res = await viaHost("/reading").catch(() => undefined);
124 const body = res === undefined ? "" : await res.text();
125 return res?.status === 200 && body === READING_V1 ? body : undefined;
126 });
127 expect(served).toBe(READING_V1);
128 });
129
130 test("a second change lands the same way and production serves that", async () => {
131 const known = new Set(listOpenPulls(app.db, repo.id).map((p) => p.number));
132 const asked = openIssue(app, alice, fresh(), { title: "Two books", body: "Say two books. READING-V2" });
133 if (!asked.ok) throw new Error(String(asked.error));
134 const assigned = assignIssue(app, alice, fresh(), asked.value, bot);
135 if (!assigned.ok) throw new Error(String(assigned.error));
136 const pull = await newPull(known);
137 await land(pull);
138 const served = await until("production to serve the second change", async () => {
139 const res = await viaHost("/reading").catch(() => undefined);
140 const body = res === undefined ? "" : await res.text();
141 return body === READING_V2 ? body : undefined;
142 });
143 expect(served).toBe(READING_V2);
144 });
test/helpers/build-standin-builder.ts added +47
@@ -0,0 +1,47 @@
1 /**
2 * What `build-standin.ts`'s stand-in does, run by `bun` in the clone it is
3 * given: edit, run the app's own tests, and only then commit and push.
4 * Argument: the file holding the prompt.
5 */
6 import { readFileSync, writeFileSync } from "node:fs";
7 import { ANCHOR, READING_V1, READING_V2, routeFor, testFor } from "./build-standin.ts";
8
9 const prompt = readFileSync(process.argv[2]!, "utf8");
10 const say = prompt.includes("READING-V2") ? READING_V2 : prompt.includes("READING-V1") ? READING_V1 : undefined;
11 const finish = (ok: boolean, result: string): void =>
12 console.log(
13 JSON.stringify({ type: "result", subtype: ok ? "success" : "error_during_execution", is_error: !ok, result, total_cost_usd: 0.01, num_turns: 1, session_id: "build-1", terminal_reason: ok ? "completed" : "tool_error" }),
14 );
15 const git = (...args: string[]): void => {
16 const r = Bun.spawnSync(["git", ...args], { stdout: "pipe", stderr: "pipe" });
17 if (r.exitCode !== 0) throw new Error(`${args.join(" ")}: ${r.stderr.toString()}`);
18 };
19
20 if (say === undefined) {
21 finish(true, "DECLINED: nothing here I know how to build.");
22 process.exit(0);
23 }
24 const routes = readFileSync("src/routes.ts", "utf8");
25 if (say === READING_V1) {
26 if (!routes.includes(ANCHOR)) {
27 finish(false, "the template's routes have changed shape");
28 process.exit(0);
29 }
30 writeFileSync("src/routes.ts", routes.replace(ANCHOR, routeFor(READING_V1) + ANCHOR));
31 } else {
32 if (!routes.includes(JSON.stringify(READING_V1))) {
33 finish(false, "there is no first change to change");
34 process.exit(0);
35 }
36 writeFileSync("src/routes.ts", routes.replace(JSON.stringify(READING_V1), JSON.stringify(READING_V2)));
37 }
38 writeFileSync("src/reading.test.ts", testFor(say));
39 const tested = Bun.spawnSync(["bun", "test"], { stdout: "pipe", stderr: "pipe" });
40 if (tested.exitCode !== 0) {
41 finish(false, `its tests failed: ${tested.stderr.toString().slice(-300)}`);
42 process.exit(0);
43 }
44 git("add", "-A");
45 git("commit", "-qm", say === READING_V1 ? "Add the reading list" : "Say two books");
46 git("push", "-q", "-u", "origin", "HEAD");
47 finish(true, `Added ${say}, tests pass.`);
test/helpers/build-standin.ts added +68
@@ -0,0 +1,68 @@
1 /**
2 * A deterministic stand-in for the model that builds on the template
3 * (#11999). It speaks stream-json as Claude Code does, but what it does is
4 * fixed: given a prompt carrying `READING-V1` it adds a `/reading` route and
5 * a test of it to the app the template made; given `READING-V2` it changes
6 * what that route says and its test. Then it runs the app's own `bun test`
7 * in the clone, and only commits and pushes when that passes; a red test
8 * ends the run as a failure, as a model that cannot make its tests pass
9 * would. No network, no login, no cost, the same bytes every time.
10 *
11 * It is the model's place in `build-journey.test.ts`.
12 */
13
14 import { chmodSync, writeFileSync } from "node:fs";
15 import { join } from "node:path";
16
17 /** What `/reading` says after the first change and after the second. */
18 export const READING_V1 = "reading list: one book";
19 export const READING_V2 = "reading list: two books";
20
21 /** The route the first change inserts, before the template's health route. */
22 export const ANCHOR = " // Up or not, for the forge and for anyone watching";
23
24 export const routeFor = (say: string): string =>
25 ` { method: "GET", path: "/reading", handle: () => text(${JSON.stringify(say)}) },\n\n`;
26
27 export const testFor = (say: string): string => `import { afterAll, expect, test } from "bun:test";
28 import { mkdtempSync, rmSync } from "node:fs";
29 import { tmpdir } from "node:os";
30 import { join } from "node:path";
31 import { serve } from "./server.ts";
32
33 const dataDir = mkdtempSync(join(tmpdir(), "reading-"));
34 const server = serve({ port: 0, dataDir, identityKey: "a-key-only-the-forge-holds" });
35 afterAll(() => {
36 server.stop(true);
37 rmSync(dataDir, { recursive: true, force: true });
38 });
39
40 test("/reading says the reading list", async () => {
41 const res = await fetch(\`http://\${server.hostname}:\${server.port}/reading\`);
42 expect(res.status).toBe(200);
43 expect(await res.text()).toBe(${JSON.stringify(say)});
44 });
45 `;
46
47 export interface BuildStandIn {
48 /** The binary to hand `startCrew` (or `FORGE_CLAUDE`). */
49 readonly fake: string;
50 }
51
52 /** Write the stand-in into `dir`: a shell script that reads the prompt and runs the builder. */
53 export function writeBuildStandIn(dir: string): BuildStandIn {
54 const builder = join(import.meta.dir, "build-standin-builder.ts");
55 const promptFile = join(dir, "build-standin.prompt");
56 const fake = join(dir, "build-claude");
57 writeFileSync(
58 fake,
59 `#!/bin/sh
60 if [ "$1" = "--version" ]; then echo "2.1.0 (build stand-in)"; exit 0; fi
61 IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}"
62 echo '{"type":"system","subtype":"init","session_id":"build-1","model":"stand-in"}'
63 exec ${JSON.stringify(process.execPath)} "${builder}" "${promptFile}"
64 `,
65 );
66 chmodSync(fake, 0o755);
67 return { fake };
68 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs