Public: anyone can read this; the crew can change it.
Real-cell proof, build half: a Small child builds a first app from the template, passes its tests and lands a second change #12000
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/11999-real-cell-proof-build-half-a-small-child → main 3 commits fork
4 files
changed +265
−0
docs/features/real-cell-proof.md added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | # Real-cell proof: the build half, in process (#11999) | |
| 2 | ||
| 3 | - `test/helpers/build-standin.ts` (its work in `build-standin-builder.ts`) is a deterministic stand-in for the model: it adds a `/reading` route and its test to the template's app (`READING-V1` in the prompt), or changes what it says (`READING-V2`), runs the app's real `bun test`, and pushes only when that passes. | |
| 4 | - `test/build-journey.test.ts` walks a first ask to production in one process: `newApp` copies the template, the crew's stand-in builds, the pull's preview run says its tests passed, a person lands it, production serves `/reading`, and a second change lands the same way. About 8 s. | |
| 5 | ||
| 6 | Not proved here, and tracked in #12001: the same walk inside a 2 GiB, 1-core cell, and a real memory breach restarting a child. |
test/build-journey.test.ts added +144
| @@ -0,0 +1,144 @@ | ||
| 1 | /** | |
| 2 | * The build half of a child's first hour (#11999), in this process: a first | |
| 3 | * ask builds an app from the template, the app's own tests pass in the | |
| 4 | * check, a person lands the pull, production serves the change, and a | |
| 5 | * second change lands the same way. The model is `build-standin.ts`, which | |
| 6 | * is deterministic and runs the app's real `bun test` before it pushes; the | |
| 7 | * template, the crew, the check, the merge and the runtime are the forge's | |
| 8 | * own. It runs in this process; the same walk in a Small cell is its own issue. | |
| 9 | */ | |
| 10 | import { afterAll, beforeAll, expect, setDefaultTimeout, test } from "bun:test"; | |
| 11 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 12 | import { tmpdir } from "node:os"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { newApp } from "../src/actions/apps.ts"; | |
| 15 | import { createAgent } from "../src/actions/crew.ts"; | |
| 16 | import { assignIssue, openIssue } from "../src/actions/issues.ts"; | |
| 17 | import { mergePull } from "../src/actions/pulls.ts"; | |
| 18 | import { createApp, type App } from "../src/app.ts"; | |
| 19 | import { AppsRuntime } from "../src/apps/runtime.ts"; | |
| 20 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 21 | import { getRepoById, type Repo } from "../src/core/repos.ts"; | |
| 22 | import { listOpenPulls, type Pull } from "../src/core/pulls.ts"; | |
| 23 | import { listRuns } from "../src/core/runs.ts"; | |
| 24 | import { createUser, type User } from "../src/core/users.ts"; | |
| 25 | import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts"; | |
| 26 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 27 | import { READING_V1, READING_V2, writeBuildStandIn } from "./helpers/build-standin.ts"; | |
| 28 | import { quietLogs } from "./helpers/globals.ts"; | |
| 29 | import "./helpers/roomy-disk.ts"; | |
| 30 | ||
| 31 | quietLogs(); | |
| 32 | // The walk takes about 8 s; the rest is room for a loaded check machine. | |
| 33 | setDefaultTimeout(60_000); | |
| 34 | ||
| 35 | const DOMAIN = "apps.test"; | |
| 36 | let dir = ""; | |
| 37 | let app: App; | |
| 38 | let servers: Servers; | |
| 39 | let runtime: AppsRuntime; | |
| 40 | let checks: ChecksHandle; | |
| 41 | let crew: CrewHandle; | |
| 42 | let alice: User; | |
| 43 | let bot: User; | |
| 44 | let repo: Repo; | |
| 45 | let slug = ""; | |
| 46 | ||
| 47 | const viaHost = (path: string) => fetch(`${servers.url}${path}`, { headers: { host: `${slug}.${DOMAIN}` }, redirect: "manual" }); | |
| 48 | ||
| 49 | async function until<T>(what: string, read: () => T | undefined | Promise<T | undefined>, ms = 30_000): Promise<T> { | |
| 50 | const deadline = Date.now() + ms; | |
| 51 | while (Date.now() < deadline) { | |
| 52 | const value = await read(); | |
| 53 | if (value !== undefined) return value; | |
| 54 | await Bun.sleep(100); // real-timer: polling a real forge, crew and runtime that run on their own clocks | |
| 55 | } | |
| 56 | throw new Error(`${what} did not happen in ${ms / 1000}s`); | |
| 57 | } | |
| 58 | ||
| 59 | beforeAll(async () => { | |
| 60 | dir = await mkdtemp(join(tmpdir(), "forge-build-journey-")); | |
| 61 | app = createApp({ dataDir: dir, port: 0 }); | |
| 62 | servers = await serve(app); | |
| 63 | runtime = new AppsRuntime(app, { domain: DOMAIN, scheme: "http", publicPort: servers.port, forgeUrl: servers.url }); | |
| 64 | await runtime.start(); | |
| 65 | checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 }); | |
| 66 | crew = await startCrew(app, { | |
| 67 | forgeUrl: servers.url, | |
| 68 | localUrl: servers.url, | |
| 69 | binary: writeBuildStandIn(dir).fake, | |
| 70 | concurrency: 1, | |
| 71 | budgetUsd: 1, | |
| 72 | runTimeoutMs: 30_000, | |
| 73 | networkRetryScale: 0, | |
| 74 | tickMs: 100, | |
| 75 | }); | |
| 76 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 77 | if (!made.ok) throw new Error(made.error); | |
| 78 | alice = made.value; | |
| 79 | const agent = await createAgent(app, alice, { username: "bot", brief: "Build what is asked." }); | |
| 80 | if (!agent.ok) throw new Error(String(agent.error)); | |
| 81 | bot = agent.value.user; | |
| 82 | }); | |
| 83 | ||
| 84 | afterAll(async () => { | |
| 85 | await crew?.stop(); | |
| 86 | await checks?.stop(); | |
| 87 | await runtime?.stop(); | |
| 88 | servers?.stop(); | |
| 89 | app?.close(); | |
| 90 | await rm(dir, { recursive: true, force: true }); | |
| 91 | }); | |
| 92 | ||
| 93 | const fresh = () => getRepoById(app.db, repo.id)!; | |
| 94 | ||
| 95 | /** The pull the crew's run opened for `issueId`'s work, once there is one that is not `known`. */ | |
| 96 | const newPull = (known: ReadonlySet<number>) => | |
| 97 | until<Pull>("the crew's pull", () => listOpenPulls(app.db, repo.id).find((p) => !known.has(p.number))); | |
| 98 | ||
| 99 | /** | |
| 100 | * Land `pull` as the person, once its preview has been built and its tests | |
| 101 | * have passed: the pull's own deploy run says "its tests passed". | |
| 102 | */ | |
| 103 | async function land(pull: Pull): Promise<void> { | |
| 104 | const built = await until("the pull's preview and its tests", () => | |
| 105 | listRuns(app.db, repo.id, { kind: "deploy", env: `pull-${pull.number}` }).find((r) => r.state === "ok" || r.state === "failed"), | |
| 106 | ); | |
| 107 | expect(built.state, built.summary ?? "").toBe("ok"); | |
| 108 | expect(built.summary ?? "").toContain("its tests passed"); | |
| 109 | const merged = await mergePull(app, alice, fresh(), pull, "squash"); | |
| 110 | if (!merged.ok) throw new Error(JSON.stringify(merged.error)); | |
| 111 | } | |
| 112 | ||
| 113 | test("a first ask builds an app from the template; its tests pass; production serves it", async () => { | |
| 114 | const made = await newApp(app, alice, { name: "reading", description: "A page that lists my reading. READING-V1", agent: "bot", visibility: "public" }); | |
| 115 | if (!made.ok) throw new Error(JSON.stringify(made.error)); | |
| 116 | repo = made.value.repo; | |
| 117 | slug = made.value.app.slug; | |
| 118 | // The template's files are the app's, whole: its own tests came with it. | |
| 119 | expect(made.value.issue).toBeDefined(); | |
| 120 | const pull = await newPull(new Set()); | |
| 121 | await land(pull); | |
| 122 | const served = await until("production to serve the change", async () => { | |
| 123 | const res = await viaHost("/reading").catch(() => undefined); | |
| 124 | const body = res === undefined ? "" : await res.text(); | |
| 125 | return res?.status === 200 && body === READING_V1 ? body : undefined; | |
| 126 | }); | |
| 127 | expect(served).toBe(READING_V1); | |
| 128 | }); | |
| 129 | ||
| 130 | test("a second change lands the same way and production serves that", async () => { | |
| 131 | const known = new Set(listOpenPulls(app.db, repo.id).map((p) => p.number)); | |
| 132 | const asked = openIssue(app, alice, fresh(), { title: "Two books", body: "Say two books. READING-V2" }); | |
| 133 | if (!asked.ok) throw new Error(String(asked.error)); | |
| 134 | const assigned = assignIssue(app, alice, fresh(), asked.value, bot); | |
| 135 | if (!assigned.ok) throw new Error(String(assigned.error)); | |
| 136 | const pull = await newPull(known); | |
| 137 | await land(pull); | |
| 138 | const served = await until("production to serve the second change", async () => { | |
| 139 | const res = await viaHost("/reading").catch(() => undefined); | |
| 140 | const body = res === undefined ? "" : await res.text(); | |
| 141 | return body === READING_V2 ? body : undefined; | |
| 142 | }); | |
| 143 | expect(served).toBe(READING_V2); | |
| 144 | }); |
test/helpers/build-standin-builder.ts added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | /** | |
| 2 | * What `build-standin.ts`'s stand-in does, run by `bun` in the clone it is | |
| 3 | * given: edit, run the app's own tests, and only then commit and push. | |
| 4 | * Argument: the file holding the prompt. | |
| 5 | */ | |
| 6 | import { readFileSync, writeFileSync } from "node:fs"; | |
| 7 | import { ANCHOR, READING_V1, READING_V2, routeFor, testFor } from "./build-standin.ts"; | |
| 8 | ||
| 9 | const prompt = readFileSync(process.argv[2]!, "utf8"); | |
| 10 | const say = prompt.includes("READING-V2") ? READING_V2 : prompt.includes("READING-V1") ? READING_V1 : undefined; | |
| 11 | const finish = (ok: boolean, result: string): void => | |
| 12 | console.log( | |
| 13 | JSON.stringify({ type: "result", subtype: ok ? "success" : "error_during_execution", is_error: !ok, result, total_cost_usd: 0.01, num_turns: 1, session_id: "build-1", terminal_reason: ok ? "completed" : "tool_error" }), | |
| 14 | ); | |
| 15 | const git = (...args: string[]): void => { | |
| 16 | const r = Bun.spawnSync(["git", ...args], { stdout: "pipe", stderr: "pipe" }); | |
| 17 | if (r.exitCode !== 0) throw new Error(`${args.join(" ")}: ${r.stderr.toString()}`); | |
| 18 | }; | |
| 19 | ||
| 20 | if (say === undefined) { | |
| 21 | finish(true, "DECLINED: nothing here I know how to build."); | |
| 22 | process.exit(0); | |
| 23 | } | |
| 24 | const routes = readFileSync("src/routes.ts", "utf8"); | |
| 25 | if (say === READING_V1) { | |
| 26 | if (!routes.includes(ANCHOR)) { | |
| 27 | finish(false, "the template's routes have changed shape"); | |
| 28 | process.exit(0); | |
| 29 | } | |
| 30 | writeFileSync("src/routes.ts", routes.replace(ANCHOR, routeFor(READING_V1) + ANCHOR)); | |
| 31 | } else { | |
| 32 | if (!routes.includes(JSON.stringify(READING_V1))) { | |
| 33 | finish(false, "there is no first change to change"); | |
| 34 | process.exit(0); | |
| 35 | } | |
| 36 | writeFileSync("src/routes.ts", routes.replace(JSON.stringify(READING_V1), JSON.stringify(READING_V2))); | |
| 37 | } | |
| 38 | writeFileSync("src/reading.test.ts", testFor(say)); | |
| 39 | const tested = Bun.spawnSync(["bun", "test"], { stdout: "pipe", stderr: "pipe" }); | |
| 40 | if (tested.exitCode !== 0) { | |
| 41 | finish(false, `its tests failed: ${tested.stderr.toString().slice(-300)}`); | |
| 42 | process.exit(0); | |
| 43 | } | |
| 44 | git("add", "-A"); | |
| 45 | git("commit", "-qm", say === READING_V1 ? "Add the reading list" : "Say two books"); | |
| 46 | git("push", "-q", "-u", "origin", "HEAD"); | |
| 47 | finish(true, `Added ${say}, tests pass.`); |
test/helpers/build-standin.ts added +68
| @@ -0,0 +1,68 @@ | ||
| 1 | /** | |
| 2 | * A deterministic stand-in for the model that builds on the template | |
| 3 | * (#11999). It speaks stream-json as Claude Code does, but what it does is | |
| 4 | * fixed: given a prompt carrying `READING-V1` it adds a `/reading` route and | |
| 5 | * a test of it to the app the template made; given `READING-V2` it changes | |
| 6 | * what that route says and its test. Then it runs the app's own `bun test` | |
| 7 | * in the clone, and only commits and pushes when that passes; a red test | |
| 8 | * ends the run as a failure, as a model that cannot make its tests pass | |
| 9 | * would. No network, no login, no cost, the same bytes every time. | |
| 10 | * | |
| 11 | * It is the model's place in `build-journey.test.ts`. | |
| 12 | */ | |
| 13 | ||
| 14 | import { chmodSync, writeFileSync } from "node:fs"; | |
| 15 | import { join } from "node:path"; | |
| 16 | ||
| 17 | /** What `/reading` says after the first change and after the second. */ | |
| 18 | export const READING_V1 = "reading list: one book"; | |
| 19 | export const READING_V2 = "reading list: two books"; | |
| 20 | ||
| 21 | /** The route the first change inserts, before the template's health route. */ | |
| 22 | export const ANCHOR = " // Up or not, for the forge and for anyone watching"; | |
| 23 | ||
| 24 | export const routeFor = (say: string): string => | |
| 25 | ` { method: "GET", path: "/reading", handle: () => text(${JSON.stringify(say)}) },\n\n`; | |
| 26 | ||
| 27 | export const testFor = (say: string): string => `import { afterAll, expect, test } from "bun:test"; | |
| 28 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 29 | import { tmpdir } from "node:os"; | |
| 30 | import { join } from "node:path"; | |
| 31 | import { serve } from "./server.ts"; | |
| 32 | ||
| 33 | const dataDir = mkdtempSync(join(tmpdir(), "reading-")); | |
| 34 | const server = serve({ port: 0, dataDir, identityKey: "a-key-only-the-forge-holds" }); | |
| 35 | afterAll(() => { | |
| 36 | server.stop(true); | |
| 37 | rmSync(dataDir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | test("/reading says the reading list", async () => { | |
| 41 | const res = await fetch(\`http://\${server.hostname}:\${server.port}/reading\`); | |
| 42 | expect(res.status).toBe(200); | |
| 43 | expect(await res.text()).toBe(${JSON.stringify(say)}); | |
| 44 | }); | |
| 45 | `; | |
| 46 | ||
| 47 | export interface BuildStandIn { | |
| 48 | /** The binary to hand `startCrew` (or `FORGE_CLAUDE`). */ | |
| 49 | readonly fake: string; | |
| 50 | } | |
| 51 | ||
| 52 | /** Write the stand-in into `dir`: a shell script that reads the prompt and runs the builder. */ | |
| 53 | export function writeBuildStandIn(dir: string): BuildStandIn { | |
| 54 | const builder = join(import.meta.dir, "build-standin-builder.ts"); | |
| 55 | const promptFile = join(dir, "build-standin.prompt"); | |
| 56 | const fake = join(dir, "build-claude"); | |
| 57 | writeFileSync( | |
| 58 | fake, | |
| 59 | `#!/bin/sh | |
| 60 | if [ "$1" = "--version" ]; then echo "2.1.0 (build stand-in)"; exit 0; fi | |
| 61 | IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}" | |
| 62 | echo '{"type":"system","subtype":"init","session_id":"build-1","model":"stand-in"}' | |
| 63 | exec ${JSON.stringify(process.execPath)} "${builder}" "${promptFile}" | |
| 64 | `, | |
| 65 | ); | |
| 66 | chmodSync(fake, 0o755); | |
| 67 | return { fake }; | |
| 68 | } |
new above
offline