trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

walkJourney in src/release/journey.ts walks a stranger's journey against any live forge URL and cleans up after itself #11813

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/11786-walkjourney-in-src-release-journey-ts-wa → main 3 commits fork

  • Merged. torque/11786-walkjourney-in-src-release-journey-ts-wa was merged into main as 79896cf.

Review

Sign in to review this change.
4 files changed +447 −0
scripts/unreached-allow.json +1
@@ -1,5 +1,6 @@
11 [
22 "src/perf/page_budget.ts",
33 "src/perf/page_budget_read.ts",
4 "src/release/journey.ts",
45 "src/spawn/pod.ts"
56 ]
src/release/journey.ts added +284
@@ -0,0 +1,284 @@
1 /**
2 * The stranger's journey, walked against a live forge (#11158, part 2): from
3 * the forge's home to a crew visibly at work, the way a person with a browser
4 * goes — the same requests `test/stranger-three-screens.test.ts` follows by hand.
5 *
6 * 1. open `/` (screen 1)
7 * 2. name a throwaway forge POST /forges/start, then wait on /forges/<name>/ready
8 * 3. choose a username + password the new forge's /claim (screen 2)
9 * 4. land in the forge (screen 3)
10 * 5. say one tiny thing to build POST /tell on the new forge
11 * 6. poll the forge's home until the crew's activity shows
12 *
13 * Plain data in, plain data out; nothing here throws for a step that went
14 * wrong — the step is named in `failure`. The budgets are `FIRST_FIVE_MINUTES`.
15 * Every request has a timeout, and no text of the caller's reaches a shell:
16 * there is no shell here, only `fetch`.
17 *
18 * The throwaway forge is removed afterwards, passed or failed: its crew run is
19 * cancelled first (best effort: the forge's removal stops the whole child and
20 * everything in it), then `POST /forges/<name>/remove` on the host with
21 * `adminToken` — the one thing the host asks an admin for. Without a token the
22 * forge cannot be removed from here, and `removed` says so; the walk's verdict
23 * is its own.
24 */
25 import { FIRST_FIVE_MINUTES } from "../design/first_five_minutes.ts";
26
27 export interface JourneyOptions {
28 /** The live forge's address, http(s). */
29 readonly base: string;
30 /** How to reach it; `fetch` when left out. A test hands in one that knows where the new forge's host is. */
31 readonly fetch?: typeof fetch;
32 /** The tiny thing to ask the crew to build. */
33 readonly ask?: string;
34 /** Milliseconds, monotonic enough to subtract; `Date.now` when left out. */
35 readonly now?: () => number;
36 /** A site admin's token on the host: removes the throwaway forge afterwards. */
37 readonly adminToken?: string;
38 /** Pause between polls, in ms (default 500); how a test keeps its clock quick. */
39 readonly pollMs?: number;
40 /** Waits `ms`; `Bun.sleep` when left out. */
41 readonly sleep?: (ms: number) => Promise<void>;
42 }
43
44 export interface JourneyResult {
45 readonly ok: boolean;
46 /** The screens a person was shown, in order, the forge last. */
47 readonly screens: readonly string[];
48 /** From naming the forge to its answering, in ms; null when it never did. */
49 readonly readyMs: number | null;
50 /** From the ask to the crew's activity on the forge's home, in ms; null when none showed. */
51 readonly firstActivityMs: number | null;
52 /** The step that failed, in words. */
53 readonly failure?: string;
54 /** The throwaway forge's name; null when none was made. */
55 readonly forge: string | null;
56 /** Whether the throwaway forge was removed afterwards (true when none was made). */
57 readonly removed: boolean;
58 }
59
60 /** The longest any one request may take. */
61 export const REQUEST_TIMEOUT_MS = 10_000;
62
63 /** The tiny ask: small enough that a probe spends no more than the first activity. */
64 export const PROBE_ASK = "A page that says hello";
65
66 /** What a forge's home says once the crew has begun on an ask (`appCrew` in `src/core/app-crew.ts`). */
67 const ACTIVITY = /\b(?:is working toward|is about to start on)\b/;
68
69 /** A run the app's runs page links to: the crew has been given the ask. */
70 const RUN_LINK = /href="\/[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+\/runs\/[A-Za-z0-9_-]+"/;
71
72 /** A name that is free to be made and removed: `probe-` and eight hex digits. */
73 const probeName = (): string => `probe-${crypto.randomUUID().replaceAll("-", "").slice(0, 8)}`;
74
75 class Step extends Error {}
76
77 /** Walks the journey against `options.base`; never throws for a step that went wrong. */
78 export async function walkJourney(options: JourneyOptions): Promise<JourneyResult> {
79 const now = options.now ?? Date.now;
80 const doFetch = options.fetch ?? globalThis.fetch;
81 const pollMs = options.pollMs ?? 500;
82 const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
83 const screens: string[] = [];
84 let forge: string | null = null;
85 let readyMs: number | null = null;
86 let firstActivityMs: number | null = null;
87 let failure: string | undefined;
88 let host: URL | undefined;
89 let child: URL | undefined;
90 let cookie = "";
91 let removed = true;
92
93 try {
94 host = new URL(options.base);
95 } catch {
96 return { ok: false, screens, readyMs, firstActivityMs, failure: "the forge's address is not a URL", forge, removed };
97 }
98 if (host.protocol !== "http:" && host.protocol !== "https:")
99 return { ok: false, screens, readyMs, firstActivityMs, failure: "the forge's address is not http or https", forge, removed };
100 const hostUrl = host;
101
102 const call = (url: URL, init: RequestInit = {}): Promise<Response> =>
103 doFetch(url.href, { redirect: "manual", signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), ...init });
104 const form = (url: URL, fields: Record<string, string>, extra: Record<string, string> = {}): Promise<Response> =>
105 call(url, {
106 method: "POST",
107 headers: { origin: url.origin, ...extra },
108 body: new URLSearchParams(fields),
109 });
110 /** Runs one step; a refusal or a network error becomes the words for that step. */
111 const step = async <T>(words: string, run: () => Promise<T>): Promise<T> => {
112 try {
113 return await run();
114 } catch (e) {
115 throw e instanceof Step ? e : new Step(`${words} (${e instanceof Error ? e.name : "failed"})`);
116 }
117 };
118 const expect = (res: Response, words: string, ...statuses: number[]): void => {
119 if (!statuses.includes(res.status)) throw new Step(`${words}: it answered ${res.status}`);
120 };
121
122 try {
123 // 1. The home.
124 await step("the forge's home did not open", async () => {
125 const res = await call(new URL("/", hostUrl));
126 expect(res, "the forge's home did not open", 200);
127 await res.text();
128 screens.push("home");
129 });
130
131 // 2. Name a throwaway forge, as the browser does (js=1: a 303 to the waiting page).
132 const name = probeName();
133 const t0 = now();
134 const claimCode = await step("the forge could not be named", async () => {
135 const res = await form(new URL("/forges/start", hostUrl), { name, js: "1" });
136 await res.body?.cancel();
137 expect(res, "the forge could not be named", 303);
138 forge = name;
139 const to = res.headers.get("location") ?? "";
140 const code = /^\/forges\/[a-z0-9-]+\/waiting#([A-Za-z0-9_-]{43})$/.exec(to)?.[1];
141 if (code === undefined) throw new Step(`the forge could not be named: it sent the person somewhere else (${to.slice(0, 40)})`);
142 return code;
143 });
144
145 // Where it will live: the waiting page names it (never the code).
146 const waiting = await step("the waiting page did not open", async () => {
147 const res = await call(new URL(`/forges/${name}/waiting`, hostUrl));
148 expect(res, "the waiting page did not open", 200);
149 return res.text();
150 });
151 const claimAt = /data-claim="([^"]+)"/.exec(waiting)?.[1]?.replaceAll("&amp;", "&");
152 let claim: URL | undefined;
153 try {
154 claim = claimAt === undefined ? undefined : new URL(claimAt);
155 } catch {}
156 const lives = (u: URL) => (u.protocol === "http:" || u.protocol === "https:") && (u.hostname.startsWith(`${name}--`) || u.hostname.startsWith(`${name}.`));
157 if (claim === undefined || !lives(claim)) throw new Step("the waiting page did not say where the forge is");
158 child = new URL("/", claim);
159 const childUrl = child;
160
161 // Ready within the budget.
162 const readyBudget = FIRST_FIVE_MINUTES.forgeReadySeconds * 1000;
163 for (;;) {
164 let ready = false;
165 try {
166 const res = await call(new URL(`/forges/${name}/ready`, hostUrl), { headers: { accept: "application/json" } });
167 if (res.status === 200) ready = ((await res.json()) as { ready?: unknown }).ready === true;
168 else await res.body?.cancel();
169 } catch {}
170 if (ready) {
171 readyMs = now() - t0;
172 break;
173 }
174 if (now() - t0 >= readyBudget) throw new Step(`the forge was not ready in ${FIRST_FIVE_MINUTES.forgeReadySeconds} s`);
175 await sleep(pollMs);
176 }
177
178 // 3. The claim form, then a username and a password.
179 await step("the account form did not open", async () => {
180 const res = await call(new URL("/claim", childUrl));
181 expect(res, "the account form did not open", 200);
182 await res.text();
183 screens.push("account");
184 });
185 const username = `probe-${crypto.randomUUID().replaceAll("-", "").slice(0, 8)}`;
186 const password = `${crypto.randomUUID()}-${crypto.randomUUID()}`;
187 const landing = await step("the account could not be made", async () => {
188 const res = await form(new URL("/claim", childUrl), { token: claimCode, username, password });
189 await res.body?.cancel();
190 expect(res, "the account could not be made", 303);
191 cookie = (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
192 if (!cookie.includes("=")) throw new Step("the account could not be made: nobody was signed in");
193 return res.headers.get("location") ?? "/";
194 });
195
196 // 4. In the forge, signed in.
197 await step("the new forge did not open", async () => {
198 const res = await call(new URL(landing, childUrl), { headers: { cookie } });
199 expect(res, "the new forge did not open", 200);
200 await res.text();
201 screens.push("forge");
202 });
203 if (screens.length > FIRST_FIVE_MINUTES.screensBeforeForge)
204 throw new Step(`it took ${screens.length} screens, more than ${FIRST_FIVE_MINUTES.screensBeforeForge}`);
205
206 // 5. One tiny ask.
207 const asked = now();
208 await step("the ask was not taken", async () => {
209 const res = await form(new URL("/tell", childUrl), { app: "new-app", text: options.ask ?? PROBE_ASK }, { cookie });
210 await res.body?.cancel();
211 expect(res, "the ask was not taken", 200, 201, 303);
212 });
213
214 // 6. The crew's first activity on the forge's home.
215 const activityBudget = FIRST_FIVE_MINUTES.firstCrewActivitySeconds * 1000;
216 for (;;) {
217 let seen = false;
218 try {
219 const res = await call(new URL("/", childUrl), { headers: { cookie } });
220 if (res.status === 200) {
221 const page = await res.text();
222 seen = ACTIVITY.test(page);
223 // An app the asker owns is not in the home's crew card; its runs page is where its first run shows.
224 const app = /<option value="([a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*)"/.exec(page)?.[1];
225 if (!seen && app !== undefined) {
226 const runs = await call(new URL(`/${app}/runs`, childUrl), { headers: { cookie } });
227 seen = runs.status === 200 && RUN_LINK.test(await runs.text());
228 }
229 } else if (res.status >= 500) throw new Step(`the forge's home answered ${res.status} while the crew started`);
230 else await res.body?.cancel();
231 } catch (e) {
232 if (e instanceof Step) throw e;
233 }
234 if (seen) {
235 firstActivityMs = now() - asked;
236 break;
237 }
238 if (now() - asked >= activityBudget) throw new Step(`the crew was not at work in ${FIRST_FIVE_MINUTES.firstCrewActivitySeconds} s`);
239 await sleep(pollMs);
240 }
241 } catch (e) {
242 failure = e instanceof Step ? e.message : "the walk stopped unexpectedly";
243 }
244
245 // Afterwards, passed or failed: stop the crew's run, then remove the forge.
246 if (forge !== null) {
247 if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
248 removed = await removeForge(doFetch, hostUrl, forge, options.adminToken);
249 }
250 const result = { ok: failure === undefined, screens, readyMs, firstActivityMs, forge, removed };
251 return failure === undefined ? result : { ...result, failure };
252 }
253
254 /** Cancels each run the forge's home links to; a miss is never an error (the forge's removal ends them anyway). */
255 async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Promise<void> {
256 try {
257 const res = await doFetch(new URL("/", child).href, { headers: { cookie }, redirect: "manual", signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) });
258 const page = res.status === 200 ? await res.text() : "";
259 const paths = new Set([...page.matchAll(/href="(\/[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+\/runs\/[A-Za-z0-9_-]+)"/g)].map((m) => m[1]!));
260 for (const path of [...paths].slice(0, 5)) {
261 const url = new URL(`${path}/cancel`, child);
262 const out = await doFetch(url.href, { method: "POST", headers: { origin: url.origin, cookie }, redirect: "manual", signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) });
263 await out.body?.cancel();
264 }
265 } catch {}
266 }
267
268 /** Removes the throwaway forge from the host; true when the host said it is gone. */
269 async function removeForge(doFetch: typeof fetch, host: URL, name: string, token: string | undefined): Promise<boolean> {
270 if (token === undefined || token === "") return false;
271 try {
272 const url = new URL(`/forges/${name}/remove`, host);
273 const res = await doFetch(url.href, {
274 method: "POST",
275 redirect: "manual",
276 signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
277 headers: { origin: url.origin, authorization: `Bearer ${token}` },
278 });
279 await res.body?.cancel();
280 return res.status === 200;
281 } catch {
282 return false;
283 }
284 }
test/release-journey.test.ts added +161
@@ -0,0 +1,161 @@
1 /**
2 * `walkJourney` (#11158, part 2) against a forge served here: the parent and
3 * the child it spawns both run for real, the child's crew is a stand-in for
4 * Claude Code (a script that speaks stream-json and stays at work). Only the
5 * child's address is a stand-in: its host name is routed to the local server.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { chmodSync, existsSync, writeFileSync } from "node:fs";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { childDirOf, childNamed } from "../src/actions/spawn.ts";
13 import { createApp, type App } from "../src/app.ts";
14 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
15 import { createToken } from "../src/core/tokens.ts";
16 import { addSubscription } from "../src/core/subscriptions.ts";
17 import { createUser } from "../src/core/users.ts";
18 import { FIRST_FIVE_MINUTES } from "../src/design/first_five_minutes.ts";
19 import { serve, type Servers } from "../src/http/server.ts";
20 import { walkJourney } from "../src/release/journey.ts";
21 import { useProbe } from "../src/spawn/door.ts";
22 import { firstBoot } from "../src/spawn/first-boot.ts";
23 import { quietLogs } from "./helpers/globals.ts";
24
25 quietLogs();
26
27 let dir = "";
28 let parent: App;
29 let parentServers: Servers;
30 let adminToken = "";
31 const kids = new Map<string, { app: App; servers: Servers; crew: CrewHandle }>();
32 /** A step that answers 503, as `METHOD path` on the child. */
33 let broken: string | undefined;
34
35 beforeAll(async () => {
36 dir = await mkdtemp(join(tmpdir(), "forge-journey-"));
37 parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 });
38 useProbe(parent, () => async () => true);
39 const alice = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
40 if (!alice.ok) throw new Error(alice.error);
41 adminToken = createToken(parent.db, alice.value.id, "journey", "admin").token;
42 parentServers = await serve(parent);
43 }, 30_000);
44
45 afterAll(async () => {
46 for (const k of kids.values()) {
47 await k.crew.stop();
48 k.servers.stop();
49 k.app.close();
50 }
51 parentServers.stop();
52 parent.close();
53 await rm(dir, { recursive: true, force: true });
54 });
55
56 /** Boots the child the parent made for `name`, with a stand-in crew, the first time its host is asked for. */
57 async function kid(name: string) {
58 const had = kids.get(name);
59 if (had) return had;
60 const made = await childNamed(parent, name);
61 if (made === undefined) throw new Error(`the parent made no forge named ${name}`);
62 const dataDir = childDirOf(parent, made.bodyId);
63 const app = createApp({ dataDir, port: 0 });
64 await firstBoot(app);
65 const sub = addSubscription(app.db, dataDir, { name: "work", token: "sk-ant-api03-journey-stand-in-0123456789abcdef" });
66 if (!sub.ok) throw new Error(sub.error);
67 const servers = await serve(app);
68 const fake = join(dir, `fake-claude-${name}`);
69 writeFileSync(
70 fake,
71 `#!/bin/sh
72 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
73 echo '{"type":"system","subtype":"init","session_id":"fake-1","model":"fake"}'
74 sleep 20
75 `,
76 );
77 chmodSync(fake, 0o755);
78 const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 1, budgetUsd: 1, runTimeoutMs: 30_000, networkRetryScale: 0, tickMs: 100 });
79 const k = { app, servers, crew };
80 kids.set(name, k);
81 return k;
82 }
83
84 /** The fetch a walker gets: the parent as it is, a child's host routed to the child's server. */
85 function routed(log: string[]): typeof fetch {
86 return (async (input: string | URL | Request, init?: RequestInit) => {
87 const url = new URL(input instanceof Request ? input.url : String(input));
88 const name = /^([a-z0-9-]+)--forge\.test$/.exec(url.hostname)?.[1];
89 if (name === undefined) {
90 // The parent believes it lives at forge.test: a browser there would send that origin.
91 const headers = new Headers(init?.headers);
92 if (headers.has("origin")) headers.set("origin", "https://forge.test");
93 return fetch(url, { ...init, headers });
94 }
95 const k = await kid(name);
96 log.push(`${init?.method ?? "GET"} ${url.pathname}`);
97 if (broken !== undefined && broken === `${init?.method ?? "GET"} ${url.pathname}`) return new Response("down", { status: 503 });
98 const headers = new Headers(init?.headers);
99 if (headers.has("origin")) headers.set("origin", k.servers.url);
100 return fetch(`${k.servers.url}${url.pathname}${url.search}`, { ...init, headers });
101 }) as typeof fetch;
102 }
103
104 const forgeGone = (name: string) => childNamed(parent, name).then((c) => c === undefined && !existsSync(join(dir, "spawn", `${name}.json`)));
105
106 test("a passing walk: the screens, the timings, and the throwaway forge removed", async () => {
107 broken = undefined;
108 const log: string[] = [];
109 let t = 0;
110 const result = await walkJourney({ base: parentServers.url, fetch: routed(log), adminToken, now: () => (t += 10), sleep: () => Bun.sleep(100), pollMs: 100 });
111 expect(result.failure).toBeUndefined();
112 expect(result.ok).toBe(true);
113 expect(result.screens).toEqual(["home", "account", "forge"]);
114 expect(result.screens.length).toBeLessThanOrEqual(FIRST_FIVE_MINUTES.screensBeforeForge);
115 expect(result.readyMs).toBeGreaterThan(0);
116 expect(result.readyMs!).toBeLessThan(FIRST_FIVE_MINUTES.forgeReadySeconds * 1000);
117 expect(result.firstActivityMs).toBeGreaterThan(0);
118 expect(result.firstActivityMs!).toBeLessThan(FIRST_FIVE_MINUTES.firstCrewActivitySeconds * 1000);
119 expect(log).toContain("POST /tell");
120 // Removed: the host no longer knows it, and its data is gone.
121 expect(result.removed).toBe(true);
122 expect(result.forge).toMatch(/^probe-[0-9a-f]{8}$/);
123 expect(await forgeGone(result.forge!)).toBe(true);
124 }, 60_000);
125
126 test("a failing walk names the step that answered 503, and still removes the forge", async () => {
127 broken = "POST /claim";
128 const log: string[] = [];
129 const result = await walkJourney({ base: parentServers.url, fetch: routed(log), adminToken, sleep: () => Bun.sleep(100), pollMs: 100 });
130 broken = undefined;
131 expect(result.ok).toBe(false);
132 expect(result.failure).toBe("the account could not be made: it answered 503");
133 expect(result.screens).toEqual(["home", "account"]);
134 expect(result.firstActivityMs).toBeNull();
135 expect(log).not.toContain("POST /tell");
136 expect(result.removed).toBe(true);
137 expect(await forgeGone(result.forge!)).toBe(true);
138 }, 60_000);
139
140 test("a forge that is never ready fails in words, with the budget in them", async () => {
141 useProbe(parent, () => async () => false);
142 let t = 0;
143 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), adminToken, now: () => (t += 5_000), sleep: async () => {}, pollMs: 0 });
144 useProbe(parent, () => async () => true);
145 expect(result.ok).toBe(false);
146 expect(result.failure).toBe(`the forge was not ready in ${FIRST_FIVE_MINUTES.forgeReadySeconds} s`);
147 expect(result.readyMs).toBeNull();
148 expect(await forgeGone(result.forge!)).toBe(true);
149 }, 30_000);
150
151 test("without an admin token the forge cannot be removed from here, and the result says so", async () => {
152 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), pollMs: 100, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} });
153 expect(result.removed).toBe(false);
154 expect(result.forge).not.toBeNull();
155 await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
156 });
157
158 test("an address that is not http(s) is refused before anything is sent", async () => {
159 const result = await walkJourney({ base: "ftp://forge.test", fetch: (() => { throw new Error("no request"); }) as unknown as typeof fetch });
160 expect(result).toMatchObject({ ok: false, forge: null, removed: true, failure: "the forge's address is not http or https" });
161 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs