Public: anyone can read this; the crew can change it.

test/steer.test.ts runs in under 10 s, every test under 1 s, proving what it proves today #10233

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/10219-test-steer-test-ts-runs-in-under-10-s-ev → main 1 commit fork

  • Merged. scout/10219-test-steer-test-ts-runs-in-under-10-s-ev was merged into main as 3a545e9.

Review

Sign in to review this change.
1 file changed +151 −135
test/steer.test.ts +151 −135
@@ -3,26 +3,37 @@
33 * hears it as its next turn. The session stays open only while something
44 * is pending; a turn nobody followed ends it.
55 */
6 import { describe, expect, test } from "bun:test";
7 import { chmodSync, writeFileSync } from "node:fs";
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { chmodSync, readFileSync, writeFileSync } from "node:fs";
88 import { mkdtemp, rm } from "node:fs/promises";
99 import { tmpdir } from "node:os";
1010 import { join } from "node:path";
11 import { addToCrew, createAgent } from "../src/actions/crew.ts";
12 import { addComment, assignIssue, openIssue } from "../src/actions/issues.ts";
13 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
14 import { createApp } from "../src/app.ts";
15 import type { Repo } from "../src/core/repos.ts";
16 import { createUser, type User } from "../src/core/users.ts";
17 import { listRuns } from "../src/core/runs.ts";
1118 import { runClaude } from "../src/crew/claude.ts";
19 import { startCrew } from "../src/crew/dispatch.ts";
20 import { serve } from "../src/http/server.ts";
1221 import { sandboxPolicy } from "../src/crew/sandbox.ts";
1322
14 describe("steering a running agent", () => {
15 test("a message sent mid-turn becomes the next turn; without one the session ends after the first", async () => {
16 const dir = await mkdtemp(join(tmpdir(), "forge-steer-"));
17 // The stand-in: answers the first message after a moment, then, if a
18 // second arrives before stdin closes, answers that too.
19 const fake = join(dir, "fake-claude");
20 writeFileSync(
21 fake,
22 `#!/bin/sh
23 /**
24 * The stand-in agent: answers the first message, but holds that first turn
25 * open until the test creates `gate` (no fixed wait, so no race and no
26 * idle seconds); then, if a second message arrives before stdin closes, it
27 * answers that too.
28 */
29 function fakeClaude(path: string, gate: string, session: string): void {
30 writeFileSync(
31 path,
32 `#!/bin/sh
33 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
2334 IFS= read -r first
24 echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}'
25 sleep 0.4
35 echo '{"type":"system","subtype":"init","session_id":"${session}","model":"fake"}'
36 while [ ! -e '${gate}' ]; do sleep 0.01; done
2637 echo '{"type":"assistant","message":{"content":[{"type":"text","text":"first turn done"}]}}'
2738 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}'
2839 if IFS= read -r second; then
@@ -30,8 +41,26 @@ if IFS= read -r second; then
3041 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok again","total_cost_usd":0.02,"num_turns":2}'
3142 fi
3243 `,
33 );
34 chmodSync(fake, 0o755);
44 );
45 chmodSync(path, 0o755);
46 }
47
48 /** Polls `ready` every 10 ms until it holds; throws past `ms`. */
49 async function until(ready: () => boolean | Promise<boolean>, what: string, ms = 10_000): Promise<void> {
50 const deadline = Date.now() + ms;
51 while (!(await ready())) {
52 if (Date.now() > deadline) throw new Error(`never: ${what}`);
53 await Bun.sleep(10);
54 }
55 }
56
57 describe("steering a running agent", () => {
58 test("a message sent mid-turn becomes the next turn; without one the session ends after the first", async () => {
59 const dir = await mkdtemp(join(tmpdir(), "forge-steer-"));
60 // Invariant: a steer sent mid-turn is heard as the next turn; without one the session ends after the first.
61 const fake = join(dir, "fake-claude");
62 const gate = join(dir, "gate");
63 fakeClaude(fake, gate, "s1");
3564 const lines: string[] = [];
3665 const common = {
3766 binary: fake,
@@ -54,9 +83,10 @@ fi
5483 send = s;
5584 },
5685 });
57 await Bun.sleep(150);
86 await until(() => lines.some((l) => l.includes("s1")), "the session opened");
5887 expect(send).toBeDefined();
5988 expect(send!("also add a test")).toBe(true);
89 writeFileSync(gate, "");
6090 const outcome = await steered;
6191 expect(outcome.ok).toBe(true);
6292 expect(outcome.turns).toBe(2);
@@ -71,136 +101,122 @@ fi
71101 expect(quiet.turns).toBe(1);
72102 expect(lines.some((l) => l.startsWith("heard:"))).toBe(false);
73103 await rm(dir, { recursive: true, force: true });
74 }, 30_000);
104 }, 10_000);
75105 });
76106
77107 describe("a comment while the agent works", () => {
78 test("reaches it as its next turn; no second run queues behind the first", async () => {
79 const { createApp } = await import("../src/app.ts");
80 const { serve } = await import("../src/http/server.ts");
81 const { startCrew } = await import("../src/crew/dispatch.ts");
82 const { createUser } = await import("../src/core/users.ts");
83 const { addToCrew, createAgent } = await import("../src/actions/crew.ts");
84 const { commitToBranch, createRepo } =
85 await import("../src/actions/repos.ts");
86 const { addComment, assignIssue, openIssue } =
87 await import("../src/actions/issues.ts");
88 const { listRuns } = await import("../src/core/runs.ts");
89 const { readFileSync } = await import("node:fs");
108 // One forge, one crew, one agent and one repository for the file.
109 let dataDir: string;
110 let app: ReturnType<typeof createApp>;
111 let servers: Awaited<ReturnType<typeof serve>>;
112 let crew: Awaited<ReturnType<typeof startCrew>>;
113 let alice: User;
114 let bot: User;
115 let repo: Repo;
116 let gate: string;
90117
91 const dataDir = await mkdtemp(join(tmpdir(), "forge-steer-crew-"));
92 const app = createApp({ dataDir, port: 0 });
93 const servers = await serve(app);
118 beforeAll(async () => {
119 dataDir = await mkdtemp(join(tmpdir(), "forge-steer-crew-"));
120 app = createApp({ dataDir, port: 0 });
121 servers = await serve(app);
94122 const fake = join(dataDir, "fake-claude");
95 writeFileSync(
96 fake,
97 `#!/bin/sh
98 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
99 IFS= read -r first
100 echo '{"type":"system","subtype":"init","session_id":"s2","model":"fake"}'
101 sleep 2.5
102 echo '{"type":"assistant","message":{"content":[{"type":"text","text":"first turn done"}]}}'
103 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}'
104 if IFS= read -r second; then
105 echo '{"type":"assistant","message":{"content":[{"type":"text","text":"heard: '"$(printf '%s' "$second" | sed 's/.*"text":"\\([^"]*\\)".*/\\1/')"'"}]}}'
106 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok again","total_cost_usd":0.02,"num_turns":2}'
107 fi
108 `,
109 );
110 chmodSync(fake, 0o755);
111 const crew = await startCrew(app, {
123 gate = join(dataDir, "gate");
124 fakeClaude(fake, gate, "s2");
125 crew = await startCrew(app, {
112126 forgeUrl: servers.url,
113127 localUrl: servers.url,
114128 binary: fake,
115129 concurrency: 1,
116130 budgetUsd: 1,
117131 runTimeoutMs: 60_000,
132 // The pump's clock: a short tick, so no test waits out the 5 s one.
133 tickMs: 20,
118134 });
119 try {
120 const a = await createUser(app.db, { origin: "seed", trusted: true,
121 username: "alice",
122 email: "[email protected]",
123 password: "a-good-passphrase",
124 adminIfFirst: true,
125 });
126 if (!a.ok) throw new Error(a.error);
127 const alice = a.value;
128 const made = await createAgent(app, alice, {
129 username: "bot",
130 brief: "Be brief.",
131 });
132 if (!made.ok) throw new Error(String(made.error));
133 const created = await createRepo(app, alice, {
134 name: "garden",
135 visibility: "private",
136 });
137 if (!created.ok) throw new Error(String(created.error));
138 const repo = created.value;
139 const seeded = await commitToBranch(app, alice, repo, {
140 message: "Start",
141 files: [{ path: "README.md", content: "# garden\n" }],
142 });
143 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
144 expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
135 const a = await createUser(app.db, { origin: "seed", trusted: true,
136 username: "alice",
137 email: "alice@forge.local",
138 password: "a-good-passphrase",
139 adminIfFirst: true,
140 });
141 if (!a.ok) throw new Error(a.error);
142 alice = a.value;
143 const made = await createAgent(app, alice, {
144 username: "bot",
145 brief: "Be brief.",
146 });
147 if (!made.ok) throw new Error(String(made.error));
148 bot = made.value.user;
149 const created = await createRepo(app, alice, {
150 name: "garden",
151 visibility: "private",
152 });
153 if (!created.ok) throw new Error(String(created.error));
154 repo = created.value;
155 const seeded = await commitToBranch(app, alice, repo, {
156 message: "Start",
157 files: [{ path: "README.md", content: "# garden\n" }],
158 });
159 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
160 if (!addToCrew(app, alice, repo, "bot").ok) throw new Error("bot not on the crew");
161 });
145162
146 const issue = openIssue(app, alice, repo, {
147 title: "Plant tulips",
148 body: "Along the fence.",
149 });
150 if (!issue.ok) throw new Error(String(issue.error));
151 expect(
152 assignIssue(app, alice, repo, issue.value, made.value.user).ok,
153 ).toBe(true);
154 // Wait for the run to be under way, then speak.
155 const running = async () => {
156 const deadline = Date.now() + 20_000;
157 while (Date.now() < deadline) {
158 const run = listRuns(app.db, repo.id, { kind: "agent" }).find(
159 (r) => r.state === "running",
160 );
161 if (run) return run;
162 await Bun.sleep(50);
163 }
164 throw new Error("the run never started");
165 };
166 const run = await running();
167 // Past the session's first line: `running` is set before the agent
168 // is spawned, so on a busy machine a fixed sleep can speak too early
169 // and queue a second run. The session line proves it can hear us.
170 const logPath = join(dataDir, "runs", `${run.id}.log`);
171 const opened = Date.now() + 20_000;
172 while (
173 Date.now() < opened &&
174 !(
175 await Bun.file(logPath)
176 .text()
177 .catch(() => "")
178 ).includes("session s2")
179 )
180 await Bun.sleep(50);
181 expect(
182 addComment(app, alice, repo, issue.value, "also the tests").ok,
183 ).toBe(true);
184 const deadline = Date.now() + 20_000;
185 while (Date.now() < deadline) {
186 const r = listRuns(app.db, repo.id, { kind: "agent" }).find(
187 (x) => x.id === run.id,
188 );
189 if (r && r.state !== "running") break;
190 await Bun.sleep(100);
191 }
192 const runs = listRuns(app.db, repo.id, { kind: "agent" });
193 expect(runs.length).toBe(1);
194 expect(runs[0]!.state).toBe("ok");
195 expect(runs[0]!.turns).toBe(2);
196 const log = readFileSync(logPath, "utf8");
197 expect(log).toContain("you: alice on #1: also the tests");
198 expect(log).toContain("heard: alice on #1: also the tests");
199 } finally {
200 await crew.stop();
201 servers.stop();
202 app.close();
203 await rm(dataDir, { recursive: true, force: true });
204 }
205 }, 60_000);
163 afterAll(async () => {
164 await crew.stop();
165 servers.stop();
166 app.close();
167 await rm(dataDir, { recursive: true, force: true });
168 });
169
170 test("reaches it as its next turn; no second run queues behind the first", async () => {
171 // Invariant: a comment on the issue an agent is running on steers that run (two turns, one run), never queues another.
172 const issue = openIssue(app, alice, repo, {
173 title: "Plant tulips",
174 body: "Along the fence.",
175 });
176 if (!issue.ok) throw new Error(String(issue.error));
177 expect(
178 assignIssue(app, alice, repo, issue.value, bot).ok,
179 ).toBe(true);
180 // Wait for the run to be under way, then speak.
181 let run: ReturnType<typeof listRuns>[number] | undefined;
182 await until(() => {
183 run = listRuns(app.db, repo.id, { kind: "agent" }).find(
184 (r) => r.state === "running",
185 );
186 return run !== undefined;
187 }, "the run started");
188 // Past the session's first line: `running` is set before the agent
189 // is spawned, so speaking too early would queue a second run. The
190 // session line proves it can hear us.
191 const logPath = join(dataDir, "runs", `${run!.id}.log`);
192 const logged = (text: string) => async () =>
193 (await Bun.file(logPath).text().catch(() => "")).includes(text);
194 await until(logged("session s2"), "the session opened");
195 expect(
196 addComment(app, alice, repo, issue.value, "also the tests").ok,
197 ).toBe(true);
198 // The steer is on its way to the agent; only now may its first turn end.
199 // The comment either reached the session or queued a run of its own.
200 await until(
201 async () =>
202 (await logged("you: alice on #1")()) ||
203 listRuns(app.db, repo.id, { kind: "agent" }).length > 1,
204 "the comment was sent as a steer or queued",
205 );
206 expect(listRuns(app.db, repo.id, { kind: "agent" }).length).toBe(1);
207 writeFileSync(gate, "");
208 await until(
209 () =>
210 listRuns(app.db, repo.id, { kind: "agent" }).find((x) => x.id === run!.id)
211 ?.state !== "running",
212 "the run settled",
213 );
214 const runs = listRuns(app.db, repo.id, { kind: "agent" });
215 expect(runs.length).toBe(1);
216 expect(runs[0]!.state).toBe("ok");
217 expect(runs[0]!.turns).toBe(2);
218 const log = readFileSync(logPath, "utf8");
219 expect(log).toContain("you: alice on #1: also the tests");
220 expect(log).toContain("heard: alice on #1: also the tests");
221 }, 10_000);
206222 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs