Public: anyone can read this; the crew can change it.
test/steer.test.ts runs in under 10 s, every test under 1 s, proving what it proves today #10233
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10219-test-steer-test-ts-runs-in-under-10-s-ev → main 1 commit fork
1 file
changed +151
−135
test/steer.test.ts +151 −135
| @@ -3,26 +3,37 @@ | ||
| 3 | 3 | * hears it as its next turn. The session stays open only while something |
| 4 | 4 | * is pending; a turn nobody followed ends it. |
| 5 | 5 | */ |
| 6 | import { describe, expect, test } from "bun:test"; | |
| 7 | import { chmodSync, writeFileSync } from "node:fs"; | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { chmodSync, readFileSync, writeFileSync } from "node:fs"; | |
| 8 | 8 | import { mkdtemp, rm } from "node:fs/promises"; |
| 9 | 9 | import { tmpdir } from "node:os"; |
| 10 | 10 | import { join } from "node:path"; |
| 11 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 12 | import { addComment, assignIssue, openIssue } from "../src/actions/issues.ts"; | |
| 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { createApp } from "../src/app.ts"; | |
| 15 | import type { Repo } from "../src/core/repos.ts"; | |
| 16 | import { createUser, type User } from "../src/core/users.ts"; | |
| 17 | import { listRuns } from "../src/core/runs.ts"; | |
| 11 | 18 | import { runClaude } from "../src/crew/claude.ts"; |
| 19 | import { startCrew } from "../src/crew/dispatch.ts"; | |
| 20 | import { serve } from "../src/http/server.ts"; | |
| 12 | 21 | import { sandboxPolicy } from "../src/crew/sandbox.ts"; |
| 13 | 22 | |
| 14 | describe("steering a running agent", () => { | |
| 15 | test("a message sent mid-turn becomes the next turn; without one the session ends after the first", async () => { | |
| 16 | const dir = await mkdtemp(join(tmpdir(), "forge-steer-")); | |
| 17 | // The stand-in: answers the first message after a moment, then, if a | |
| 18 | // second arrives before stdin closes, answers that too. | |
| 19 | const fake = join(dir, "fake-claude"); | |
| 20 | writeFileSync( | |
| 21 | fake, | |
| 22 | `#!/bin/sh | |
| 23 | /** | |
| 24 | * The stand-in agent: answers the first message, but holds that first turn | |
| 25 | * open until the test creates `gate` (no fixed wait, so no race and no | |
| 26 | * idle seconds); then, if a second message arrives before stdin closes, it | |
| 27 | * answers that too. | |
| 28 | */ | |
| 29 | function fakeClaude(path: string, gate: string, session: string): void { | |
| 30 | writeFileSync( | |
| 31 | path, | |
| 32 | `#!/bin/sh | |
| 33 | if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi | |
| 23 | 34 | IFS= read -r first |
| 24 | echo '{"type":"system","subtype":"init","session_id":"s1","model":"fake"}' | |
| 25 | sleep 0.4 | |
| 35 | echo '{"type":"system","subtype":"init","session_id":"${session}","model":"fake"}' | |
| 36 | while [ ! -e '${gate}' ]; do sleep 0.01; done | |
| 26 | 37 | echo '{"type":"assistant","message":{"content":[{"type":"text","text":"first turn done"}]}}' |
| 27 | 38 | echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}' |
| 28 | 39 | if IFS= read -r second; then |
| @@ -30,8 +41,26 @@ if IFS= read -r second; then | ||
| 30 | 41 | echo '{"type":"result","subtype":"success","is_error":false,"result":"ok again","total_cost_usd":0.02,"num_turns":2}' |
| 31 | 42 | fi |
| 32 | 43 | `, |
| 33 | ); | |
| 34 | chmodSync(fake, 0o755); | |
| 44 | ); | |
| 45 | chmodSync(path, 0o755); | |
| 46 | } | |
| 47 | ||
| 48 | /** Polls `ready` every 10 ms until it holds; throws past `ms`. */ | |
| 49 | async function until(ready: () => boolean | Promise<boolean>, what: string, ms = 10_000): Promise<void> { | |
| 50 | const deadline = Date.now() + ms; | |
| 51 | while (!(await ready())) { | |
| 52 | if (Date.now() > deadline) throw new Error(`never: ${what}`); | |
| 53 | await Bun.sleep(10); | |
| 54 | } | |
| 55 | } | |
| 56 | ||
| 57 | describe("steering a running agent", () => { | |
| 58 | test("a message sent mid-turn becomes the next turn; without one the session ends after the first", async () => { | |
| 59 | const dir = await mkdtemp(join(tmpdir(), "forge-steer-")); | |
| 60 | // Invariant: a steer sent mid-turn is heard as the next turn; without one the session ends after the first. | |
| 61 | const fake = join(dir, "fake-claude"); | |
| 62 | const gate = join(dir, "gate"); | |
| 63 | fakeClaude(fake, gate, "s1"); | |
| 35 | 64 | const lines: string[] = []; |
| 36 | 65 | const common = { |
| 37 | 66 | binary: fake, |
| @@ -54,9 +83,10 @@ fi | ||
| 54 | 83 | send = s; |
| 55 | 84 | }, |
| 56 | 85 | }); |
| 57 | await Bun.sleep(150); | |
| 86 | await until(() => lines.some((l) => l.includes("s1")), "the session opened"); | |
| 58 | 87 | expect(send).toBeDefined(); |
| 59 | 88 | expect(send!("also add a test")).toBe(true); |
| 89 | writeFileSync(gate, ""); | |
| 60 | 90 | const outcome = await steered; |
| 61 | 91 | expect(outcome.ok).toBe(true); |
| 62 | 92 | expect(outcome.turns).toBe(2); |
| @@ -71,136 +101,122 @@ fi | ||
| 71 | 101 | expect(quiet.turns).toBe(1); |
| 72 | 102 | expect(lines.some((l) => l.startsWith("heard:"))).toBe(false); |
| 73 | 103 | await rm(dir, { recursive: true, force: true }); |
| 74 | }, 30_000); | |
| 104 | }, 10_000); | |
| 75 | 105 | }); |
| 76 | 106 | |
| 77 | 107 | describe("a comment while the agent works", () => { |
| 78 | test("reaches it as its next turn; no second run queues behind the first", async () => { | |
| 79 | const { createApp } = await import("../src/app.ts"); | |
| 80 | const { serve } = await import("../src/http/server.ts"); | |
| 81 | const { startCrew } = await import("../src/crew/dispatch.ts"); | |
| 82 | const { createUser } = await import("../src/core/users.ts"); | |
| 83 | const { addToCrew, createAgent } = await import("../src/actions/crew.ts"); | |
| 84 | const { commitToBranch, createRepo } = | |
| 85 | await import("../src/actions/repos.ts"); | |
| 86 | const { addComment, assignIssue, openIssue } = | |
| 87 | await import("../src/actions/issues.ts"); | |
| 88 | const { listRuns } = await import("../src/core/runs.ts"); | |
| 89 | const { readFileSync } = await import("node:fs"); | |
| 108 | // One forge, one crew, one agent and one repository for the file. | |
| 109 | let dataDir: string; | |
| 110 | let app: ReturnType<typeof createApp>; | |
| 111 | let servers: Awaited<ReturnType<typeof serve>>; | |
| 112 | let crew: Awaited<ReturnType<typeof startCrew>>; | |
| 113 | let alice: User; | |
| 114 | let bot: User; | |
| 115 | let repo: Repo; | |
| 116 | let gate: string; | |
| 90 | 117 | |
| 91 | const dataDir = await mkdtemp(join(tmpdir(), "forge-steer-crew-")); | |
| 92 | const app = createApp({ dataDir, port: 0 }); | |
| 93 | const servers = await serve(app); | |
| 118 | beforeAll(async () => { | |
| 119 | dataDir = await mkdtemp(join(tmpdir(), "forge-steer-crew-")); | |
| 120 | app = createApp({ dataDir, port: 0 }); | |
| 121 | servers = await serve(app); | |
| 94 | 122 | const fake = join(dataDir, "fake-claude"); |
| 95 | writeFileSync( | |
| 96 | fake, | |
| 97 | `#!/bin/sh | |
| 98 | if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi | |
| 99 | IFS= read -r first | |
| 100 | echo '{"type":"system","subtype":"init","session_id":"s2","model":"fake"}' | |
| 101 | sleep 2.5 | |
| 102 | echo '{"type":"assistant","message":{"content":[{"type":"text","text":"first turn done"}]}}' | |
| 103 | echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}' | |
| 104 | if IFS= read -r second; then | |
| 105 | echo '{"type":"assistant","message":{"content":[{"type":"text","text":"heard: '"$(printf '%s' "$second" | sed 's/.*"text":"\\([^"]*\\)".*/\\1/')"'"}]}}' | |
| 106 | echo '{"type":"result","subtype":"success","is_error":false,"result":"ok again","total_cost_usd":0.02,"num_turns":2}' | |
| 107 | fi | |
| 108 | `, | |
| 109 | ); | |
| 110 | chmodSync(fake, 0o755); | |
| 111 | const crew = await startCrew(app, { | |
| 123 | gate = join(dataDir, "gate"); | |
| 124 | fakeClaude(fake, gate, "s2"); | |
| 125 | crew = await startCrew(app, { | |
| 112 | 126 | forgeUrl: servers.url, |
| 113 | 127 | localUrl: servers.url, |
| 114 | 128 | binary: fake, |
| 115 | 129 | concurrency: 1, |
| 116 | 130 | budgetUsd: 1, |
| 117 | 131 | runTimeoutMs: 60_000, |
| 132 | // The pump's clock: a short tick, so no test waits out the 5 s one. | |
| 133 | tickMs: 20, | |
| 118 | 134 | }); |
| 119 | try { | |
| 120 | const a = await createUser(app.db, { origin: "seed", trusted: true, | |
| 121 | username: "alice", | |
| 122 | email: "[email protected]", | |
| 123 | password: "a-good-passphrase", | |
| 124 | adminIfFirst: true, | |
| 125 | }); | |
| 126 | if (!a.ok) throw new Error(a.error); | |
| 127 | const alice = a.value; | |
| 128 | const made = await createAgent(app, alice, { | |
| 129 | username: "bot", | |
| 130 | brief: "Be brief.", | |
| 131 | }); | |
| 132 | if (!made.ok) throw new Error(String(made.error)); | |
| 133 | const created = await createRepo(app, alice, { | |
| 134 | name: "garden", | |
| 135 | visibility: "private", | |
| 136 | }); | |
| 137 | if (!created.ok) throw new Error(String(created.error)); | |
| 138 | const repo = created.value; | |
| 139 | const seeded = await commitToBranch(app, alice, repo, { | |
| 140 | message: "Start", | |
| 141 | files: [{ path: "README.md", content: "# garden\n" }], | |
| 142 | }); | |
| 143 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 144 | expect(addToCrew(app, alice, repo, "bot").ok).toBe(true); | |
| 135 | const a = await createUser(app.db, { origin: "seed", trusted: true, | |
| 136 | username: "alice", | |
| 137 | email: "alice@forge.local", | |
| 138 | password: "a-good-passphrase", | |
| 139 | adminIfFirst: true, | |
| 140 | }); | |
| 141 | if (!a.ok) throw new Error(a.error); | |
| 142 | alice = a.value; | |
| 143 | const made = await createAgent(app, alice, { | |
| 144 | username: "bot", | |
| 145 | brief: "Be brief.", | |
| 146 | }); | |
| 147 | if (!made.ok) throw new Error(String(made.error)); | |
| 148 | bot = made.value.user; | |
| 149 | const created = await createRepo(app, alice, { | |
| 150 | name: "garden", | |
| 151 | visibility: "private", | |
| 152 | }); | |
| 153 | if (!created.ok) throw new Error(String(created.error)); | |
| 154 | repo = created.value; | |
| 155 | const seeded = await commitToBranch(app, alice, repo, { | |
| 156 | message: "Start", | |
| 157 | files: [{ path: "README.md", content: "# garden\n" }], | |
| 158 | }); | |
| 159 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 160 | if (!addToCrew(app, alice, repo, "bot").ok) throw new Error("bot not on the crew"); | |
| 161 | }); | |
| 145 | 162 | |
| 146 | const issue = openIssue(app, alice, repo, { | |
| 147 | title: "Plant tulips", | |
| 148 | body: "Along the fence.", | |
| 149 | }); | |
| 150 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 151 | expect( | |
| 152 | assignIssue(app, alice, repo, issue.value, made.value.user).ok, | |
| 153 | ).toBe(true); | |
| 154 | // Wait for the run to be under way, then speak. | |
| 155 | const running = async () => { | |
| 156 | const deadline = Date.now() + 20_000; | |
| 157 | while (Date.now() < deadline) { | |
| 158 | const run = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 159 | (r) => r.state === "running", | |
| 160 | ); | |
| 161 | if (run) return run; | |
| 162 | await Bun.sleep(50); | |
| 163 | } | |
| 164 | throw new Error("the run never started"); | |
| 165 | }; | |
| 166 | const run = await running(); | |
| 167 | // Past the session's first line: `running` is set before the agent | |
| 168 | // is spawned, so on a busy machine a fixed sleep can speak too early | |
| 169 | // and queue a second run. The session line proves it can hear us. | |
| 170 | const logPath = join(dataDir, "runs", `${run.id}.log`); | |
| 171 | const opened = Date.now() + 20_000; | |
| 172 | while ( | |
| 173 | Date.now() < opened && | |
| 174 | !( | |
| 175 | await Bun.file(logPath) | |
| 176 | .text() | |
| 177 | .catch(() => "") | |
| 178 | ).includes("session s2") | |
| 179 | ) | |
| 180 | await Bun.sleep(50); | |
| 181 | expect( | |
| 182 | addComment(app, alice, repo, issue.value, "also the tests").ok, | |
| 183 | ).toBe(true); | |
| 184 | const deadline = Date.now() + 20_000; | |
| 185 | while (Date.now() < deadline) { | |
| 186 | const r = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 187 | (x) => x.id === run.id, | |
| 188 | ); | |
| 189 | if (r && r.state !== "running") break; | |
| 190 | await Bun.sleep(100); | |
| 191 | } | |
| 192 | const runs = listRuns(app.db, repo.id, { kind: "agent" }); | |
| 193 | expect(runs.length).toBe(1); | |
| 194 | expect(runs[0]!.state).toBe("ok"); | |
| 195 | expect(runs[0]!.turns).toBe(2); | |
| 196 | const log = readFileSync(logPath, "utf8"); | |
| 197 | expect(log).toContain("you: alice on #1: also the tests"); | |
| 198 | expect(log).toContain("heard: alice on #1: also the tests"); | |
| 199 | } finally { | |
| 200 | await crew.stop(); | |
| 201 | servers.stop(); | |
| 202 | app.close(); | |
| 203 | await rm(dataDir, { recursive: true, force: true }); | |
| 204 | } | |
| 205 | }, 60_000); | |
| 163 | afterAll(async () => { | |
| 164 | await crew.stop(); | |
| 165 | servers.stop(); | |
| 166 | app.close(); | |
| 167 | await rm(dataDir, { recursive: true, force: true }); | |
| 168 | }); | |
| 169 | ||
| 170 | test("reaches it as its next turn; no second run queues behind the first", async () => { | |
| 171 | // Invariant: a comment on the issue an agent is running on steers that run (two turns, one run), never queues another. | |
| 172 | const issue = openIssue(app, alice, repo, { | |
| 173 | title: "Plant tulips", | |
| 174 | body: "Along the fence.", | |
| 175 | }); | |
| 176 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 177 | expect( | |
| 178 | assignIssue(app, alice, repo, issue.value, bot).ok, | |
| 179 | ).toBe(true); | |
| 180 | // Wait for the run to be under way, then speak. | |
| 181 | let run: ReturnType<typeof listRuns>[number] | undefined; | |
| 182 | await until(() => { | |
| 183 | run = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 184 | (r) => r.state === "running", | |
| 185 | ); | |
| 186 | return run !== undefined; | |
| 187 | }, "the run started"); | |
| 188 | // Past the session's first line: `running` is set before the agent | |
| 189 | // is spawned, so speaking too early would queue a second run. The | |
| 190 | // session line proves it can hear us. | |
| 191 | const logPath = join(dataDir, "runs", `${run!.id}.log`); | |
| 192 | const logged = (text: string) => async () => | |
| 193 | (await Bun.file(logPath).text().catch(() => "")).includes(text); | |
| 194 | await until(logged("session s2"), "the session opened"); | |
| 195 | expect( | |
| 196 | addComment(app, alice, repo, issue.value, "also the tests").ok, | |
| 197 | ).toBe(true); | |
| 198 | // The steer is on its way to the agent; only now may its first turn end. | |
| 199 | // The comment either reached the session or queued a run of its own. | |
| 200 | await until( | |
| 201 | async () => | |
| 202 | (await logged("you: alice on #1")()) || | |
| 203 | listRuns(app.db, repo.id, { kind: "agent" }).length > 1, | |
| 204 | "the comment was sent as a steer or queued", | |
| 205 | ); | |
| 206 | expect(listRuns(app.db, repo.id, { kind: "agent" }).length).toBe(1); | |
| 207 | writeFileSync(gate, ""); | |
| 208 | await until( | |
| 209 | () => | |
| 210 | listRuns(app.db, repo.id, { kind: "agent" }).find((x) => x.id === run!.id) | |
| 211 | ?.state !== "running", | |
| 212 | "the run settled", | |
| 213 | ); | |
| 214 | const runs = listRuns(app.db, repo.id, { kind: "agent" }); | |
| 215 | expect(runs.length).toBe(1); | |
| 216 | expect(runs[0]!.state).toBe("ok"); | |
| 217 | expect(runs[0]!.turns).toBe(2); | |
| 218 | const log = readFileSync(logPath, "utf8"); | |
| 219 | expect(log).toContain("you: alice on #1: also the tests"); | |
| 220 | expect(log).toContain("heard: alice on #1: also the tests"); | |
| 221 | }, 10_000); | |
| 206 | 222 | }); |