Public: anyone can read this; the crew can change it.

The test that parks other people's pulls: a 60s sleep against a 60s deadline #135

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/124-the-test-that-parks-other-people-s-pulls → main 1 commit fork

  • Merged. scout/124-the-test-that-parks-other-people-s-pulls was merged into main as 70dcb07.

Review

Sign in to review this change.
2 files changed +44 −14
src/crew/claude.ts +16 −2
@@ -139,6 +139,10 @@ export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> {
139139 stdin: "pipe",
140140 stdout: "pipe",
141141 stderr: "pipe",
142 // Its own session, so a cancel can reach whatever it spawned in turn
143 // (a tool call left running) and not just the agent itself: a signal
144 // to one pid never touches a grandchild holding stdout open.
145 detached: true,
142146 });
143147 // What the agent hears: the prompt first, then whatever a person says
144148 // while it works. Each is a user turn; the session ends when the forge
@@ -243,13 +247,23 @@ export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> {
243247 // `user` records are tool results: long and mostly file contents.
244248 };
245249
250 // Signals the whole process group (a negative pid), so anything the
251 // agent spawned in turn goes with it; falls back to the process alone
252 // if the group is already gone.
253 const signalGroup = (sig: "SIGTERM" | "SIGKILL"): void => {
254 try {
255 process.kill(-proc.pid, sig);
256 } catch {
257 proc.kill(sig);
258 }
259 };
246260 let why: "timeout" | "cancelled" | undefined;
247261 const stopChild = (): void => {
248262 closeInput();
249 proc.kill("SIGTERM");
263 signalGroup("SIGTERM");
250264 setTimeout(() => {
251265 if (proc.exitCode === null && proc.signalCode === null)
252 proc.kill("SIGKILL");
266 signalGroup("SIGKILL");
253267 }, 5_000);
254268 };
255269 const timer = setTimeout(() => {
test/landing.test.ts +28 −12
@@ -11,6 +11,9 @@ import { join } from "node:path";
1111 */
1212 describe("a pull closed under a running agent", () => {
1313 test("the run is cancelled, and its clone goes after it, not before", async () => {
14 // Cancellation must beat the fake agent's think by a wide margin, not
15 // by luck: the think, the run timeout and this test's own deadline are
16 // three different numbers, none of them close to another.
1417 const { createApp } = await import("../src/app.ts");
1518 const { serve } = await import("../src/http/server.ts");
1619 const { startCrew } = await import("../src/crew/dispatch.ts");
@@ -28,7 +31,9 @@ describe("a pull closed under a running agent", () => {
2831 const dataDir = await mkdtemp(join(tmpdir(), "forge-landing-"));
2932 const app = createApp({ dataDir, port: 0 });
3033 const servers = await serve(app);
31 // An agent that settles in for a long think.
34 // An agent that settles in for a long think — long enough that it
35 // cannot possibly finish on its own before the test does, so the only
36 // way the run ends is by being cancelled.
3237 const fake = join(dataDir, "fake-claude");
3338 writeFileSync(
3439 fake,
@@ -36,7 +41,7 @@ describe("a pull closed under a running agent", () => {
3641 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
3742 IFS= read -r first
3843 echo '{"type":"system","subtype":"init","session_id":"s3","model":"fake"}'
39 sleep 60
44 sleep 1800
4045 `,
4146 );
4247 chmodSync(fake, 0o755);
@@ -46,12 +51,14 @@ sleep 60
4651 binary: fake,
4752 concurrency: 1,
4853 budgetUsd: 1,
49 runTimeoutMs: 60_000,
54 // Far past anything this test waits for: the run's own timeout must
55 // never be what ends it here, only the pull closing.
56 runTimeoutMs: 10 * 60_000,
5057 });
5158 const until = async <T>(
5259 read: () => T | undefined,
5360 what: string,
54 ms = 20_000,
61 ms = 10_000,
5562 ): Promise<T> => {
5663 const deadline = Date.now() + ms;
5764 while (Date.now() < deadline) {
@@ -124,20 +131,29 @@ sleep 60
124131 const clone = workdirFor(dataDir, bot, repo, 1);
125132 await until(() => (existsSync(clone) ? true : undefined), "the clone");
126133
127 // The pull closes under it.
134 // The pull closes under it. Cancelling is a signal, not a wait for
135 // the agent to notice on its own — it must land in a few seconds,
136 // not sometime before the fake agent's 1800-second think ends.
137 const closedAt = Date.now();
128138 expect(setIssueState(app, alice, repo, asIssue, "closed").ok).toBe(true);
129 const ended = await until(() => {
130 const r = listRuns(app.db, repo.id, { kind: "agent" }).find(
131 (x) => x.id === run.id,
132 );
133 return r && r.state !== "running" ? r : undefined;
134 }, "the run's end");
139 const ended = await until(
140 () => {
141 const r = listRuns(app.db, repo.id, { kind: "agent" }).find(
142 (x) => x.id === run.id,
143 );
144 return r && r.state !== "running" ? r : undefined;
145 },
146 "the run's end",
147 5_000,
148 );
149 expect(Date.now() - closedAt).toBeLessThan(5_000);
135150 expect(ended.state).toBe("cancelled");
136151 expect(ended.summary).toBe("the pull closed");
137152 // The clone went with the run, once the run was over.
138153 await until(
139154 () => (existsSync(clone) ? undefined : true),
140155 "the clone's end",
156 5_000,
141157 );
142158 const log = readFileSync(join(dataDir, "runs", `${run.id}.log`), "utf8");
143159 expect(log).not.toContain("the run failed");
@@ -154,5 +170,5 @@ sleep 60
154170 app.close();
155171 await rm(dataDir, { recursive: true, force: true });
156172 }
157 });
173 }, 25_000);
158174 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs