trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns3g rEventsFlows

test/crew-interrupted.test.ts runs in under 10 s #11062

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/11053-test-crew-interrupted-test-ts-runs-in-un → main 2 commits fork 2 threads · 2 unresolved

  • Merged. mason/11053-test-crew-interrupted-test-ts-runs-in-un was merged into main as b9b9b28.

Review

Sign in to review this change.
1 file changed +86 −44
test/crew-interrupted.test.ts +86 −44
@@ -7,7 +7,6 @@ import { addToCrew, createAgent } from "../src/actions/crew.ts";
77 import { addComment, assignIssue, openIssue } from "../src/actions/issues.ts";
88 import { listComments } from "../src/core/issues.ts";
99 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
10 import { getRepo } from "../src/core/repos.ts";
1110 import { createApp, type App } from "../src/app.ts";
1211 import {
1312 cancelRun,
@@ -26,6 +25,11 @@ import { createUser, type User } from "../src/core/users.ts";
2625 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
2726 import { serve, type Servers } from "../src/http/server.ts";
2827
28 /** The crew's tick, injected short: every wait below is for the next tick (#11053). */
29 const TICK_MS = 50;
30 /** How often a test looks again at a run it waits on. */
31 const POLL_MS = 20;
32
2933 describe("a run the last forge left running", () => {
3034 let dataDir: string;
3135 let app: App;
@@ -60,14 +64,61 @@ describe("a run the last forge left running", () => {
6064 await rm(dataDir, { recursive: true, force: true });
6165 });
6266
63 test("is cancelled at boot and queued again under the same reason", async () => {
67 /**
68 * No crew left up and no run left queued or running by the test before:
69 * with no crew up, a running run is a leftover, and the next crew's tick
70 * would requeue it once its lease lapsed or count it against a slot. Each
71 * test starts from rest, so they pass in any order (`--randomize`).
72 */
73 const atRest = async () => {
74 await crew?.stop();
75 crew = undefined;
76 const left = app.db
77 .query<{ id: string }, []>(
78 "SELECT id FROM run WHERE state IN ('queued', 'running')",
79 )
80 .all();
81 for (const { id } of left) cancelRun(app.db, id, "cleared for the next test");
82 };
83
84 /** A repository of the test's own, with the bot on its crew: no case shares one. */
85 const crewedRepo = async (name: string) => {
6486 const created = await createRepo(app, alice, {
65 name: "garden",
87 name,
6688 visibility: "private",
6789 });
6890 if (!created.ok) throw new Error(String(created.error));
69 const repo = created.value;
70 expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
91 expect(addToCrew(app, alice, created.value, "bot").ok).toBe(true);
92 return created.value;
93 };
94
95 /** Poll a run until it ends (ok, failed or cancelled), or the deadline passes. */
96 const settled = async (id: string, withinMs: number) => {
97 const deadline = Date.now() + withinMs;
98 while (
99 !["ok", "failed", "cancelled"].includes(getRun(app.db, id)!.state) &&
100 Date.now() < deadline
101 )
102 await Bun.sleep(POLL_MS);
103 return getRun(app.db, id)!;
104 };
105
106 /** Concurrency 0: the crew boots, tidies, and starts nothing. */
107 const idleCrew = () =>
108 startCrew(app, {
109 forgeUrl: servers.url,
110 localUrl: servers.url,
111 binary: join(dataDir, "no-such-claude"),
112 concurrency: 0,
113 budgetUsd: 1,
114 runTimeoutMs: 60_000,
115 tickMs: TICK_MS,
116 networkRetryScale: 0,
117 });
118
119 test("is cancelled at boot and queued again under the same reason", async () => {
120 await atRest();
121 const repo = await crewedRepo("garden");
71122 const issued = openIssue(app, alice, repo, {
72123 title: "Plant something",
73124 body: "",
@@ -103,15 +154,7 @@ describe("a run the last forge left running", () => {
103154 });
104155 claimRun(app.db, alive.id);
105156
106 // Concurrency 0: the crew boots, tidies, and starts nothing.
107 crew = await startCrew(app, {
108 forgeUrl: servers.url,
109 localUrl: servers.url,
110 binary: join(dataDir, "no-such-claude"),
111 concurrency: 0,
112 budgetUsd: 1,
113 runTimeoutMs: 60_000,
114 });
157 crew = await idleCrew();
115158
116159 const after = getRun(app.db, orphan.id)!;
117160 expect(after.state).toBe("cancelled");
@@ -125,10 +168,13 @@ describe("a run the last forge left running", () => {
125168 expect(getRun(app.db, alive.id)?.state).toBe("running");
126169 // The other forge finishes it; here, it is simply out of the way.
127170 cancelRun(app.db, alive.id, "finished elsewhere");
128 });
171 }, 30_000);
129172
130173 test("a run whose lease lapses while the crew is up is queued again on the next tick", async () => {
131 const repo = getRepo(app.db, "alice", "garden")!;
174 await atRest();
175 const repo = await crewedRepo("allotment");
176 // The crew is up before the lease lapses: only a tick can see it.
177 crew = await idleCrew();
132178 const issued = openIssue(app, alice, repo, {
133179 title: "Water it",
134180 body: "",
@@ -156,7 +202,7 @@ describe("a run the last forge left running", () => {
156202 getRun(app.db, orphan.id)?.state === "running" &&
157203 Date.now() < deadline
158204 )
159 await Bun.sleep(200);
205 await Bun.sleep(POLL_MS);
160206 const after = getRun(app.db, orphan.id)!;
161207 expect(after.state).toBe("cancelled");
162208 expect(after.summary).toContain("interrupted by a restart");
@@ -168,8 +214,7 @@ describe("a run the last forge left running", () => {
168214 }, 30_000);
169215
170216 test("a run in flight when the crew stops goes back on the queue", async () => {
171 await crew?.stop();
172 crew = undefined;
217 await atRest();
173218 const created = await createRepo(app, alice, {
174219 name: "orchard",
175220 visibility: "private",
@@ -215,13 +260,15 @@ sleep 120
215260 concurrency: 1,
216261 budgetUsd: 1,
217262 runTimeoutMs: 60_000,
263 tickMs: TICK_MS,
264 networkRetryScale: 0,
218265 });
219266 const deadline = Date.now() + 20_000;
220267 while (
221268 getRun(app.db, queuedRun.id)?.state !== "running" &&
222269 Date.now() < deadline
223270 )
224 await Bun.sleep(100);
271 await Bun.sleep(POLL_MS);
225272 expect(getRun(app.db, queuedRun.id)?.state).toBe("running");
226273
227274 await crew.stop();
@@ -240,10 +287,7 @@ sleep 120
240287 }, 40_000);
241288
242289 test("a draining crew hands a long run back queued within its budget (#7799)", async () => {
243 await crew?.stop();
244 crew = undefined;
245 // The run the test before put back on the queue would take the one slot.
246 for (const left of listQueued(app.db, "agent")) cancelRun(app.db, left.id, "cleared for the next test");
290 await atRest();
247291 const created = await createRepo(app, alice, { name: "thicket", visibility: "private" });
248292 if (!created.ok) throw new Error(String(created.error));
249293 const repo = created.value;
@@ -281,13 +325,15 @@ sleep 120
281325 concurrency: 1,
282326 budgetUsd: 1,
283327 runTimeoutMs: 60_000,
328 tickMs: TICK_MS,
329 networkRetryScale: 0,
284330 });
285331 const deadline = Date.now() + 20_000;
286 while (getRun(app.db, queuedRun.id)?.state !== "running" && Date.now() < deadline) await Bun.sleep(100);
332 while (getRun(app.db, queuedRun.id)?.state !== "running" && Date.now() < deadline) await Bun.sleep(POLL_MS);
287333 expect(getRun(app.db, queuedRun.id)?.state).toBe("running");
288334
289335 const started = Date.now();
290 await crew.stop({ drain: true, drainMs: 1_000 });
336 await crew.stop({ drain: true, drainMs: 250 });
291337 crew = undefined;
292338 expect(Date.now() - started).toBeLessThan(15_000);
293339
@@ -299,8 +345,7 @@ sleep 120
299345 }, 40_000);
300346
301347 test("a draining crew lets a run in flight finish and land; nothing is queued again", async () => {
302 await crew?.stop();
303 crew = undefined;
348 await atRest();
304349 const created = await createRepo(app, alice, {
305350 name: "grove",
306351 visibility: "private",
@@ -324,7 +369,7 @@ sleep 120
324369 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
325370 cat > /dev/null &
326371 echo '{"type":"system","subtype":"init","session_id":"brief-1","model":"fake"}'
327 sleep 2
372 sleep 0.5
328373 echo "drained work" >> CREW.md
329374 git add -A && git commit -qm "drained" && git push -q -u origin HEAD
330375 echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while draining.","total_cost_usd":0.01,"num_turns":2,"session_id":"brief-1","terminal_reason":"completed"}'
@@ -345,19 +390,22 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while
345390 concurrency: 1,
346391 budgetUsd: 1,
347392 runTimeoutMs: 60_000,
393 tickMs: TICK_MS,
394 networkRetryScale: 0,
348395 });
349396 const deadline = Date.now() + 20_000;
350397 while (
351398 getRun(app.db, queuedRun.id)?.state !== "running" &&
352399 Date.now() < deadline
353400 )
354 await Bun.sleep(100);
401 await Bun.sleep(POLL_MS);
355402 expect(getRun(app.db, queuedRun.id)?.state).toBe("running");
356403
357404 await crew.stop({ drain: true, drainMs: 30_000 });
358405 crew = undefined;
359406
360 const ended = getRun(app.db, queuedRun.id)!;
407 // The run's end, however long the machine took to reach it.
408 const ended = await settled(queuedRun.id, 20_000);
361409 expect(ended.state).toBe("ok");
362410 expect(ended.summary).toMatch(/^opened pull #\d+/);
363411 expect(
@@ -366,8 +414,7 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while
366414 }, 40_000);
367415
368416 const keptClone = async (name: string) => {
369 await crew?.stop();
370 crew = undefined;
417 await atRest();
371418 const created = await createRepo(app, alice, {
372419 name,
373420 visibility: "private",
@@ -403,6 +450,8 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done.","to
403450 concurrency: 1,
404451 budgetUsd: 1,
405452 runTimeoutMs: 60_000,
453 tickMs: TICK_MS,
454 networkRetryScale: 0,
406455 };
407456 const first = createRun(app.db, {
408457 repoId: repo.id,
@@ -412,16 +461,7 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done.","to
412461 reason: "assigned",
413462 });
414463 crew = await startCrew(app, options);
415 const wait = async (id: string) => {
416 const deadline = Date.now() + 25_000;
417 while (
418 !["ok", "failed", "cancelled"].includes(getRun(app.db, id)!.state) &&
419 Date.now() < deadline
420 )
421 await Bun.sleep(100);
422 return getRun(app.db, id)!;
423 };
424 expect((await wait(first.id)).state).toBe("ok");
464 expect((await settled(first.id, 25_000)).state).toBe("ok");
425465
426466 // The clone is kept while the pull is open; a cut-off run left a lock.
427467 const agent = getAgent(app.db, bot.id)!;
@@ -443,7 +483,7 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done.","to
443483 while (Date.now() < deadline) {
444484 next = listRuns(app.db, repo.id, { issueId: pull!.id })[0];
445485 if (next && ["ok", "failed", "cancelled"].includes(next.state)) break;
446 await Bun.sleep(100);
486 await Bun.sleep(POLL_MS);
447487 }
448488 expect(next).toBeDefined();
449489 return next!;
@@ -566,6 +606,8 @@ describe("an issue handed to someone else", () => {
566606 concurrency: 0,
567607 budgetUsd: 1,
568608 runTimeoutMs: 60_000,
609 tickMs: TICK_MS,
610 networkRetryScale: 0,
569611 });
570612 });
571613

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs