Public: anyone can read this; the crew can change it.

The planner-run, review-loop and migration-43 tests leave test/crew-parks.test.ts for their own file #3706

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/3602-the-planner-run-review-loop-and-migratio → main 2 commits fork

  • Merged. mason/3602-the-planner-run-review-loop-and-migratio was merged into main as 54043bb.

Review

Sign in to review this change.
3 files changed +218 −201
test/crew-parks-planner.test.ts added +212
@@ -0,0 +1,212 @@
1 /**
2 * The planner's side of a park, end to end: a stand-in Claude Code is handed
3 * the issue parked twice and why, and a pull whose review loop is spent; and
4 * migration 43 rewords old outage parks so they no longer count. Split from
5 * `test/crew-parks.test.ts`, whose `engine` it shares.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { Database } from "bun:sqlite";
9 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
10 import { join } from "node:path";
11 import { addToCrew, createAgent } from "../src/actions/crew.ts";
12 import { openPull } from "../src/actions/pulls.ts";
13 import { commitToBranch } from "../src/actions/repos.ts";
14 import { setCrewRole } from "../src/core/agents.ts";
15 import { listRuns } from "../src/core/runs.ts";
16 import { migrations } from "../src/core/schema.ts";
17 import { submitReview } from "../src/core/reviews.ts";
18 import { getUserByName } from "../src/core/users.ts";
19 import { handedOverPulls } from "../src/crew/guard.ts";
20 import { driveRepo } from "../src/crew/autonomy.ts";
21 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
22 import { engine, tick, type Engine } from "./helpers/parks-engine.ts";
23
24 describe("the planner's run, with a stand-in Claude Code", () => {
25 let e: Engine;
26 let crew: CrewHandle;
27 let promptFile: string;
28
29 beforeAll(async () => {
30 e = await engine("forge-parked-brief-");
31 const seeded = await commitToBranch(e.app, e.alice, e.repo, {
32 message: "Start",
33 files: [{ path: "README.md", content: "# engine\n" }],
34 });
35 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
36 promptFile = join(e.dataDir, "fake.prompt");
37 // The stand-in: records what it was told, plans nothing, pushes nothing.
38 const fake = join(e.dataDir, "fake-claude");
39 writeFileSync(
40 fake,
41 `#!/bin/sh
42 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
43 IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}"
44 echo '{"type":"system","subtype":"init","session_id":"fake-plan","model":"fake"}'
45 echo '{"type":"result","subtype":"success","is_error":false,"result":"Nothing to add.","total_cost_usd":0.01,"num_turns":1,"session_id":"fake-plan","terminal_reason":"completed"}'
46 `,
47 );
48 chmodSync(fake, 0o755);
49 crew = await startCrew(e.app, {
50 forgeUrl: e.servers.url,
51 localUrl: e.servers.url,
52 binary: fake,
53 concurrency: 1,
54 budgetUsd: 1,
55 runTimeoutMs: 60_000,
56 tickMs: 50,
57 });
58 }, 30_000);
59
60 afterAll(async () => {
61 await crew.stop();
62 await e.close();
63 });
64
65 test("second park, next tick, and the planning run's task carries the issue and why", async () => {
66 const work = e.issue("Scene graph");
67 e.ended(work, "no changes pushed: DECLINED: out of scope");
68 await tick();
69 e.ended(work, "nothing to propose");
70 await tick();
71 expect(await driveRepo(e.app, e.fresh())).toBe("planned");
72 const direction = e.fresh().directionIssueId!;
73 const deadline = Date.now() + 30_000;
74 const settled = () =>
75 listRuns(e.app.db, e.repo.id, { kind: "agent" }).find(
76 (r) =>
77 r.issueId === direction &&
78 (r.state === "ok" || r.state === "failed" || r.state === "cancelled"),
79 );
80 while (settled() === undefined && Date.now() < deadline)
81 await Bun.sleep(100);
82 expect(settled()?.actorId).toBe(e.atlas.id);
83 expect(existsSync(promptFile)).toBe(true);
84 const told = readFileSync(promptFile, "utf8");
85 expect(told).toContain(`- Parked twice: #${work.number} Scene graph`);
86 expect(told).toContain(
87 "1. the run ended without a pull: no changes pushed: DECLINED: out of scope",
88 );
89 expect(told).toContain(
90 "2. the run ended without a pull: nothing to propose",
91 );
92 }, 60_000);
93 });
94
95 describe("a pull whose review loop is spent is the planner's (#1655)", () => {
96 let e: Engine;
97 let crew: CrewHandle;
98 let promptFile: string;
99
100 beforeAll(async () => {
101 e = await engine("forge-spent-pull-");
102 const seeded = await commitToBranch(e.app, e.alice, e.repo, {
103 message: "Start",
104 files: [{ path: "README.md", content: "# engine\n" }],
105 });
106 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
107 promptFile = join(e.dataDir, "fake.prompt");
108 const fake = join(e.dataDir, "fake-claude");
109 writeFileSync(
110 fake,
111 `#!/bin/sh
112 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
113 IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}"
114 echo '{"type":"system","subtype":"init","session_id":"fake-plan","model":"fake"}'
115 echo '{"type":"result","subtype":"success","is_error":false,"result":"Nothing to add.","total_cost_usd":0.01,"num_turns":1,"session_id":"fake-plan","terminal_reason":"completed"}'
116 `,
117 );
118 chmodSync(fake, 0o755);
119 crew = await startCrew(e.app, {
120 forgeUrl: e.servers.url,
121 localUrl: e.servers.url,
122 binary: fake,
123 concurrency: 1,
124 budgetUsd: 1,
125 runTimeoutMs: 60_000,
126 tickMs: 50,
127 });
128 }, 30_000);
129
130 afterAll(async () => {
131 await crew.stop();
132 await e.close();
133 });
134
135 test("the planner is asked for it and its brief says three reviews asked for changes", async () => {
136 const agent = await createAgent(e.app, e.alice, { username: "sentinel" });
137 if (!agent.ok) throw new Error(String(agent.error));
138 expect(addToCrew(e.app, e.alice, e.fresh(), "sentinel").ok).toBe(true);
139 const sentinel = getUserByName(e.app.db, "sentinel")!;
140 setCrewRole(e.app.db, e.repo.id, sentinel.id, "reviewer");
141 const branch = await commitToBranch(e.app, e.alice, e.fresh(), {
142 branch: "feature/spent",
143 base: "main",
144 message: "spent",
145 files: [{ path: "spent.txt", content: "spent\n" }],
146 });
147 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
148 const opened = await openPull(e.app, e.alice, e.fresh(), {
149 title: "Spent loop",
150 baseBranch: "main",
151 headBranch: "feature/spent",
152 });
153 if (!opened.ok) throw new Error(String(opened.error));
154 const pull = opened.value;
155 e.app.db.query("UPDATE issue SET author_id = ?2 WHERE id = ?1").run(pull.id, e.mason.id);
156 for (const n of [1, 2, 3])
157 submitReview(e.app.db, {
158 issueId: pull.id,
159 authorId: sentinel.id,
160 state: "changes_requested",
161 body: `Round ${n}`,
162 commitOid: pull.headOid,
163 });
164 expect(handedOverPulls(e.app.db, e.fresh()).map((h) => h.number)).toEqual([pull.number]);
165 expect(await driveRepo(e.app, e.fresh())).toBe("planned");
166 const direction = e.fresh().directionIssueId!;
167 const deadline = Date.now() + 30_000;
168 const settled = () =>
169 listRuns(e.app.db, e.repo.id, { kind: "agent" }).find(
170 (r) => r.issueId === direction && ["ok", "failed", "cancelled"].includes(r.state),
171 );
172 while (settled() === undefined && Date.now() < deadline) await Bun.sleep(100);
173 expect(existsSync(promptFile)).toBe(true);
174 const told = readFileSync(promptFile, "utf8");
175 expect(told).toContain(`Three reviews asked for changes, pull #${pull.number} Spent loop`);
176 expect(told).toContain("Round 3");
177 expect(told).toContain("A pull above is not closed by splitting an issue");
178 }, 60_000);
179 });
180
181 describe("migration 43", () => {
182 test("rewords old outage parks so they no longer count, and leaves real parks", () => {
183 const db = new Database(":memory:");
184 db.exec(
185 "CREATE TABLE comment (id TEXT PRIMARY KEY, issue_id TEXT, author_id TEXT, body TEXT)",
186 );
187 const bodies = [
188 "Parked by the crew: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
189 "Parked by the crew: the run ended without a pull: the run failed: Error: API unreachable",
190 "Parked by the crew: the run ended without a pull: cancelled",
191 "Parked by the crew: the run ended without a pull: nothing to propose",
192 "Parked by the crew: the check stayed red past its tries.",
193 ];
194 bodies.forEach((body, n) =>
195 db
196 .query("INSERT INTO comment VALUES (?1, 'i', 'u', ?2)")
197 .run(String(n), body),
198 );
199 db.exec(migrations[42]!);
200 const after = db
201 .query<{ body: string }, []>("SELECT body FROM comment ORDER BY id")
202 .all()
203 .map((r) => r.body);
204 expect(after[0]).toBe(
205 "Interrupted, not parked: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
206 );
207 expect(after[1]).toStartWith("Interrupted, not parked: ");
208 expect(after[2]).toStartWith("Interrupted, not parked: ");
209 expect(after[3]).toBe(bodies[3]!);
210 expect(after[4]).toBe(bodies[4]!);
211 });
212 });
test/crew-parks.test.ts +4 −199
@@ -11,13 +11,11 @@
1111 * replaced or deferred.
1212 *
1313 * An issue found already done is not a park: `test/crew-parks-done.test.ts`.
14 * The planner's run with a stand-in Claude Code, a spent review loop and
15 * migration 43: `test/crew-parks-planner.test.ts`.
1416 */
1517 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
16 import { Database } from "bun:sqlite";
17 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
18 import { join } from "node:path";
1918 import { type App } from "../src/app.ts";
20 import { addToCrew, createAgent } from "../src/actions/crew.ts";
2119 import { linkIssue } from "../src/actions/dependencies.ts";
2220 import {
2321 addComment,
@@ -26,7 +24,7 @@ import {
2624 } from "../src/actions/issues.ts";
2725 import { openPull } from "../src/actions/pulls.ts";
2826 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
29 import { getAgentByName, setCrewRole } from "../src/core/agents.ts";
27 import { getAgentByName } from "../src/core/agents.ts";
3028 import { createApp as insertApp } from "../src/core/apps.ts";
3129 import { getIssueById, listComments, type Issue } from "../src/core/issues.ts";
3230 import { type Repo } from "../src/core/repos.ts";
@@ -34,18 +32,13 @@ import {
3432 claimRun,
3533 createRun,
3634 finishRun,
37 listRuns,
3835 } from "../src/core/runs.ts";
39 import { migrations } from "../src/core/schema.ts";
40 import { submitReview } from "../src/core/reviews.ts";
4136 import { createToken } from "../src/core/tokens.ts";
42 import { createUser, getUserByName, type User } from "../src/core/users.ts";
37 import { createUser, type User } from "../src/core/users.ts";
4338 import { type Servers } from "../src/http/server.ts";
44 import { handedOverPulls } from "../src/crew/guard.ts";
4539 import { driveRepo, parkedCount, settleAgentRun } from "../src/crew/autonomy.ts";
4640 import { taskPrompt } from "../src/crew/brief.ts";
4741 import { breakerTripped } from "../src/crew/breaker.ts";
48 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
4942 import {
5043 interruptedSummary,
5144 interruptionOf,
@@ -999,191 +992,3 @@ describe("an issue parked twice is the planner's", () => {
999992 expect(listed()).toEqual([again.number]);
1000993 });
1001994 });
1002
1003 describe("the planner's run, with a stand-in Claude Code", () => {
1004 let e: Engine;
1005 let crew: CrewHandle;
1006 let promptFile: string;
1007
1008 beforeAll(async () => {
1009 e = await engine("forge-parked-brief-");
1010 const seeded = await commitToBranch(e.app, e.alice, e.repo, {
1011 message: "Start",
1012 files: [{ path: "README.md", content: "# engine\n" }],
1013 });
1014 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
1015 promptFile = join(e.dataDir, "fake.prompt");
1016 // The stand-in: records what it was told, plans nothing, pushes nothing.
1017 const fake = join(e.dataDir, "fake-claude");
1018 writeFileSync(
1019 fake,
1020 `#!/bin/sh
1021 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
1022 IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}"
1023 echo '{"type":"system","subtype":"init","session_id":"fake-plan","model":"fake"}'
1024 echo '{"type":"result","subtype":"success","is_error":false,"result":"Nothing to add.","total_cost_usd":0.01,"num_turns":1,"session_id":"fake-plan","terminal_reason":"completed"}'
1025 `,
1026 );
1027 chmodSync(fake, 0o755);
1028 crew = await startCrew(e.app, {
1029 forgeUrl: e.servers.url,
1030 localUrl: e.servers.url,
1031 binary: fake,
1032 concurrency: 1,
1033 budgetUsd: 1,
1034 runTimeoutMs: 60_000,
1035 });
1036 }, 30_000);
1037
1038 afterAll(async () => {
1039 await crew.stop();
1040 await e.close();
1041 });
1042
1043 test("second park, next tick, and the planning run's task carries the issue and why", async () => {
1044 const work = e.issue("Scene graph");
1045 e.ended(work, "no changes pushed: DECLINED: out of scope");
1046 await tick();
1047 e.ended(work, "nothing to propose");
1048 await tick();
1049 expect(await driveRepo(e.app, e.fresh())).toBe("planned");
1050 const direction = e.fresh().directionIssueId!;
1051 const deadline = Date.now() + 30_000;
1052 const settled = () =>
1053 listRuns(e.app.db, e.repo.id, { kind: "agent" }).find(
1054 (r) =>
1055 r.issueId === direction &&
1056 (r.state === "ok" || r.state === "failed" || r.state === "cancelled"),
1057 );
1058 while (settled() === undefined && Date.now() < deadline)
1059 await Bun.sleep(100);
1060 expect(settled()?.actorId).toBe(e.atlas.id);
1061 expect(existsSync(promptFile)).toBe(true);
1062 const told = readFileSync(promptFile, "utf8");
1063 expect(told).toContain(`- Parked twice: #${work.number} Scene graph`);
1064 expect(told).toContain(
1065 "1. the run ended without a pull: no changes pushed: DECLINED: out of scope",
1066 );
1067 expect(told).toContain(
1068 "2. the run ended without a pull: nothing to propose",
1069 );
1070 }, 60_000);
1071 });
1072
1073 describe("a pull whose review loop is spent is the planner's (#1655)", () => {
1074 let e: Engine;
1075 let crew: CrewHandle;
1076 let promptFile: string;
1077
1078 beforeAll(async () => {
1079 e = await engine("forge-spent-pull-");
1080 const seeded = await commitToBranch(e.app, e.alice, e.repo, {
1081 message: "Start",
1082 files: [{ path: "README.md", content: "# engine\n" }],
1083 });
1084 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
1085 promptFile = join(e.dataDir, "fake.prompt");
1086 const fake = join(e.dataDir, "fake-claude");
1087 writeFileSync(
1088 fake,
1089 `#!/bin/sh
1090 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
1091 IFS= read -r first; printf '%s\\n' "$first" > "${promptFile}"
1092 echo '{"type":"system","subtype":"init","session_id":"fake-plan","model":"fake"}'
1093 echo '{"type":"result","subtype":"success","is_error":false,"result":"Nothing to add.","total_cost_usd":0.01,"num_turns":1,"session_id":"fake-plan","terminal_reason":"completed"}'
1094 `,
1095 );
1096 chmodSync(fake, 0o755);
1097 crew = await startCrew(e.app, {
1098 forgeUrl: e.servers.url,
1099 localUrl: e.servers.url,
1100 binary: fake,
1101 concurrency: 1,
1102 budgetUsd: 1,
1103 runTimeoutMs: 60_000,
1104 });
1105 }, 30_000);
1106
1107 afterAll(async () => {
1108 await crew.stop();
1109 await e.close();
1110 });
1111
1112 test("the planner is asked for it and its brief says three reviews asked for changes", async () => {
1113 const agent = await createAgent(e.app, e.alice, { username: "sentinel" });
1114 if (!agent.ok) throw new Error(String(agent.error));
1115 expect(addToCrew(e.app, e.alice, e.fresh(), "sentinel").ok).toBe(true);
1116 const sentinel = getUserByName(e.app.db, "sentinel")!;
1117 setCrewRole(e.app.db, e.repo.id, sentinel.id, "reviewer");
1118 const branch = await commitToBranch(e.app, e.alice, e.fresh(), {
1119 branch: "feature/spent",
1120 base: "main",
1121 message: "spent",
1122 files: [{ path: "spent.txt", content: "spent\n" }],
1123 });
1124 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
1125 const opened = await openPull(e.app, e.alice, e.fresh(), {
1126 title: "Spent loop",
1127 baseBranch: "main",
1128 headBranch: "feature/spent",
1129 });
1130 if (!opened.ok) throw new Error(String(opened.error));
1131 const pull = opened.value;
1132 e.app.db.query("UPDATE issue SET author_id = ?2 WHERE id = ?1").run(pull.id, e.mason.id);
1133 for (const n of [1, 2, 3])
1134 submitReview(e.app.db, {
1135 issueId: pull.id,
1136 authorId: sentinel.id,
1137 state: "changes_requested",
1138 body: `Round ${n}`,
1139 commitOid: pull.headOid,
1140 });
1141 expect(handedOverPulls(e.app.db, e.fresh()).map((h) => h.number)).toEqual([pull.number]);
1142 expect(await driveRepo(e.app, e.fresh())).toBe("planned");
1143 const direction = e.fresh().directionIssueId!;
1144 const deadline = Date.now() + 30_000;
1145 const settled = () =>
1146 listRuns(e.app.db, e.repo.id, { kind: "agent" }).find(
1147 (r) => r.issueId === direction && ["ok", "failed", "cancelled"].includes(r.state),
1148 );
1149 while (settled() === undefined && Date.now() < deadline) await Bun.sleep(100);
1150 expect(existsSync(promptFile)).toBe(true);
1151 const told = readFileSync(promptFile, "utf8");
1152 expect(told).toContain(`Three reviews asked for changes, pull #${pull.number} Spent loop`);
1153 expect(told).toContain("Round 3");
1154 expect(told).toContain("A pull above is not closed by splitting an issue");
1155 }, 60_000);
1156 });
1157
1158 describe("migration 43", () => {
1159 test("rewords old outage parks so they no longer count, and leaves real parks", () => {
1160 const db = new Database(":memory:");
1161 db.exec(
1162 "CREATE TABLE comment (id TEXT PRIMARY KEY, issue_id TEXT, author_id TEXT, body TEXT)",
1163 );
1164 const bodies = [
1165 "Parked by the crew: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
1166 "Parked by the crew: the run ended without a pull: the run failed: Error: API unreachable",
1167 "Parked by the crew: the run ended without a pull: cancelled",
1168 "Parked by the crew: the run ended without a pull: nothing to propose",
1169 "Parked by the crew: the check stayed red past its tries.",
1170 ];
1171 bodies.forEach((body, n) =>
1172 db
1173 .query("INSERT INTO comment VALUES (?1, 'i', 'u', ?2)")
1174 .run(String(n), body),
1175 );
1176 db.exec(migrations[42]!);
1177 const after = db
1178 .query<{ body: string }, []>("SELECT body FROM comment ORDER BY id")
1179 .all()
1180 .map((r) => r.body);
1181 expect(after[0]).toBe(
1182 "Interrupted, not parked: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
1183 );
1184 expect(after[1]).toStartWith("Interrupted, not parked: ");
1185 expect(after[2]).toStartWith("Interrupted, not parked: ");
1186 expect(after[3]).toBe(bodies[3]!);
1187 expect(after[4]).toBe(bodies[4]!);
1188 });
1189 });
test/helpers/parks-engine.ts +2 −2
@@ -18,8 +18,8 @@ import { interruptedSummary } from "../../src/crew/interrupted.ts";
1818 /**
1919 * One forge with alice's `engine` on its own: atlas plans, mason builds.
2020 * `ended` is a builder's run on an issue that ends without a pull, settled
21 * as the crew would settle it. Shared by `test/crew-parks.test.ts` and
22 * `test/crew-parks-done.test.ts`.
21 * as the crew would settle it. Shared by `test/crew-parks.test.ts`,
22 * `test/crew-parks-done.test.ts` and `test/crew-parks-planner.test.ts`.
2323 */
2424 export async function engine(
2525 prefix: string,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs