Public: anyone can read this; the crew can change it.

test/landing.test.ts runs in under 10 s, every test under 1 s, proving what it proves today #10255

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/10240-test-landing-test-ts-runs-in-under-10-s → main 1 commit fork

  • Merged. mason/10240-test-landing-test-ts-runs-in-under-10-s was merged into main as 2120c53.

Review

Sign in to review this change.
1 file changed +168 −150
test/landing.test.ts +168 −150
@@ -1,8 +1,21 @@
1 import { describe, expect, test } from "bun:test";
1 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
22 import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
33 import { mkdtemp, rm } from "node:fs/promises";
44 import { tmpdir } from "node:os";
55 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
9 import { createUser, type User } from "../src/core/users.ts";
10 import { addToCrew, createAgent } from "../src/actions/crew.ts";
11 import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
12 import { addComment, setIssueState } from "../src/actions/issues.ts";
13 import { openPull } from "../src/actions/pulls.ts";
14 import { getIssueById, type Issue } from "../src/core/issues.ts";
15 import { listRuns } from "../src/core/runs.ts";
16 import { workdirFor } from "../src/crew/workdir.ts";
17 import type { Agent } from "../src/core/agents.ts";
18 import type { Repo } from "../src/core/repos.ts";
619
720 /**
821 * A pull that lands or closes while an agent still works on it: the run is
@@ -10,25 +23,24 @@ import { join } from "node:path";
1023 * run is over, never from under a git call.
1124 */
1225 describe("a pull closed under a running agent", () => {
13 test("the run is cancelled, and its clone goes after it, not before", async () => {
14 const { createApp } = await import("../src/app.ts");
15 const { serve } = await import("../src/http/server.ts");
16 const { startCrew } = await import("../src/crew/dispatch.ts");
17 const { createUser } = await import("../src/core/users.ts");
18 const { addToCrew, createAgent } = await import("../src/actions/crew.ts");
19 const { commitToBranch, createRepo, forkRepo } =
20 await import("../src/actions/repos.ts");
21 const { addComment, setIssueState } =
22 await import("../src/actions/issues.ts");
23 const { openPull } = await import("../src/actions/pulls.ts");
24 const { getIssueById } = await import("../src/core/issues.ts");
25 const { listRuns } = await import("../src/core/runs.ts");
26 const { workdirFor } = await import("../src/crew/workdir.ts");
26 // Built once for the file: the forge, its crew, and the agent's pull in
27 // its fork. Starting the crew and forking are the slow part, and the test
28 // is about neither.
29 let dataDir: string;
30 let app: App;
31 let servers: Servers;
32 let crew: CrewHandle;
33 let alice: User;
34 let bot: Agent;
35 let repo: Repo;
36 let asIssue: Issue;
2737
28 const dataDir = await mkdtemp(join(tmpdir(), "forge-landing-"));
29 const app = createApp({ dataDir, port: 0 });
30 const servers = await serve(app);
31 // An agent that settles in for a long think.
38 beforeAll(async () => {
39 dataDir = await mkdtemp(join(tmpdir(), "forge-landing-"));
40 app = createApp({ dataDir, port: 0 });
41 servers = await serve(app);
42 // An agent that settles in for a long think: it never ends on its own,
43 // only when the crew stops it (the whole process group goes).
3244 const fake = join(dataDir, "fake-claude");
3345 writeFileSync(
3446 fake,
@@ -40,7 +52,9 @@ sleep 60
4052 `,
4153 );
4254 chmodSync(fake, 0o755);
43 const crew = await startCrew(app, {
55 // The crew ticks every 50 ms (`tickMs`), not every five seconds, so
56 // nothing waits on the default clock.
57 crew = await startCrew(app, {
4458 forgeUrl: servers.url,
4559 localUrl: servers.url,
4660 binary: fake,
@@ -49,136 +63,140 @@ sleep 60
4963 runTimeoutMs: 60_000,
5064 tickMs: 50,
5165 });
52 // Where the run stands, for a failure to name: its state and summary,
53 // whether the clone exists, the tail of its log.
54 const seen = (repoId: string, clone?: string): string => {
55 const runs = listRuns(app.db, repoId, { kind: "agent" });
56 const last = runs[0];
57 let tail = "(no run log)";
58 if (last) {
59 try {
60 tail = readFileSync(join(dataDir, "runs", `${last.id}.log`), "utf8")
61 .split("\n")
62 .slice(-20)
63 .join("\n");
64 } catch {
65 // no log yet
66 }
67 }
68 return `runs=${JSON.stringify(runs.map((r) => [r.reason, r.state, r.summary]))} clone=${clone === undefined ? "n/a" : existsSync(clone)}\nlog tail:\n${tail}`;
69 };
70 const until = async <T>(
71 read: () => T | undefined,
72 what: string,
73 state: () => string,
74 ms = 20_000,
75 ): Promise<T> => {
76 const deadline = Date.now() + ms;
77 while (Date.now() < deadline) {
78 const value = read();
79 if (value !== undefined) return value;
80 await Bun.sleep(10);
81 }
82 throw new Error(`${what}: did not happen in ${ms} ms; saw ${state()}`);
83 };
84 try {
85 const a = await createUser(app.db, { origin: "seed", trusted: true,
86 username: "alice",
87 email: "[email protected]",
88 password: "a-good-passphrase",
89 adminIfFirst: true,
90 });
91 if (!a.ok) throw new Error(a.error);
92 const alice = a.value;
93 const made = await createAgent(app, alice, {
94 username: "bot",
95 brief: "Be brief.",
96 });
97 if (!made.ok) throw new Error(String(made.error));
98 const bot = made.value;
99 const created = await createRepo(app, alice, {
100 name: "garden",
101 visibility: "private",
102 });
103 if (!created.ok) throw new Error(String(created.error));
104 const repo = created.value;
105 const seeded = await commitToBranch(app, alice, repo, {
106 message: "Start",
107 files: [{ path: "README.md", content: "# garden\n" }],
108 });
109 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
110 expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
66 const a = await createUser(app.db, { origin: "seed", trusted: true,
67 username: "alice",
68 email: "[email protected]",
69 password: "a-good-passphrase",
70 adminIfFirst: true,
71 });
72 if (!a.ok) throw new Error(a.error);
73 alice = a.value;
74 const made = await createAgent(app, alice, {
75 username: "bot",
76 brief: "Be brief.",
77 });
78 if (!made.ok) throw new Error(String(made.error));
79 bot = made.value;
80 const created = await createRepo(app, alice, {
81 name: "garden",
82 visibility: "private",
83 });
84 if (!created.ok) throw new Error(String(created.error));
85 repo = created.value;
86 const seeded = await commitToBranch(app, alice, repo, {
87 message: "Start",
88 files: [{ path: "README.md", content: "# garden\n" }],
89 });
90 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
91 if (!addToCrew(app, alice, repo, "bot").ok)
92 throw new Error("bot did not join the crew");
11193
112 // The agent's branch for issue 1 in its fork, its pull, and the
113 // author, the agent.
114 const forked = await forkRepo(app, bot.user, repo);
115 if (!forked.ok) throw new Error(String(forked.error));
116 const branched = await commitToBranch(app, bot.user, forked.value, {
117 branch: "bot/1-tulips",
118 message: "Tulips",
119 files: [{ path: "TULIPS.md", content: "tulips\n" }],
120 });
121 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
122 const opened = await openPull(app, bot.user, repo, {
123 title: "Tulips",
124 baseBranch: repo.defaultBranch,
125 headBranch: "bot/1-tulips",
126 headRepo: forked.value,
127 });
128 if (!opened.ok) throw new Error(String(opened.error));
129 const pull = opened.value;
130 const asIssue = getIssueById(app.db, pull.id);
131 if (!asIssue) throw new Error("the pull is not an issue");
132 // A word on the pull sends its author back to it.
133 expect(
134 addComment(app, alice, repo, asIssue, "also the daffodils").ok,
135 ).toBe(true);
136 const run = await until(
137 () =>
138 listRuns(app.db, repo.id, { kind: "agent" }).find(
139 (r) => r.state === "running",
140 ),
141 "the run starting",
142 () => seen(repo.id),
143 );
144 expect(run.reason).toBe("feedback");
145 const clone = workdirFor(dataDir, bot, repo, 1);
146 await until(() => (existsSync(clone) ? true : undefined), "the clone appearing", () =>
147 seen(repo.id, clone),
148 );
149
150 // The pull closes under it.
151 expect(setIssueState(app, alice, repo, asIssue, "closed").ok).toBe(true);
152 const ended = await until(() => {
153 const r = listRuns(app.db, repo.id, { kind: "agent" }).find(
154 (x) => x.id === run.id,
155 );
156 return r && r.state !== "running" ? r : undefined;
157 }, "the run's end", () => seen(repo.id, clone));
158 expect(ended.state).toBe("cancelled");
159 expect(ended.summary).toBe("the pull closed");
160 // The clone went with the run, once the run was over.
161 await until(
162 () => (existsSync(clone) ? undefined : true),
163 "the clone's removal",
164 () => seen(repo.id, clone),
165 );
166 const log = readFileSync(join(dataDir, "runs", `${run.id}.log`), "utf8");
167 expect(log).not.toContain("the run failed");
168 expect(log).not.toContain("ENOENT");
169 // Nothing queued behind it for a pull that is gone.
170 expect(
171 listRuns(app.db, repo.id, { kind: "agent" }).filter(
172 (r) => r.state === "queued",
173 ),
174 ).toHaveLength(0);
175 } finally {
176 await crew.stop();
177 servers.stop();
178 app.close();
179 await rm(dataDir, { recursive: true, force: true });
180 }
181 // The crew ticks every 50 ms here (`tickMs`), not every five seconds, so
182 // nothing waits on the default clock.
94 // The agent's branch for issue 1 in its fork, its pull, and the
95 // author, the agent.
96 const forked = await forkRepo(app, bot.user, repo);
97 if (!forked.ok) throw new Error(String(forked.error));
98 const branched = await commitToBranch(app, bot.user, forked.value, {
99 branch: "bot/1-tulips",
100 message: "Tulips",
101 files: [{ path: "TULIPS.md", content: "tulips\n" }],
102 });
103 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
104 const opened = await openPull(app, bot.user, repo, {
105 title: "Tulips",
106 baseBranch: repo.defaultBranch,
107 headBranch: "bot/1-tulips",
108 headRepo: forked.value,
109 });
110 if (!opened.ok) throw new Error(String(opened.error));
111 const issue = getIssueById(app.db, opened.value.id);
112 if (!issue) throw new Error("the pull is not an issue");
113 asIssue = issue;
183114 }, 30_000);
115
116 afterAll(async () => {
117 await crew?.stop();
118 servers?.stop();
119 app?.close();
120 if (dataDir) await rm(dataDir, { recursive: true, force: true });
121 });
122
123 // Where the run stands, for a failure to name: its state and summary,
124 // whether the clone exists, the tail of its log.
125 const seen = (clone?: string): string => {
126 const runs = listRuns(app.db, repo.id, { kind: "agent" });
127 const last = runs[0];
128 let tail = "(no run log)";
129 if (last) {
130 try {
131 tail = readFileSync(join(dataDir, "runs", `${last.id}.log`), "utf8")
132 .split("\n")
133 .slice(-20)
134 .join("\n");
135 } catch {
136 // no log yet
137 }
138 }
139 return `runs=${JSON.stringify(runs.map((r) => [r.reason, r.state, r.summary]))} clone=${clone === undefined ? "n/a" : existsSync(clone)}\nlog tail:\n${tail}`;
140 };
141 const until = async <T>(
142 read: () => T | undefined,
143 what: string,
144 state: () => string,
145 ms = 5_000,
146 ): Promise<T> => {
147 const deadline = Date.now() + ms;
148 while (Date.now() < deadline) {
149 const value = read();
150 if (value !== undefined) return value;
151 await Bun.sleep(10);
152 }
153 throw new Error(`${what}: did not happen in ${ms} ms; saw ${state()}`);
154 };
155
156 test("the run is cancelled, and its clone goes after it, not before", async () => {
157 // Invariant: closing a pull under its author's running run cancels the run ("the pull closed"), removes its clone only after the run ends, and queues nothing after it.
158 // A word on the pull sends its author back to it.
159 expect(
160 addComment(app, alice, repo, asIssue, "also the daffodils").ok,
161 ).toBe(true);
162 const run = await until(
163 () =>
164 listRuns(app.db, repo.id, { kind: "agent" }).find(
165 (r) => r.state === "running",
166 ),
167 "the run starting",
168 () => seen(),
169 );
170 expect(run.reason).toBe("feedback");
171 const clone = workdirFor(dataDir, bot, repo, 1);
172 await until(() => (existsSync(clone) ? true : undefined), "the clone appearing", () =>
173 seen(clone),
174 );
175
176 // The pull closes under it.
177 expect(setIssueState(app, alice, repo, asIssue, "closed").ok).toBe(true);
178 const ended = await until(() => {
179 const r = listRuns(app.db, repo.id, { kind: "agent" }).find(
180 (x) => x.id === run.id,
181 );
182 return r && r.state !== "running" ? r : undefined;
183 }, "the run's end", () => seen(clone));
184 expect(ended.state).toBe("cancelled");
185 expect(ended.summary).toBe("the pull closed");
186 // The clone went with the run, once the run was over.
187 await until(
188 () => (existsSync(clone) ? undefined : true),
189 "the clone's removal",
190 () => seen(clone),
191 );
192 const log = readFileSync(join(dataDir, "runs", `${run.id}.log`), "utf8");
193 expect(log).not.toContain("the run failed");
194 expect(log).not.toContain("ENOENT");
195 // Nothing queued behind it for a pull that is gone.
196 expect(
197 listRuns(app.db, repo.id, { kind: "agent" }).filter(
198 (r) => r.state === "queued",
199 ),
200 ).toHaveLength(0);
201 }, 10_000);
184202 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs