Public: anyone can read this; the crew can change it.
test/landing.test.ts runs in under 10 s, every test under 1 s, proving what it proves today #10255
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10240-test-landing-test-ts-runs-in-under-10-s → main 1 commit fork
1 file
changed +168
−150
test/landing.test.ts +168 −150
| @@ -1,8 +1,21 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 1 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 2 | 2 | import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; |
| 3 | 3 | import { mkdtemp, rm } from "node:fs/promises"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { join } from "node:path"; |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 8 | import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts"; | |
| 9 | import { createUser, type User } from "../src/core/users.ts"; | |
| 10 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 11 | import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts"; | |
| 12 | import { addComment, setIssueState } from "../src/actions/issues.ts"; | |
| 13 | import { openPull } from "../src/actions/pulls.ts"; | |
| 14 | import { getIssueById, type Issue } from "../src/core/issues.ts"; | |
| 15 | import { listRuns } from "../src/core/runs.ts"; | |
| 16 | import { workdirFor } from "../src/crew/workdir.ts"; | |
| 17 | import type { Agent } from "../src/core/agents.ts"; | |
| 18 | import type { Repo } from "../src/core/repos.ts"; | |
| 6 | 19 | |
| 7 | 20 | /** |
| 8 | 21 | * A pull that lands or closes while an agent still works on it: the run is |
| @@ -10,25 +23,24 @@ import { join } from "node:path"; | ||
| 10 | 23 | * run is over, never from under a git call. |
| 11 | 24 | */ |
| 12 | 25 | describe("a pull closed under a running agent", () => { |
| 13 | test("the run is cancelled, and its clone goes after it, not before", async () => { | |
| 14 | const { createApp } = await import("../src/app.ts"); | |
| 15 | const { serve } = await import("../src/http/server.ts"); | |
| 16 | const { startCrew } = await import("../src/crew/dispatch.ts"); | |
| 17 | const { createUser } = await import("../src/core/users.ts"); | |
| 18 | const { addToCrew, createAgent } = await import("../src/actions/crew.ts"); | |
| 19 | const { commitToBranch, createRepo, forkRepo } = | |
| 20 | await import("../src/actions/repos.ts"); | |
| 21 | const { addComment, setIssueState } = | |
| 22 | await import("../src/actions/issues.ts"); | |
| 23 | const { openPull } = await import("../src/actions/pulls.ts"); | |
| 24 | const { getIssueById } = await import("../src/core/issues.ts"); | |
| 25 | const { listRuns } = await import("../src/core/runs.ts"); | |
| 26 | const { workdirFor } = await import("../src/crew/workdir.ts"); | |
| 26 | // Built once for the file: the forge, its crew, and the agent's pull in | |
| 27 | // its fork. Starting the crew and forking are the slow part, and the test | |
| 28 | // is about neither. | |
| 29 | let dataDir: string; | |
| 30 | let app: App; | |
| 31 | let servers: Servers; | |
| 32 | let crew: CrewHandle; | |
| 33 | let alice: User; | |
| 34 | let bot: Agent; | |
| 35 | let repo: Repo; | |
| 36 | let asIssue: Issue; | |
| 27 | 37 | |
| 28 | const dataDir = await mkdtemp(join(tmpdir(), "forge-landing-")); | |
| 29 | const app = createApp({ dataDir, port: 0 }); | |
| 30 | const servers = await serve(app); | |
| 31 | // An agent that settles in for a long think. | |
| 38 | beforeAll(async () => { | |
| 39 | dataDir = await mkdtemp(join(tmpdir(), "forge-landing-")); | |
| 40 | app = createApp({ dataDir, port: 0 }); | |
| 41 | servers = await serve(app); | |
| 42 | // An agent that settles in for a long think: it never ends on its own, | |
| 43 | // only when the crew stops it (the whole process group goes). | |
| 32 | 44 | const fake = join(dataDir, "fake-claude"); |
| 33 | 45 | writeFileSync( |
| 34 | 46 | fake, |
| @@ -40,7 +52,9 @@ sleep 60 | ||
| 40 | 52 | `, |
| 41 | 53 | ); |
| 42 | 54 | chmodSync(fake, 0o755); |
| 43 | const crew = await startCrew(app, { | |
| 55 | // The crew ticks every 50 ms (`tickMs`), not every five seconds, so | |
| 56 | // nothing waits on the default clock. | |
| 57 | crew = await startCrew(app, { | |
| 44 | 58 | forgeUrl: servers.url, |
| 45 | 59 | localUrl: servers.url, |
| 46 | 60 | binary: fake, |
| @@ -49,136 +63,140 @@ sleep 60 | ||
| 49 | 63 | runTimeoutMs: 60_000, |
| 50 | 64 | tickMs: 50, |
| 51 | 65 | }); |
| 52 | // Where the run stands, for a failure to name: its state and summary, | |
| 53 | // whether the clone exists, the tail of its log. | |
| 54 | const seen = (repoId: string, clone?: string): string => { | |
| 55 | const runs = listRuns(app.db, repoId, { kind: "agent" }); | |
| 56 | const last = runs[0]; | |
| 57 | let tail = "(no run log)"; | |
| 58 | if (last) { | |
| 59 | try { | |
| 60 | tail = readFileSync(join(dataDir, "runs", `${last.id}.log`), "utf8") | |
| 61 | .split("\n") | |
| 62 | .slice(-20) | |
| 63 | .join("\n"); | |
| 64 | } catch { | |
| 65 | // no log yet | |
| 66 | } | |
| 67 | } | |
| 68 | return `runs=${JSON.stringify(runs.map((r) => [r.reason, r.state, r.summary]))} clone=${clone === undefined ? "n/a" : existsSync(clone)}\nlog tail:\n${tail}`; | |
| 69 | }; | |
| 70 | const until = async <T>( | |
| 71 | read: () => T | undefined, | |
| 72 | what: string, | |
| 73 | state: () => string, | |
| 74 | ms = 20_000, | |
| 75 | ): Promise<T> => { | |
| 76 | const deadline = Date.now() + ms; | |
| 77 | while (Date.now() < deadline) { | |
| 78 | const value = read(); | |
| 79 | if (value !== undefined) return value; | |
| 80 | await Bun.sleep(10); | |
| 81 | } | |
| 82 | throw new Error(`${what}: did not happen in ${ms} ms; saw ${state()}`); | |
| 83 | }; | |
| 84 | try { | |
| 85 | const a = await createUser(app.db, { origin: "seed", trusted: true, | |
| 86 | username: "alice", | |
| 87 | email: "[email protected]", | |
| 88 | password: "a-good-passphrase", | |
| 89 | adminIfFirst: true, | |
| 90 | }); | |
| 91 | if (!a.ok) throw new Error(a.error); | |
| 92 | const alice = a.value; | |
| 93 | const made = await createAgent(app, alice, { | |
| 94 | username: "bot", | |
| 95 | brief: "Be brief.", | |
| 96 | }); | |
| 97 | if (!made.ok) throw new Error(String(made.error)); | |
| 98 | const bot = made.value; | |
| 99 | const created = await createRepo(app, alice, { | |
| 100 | name: "garden", | |
| 101 | visibility: "private", | |
| 102 | }); | |
| 103 | if (!created.ok) throw new Error(String(created.error)); | |
| 104 | const repo = created.value; | |
| 105 | const seeded = await commitToBranch(app, alice, repo, { | |
| 106 | message: "Start", | |
| 107 | files: [{ path: "README.md", content: "# garden\n" }], | |
| 108 | }); | |
| 109 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 110 | expect(addToCrew(app, alice, repo, "bot").ok).toBe(true); | |
| 66 | const a = await createUser(app.db, { origin: "seed", trusted: true, | |
| 67 | username: "alice", | |
| 68 | email: "[email protected]", | |
| 69 | password: "a-good-passphrase", | |
| 70 | adminIfFirst: true, | |
| 71 | }); | |
| 72 | if (!a.ok) throw new Error(a.error); | |
| 73 | alice = a.value; | |
| 74 | const made = await createAgent(app, alice, { | |
| 75 | username: "bot", | |
| 76 | brief: "Be brief.", | |
| 77 | }); | |
| 78 | if (!made.ok) throw new Error(String(made.error)); | |
| 79 | bot = made.value; | |
| 80 | const created = await createRepo(app, alice, { | |
| 81 | name: "garden", | |
| 82 | visibility: "private", | |
| 83 | }); | |
| 84 | if (!created.ok) throw new Error(String(created.error)); | |
| 85 | repo = created.value; | |
| 86 | const seeded = await commitToBranch(app, alice, repo, { | |
| 87 | message: "Start", | |
| 88 | files: [{ path: "README.md", content: "# garden\n" }], | |
| 89 | }); | |
| 90 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 91 | if (!addToCrew(app, alice, repo, "bot").ok) | |
| 92 | throw new Error("bot did not join the crew"); | |
| 111 | 93 | |
| 112 | // The agent's branch for issue 1 in its fork, its pull, and the | |
| 113 | // author, the agent. | |
| 114 | const forked = await forkRepo(app, bot.user, repo); | |
| 115 | if (!forked.ok) throw new Error(String(forked.error)); | |
| 116 | const branched = await commitToBranch(app, bot.user, forked.value, { | |
| 117 | branch: "bot/1-tulips", | |
| 118 | message: "Tulips", | |
| 119 | files: [{ path: "TULIPS.md", content: "tulips\n" }], | |
| 120 | }); | |
| 121 | if (!branched.ok) throw new Error(JSON.stringify(branched.error)); | |
| 122 | const opened = await openPull(app, bot.user, repo, { | |
| 123 | title: "Tulips", | |
| 124 | baseBranch: repo.defaultBranch, | |
| 125 | headBranch: "bot/1-tulips", | |
| 126 | headRepo: forked.value, | |
| 127 | }); | |
| 128 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 129 | const pull = opened.value; | |
| 130 | const asIssue = getIssueById(app.db, pull.id); | |
| 131 | if (!asIssue) throw new Error("the pull is not an issue"); | |
| 132 | // A word on the pull sends its author back to it. | |
| 133 | expect( | |
| 134 | addComment(app, alice, repo, asIssue, "also the daffodils").ok, | |
| 135 | ).toBe(true); | |
| 136 | const run = await until( | |
| 137 | () => | |
| 138 | listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 139 | (r) => r.state === "running", | |
| 140 | ), | |
| 141 | "the run starting", | |
| 142 | () => seen(repo.id), | |
| 143 | ); | |
| 144 | expect(run.reason).toBe("feedback"); | |
| 145 | const clone = workdirFor(dataDir, bot, repo, 1); | |
| 146 | await until(() => (existsSync(clone) ? true : undefined), "the clone appearing", () => | |
| 147 | seen(repo.id, clone), | |
| 148 | ); | |
| 149 | ||
| 150 | // The pull closes under it. | |
| 151 | expect(setIssueState(app, alice, repo, asIssue, "closed").ok).toBe(true); | |
| 152 | const ended = await until(() => { | |
| 153 | const r = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 154 | (x) => x.id === run.id, | |
| 155 | ); | |
| 156 | return r && r.state !== "running" ? r : undefined; | |
| 157 | }, "the run's end", () => seen(repo.id, clone)); | |
| 158 | expect(ended.state).toBe("cancelled"); | |
| 159 | expect(ended.summary).toBe("the pull closed"); | |
| 160 | // The clone went with the run, once the run was over. | |
| 161 | await until( | |
| 162 | () => (existsSync(clone) ? undefined : true), | |
| 163 | "the clone's removal", | |
| 164 | () => seen(repo.id, clone), | |
| 165 | ); | |
| 166 | const log = readFileSync(join(dataDir, "runs", `${run.id}.log`), "utf8"); | |
| 167 | expect(log).not.toContain("the run failed"); | |
| 168 | expect(log).not.toContain("ENOENT"); | |
| 169 | // Nothing queued behind it for a pull that is gone. | |
| 170 | expect( | |
| 171 | listRuns(app.db, repo.id, { kind: "agent" }).filter( | |
| 172 | (r) => r.state === "queued", | |
| 173 | ), | |
| 174 | ).toHaveLength(0); | |
| 175 | } finally { | |
| 176 | await crew.stop(); | |
| 177 | servers.stop(); | |
| 178 | app.close(); | |
| 179 | await rm(dataDir, { recursive: true, force: true }); | |
| 180 | } | |
| 181 | // The crew ticks every 50 ms here (`tickMs`), not every five seconds, so | |
| 182 | // nothing waits on the default clock. | |
| 94 | // The agent's branch for issue 1 in its fork, its pull, and the | |
| 95 | // author, the agent. | |
| 96 | const forked = await forkRepo(app, bot.user, repo); | |
| 97 | if (!forked.ok) throw new Error(String(forked.error)); | |
| 98 | const branched = await commitToBranch(app, bot.user, forked.value, { | |
| 99 | branch: "bot/1-tulips", | |
| 100 | message: "Tulips", | |
| 101 | files: [{ path: "TULIPS.md", content: "tulips\n" }], | |
| 102 | }); | |
| 103 | if (!branched.ok) throw new Error(JSON.stringify(branched.error)); | |
| 104 | const opened = await openPull(app, bot.user, repo, { | |
| 105 | title: "Tulips", | |
| 106 | baseBranch: repo.defaultBranch, | |
| 107 | headBranch: "bot/1-tulips", | |
| 108 | headRepo: forked.value, | |
| 109 | }); | |
| 110 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 111 | const issue = getIssueById(app.db, opened.value.id); | |
| 112 | if (!issue) throw new Error("the pull is not an issue"); | |
| 113 | asIssue = issue; | |
| 183 | 114 | }, 30_000); |
| 115 | ||
| 116 | afterAll(async () => { | |
| 117 | await crew?.stop(); | |
| 118 | servers?.stop(); | |
| 119 | app?.close(); | |
| 120 | if (dataDir) await rm(dataDir, { recursive: true, force: true }); | |
| 121 | }); | |
| 122 | ||
| 123 | // Where the run stands, for a failure to name: its state and summary, | |
| 124 | // whether the clone exists, the tail of its log. | |
| 125 | const seen = (clone?: string): string => { | |
| 126 | const runs = listRuns(app.db, repo.id, { kind: "agent" }); | |
| 127 | const last = runs[0]; | |
| 128 | let tail = "(no run log)"; | |
| 129 | if (last) { | |
| 130 | try { | |
| 131 | tail = readFileSync(join(dataDir, "runs", `${last.id}.log`), "utf8") | |
| 132 | .split("\n") | |
| 133 | .slice(-20) | |
| 134 | .join("\n"); | |
| 135 | } catch { | |
| 136 | // no log yet | |
| 137 | } | |
| 138 | } | |
| 139 | return `runs=${JSON.stringify(runs.map((r) => [r.reason, r.state, r.summary]))} clone=${clone === undefined ? "n/a" : existsSync(clone)}\nlog tail:\n${tail}`; | |
| 140 | }; | |
| 141 | const until = async <T>( | |
| 142 | read: () => T | undefined, | |
| 143 | what: string, | |
| 144 | state: () => string, | |
| 145 | ms = 5_000, | |
| 146 | ): Promise<T> => { | |
| 147 | const deadline = Date.now() + ms; | |
| 148 | while (Date.now() < deadline) { | |
| 149 | const value = read(); | |
| 150 | if (value !== undefined) return value; | |
| 151 | await Bun.sleep(10); | |
| 152 | } | |
| 153 | throw new Error(`${what}: did not happen in ${ms} ms; saw ${state()}`); | |
| 154 | }; | |
| 155 | ||
| 156 | test("the run is cancelled, and its clone goes after it, not before", async () => { | |
| 157 | // Invariant: closing a pull under its author's running run cancels the run ("the pull closed"), removes its clone only after the run ends, and queues nothing after it. | |
| 158 | // A word on the pull sends its author back to it. | |
| 159 | expect( | |
| 160 | addComment(app, alice, repo, asIssue, "also the daffodils").ok, | |
| 161 | ).toBe(true); | |
| 162 | const run = await until( | |
| 163 | () => | |
| 164 | listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 165 | (r) => r.state === "running", | |
| 166 | ), | |
| 167 | "the run starting", | |
| 168 | () => seen(), | |
| 169 | ); | |
| 170 | expect(run.reason).toBe("feedback"); | |
| 171 | const clone = workdirFor(dataDir, bot, repo, 1); | |
| 172 | await until(() => (existsSync(clone) ? true : undefined), "the clone appearing", () => | |
| 173 | seen(clone), | |
| 174 | ); | |
| 175 | ||
| 176 | // The pull closes under it. | |
| 177 | expect(setIssueState(app, alice, repo, asIssue, "closed").ok).toBe(true); | |
| 178 | const ended = await until(() => { | |
| 179 | const r = listRuns(app.db, repo.id, { kind: "agent" }).find( | |
| 180 | (x) => x.id === run.id, | |
| 181 | ); | |
| 182 | return r && r.state !== "running" ? r : undefined; | |
| 183 | }, "the run's end", () => seen(clone)); | |
| 184 | expect(ended.state).toBe("cancelled"); | |
| 185 | expect(ended.summary).toBe("the pull closed"); | |
| 186 | // The clone went with the run, once the run was over. | |
| 187 | await until( | |
| 188 | () => (existsSync(clone) ? undefined : true), | |
| 189 | "the clone's removal", | |
| 190 | () => seen(clone), | |
| 191 | ); | |
| 192 | const log = readFileSync(join(dataDir, "runs", `${run.id}.log`), "utf8"); | |
| 193 | expect(log).not.toContain("the run failed"); | |
| 194 | expect(log).not.toContain("ENOENT"); | |
| 195 | // Nothing queued behind it for a pull that is gone. | |
| 196 | expect( | |
| 197 | listRuns(app.db, repo.id, { kind: "agent" }).filter( | |
| 198 | (r) => r.state === "queued", | |
| 199 | ), | |
| 200 | ).toHaveLength(0); | |
| 201 | }, 10_000); | |
| 184 | 202 | }); |