trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns6g rEventsFlows

test/checks.test.ts runs in under 10 s, split where it covers more than one concept #10878

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/10866-test-checks-test-ts-runs-in-under-10-s-s → main 2 commits fork

  • Merged. mason/10866-test-checks-test-ts-runs-in-under-10-s-s was merged into main as df3c612.

Review

Sign in to review this change.
6 files changed +1222 −1127
scripts/test-times.json +4 −1
@@ -14,7 +14,10 @@
1414 "test/asks.test.ts": 7.3,
1515 "test/autonomy.test.ts": 18,
1616 "test/board-windows.test.ts": 6,
17 "test/checks.test.ts": 26.3,
17 "test/checks.test.ts": 2.7,
18 "test/checks-failure-summary.test.ts": 0.2,
19 "test/checks-queue.test.ts": 2.3,
20 "test/checks-restart.test.ts": 1.9,
1821 "test/cloud.test.ts": 7.1,
1922 "test/crew-already-done.test.ts": 1.2,
2023 "test/crew-check-failed.test.ts": 1.8,
test/checks-failure-summary.test.ts added +277
@@ -0,0 +1,277 @@
1 /**
2 * What a failed check's summary says: `parseBunTestFailure` reading bun
3 * test's own tail, and `summariseCheckFailure` choosing the line a person
4 * reads first, a sharded gate's log included. Plain lines in, plain data out.
5 */
6
7 import { describe, expect, test } from "bun:test";
8 import { parseBunTestFailure } from "../src/checks/bun-test.ts";
9 import { summariseCheckFailure } from "../src/checks/summary.ts";
10 import { SLOW_SECONDS, summaryLines } from "../scripts/gate-summary.ts";
11 import { tally } from "../scripts/shards.ts";
12
13 describe("parseBunTestFailure: a convenience over one known runner, not a general protocol", () => {
14 test("bun test's own tail: the counts, and the failing names bounded to a few", () => {
15 const lines = [
16 "bun test v1.3.14 (0d9b296a)",
17 "",
18 "check.test.ts:",
19 "(fail) first [0.16ms]",
20 "(fail) second [0.07ms]",
21 "(fail) third [0.05ms]",
22 "(fail) fourth [0.04ms]",
23 "",
24 " 3 pass",
25 " 4 fail",
26 " 7 expect() calls",
27 "Ran 7 tests across 1 file. [51.00ms]",
28 ];
29 expect(parseBunTestFailure(lines)).toEqual({
30 passed: 3,
31 failed: 4,
32 names: ["first", "second", "third"],
33 });
34 });
35
36 test("a name arbitrary text from a subprocess, kept verbatim — escaping is the page's job, not the parser's", () => {
37 const lines = [
38 "(fail) <script>alert(1)</script> [1.00ms]",
39 " 0 pass",
40 " 1 fail",
41 "Ran 1 test across 1 file. [1.00ms]",
42 ];
43 expect(parseBunTestFailure(lines)?.names).toEqual([
44 "<script>alert(1)</script>",
45 ]);
46 });
47
48 test("a failing test bun printed without a duration — as it does under CI for one that took no time — is still named", () => {
49 const lines = [
50 "(pass) a passing test",
51 "(fail) first failing test [0.10ms]",
52 "(fail) second failing test",
53 " 1 pass",
54 " 2 fail",
55 "Ran 3 tests across 1 file. [9.00ms]",
56 ];
57 expect(parseBunTestFailure(lines)).toEqual({
58 passed: 1,
59 failed: 2,
60 names: ["first failing test", "second failing test"],
61 });
62 });
63
64 test("a shard's named failure leads; the (unnamed) fallout behind it is not named", () => {
65 const lines = [
66 "── shard 1/1 (exit 1, 3s): test/a.test.ts",
67 "test/a.test.ts:",
68 "(fail) a > writes the row [2ms]",
69 "error: database is closed",
70 "(fail) (unnamed) [1ms]",
71 " 4 pass",
72 " 2 fail",
73 ];
74 expect(parseBunTestFailure(lines)?.names).toEqual(["a > writes the row"]);
75 // Only a shard that named nothing else is named by its fallout.
76 const only = ["test/b.test.ts:", "error: boom", "(fail) (unnamed)", " 0 pass", " 1 fail"];
77 expect(parseBunTestFailure(only)?.names).toEqual(["test/b.test.ts > (unnamed): error: boom"]);
78 });
79
80 test("shards that finished out of order are named in shard order, each shard's first failure first", () => {
81 const lines = [
82 "── shard 3/3 (exit 1, 2s): test/c.test.ts",
83 "test/c.test.ts:",
84 "(fail) c one [1ms]",
85 "(fail) c two [1ms]",
86 " 1 pass",
87 " 2 fail",
88 "── shard 1/3 (exit 1, 9s): test/a.test.ts",
89 "test/a.test.ts:",
90 "error: boom",
91 "(fail) (unnamed) [1ms]",
92 " 3 pass",
93 " 1 fail",
94 "── shard 2/3 (exit 0, 5s): test/b.test.ts",
95 " 7 pass",
96 " 0 fail",
97 "2 of 3 shards failed:",
98 "(fail) test/a.test.ts > (unnamed): error: boom [1ms] [shard 1]",
99 "(fail) c one [1ms] [shard 3]",
100 "(fail) c two [1ms] [shard 3]",
101 ];
102 expect(parseBunTestFailure(lines)).toEqual({
103 passed: 11,
104 failed: 3,
105 names: ["test/a.test.ts > (unnamed): error: boom", "c one", "c two"],
106 });
107 });
108
109 test("output that is not bun test's own shape is not guessed at", () => {
110 expect(
111 parseBunTestFailure([
112 "$ test -f NOPE.md",
113 "test: NOPE.md: No such file or directory",
114 ]),
115 ).toBeUndefined();
116 // A `pass` line with no `fail` line beside it is not this shape either.
117 expect(
118 parseBunTestFailure([" 5 pass", "Ran 5 tests across 1 file."]),
119 ).toBeUndefined();
120 // All green: nothing failed, so there is nothing to summarise.
121 expect(
122 parseBunTestFailure([" 5 pass", " 0 fail", "Ran 5 tests across 1 file."]),
123 ).toBeUndefined();
124 });
125 });
126
127 describe("summariseCheckFailure: what a failed check's summary says first", () => {
128 test("real bun test output with one failure yields the name and the counts", () => {
129 const lines = [
130 "bun test v1.3.14 (0d9b296a)",
131 "",
132 "check.test.ts:",
133 "1 tests failed:",
134 "(fail) the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed [30000.49ms]",
135 " ^ this test timed out after 30000ms.",
136 "",
137 " 409 pass",
138 " 1 fail",
139 "Ran 410 tests across 1 file. [30.05s]",
140 ];
141 expect(summariseCheckFailure(lines, "check exited with code 1")).toEqual({
142 summary:
143 "1 of 410 tests — \"the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed\"",
144 detail: {
145 passed: 409,
146 failed: 1,
147 names: [
148 "the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed",
149 ],
150 },
151 });
152 });
153
154 test("output with no bun test shape yields the last non-empty line, not the bare exit code", () => {
155 const lines = ["src/foo.ts(10,5): error TS2345: not assignable", ""];
156 expect(summariseCheckFailure(lines, "check exited with code 2")).toEqual({
157 summary: "src/foo.ts(10,5): error TS2345: not assignable",
158 detail: undefined,
159 });
160 });
161
162 test("no output at all falls back to the bare exit code — nothing was there to name", () => {
163 expect(summariseCheckFailure([], "check exited with code 1")).toEqual({
164 summary: "check exited with code 1",
165 detail: undefined,
166 });
167 });
168
169 const failing = (n: number) => [
170 ...Array.from({ length: n }, (_, i) => `(fail) t${i + 1} [1.00ms]`),
171 " 5 pass",
172 ` ${n} fail`,
173 ];
174
175 test("three failures name all three; five name three and how many more", () => {
176 expect(summariseCheckFailure(failing(3), "x").summary).toBe(
177 '3 of 8 tests — "t1", "t2" and "t3"',
178 );
179 expect(summariseCheckFailure(failing(5), "x").summary).toBe(
180 '5 of 10 tests — "t1", "t2", "t3" and 2 more',
181 );
182 });
183
184 test("a name list longer than the limit is still bounded to one line", () => {
185 const long = "y".repeat(100);
186 const lines = ["a", "b", "c", "d"].map((c) => `(fail) ${c}${long} [1.00ms]`);
187 const { summary } = summariseCheckFailure([...lines, " 0 pass", " 4 fail"], "x");
188 expect(summary.length).toBe(200);
189 expect(summary.endsWith("…")).toBe(true);
190 });
191
192 test("a very long name is bounded to one line", () => {
193 const long = "x".repeat(500);
194 const lines = [`(fail) ${long} [1.00ms]`, " 0 pass", " 1 fail"];
195 const { summary } = summariseCheckFailure(
196 lines,
197 "check exited with code 1",
198 );
199 expect(summary.length).toBeLessThanOrEqual(200);
200 expect(summary.endsWith("…")).toBe(true);
201 });
202 });
203
204 describe("summariseCheckFailure: a red gate is summarised by its red test, never a warning line (#2271)", () => {
205 // The sharded gate's log, as `scripts/test-shards.ts` prints it: each
206 // shard's own bun test output whole, the red one first, then the tally,
207 // then the gate summary block with its slow-file warning.
208 const red =
209 "bun test v1.3.14\n\ntest/standards.test.ts:\n(pass) the README names every script [1.00ms]\n" +
210 "(fail) releases.json is the shipped releases > every entry is a release [3.00ms]\n\n" +
211 " 11 pass\n 1 fail\n 40 expect() calls\nRan 12 tests across 1 file. [80.00ms]\n";
212 const slow =
213 "bun test v1.3.14\n\ntest/crew.test.ts:\n(pass) a crew is listed [2.00ms]\n\n" +
214 " 30 pass\n 0 fail\n 90 expect() calls\nRan 30 tests across 1 file. [52.40s]\n";
215 const results = [
216 { files: ["test/standards.test.ts"], code: 1, output: red, seconds: 1.2 },
217 { files: ["test/crew.test.ts"], code: 0, output: slow, seconds: 52.5 },
218 ];
219 const block = (times: { file: string; seconds: number }[]): string[] =>
220 summaryLines({
221 shards: 2,
222 cpus: 16,
223 shardsMax: 16,
224 files: 2,
225 override: undefined,
226 memoryBytes: 8 * 1024 ** 3,
227 workerBytes: 1024 ** 3,
228 memory: { limit: undefined, peak: undefined },
229 wallSeconds: 52.6,
230 times,
231 });
232 const log = (shards: typeof results): string[] => [
233 ...shards.flatMap((r, i) => [`── shard ${i + 1}/2 (exit ${r.code}, ${r.seconds}s): ${r.files[0]}`, ...r.output.split("\n")]),
234 ...tally(shards, 52.6).lines,
235 "",
236 ...block(shards.map((r) => ({ file: r.files[0]!, seconds: r.seconds }))),
237 ];
238
239 test("one red file and one slow file: the summary is the red file's test", () => {
240 const lines = log(results);
241 expect(lines.at(-1)).toBe(`test-shards: over ${SLOW_SECONDS} s: test/crew.test.ts (52.5s)`);
242 expect(summariseCheckFailure(lines, "check exited with code 1")).toEqual({
243 summary: '1 of 42 tests — "releases.json is the shipped releases > every entry is a release"',
244 detail: {
245 passed: 41,
246 failed: 1,
247 names: ["releases.json is the shipped releases > every entry is a release"],
248 },
249 });
250 });
251
252 test("a red shard that named no test is named by its files, not by the gate summary block", () => {
253 const crashed = [
254 { files: ["test/standards.test.ts"], code: 137, output: "bun test v1.3.14\n", seconds: 1.2 },
255 results[1]!,
256 ];
257 expect(summariseCheckFailure(log(crashed), "check exited with code 1").summary).toBe(
258 "shard 1 exited 137 without naming a test: test/standards.test.ts",
259 );
260 });
261
262 test("a killed shard's line is the summary, and the old line is still recognised", () => {
263 expect(
264 summariseCheckFailure(["shard 258 killed after 412 s, running test/x.test.ts"], "check exited with code 1").summary,
265 ).toBe("shard 258 killed after 412 s, running test/x.test.ts");
266 expect(
267 summariseCheckFailure(["shard 258 exited 143 without naming a test: a b"], "check exited with code 1").summary,
268 ).toBe("shard 258 exited 143 without naming a test: a b");
269 });
270
271 test("without names or a tally, the last line before the gate summary block, never a line of it", () => {
272 const lines = ["test-shards: something went wrong", "", ...block([{ file: "test/crew.test.ts", seconds: 52.5 }])];
273 expect(summariseCheckFailure(lines, "check exited with code 1").summary).toBe(
274 "test-shards: something went wrong",
275 );
276 });
277 });
test/checks-queue.test.ts added +381
@@ -0,0 +1,381 @@
1 /**
2 * The check lane: one slot, what waits in it and what leaves it. A newer
3 * commit supersedes a waiting check, a new head stops its old head's, a pull
4 * that lands or closes leaves nothing behind, and a freed slot takes the
5 * newest (`pickNext`) unless one has starved.
6 */
7
8 import { describe, expect, test } from "bun:test";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp } from "../src/app.ts";
13 import { mergePull, openPull } from "../src/actions/pulls.ts";
14 import { setIssueState } from "../src/actions/issues.ts";
15 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
16 import { holdsLastSlot, pickNext, STARVATION_MS } from "../src/checks/service.ts";
17 import {
18 cancelRun,
19 cancelRunsForPull,
20 claimRun,
21 createRun,
22 getRun,
23 listRuns,
24 type Run,
25 } from "../src/core/runs.ts";
26 import { createUser } from "../src/core/users.ts";
27 import { readRunLog } from "../src/runs/log.ts";
28 import { checkJson, checksForge } from "./helpers/checks-forge.ts";
29
30 describe("cancelRunsForPull: a landed or closed pull leaves nothing in the lane", () => {
31 test("queued and running checks and queued deploys of that pull are cancelled, building deploys and other pulls kept", async () => {
32 // Invariant: cancelRunsForPull cancels that pull's queued and running checks and queued deploys, and nothing else.
33 const dir = await mkdtemp(join(tmpdir(), "forge-cancel-"));
34 const db = createApp({ dataDir: dir, port: 0 });
35 try {
36 const u = await createUser(db.db, {
37 origin: "seed",
38 trusted: true,
39 username: "alice",
40 email: "[email protected]",
41 password: "a-good-passphrase",
42 adminIfFirst: true,
43 });
44 if (!u.ok) throw new Error(u.error);
45 const made = await createRepo(db, u.value, { name: "lib" });
46 if (!made.ok) throw new Error(String(made.error));
47 const repoId = made.value.id;
48 const first = await commitToBranch(db, u.value, made.value, {
49 message: "Start",
50 files: [{ path: "README.md", content: "x\n" }],
51 });
52 if (!first.ok) throw new Error(JSON.stringify(first.error));
53 const pullOn = async (branch: string): Promise<string> => {
54 const c = await commitToBranch(db, u.value, made.value, {
55 branch,
56 message: branch,
57 files: [{ path: `${branch}.md`, content: "x\n" }],
58 });
59 if (!c.ok) throw new Error(JSON.stringify(c.error));
60 const o = await openPull(db, u.value, made.value, {
61 title: branch,
62 baseBranch: "main",
63 headBranch: branch,
64 });
65 if (!o.ok) throw new Error(String(o.error));
66 return o.value.id;
67 };
68 const p1 = await pullOn("one");
69 const p2 = await pullOn("two");
70 const mk = (kind: "check" | "deploy", issueId: string) =>
71 createRun(db.db, { repoId, kind, env: "pull-1", issueId });
72 const queued = mk("check", p1);
73 const running = mk("check", p1);
74 claimRun(db.db, running.id);
75 const deploy = mk("deploy", p1);
76 const building = mk("deploy", p1);
77 claimRun(db.db, building.id);
78 const other = mk("check", p2);
79 const stopped = cancelRunsForPull(db.db, p1, "pull landed");
80 expect(stopped.map((r) => r.id).sort()).toEqual(
81 [queued.id, running.id, deploy.id].sort(),
82 );
83 // A building deploy is the runtime's to stop; the row must not say otherwise.
84 expect(getRun(db.db, building.id)?.state).toBe("running");
85 for (const r of [queued, running, deploy]) {
86 expect(getRun(db.db, r.id)?.state).toBe("cancelled");
87 expect(getRun(db.db, r.id)?.summary).toBe("pull landed");
88 }
89 expect(getRun(db.db, other.id)?.state).toBe("queued");
90 } finally {
91 db.close();
92 await rm(dir, { recursive: true, force: true });
93 }
94 });
95 });
96
97 describe("checks", () => {
98 const f = checksForge("forge-checks-queue-");
99
100 test("a newer commit supersedes a check still waiting for the same environment", async () => {
101 // Invariant: a queued check is cancelled as superseded when a newer commit asks for the same environment.
102 const slow = await commitToBranch(f.app, f.alice, f.repo, {
103 message: "A slow check",
104 files: [{ path: "forge.json", content: checkJson(f.held("supersede")) }],
105 });
106 if (!slow.ok) throw new Error(JSON.stringify(slow.error));
107 // While that one holds the only slot, two more arrive: the first of
108 // them is superseded by the second before it ever runs.
109 const one = await f.checks.request(
110 f.repo,
111 "main",
112 slow.value.commitOid,
113 "push",
114 );
115 expect(one).toBeDefined();
116 const two = await commitToBranch(f.app, f.alice, f.repo, {
117 message: "Two",
118 files: [{ path: "TWO.md", content: "2\n" }],
119 });
120 if (!two.ok) throw new Error(JSON.stringify(two.error));
121 await f.until(() => f.checksOf(two.value.commitOid).length > 0, "two queued");
122 const three = await commitToBranch(f.app, f.alice, f.repo, {
123 message: "Three",
124 files: [{ path: "THREE.md", content: "3\n" }],
125 });
126 if (!three.ok) throw new Error(JSON.stringify(three.error));
127 await f.until(() => f.checksOf(three.value.commitOid).length > 0, "three queued");
128 f.release("supersede");
129 const last = await f.settled((r) => r.commitOid === three.value.commitOid);
130 expect(last.state).toBe("ok");
131 const middle = f.checksOf(two.value.commitOid)[0];
132 expect(middle?.state).toBe("cancelled");
133 expect(middle?.summary).toContain("superseded by");
134 });
135
136 test("a new head on a pull stops its running check for the old head, superseded not failed", async () => {
137 // Invariant: a pull's new head cuts off the running check of its old head, which ends cancelled as superseded.
138 const slow = await commitToBranch(f.app, f.alice, f.repo, {
139 branch: "supersede",
140 message: "A check that never ends on its own",
141 files: [
142 { path: "forge.json", content: checkJson(f.held("never")) },
143 { path: "S.md", content: "s\n" },
144 ],
145 });
146 if (!slow.ok) throw new Error(JSON.stringify(slow.error));
147 const opened = await openPull(f.app, f.alice, f.repo, {
148 title: "supersede",
149 baseBranch: "main",
150 headBranch: "supersede",
151 });
152 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
153 const env = `pull-${opened.value.number}`;
154 const old = await f.checks.request(
155 f.repo,
156 env,
157 slow.value.commitOid,
158 "push",
159 opened.value.id,
160 );
161 if (!old) throw new Error("no run");
162 await f.until(() => getRun(f.app.db, old.id)?.state === "running", "old head running");
163 const next = await commitToBranch(f.app, f.alice, f.repo, {
164 branch: "supersede",
165 message: "Fast",
166 files: [
167 { path: "forge.json", content: checkJson("test -f S.md") },
168 ],
169 });
170 if (!next.ok) throw new Error(JSON.stringify(next.error));
171 const fresh = await f.settled((r) => r.commitOid === next.value.commitOid);
172 expect(fresh.state).toBe("ok");
173 const stopped = getRun(f.app.db, old.id);
174 expect(stopped?.state).toBe("cancelled");
175 expect(stopped?.summary).toContain("superseded by");
176 // Stopped, not only marked: the old check, which never ends on its own,
177 // was cut off and wrote its end.
178 await f.until(
179 () => readRunLog(f.dataDir, old.id).lines.join("\n").includes("superseded"),
180 "the old head's check cut off",
181 );
182 });
183
184 for (const ending of ["merge", "close"] as const)
185 test(`${ending === "merge" ? "merging" : "closing"} a pull leaves no queued or running run for it`, async () => {
186 // Invariant: a pull merged or closed has its queued checks cancelled with that reason.
187 const blocker = await commitToBranch(f.app, f.alice, f.repo, {
188 message: "Hold the lane",
189 files: [{ path: "forge.json", content: checkJson(f.held(`lane-${ending}`)) }],
190 });
191 if (!blocker.ok) throw new Error(JSON.stringify(blocker.error));
192 await f.checks.request(f.repo, "lane", blocker.value.commitOid, "push");
193 const branched = await commitToBranch(f.app, f.alice, f.repo, {
194 branch: `lands-${ending}`,
195 message: "Lands before its check runs",
196 files: [{ path: "LANDS.md", content: "x\n" }],
197 });
198 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
199 const opened = await openPull(f.app, f.alice, f.repo, {
200 title: "Lands soon",
201 baseBranch: "main",
202 headBranch: `lands-${ending}`,
203 });
204 if (!opened.ok) throw new Error(String(opened.error));
205 const pull = opened.value;
206 const mine = (): Run[] =>
207 listRuns(f.app.db, f.repo.id, { kind: "check" }).filter(
208 (r) => r.issueId === pull.id,
209 );
210 await f.until(() => mine().length > 0, "the pull's check queued");
211 expect(mine().some((r) => r.state === "queued")).toBe(true);
212 if (ending === "merge") {
213 const landed = await mergePull(f.app, f.alice, f.repo, pull, "merge");
214 if (!landed.ok) throw new Error(String(landed.error));
215 } else {
216 const closed = setIssueState(f.app, f.alice, f.repo, pull, "closed");
217 if (!closed.ok) throw new Error(String(closed.error));
218 }
219 await f.until(() => mine().every((r) => r.state !== "queued"), "the pull's check gone");
220 expect(mine().filter((r) => r.state === "queued")).toEqual([]);
221 expect(
222 mine().some(
223 (r) =>
224 r.summary === (ending === "merge" ? "pull landed" : "pull closed"),
225 ),
226 ).toBe(true);
227 // The lane is let go, and left empty for the next test.
228 f.release(`lane-${ending}`);
229 await f.until(
230 () => f.checksOf(blocker.value.commitOid).every((r) => r.state !== "queued" && r.state !== "running"),
231 "the lane's holder done",
232 );
233 });
234
235 test("a queued gate keeps the last free slot from pull checks", () => {
236 // One slot free, a gate queued: the pull check passes, the gate starts.
237 expect(holdsLastSlot(1, true)).toBe(true);
238 // With no gate queued the pull check takes it, as every slot.
239 expect(holdsLastSlot(1, false)).toBe(false);
240 // More than one free: the pull check starts, one is still left.
241 expect(holdsLastSlot(2, true)).toBe(false);
242 });
243
244 test("pickNext takes the newest queued check, unless one has starved", () => {
245 const a = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "a" });
246 const b = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "b" });
247 const c = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "c" });
248 // ids are creation-ordered, so this is oldest-to-newest already.
249 expect(pickNext([a, b, c])?.id).toBe(c.id);
250 expect(pickNext([c, a, b])?.id).toBe(c.id);
251 expect(pickNext([a])?.id).toBe(a.id);
252 expect(pickNext([])).toBeUndefined();
253
254 // `a` has waited past the starvation bound: it outranks `c`, however new.
255 f.app.db
256 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
257 .run(a.id, new Date(Date.now() - STARVATION_MS - 1_000).toISOString());
258 const starved = getRun(f.app.db, a.id)!;
259 expect(pickNext([starved, b, c])?.id).toBe(starved.id);
260 // Just short of the bound, the newest still wins.
261 f.app.db
262 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
263 .run(a.id, new Date(Date.now() - STARVATION_MS + 1_000).toISOString());
264 expect(pickNext([getRun(f.app.db, a.id)!, b, c])?.id).toBe(c.id);
265
266 // These never named a real commit: cancel them rather than leave them
267 // for the live checks service running beside this test to trip over.
268 for (const run of [a, b, c]) cancelRun(f.app.db, run.id);
269 });
270
271 test("pickNext puts an approved pull's check first, newest among them, and the starved before all", () => {
272 const a = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "a" });
273 const b = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "b" });
274 const c = createRun(f.app.db, { repoId: f.repo.id, kind: "check", env: "c" });
275 const now = Date.parse(c.createdAt);
276 const only = (...ids: string[]) => (run: { id: string }) => ids.includes(run.id);
277 // An older approved check beats a newer unapproved one.
278 expect(pickNext([a, b, c], now, only(a.id))?.id).toBe(a.id);
279 // Newest first among the approved.
280 expect(pickNext([a, b, c], now, only(a.id, b.id))?.id).toBe(b.id);
281 // With none approved, the order is unchanged.
282 expect(pickNext([a, b, c], now, () => false)?.id).toBe(c.id);
283 // A starved unapproved check still goes first.
284 const later = now + STARVATION_MS + 1_000;
285 const old = { ...a, createdAt: new Date(now - 1_000).toISOString() };
286 const fresh = { ...b, createdAt: new Date(later).toISOString() };
287 const approvedFresh = { ...c, createdAt: new Date(later + 1).toISOString() };
288 expect(pickNext([fresh, approvedFresh, old], later, only(approvedFresh.id))?.id).toBe(old.id);
289 for (const run of [a, b, c]) cancelRun(f.app.db, run.id);
290 });
291
292 test("when a slot frees, the newest of several queued checks runs first", async () => {
293 // Invariant: of several checks queued behind a held slot, the newest starts first.
294 // Three checks, on three environments so none supersedes another. Their
295 // commits are written before the slot is taken: written after, on a
296 // loaded machine they can outlast the holder, and the oldest then finds
297 // the slot free instead of queueing.
298 const oldest = await commitToBranch(f.app, f.alice, f.repo, {
299 branch: "order-a",
300 message: "A",
301 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
302 });
303 if (!oldest.ok) throw new Error(JSON.stringify(oldest.error));
304 const middle = await commitToBranch(f.app, f.alice, f.repo, {
305 branch: "order-b",
306 message: "B",
307 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
308 });
309 if (!middle.ok) throw new Error(JSON.stringify(middle.error));
310 const newest = await commitToBranch(f.app, f.alice, f.repo, {
311 branch: "order-c",
312 message: "C",
313 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
314 });
315 if (!newest.ok) throw new Error(JSON.stringify(newest.error));
316
317 // A slow check on its own environment holds the only slot, and the
318 // three queue behind it — oldest requested first.
319 const holding = await commitToBranch(f.app, f.alice, f.repo, {
320 branch: "order-hold",
321 message: "Hold the slot",
322 files: [
323 { path: "forge.json", content: checkJson(f.held("order")) },
324 ],
325 });
326 if (!holding.ok) throw new Error(JSON.stringify(holding.error));
327 const holder = await f.checks.request(
328 f.repo,
329 "order-hold",
330 holding.value.commitOid,
331 "push",
332 );
333 expect(holder).toBeDefined();
334
335 const oldestRun = await f.checks.request(
336 f.repo,
337 "order-a",
338 oldest.value.commitOid,
339 "push",
340 );
341 const middleRun = await f.checks.request(
342 f.repo,
343 "order-b",
344 middle.value.commitOid,
345 "push",
346 );
347 const newestRun = await f.checks.request(
348 f.repo,
349 "order-c",
350 newest.value.commitOid,
351 "push",
352 );
353 expect(oldestRun && middleRun && newestRun).toBeTruthy();
354 expect(oldestRun?.state).toBe("queued");
355 expect(middleRun?.state).toBe("queued");
356 expect(newestRun?.state).toBe("queued");
357
358 f.release("order");
359 await f.settled((r) => r.id === newestRun!.id);
360 await f.settled((r) => r.id === middleRun!.id);
361 await f.settled((r) => r.id === oldestRun!.id);
362
363 const runs = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 });
364 const startedAt = (id: string): number =>
365 Date.parse(runs.find((r) => r.id === id)!.startedAt!);
366 expect(startedAt(newestRun!.id)).toBeLessThan(startedAt(middleRun!.id));
367 expect(startedAt(middleRun!.id)).toBeLessThan(startedAt(oldestRun!.id));
368 });
369 test("a check whose pull lands while it runs ends as superseded, not failed", async () => {
370 // Invariant: a running check whose row is cancelled under it logs superseded with the reason, never failed, even when its command fails.
371 const id = await f.slowCheck(f.held("overtaken", "false"), "A check overtaken");
372 cancelRun(f.app.db, id, "pull landed");
373 f.release("overtaken");
374 const log = (): string => readRunLog(f.dataDir, id).lines.join("\n");
375 await f.until(() => log().includes("superseded"), "superseded in the log");
376 const lines = log();
377 expect(lines).toContain("superseded: pull landed");
378 expect(lines).not.toContain("failed:");
379 expect(getRun(f.app.db, id)?.state).toBe("cancelled");
380 });
381 });
test/checks-restart.test.ts added +338
@@ -0,0 +1,338 @@
1 /**
2 * A check across the forge's own stops: a release that stops the service
3 * (queued again, or drained to its verdict), a person's stop, a boot that
4 * finds an open pull never checked, a check whose forge died under it
5 * (`requeueAbandonedChecks`), and the boot sweep of checkouts nobody runs.
6 */
7
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
9 import { existsSync, mkdirSync } from "node:fs";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp, type App } from "../src/app.ts";
14 import { openPull } from "../src/actions/pulls.ts";
15 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
16 import { requeueAbandonedChecks, startChecks, sweepSoon } from "../src/checks/service.ts";
17 import {
18 cancelRun,
19 CHECK_RESTART_LIMIT,
20 claimRun,
21 createRun,
22 getRun,
23 listRuns,
24 REQUEUED_SUMMARY,
25 type Run,
26 } from "../src/core/runs.ts";
27 import type { Pull } from "../src/core/pulls.ts";
28 import { repoDir } from "../src/git/repos.ts";
29 import { STALE_AFTER_MS } from "../src/runs/lease.ts";
30 import { pullWord, servingHistory } from "../src/web/studio/words.ts";
31 import { createUser, type User } from "../src/core/users.ts";
32 import type { Repo } from "../src/core/repos.ts";
33 import { checkJson, checksForge } from "./helpers/checks-forge.ts";
34
35 describe("checks", () => {
36 const f = checksForge("forge-checks-restart-");
37
38 test("a check in flight when the forge stops is queued again, and the next forge runs it", async () => {
39 // Invariant: stopping the service cancels a running check as queued again, attempt + 1, and the next service runs it to the end.
40 // Hangs the first time it runs, passes the second: no wait to tune.
41 const id = await f.slowCheck(
42 `if [ -f '${f.dataDir}/ran-once' ]; then test -f OK.md; else touch '${f.dataDir}/ran-once'; ${f.held("never")}; fi`,
43 "A slower check",
44 );
45 // Running is said before the command begins: stop only once it has.
46 await f.until(() => existsSync(join(f.dataDir, "ran-once")), "the command begun");
47 const oid = getRun(f.app.db, id)!.commitOid;
48 const cut = getRun(f.app.db, id);
49
50 // This forge stops — a release — with the check under way.
51 await f.checks.stop();
52 const runs = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 });
53 const interrupted = runs.find((r) => r.id === cut!.id);
54 expect(interrupted?.state).toBe("cancelled");
55 expect(interrupted?.summary).toContain("queued again");
56 const again = runs.find((r) => r.commitOid === oid && r.state === "queued");
57 expect(again?.attempt).toBe(1);
58
59 // The forge that carries on finds it queued and runs it to the end.
60 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
61 const done = await f.settled((r) => r.id === again!.id);
62 expect(done.state).toBe("ok");
63 });
64 test("a roll waits for a running check to finish, within its budget, instead of throwing it away", async () => {
65 // Invariant: stop with a drain budget lets a running check finish to its verdict, leaving nothing queued.
66 await f.checks.stop();
67 f.checks = startChecks(f.app, {
68 forgeUrl: f.servers.url,
69 concurrency: 1,
70 drainMs: 30_000,
71 });
72 const id = await f.slowCheck(f.held("drained"), "A drained check");
73 // Let go only once the stop is under way: a stop that did not wait
74 // would have cut it off already.
75 const stopping = f.checks.stop();
76 f.release("drained");
77 await stopping;
78 const run = getRun(f.app.db, id);
79 expect(run?.state).toBe("ok");
80 expect(
81 listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).some(
82 (r) => r.state === "queued",
83 ),
84 ).toBe(false);
85 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
86 });
87
88 test("a person's stop cuts a running check off at once, and queues it again", async () => {
89 // Invariant: stop without drain cancels a running check at once as queued again, and the next service runs it.
90 await f.checks.stop();
91 f.checks = startChecks(f.app, {
92 forgeUrl: f.servers.url,
93 concurrency: 1,
94 drainMs: 30_000,
95 });
96 const id = await f.slowCheck(f.held("cut"), "A cut-off check");
97 await f.checks.stop({ drain: false });
98 const run = getRun(f.app.db, id);
99 expect(run?.state).toBe("cancelled");
100 expect(run?.summary).toContain("queued again");
101 const again = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).find(
102 (r) => r.commitOid === run?.commitOid && r.state === "queued",
103 );
104 expect(again).toBeDefined();
105 f.release("cut");
106 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
107 const done = await f.settled((r) => r.id === again?.id);
108 expect(done.state).toBe("ok");
109 });
110 test("an open pull whose head was never checked is checked when the service boots", async () => {
111 // Invariant: a booting service checks the head of an open pull that has no check, with reason boot.
112 await f.checks.stop();
113 // The pull is checked by the base's check, so the base must ask for it.
114 const gate = await commitToBranch(f.app, f.alice, f.repo, {
115 message: "The base's check wants QUIET.md",
116 files: [{ path: "forge.json", content: checkJson("test -f QUIET.md") }],
117 });
118 if (!gate.ok) throw new Error(JSON.stringify(gate.error));
119 const branch = await commitToBranch(f.app, f.alice, f.repo, {
120 branch: "quiet",
121 message: "Quiet",
122 files: [{ path: "QUIET.md", content: "quiet\n" }],
123 });
124 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
125 // The pull's check is the base's (#1275): give main a valid one.
126 const base = await commitToBranch(f.app, f.alice, f.repo, {
127 branch: "main",
128 message: "The base's check",
129 files: [{ path: "forge.json", content: checkJson("test -f QUIET.md") }],
130 });
131 if (!base.ok) throw new Error(JSON.stringify(base.error));
132 // Opened while no checks service was listening: nothing asked for a check.
133 const opened = await openPull(f.app, f.alice, f.repo, {
134 title: "Quiet",
135 baseBranch: "main",
136 headBranch: "quiet",
137 });
138 if (!opened.ok) throw new Error(String(opened.error));
139 expect(
140 listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter(
141 (r) => r.commitOid === opened.value.headOid,
142 ),
143 ).toHaveLength(0);
144 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
145 const done = await f.settled((r) => r.commitOid === opened.value.headOid);
146 expect(done.reason).toBe("boot");
147 expect(done.state).toBe("ok");
148 });
149 });
150
151 describe("a check its forge died under", () => {
152 let dataDir: string;
153 let app: App;
154 let alice: User;
155 let repo: Repo;
156 let pull: Pull;
157 let stale: string;
158
159 beforeAll(async () => {
160 dataDir = await mkdtemp(join(tmpdir(), "forge-checks-abandoned-"));
161 app = createApp({ dataDir, port: 0 });
162 const a = await createUser(app.db, {
163 origin: "seed",
164 trusted: true,
165 username: "alice",
166 email: "[email protected]",
167 password: "a-good-passphrase",
168 adminIfFirst: true,
169 });
170 if (!a.ok) throw new Error(a.error);
171 alice = a.value;
172 const made = await createRepo(app, alice, { name: "lib" });
173 if (!made.ok) throw new Error(String(made.error));
174 repo = made.value;
175 const seeded = await commitToBranch(app, alice, repo, {
176 message: "Start",
177 files: [{ path: "README.md", content: "# lib\n" }],
178 });
179 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
180 const first = await commitToBranch(app, alice, repo, {
181 branch: "feature",
182 message: "First",
183 files: [{ path: "A.md", content: "a\n" }],
184 });
185 if (!first.ok) throw new Error(JSON.stringify(first.error));
186 stale = first.value.commitOid;
187 const second = await commitToBranch(app, alice, repo, {
188 branch: "feature",
189 message: "Second",
190 files: [{ path: "B.md", content: "b\n" }],
191 });
192 if (!second.ok) throw new Error(JSON.stringify(second.error));
193 const opened = await openPull(app, alice, repo, {
194 title: "Feature",
195 baseBranch: "main",
196 headBranch: "feature",
197 });
198 if (!opened.ok) throw new Error(String(opened.error));
199 pull = opened.value;
200 });
201
202 afterAll(async () => {
203 app.close();
204 await rm(dataDir, { recursive: true, force: true });
205 });
206
207 /** A check that was running when its forge died: claimed, lease long lapsed. */
208 const abandoned = (commitOid: string, attempt = 0): Run => {
209 const run = createRun(app.db, {
210 repoId: repo.id,
211 kind: "check",
212 env: `pull-${pull.number}`,
213 commitOid,
214 reason: "push",
215 issueId: pull.id,
216 attempt,
217 });
218 claimRun(app.db, run.id);
219 app.db
220 .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1")
221 .run(run.id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString());
222 return getRun(app.db, run.id)!;
223 };
224
225 const word = () =>
226 pullWord(
227 {
228 db: app.db,
229 dir: repoDir(app.reposRoot, repo.ownerName, repo.name),
230 runtime: undefined,
231 record: undefined,
232 history: servingHistory(),
233 viewer: undefined,
234 },
235 pull,
236 );
237
238 const queuedFor = (commitOid: string): Run[] =>
239 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
240 (r) => r.commitOid === commitOid && r.state === "queued",
241 );
242
243 test("a check for the head of an open pull is queued again, attempt + 1, and the change reads working", async () => {
244 const run = abandoned(pull.headOid);
245 expect((await word()).word).toBe("working");
246 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
247 // A second forge sweeping the same lapse finds nothing left to move.
248 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(0);
249 const cut = getRun(app.db, run.id)!;
250 expect(cut.state).toBe("cancelled");
251 expect(cut.summary).toBe(REQUEUED_SUMMARY);
252 const again = queuedFor(pull.headOid);
253 expect(again).toHaveLength(1);
254 expect(again[0]!.attempt).toBe(1);
255 expect(again[0]!.issueId).toBe(pull.id);
256 expect(again[0]!.env).toBe(`pull-${pull.number}`);
257 expect(await word()).toEqual({ word: "working", note: "queued to check" });
258 cancelRun(app.db, again[0]!.id);
259 });
260
261 test("a check whose lease is still held is left alone", () => {
262 const run = createRun(app.db, {
263 repoId: repo.id,
264 kind: "check",
265 env: `pull-${pull.number}`,
266 commitOid: pull.headOid,
267 issueId: pull.id,
268 });
269 claimRun(app.db, run.id);
270 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(0);
271 expect(getRun(app.db, run.id)?.state).toBe("running");
272 cancelRun(app.db, run.id);
273 });
274
275 test("at the ceiling the check fails once, saying the forge kept going away", async () => {
276 const run = abandoned(pull.headOid, CHECK_RESTART_LIMIT);
277 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
278 const failed = getRun(app.db, run.id)!;
279 expect(failed.state).toBe("failed");
280 expect(failed.summary).toContain(
281 "the forge kept going away under this check",
282 );
283 expect(queuedFor(pull.headOid)).toHaveLength(0);
284 expect((await word()).word).toBe("stopped");
285 });
286
287 test("a check for a commit that no longer heads an open pull is cancelled, not queued again", () => {
288 const run = abandoned(stale);
289 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
290 const cut = getRun(app.db, run.id)!;
291 expect(cut.state).toBe("cancelled");
292 expect(cut.summary).toContain("no longer heads an open pull");
293 expect(queuedFor(stale)).toHaveLength(0);
294 });
295 });
296
297 describe("startChecks: a booting forge removes only checkouts nobody is running (#1884)", () => {
298 test("a live run's checkout survives, a lapsed run's and a stranger's go", async () => {
299 // Invariant: the boot sweep removes checkouts of lapsed or unknown runs and keeps a live run's.
300 const dir = await mkdtemp(join(tmpdir(), "forge-sweep-"));
301 const db = createApp({ dataDir: dir, port: 0 });
302 try {
303 const u = await createUser(db.db, {
304 origin: "seed",
305 trusted: true,
306 username: "alice",
307 email: "[email protected]",
308 password: "a-good-passphrase",
309 adminIfFirst: true,
310 });
311 if (!u.ok) throw new Error(u.error);
312 const made = await createRepo(db, u.value, { name: "lib" });
313 if (!made.ok) throw new Error(String(made.error));
314 const mk = (): Run => {
315 const run = createRun(db.db, { repoId: made.value.id, kind: "check", env: "x" });
316 claimRun(db.db, run.id);
317 return run;
318 };
319 const live = mk();
320 const lapsed = mk();
321 db.db
322 .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1")
323 .run(lapsed.id, new Date(Date.now() - STALE_AFTER_MS - 5_000).toISOString());
324 const where = (id: string): string => join(dir, "checks", id);
325 for (const id of [live.id, lapsed.id, "no-such-run"])
326 mkdirSync(where(id), { recursive: true });
327 const handle = startChecks(db, { forgeUrl: "http://localhost" });
328 await sweepSoon(db);
329 await handle.stop();
330 expect(existsSync(where(live.id))).toBe(true);
331 expect(existsSync(where(lapsed.id))).toBe(false);
332 expect(existsSync(where("no-such-run"))).toBe(false);
333 } finally {
334 db.close();
335 await rm(dir, { recursive: true, force: true });
336 }
337 });
338 });
test/checks.test.ts +70 −1126
@@ -1,512 +1,54 @@
11 /**
22 * A repository's own check, run by the forge: `check` in forge.json, on
33 * every push a pull or the default branch takes, the verdict a run beside
4 * the pull — the same place a preview's deploy reports.
4 * the pull — the same place a preview's deploy reports. This file: what
5 * asks for a check and what its verdict says. The lane's order is
6 * `checks-queue.test.ts`; a stopping or dying forge, `checks-restart.test.ts`;
7 * the failure summary's parsing, `checks-failure-summary.test.ts`.
58 */
69
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
10 import { describe, expect, test } from "bun:test";
911 import { mkdtemp, rm } from "node:fs/promises";
1012 import { tmpdir } from "node:os";
1113 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { mergePull, openPull, submitReview } from "../src/actions/pulls.ts";
14 import { setIssueState } from "../src/actions/issues.ts";
14 import { createApp } from "../src/app.ts";
15 import { openPull, submitReview } from "../src/actions/pulls.ts";
1516 import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
16 import { parseBunTestFailure } from "../src/checks/bun-test.ts";
17 import { summariseCheckFailure } from "../src/checks/summary.ts";
18 import { SLOW_SECONDS, summaryLines } from "../scripts/gate-summary.ts";
19 import { tally } from "../scripts/shards.ts";
20 import {
21 holdsLastSlot,
22 pickNext,
23 requeueAbandonedChecks,
24 startChecks,
25 sweepSoon,
26 STARVATION_MS,
27 type ChecksHandle,
28 } from "../src/checks/service.ts";
29 import {
30 cancelRun,
31 cancelRunsForPull,
32 CHECK_RESTART_LIMIT,
33 checkSummary,
34 checksForHead,
35 claimRun,
36 createRun,
37 getRun,
38 listRuns,
39 REQUEUED_SUMMARY,
40 type Run,
41 } from "../src/core/runs.ts";
42 import type { Pull } from "../src/core/pulls.ts";
17 import { startChecks } from "../src/checks/service.ts";
18 import { checkSummary, checksForHead, listRuns, type Run } from "../src/core/runs.ts";
4319 import { checksText } from "../src/tui/screens/pull.ts";
4420 import { checksLine } from "../src/web/pages/pull.ts";
45 import { repoDir } from "../src/git/repos.ts";
46 import { STALE_AFTER_MS } from "../src/runs/lease.ts";
47 import { pullWord, servingHistory } from "../src/web/studio/words.ts";
48 import { createUser, type User } from "../src/core/users.ts";
49 import { getRepo, type Repo } from "../src/core/repos.ts";
50 import { serve, type Servers } from "../src/http/server.ts";
21 import { createUser } from "../src/core/users.ts";
22 import { getRepo } from "../src/core/repos.ts";
23 import { serve } from "../src/http/server.ts";
5124 import { listEvents } from "../src/core/events.ts";
5225 import { readRunLog } from "../src/runs/log.ts";
53 import { promote } from "./helpers/members.ts";
54
55 const checkJson = (run: string, extra = ""): string =>
56 `{"check": {"run": ${JSON.stringify(run)}, "install": ""${extra}}}\n`;
57
58 describe("cancelRunsForPull: a landed or closed pull leaves nothing in the lane", () => {
59 test("queued and running checks and queued deploys of that pull are cancelled, building deploys and other pulls kept", async () => {
60 // Invariant: cancelRunsForPull cancels that pull's queued and running checks and queued deploys, and nothing else.
61 const dir = await mkdtemp(join(tmpdir(), "forge-cancel-"));
62 const db = createApp({ dataDir: dir, port: 0 });
63 try {
64 const u = await createUser(db.db, {
65 origin: "seed",
66 trusted: true,
67 username: "alice",
68 email: "[email protected]",
69 password: "a-good-passphrase",
70 adminIfFirst: true,
71 });
72 if (!u.ok) throw new Error(u.error);
73 const made = await createRepo(db, u.value, { name: "lib" });
74 if (!made.ok) throw new Error(String(made.error));
75 const repoId = made.value.id;
76 const first = await commitToBranch(db, u.value, made.value, {
77 message: "Start",
78 files: [{ path: "README.md", content: "x\n" }],
79 });
80 if (!first.ok) throw new Error(JSON.stringify(first.error));
81 const pullOn = async (branch: string): Promise<string> => {
82 const c = await commitToBranch(db, u.value, made.value, {
83 branch,
84 message: branch,
85 files: [{ path: `${branch}.md`, content: "x\n" }],
86 });
87 if (!c.ok) throw new Error(JSON.stringify(c.error));
88 const o = await openPull(db, u.value, made.value, {
89 title: branch,
90 baseBranch: "main",
91 headBranch: branch,
92 });
93 if (!o.ok) throw new Error(String(o.error));
94 return o.value.id;
95 };
96 const p1 = await pullOn("one");
97 const p2 = await pullOn("two");
98 const mk = (kind: "check" | "deploy", issueId: string) =>
99 createRun(db.db, { repoId, kind, env: "pull-1", issueId });
100 const queued = mk("check", p1);
101 const running = mk("check", p1);
102 claimRun(db.db, running.id);
103 const deploy = mk("deploy", p1);
104 const building = mk("deploy", p1);
105 claimRun(db.db, building.id);
106 const other = mk("check", p2);
107 const stopped = cancelRunsForPull(db.db, p1, "pull landed");
108 expect(stopped.map((r) => r.id).sort()).toEqual(
109 [queued.id, running.id, deploy.id].sort(),
110 );
111 // A building deploy is the runtime's to stop; the row must not say otherwise.
112 expect(getRun(db.db, building.id)?.state).toBe("running");
113 for (const r of [queued, running, deploy]) {
114 expect(getRun(db.db, r.id)?.state).toBe("cancelled");
115 expect(getRun(db.db, r.id)?.summary).toBe("pull landed");
116 }
117 expect(getRun(db.db, other.id)?.state).toBe("queued");
118 } finally {
119 db.close();
120 await rm(dir, { recursive: true, force: true });
121 }
122 });
123 });
124
125 describe("parseBunTestFailure: a convenience over one known runner, not a general protocol", () => {
126 test("bun test's own tail: the counts, and the failing names bounded to a few", () => {
127 const lines = [
128 "bun test v1.3.14 (0d9b296a)",
129 "",
130 "check.test.ts:",
131 "(fail) first [0.16ms]",
132 "(fail) second [0.07ms]",
133 "(fail) third [0.05ms]",
134 "(fail) fourth [0.04ms]",
135 "",
136 " 3 pass",
137 " 4 fail",
138 " 7 expect() calls",
139 "Ran 7 tests across 1 file. [51.00ms]",
140 ];
141 expect(parseBunTestFailure(lines)).toEqual({
142 passed: 3,
143 failed: 4,
144 names: ["first", "second", "third"],
145 });
146 });
147
148 test("a name arbitrary text from a subprocess, kept verbatim — escaping is the page's job, not the parser's", () => {
149 const lines = [
150 "(fail) <script>alert(1)</script> [1.00ms]",
151 " 0 pass",
152 " 1 fail",
153 "Ran 1 test across 1 file. [1.00ms]",
154 ];
155 expect(parseBunTestFailure(lines)?.names).toEqual([
156 "<script>alert(1)</script>",
157 ]);
158 });
159
160 test("a failing test bun printed without a duration — as it does under CI for one that took no time — is still named", () => {
161 const lines = [
162 "(pass) a passing test",
163 "(fail) first failing test [0.10ms]",
164 "(fail) second failing test",
165 " 1 pass",
166 " 2 fail",
167 "Ran 3 tests across 1 file. [9.00ms]",
168 ];
169 expect(parseBunTestFailure(lines)).toEqual({
170 passed: 1,
171 failed: 2,
172 names: ["first failing test", "second failing test"],
173 });
174 });
175
176 test("a shard's named failure leads; the (unnamed) fallout behind it is not named", () => {
177 const lines = [
178 "── shard 1/1 (exit 1, 3s): test/a.test.ts",
179 "test/a.test.ts:",
180 "(fail) a > writes the row [2ms]",
181 "error: database is closed",
182 "(fail) (unnamed) [1ms]",
183 " 4 pass",
184 " 2 fail",
185 ];
186 expect(parseBunTestFailure(lines)?.names).toEqual(["a > writes the row"]);
187 // Only a shard that named nothing else is named by its fallout.
188 const only = ["test/b.test.ts:", "error: boom", "(fail) (unnamed)", " 0 pass", " 1 fail"];
189 expect(parseBunTestFailure(only)?.names).toEqual(["test/b.test.ts > (unnamed): error: boom"]);
190 });
191
192 test("shards that finished out of order are named in shard order, each shard's first failure first", () => {
193 const lines = [
194 "── shard 3/3 (exit 1, 2s): test/c.test.ts",
195 "test/c.test.ts:",
196 "(fail) c one [1ms]",
197 "(fail) c two [1ms]",
198 " 1 pass",
199 " 2 fail",
200 "── shard 1/3 (exit 1, 9s): test/a.test.ts",
201 "test/a.test.ts:",
202 "error: boom",
203 "(fail) (unnamed) [1ms]",
204 " 3 pass",
205 " 1 fail",
206 "── shard 2/3 (exit 0, 5s): test/b.test.ts",
207 " 7 pass",
208 " 0 fail",
209 "2 of 3 shards failed:",
210 "(fail) test/a.test.ts > (unnamed): error: boom [1ms] [shard 1]",
211 "(fail) c one [1ms] [shard 3]",
212 "(fail) c two [1ms] [shard 3]",
213 ];
214 expect(parseBunTestFailure(lines)).toEqual({
215 passed: 11,
216 failed: 3,
217 names: ["test/a.test.ts > (unnamed): error: boom", "c one", "c two"],
218 });
219 });
220
221 test("output that is not bun test's own shape is not guessed at", () => {
222 expect(
223 parseBunTestFailure([
224 "$ test -f NOPE.md",
225 "test: NOPE.md: No such file or directory",
226 ]),
227 ).toBeUndefined();
228 // A `pass` line with no `fail` line beside it is not this shape either.
229 expect(
230 parseBunTestFailure([" 5 pass", "Ran 5 tests across 1 file."]),
231 ).toBeUndefined();
232 // All green: nothing failed, so there is nothing to summarise.
233 expect(
234 parseBunTestFailure([" 5 pass", " 0 fail", "Ran 5 tests across 1 file."]),
235 ).toBeUndefined();
236 });
237 });
238
239 describe("summariseCheckFailure: what a failed check's summary says first", () => {
240 test("real bun test output with one failure yields the name and the counts", () => {
241 const lines = [
242 "bun test v1.3.14 (0d9b296a)",
243 "",
244 "check.test.ts:",
245 "1 tests failed:",
246 "(fail) the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed [30000.49ms]",
247 " ^ this test timed out after 30000ms.",
248 "",
249 " 409 pass",
250 " 1 fail",
251 "Ran 410 tests across 1 file. [30.05s]",
252 ];
253 expect(summariseCheckFailure(lines, "check exited with code 1")).toEqual({
254 summary:
255 "1 of 410 tests — \"the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed\"",
256 detail: {
257 passed: 409,
258 failed: 1,
259 names: [
260 "the forge's own restart never reads as this deploy's failure (issue #255) > an install still in flight when the runtime stops finishes cancelled, not failed",
261 ],
262 },
263 });
264 });
265
266 test("output with no bun test shape yields the last non-empty line, not the bare exit code", () => {
267 const lines = ["src/foo.ts(10,5): error TS2345: not assignable", ""];
268 expect(summariseCheckFailure(lines, "check exited with code 2")).toEqual({
269 summary: "src/foo.ts(10,5): error TS2345: not assignable",
270 detail: undefined,
271 });
272 });
273
274 test("no output at all falls back to the bare exit code — nothing was there to name", () => {
275 expect(summariseCheckFailure([], "check exited with code 1")).toEqual({
276 summary: "check exited with code 1",
277 detail: undefined,
278 });
279 });
280
281 const failing = (n: number) => [
282 ...Array.from({ length: n }, (_, i) => `(fail) t${i + 1} [1.00ms]`),
283 " 5 pass",
284 ` ${n} fail`,
285 ];
286
287 test("three failures name all three; five name three and how many more", () => {
288 expect(summariseCheckFailure(failing(3), "x").summary).toBe(
289 '3 of 8 tests — "t1", "t2" and "t3"',
290 );
291 expect(summariseCheckFailure(failing(5), "x").summary).toBe(
292 '5 of 10 tests — "t1", "t2", "t3" and 2 more',
293 );
294 });
295
296 test("a name list longer than the limit is still bounded to one line", () => {
297 const long = "y".repeat(100);
298 const lines = ["a", "b", "c", "d"].map((c) => `(fail) ${c}${long} [1.00ms]`);
299 const { summary } = summariseCheckFailure([...lines, " 0 pass", " 4 fail"], "x");
300 expect(summary.length).toBe(200);
301 expect(summary.endsWith("…")).toBe(true);
302 });
303
304 test("a very long name is bounded to one line", () => {
305 const long = "x".repeat(500);
306 const lines = [`(fail) ${long} [1.00ms]`, " 0 pass", " 1 fail"];
307 const { summary } = summariseCheckFailure(
308 lines,
309 "check exited with code 1",
310 );
311 expect(summary.length).toBeLessThanOrEqual(200);
312 expect(summary.endsWith("…")).toBe(true);
313 });
314 });
315
316 describe("summariseCheckFailure: a red gate is summarised by its red test, never a warning line (#2271)", () => {
317 // The sharded gate's log, as `scripts/test-shards.ts` prints it: each
318 // shard's own bun test output whole, the red one first, then the tally,
319 // then the gate summary block with its slow-file warning.
320 const red =
321 "bun test v1.3.14\n\ntest/standards.test.ts:\n(pass) the README names every script [1.00ms]\n" +
322 "(fail) releases.json is the shipped releases > every entry is a release [3.00ms]\n\n" +
323 " 11 pass\n 1 fail\n 40 expect() calls\nRan 12 tests across 1 file. [80.00ms]\n";
324 const slow =
325 "bun test v1.3.14\n\ntest/crew.test.ts:\n(pass) a crew is listed [2.00ms]\n\n" +
326 " 30 pass\n 0 fail\n 90 expect() calls\nRan 30 tests across 1 file. [52.40s]\n";
327 const results = [
328 { files: ["test/standards.test.ts"], code: 1, output: red, seconds: 1.2 },
329 { files: ["test/crew.test.ts"], code: 0, output: slow, seconds: 52.5 },
330 ];
331 const block = (times: { file: string; seconds: number }[]): string[] =>
332 summaryLines({
333 shards: 2,
334 cpus: 16,
335 shardsMax: 16,
336 files: 2,
337 override: undefined,
338 memoryBytes: 8 * 1024 ** 3,
339 workerBytes: 1024 ** 3,
340 memory: { limit: undefined, peak: undefined },
341 wallSeconds: 52.6,
342 times,
343 });
344 const log = (shards: typeof results): string[] => [
345 ...shards.flatMap((r, i) => [`── shard ${i + 1}/2 (exit ${r.code}, ${r.seconds}s): ${r.files[0]}`, ...r.output.split("\n")]),
346 ...tally(shards, 52.6).lines,
347 "",
348 ...block(shards.map((r) => ({ file: r.files[0]!, seconds: r.seconds }))),
349 ];
350
351 test("one red file and one slow file: the summary is the red file's test", () => {
352 const lines = log(results);
353 expect(lines.at(-1)).toBe(`test-shards: over ${SLOW_SECONDS} s: test/crew.test.ts (52.5s)`);
354 expect(summariseCheckFailure(lines, "check exited with code 1")).toEqual({
355 summary: '1 of 42 tests — "releases.json is the shipped releases > every entry is a release"',
356 detail: {
357 passed: 41,
358 failed: 1,
359 names: ["releases.json is the shipped releases > every entry is a release"],
360 },
361 });
362 });
363
364 test("a red shard that named no test is named by its files, not by the gate summary block", () => {
365 const crashed = [
366 { files: ["test/standards.test.ts"], code: 137, output: "bun test v1.3.14\n", seconds: 1.2 },
367 results[1]!,
368 ];
369 expect(summariseCheckFailure(log(crashed), "check exited with code 1").summary).toBe(
370 "shard 1 exited 137 without naming a test: test/standards.test.ts",
371 );
372 });
373
374 test("a killed shard's line is the summary, and the old line is still recognised", () => {
375 expect(
376 summariseCheckFailure(["shard 258 killed after 412 s, running test/x.test.ts"], "check exited with code 1").summary,
377 ).toBe("shard 258 killed after 412 s, running test/x.test.ts");
378 expect(
379 summariseCheckFailure(["shard 258 exited 143 without naming a test: a b"], "check exited with code 1").summary,
380 ).toBe("shard 258 exited 143 without naming a test: a b");
381 });
382
383 test("without names or a tally, the last line before the gate summary block, never a line of it", () => {
384 const lines = ["test-shards: something went wrong", "", ...block([{ file: "test/crew.test.ts", seconds: 52.5 }])];
385 expect(summariseCheckFailure(lines, "check exited with code 1").summary).toBe(
386 "test-shards: something went wrong",
387 );
388 });
389 });
26 import { checkJson, checksForge } from "./helpers/checks-forge.ts";
39027
39128 describe("checks", () => {
392 let dataDir: string;
393 let app: App;
394 let servers: Servers;
395 let checks: ChecksHandle;
396 let alice: User;
397 let bob: User;
398 let repo: Repo;
399
400 async function settled(
401 where: (run: Run) => boolean,
402 timeoutMs = 4_000,
403 ): Promise<Run> {
404 const deadline = Date.now() + timeoutMs;
405 while (Date.now() < deadline) {
406 const run = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find(
407 where,
408 );
409 if (run && (run.state === "ok" || run.state === "failed")) return run;
410 await Bun.sleep(10);
411 }
412 throw new Error("check did not settle");
413 }
414
415 /** Waits until `done` holds, polling the database; a check's own pace, not a timer's. */
416 async function until(done: () => boolean, what: string): Promise<void> {
417 const deadline = Date.now() + 4_000;
418 while (!done()) {
419 if (Date.now() > deadline) throw new Error(`never came: ${what}`);
420 await Bun.sleep(10);
421 }
422 }
423
424 /**
425 * A check command that holds its slot until the test lets it go
426 * (`release`), then runs `then`: the test is the clock, so no wait is
427 * guessed at. Never released, it runs until it is cut off.
428 */
429 const held = (name: string, then = "test -f OK.md"): string =>
430 `while [ ! -f '${dataDir}/go-${name}' ]; do sleep 0.01; done; ${then}`;
431 const release = (name: string): void =>
432 writeFileSync(join(dataDir, `go-${name}`), "");
433
434 /** Waits until a fork pull's new head is announced on the bus as `pull.pushed`. */
435 const headAnnounced = (pull: Pull, headOid: string): Promise<void> =>
436 until(
437 () =>
438 listEvents(app.db, { repoId: repo.id, kind: "pull.pushed" }).some(
439 (e) => e.payload["number"] === pull.number && e.payload["headOid"] === headOid,
440 ),
441 "the pull's new head announced",
442 );
443
444 const checksOf = (commitOid: string): Run[] =>
445 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
446 (r) => r.commitOid === commitOid,
447 );
448
449 beforeAll(async () => {
450 dataDir = await mkdtemp(join(tmpdir(), "forge-checks-"));
451 app = createApp({ dataDir, port: 0 });
452 servers = await serve(app);
453 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
454 const a = await createUser(app.db, {
455 origin: "seed",
456 trusted: true,
457 username: "alice",
458 email: "[email protected]",
459 password: "a-good-passphrase",
460 adminIfFirst: true,
461 });
462 if (!a.ok) throw new Error(a.error);
463 alice = a.value;
464 const b = await createUser(app.db, {
465 origin: "seed",
466 trusted: true,
467 username: "bob",
468 email: "[email protected]",
469 password: "a-good-passphrase",
470 });
471 if (!b.ok) throw new Error(b.error);
472 bob = b.value;
473 promote(app.db, bob);
474 const made = await createRepo(app, alice, {
475 name: "lib",
476 visibility: "public",
477 });
478 if (!made.ok) throw new Error(String(made.error));
479 repo = made.value;
480 });
481
482 afterAll(async () => {
483 await checks.stop();
484 servers.stop();
485 app.close();
486 await rm(dataDir, { recursive: true, force: true });
487 });
29 const f = checksForge("forge-checks-");
48830
48931 test("a commit that declares no check is not checked", async () => {
49032 // Invariant: a push whose commit declares no check queues no run; the next push that declares one is the only run.
491 const seeded = await commitToBranch(app, alice, repo, {
33 const seeded = await commitToBranch(f.app, f.alice, f.repo, {
49234 message: "Start",
49335 files: [{ path: "README.md", content: "# lib\n" }],
49436 });
49537 expect(seeded.ok).toBe(true);
49638 // The push after it declares a check: once that one settles, the
49739 // first push's handler (begun earlier, doing no more) has answered too.
498 const declared = await commitToBranch(app, alice, repo, {
40 const declared = await commitToBranch(f.app, f.alice, f.repo, {
49941 message: "Declare a passing check",
50042 files: [{ path: "forge.json", content: checkJson("true # first") }],
50143 });
50244 if (!declared.ok) throw new Error(JSON.stringify(declared.error));
503 const run = await settled((r) => r.commitOid === declared.value.commitOid);
504 expect(listRuns(app.db, repo.id, { kind: "check" }).map((r) => r.id)).toEqual([run.id]);
45 const run = await f.settled((r) => r.commitOid === declared.value.commitOid);
46 expect(listRuns(f.app.db, f.repo.id, { kind: "check" }).map((r) => r.id)).toEqual([run.id]);
50547 });
50648
50749 test("the default branch is checked on push, with the log kept", async () => {
50850 // Invariant: a push to the default branch that declares a check runs it, env main, reason push, its commands in the log.
509 const declared = await commitToBranch(app, alice, repo, {
51 const declared = await commitToBranch(f.app, f.alice, f.repo, {
51052 message: "Declare a check",
51153 files: [
51254 { path: "forge.json", content: checkJson("test -f OK.md") },
@@ -514,12 +56,12 @@ describe("checks", () => {
51456 ],
51557 });
51658 if (!declared.ok) throw new Error(JSON.stringify(declared.error));
517 const run = await settled((r) => r.commitOid === declared.value.commitOid);
59 const run = await f.settled((r) => r.commitOid === declared.value.commitOid);
51860 expect(run.state).toBe("ok");
51961 expect(run.env).toBe("main");
52062 expect(run.reason).toBe("push");
52163 expect(run.summary).toMatch(/^check passed in \d+s$/);
522 const log = readRunLog(dataDir, run.id).lines.join("\n");
64 const log = readRunLog(f.dataDir, run.id).lines.join("\n");
52365 expect(log).toContain("$ test -f OK.md");
52466 });
52567
@@ -527,7 +69,7 @@ describe("checks", () => {
52769 // Invariant: a check's command sees FORGE_CELLS=off.
52870 // A forge the tests boot inside a check cell must not wrap its apps in
52971 // nested cells (they never answer in time); it is told so in its env.
530 const declared = await commitToBranch(app, alice, repo, {
72 const declared = await commitToBranch(f.app, f.alice, f.repo, {
53173 message: "A check that reads its cell",
53274 files: [
53375 {
@@ -539,9 +81,9 @@ describe("checks", () => {
53981 ],
54082 });
54183 if (!declared.ok) throw new Error(JSON.stringify(declared.error));
542 const run = await settled((r) => r.commitOid === declared.value.commitOid);
84 const run = await f.settled((r) => r.commitOid === declared.value.commitOid);
54385 expect(run.state).toBe("ok");
544 expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain(
86 expect(readRunLog(f.dataDir, run.id).lines.join("\n")).toContain(
54587 "cells: off",
54688 );
54789 });
@@ -549,25 +91,25 @@ describe("checks", () => {
54991 test("a pull is checked when it opens and each time its branch moves; a failure names the exit", async () => {
55092 // Invariant: a branch heading no pull is not checked; opening a pull checks its head with the base's check, and each push to it again.
55193 // A pull is checked by the base's check, not its own.
552 const gate = await commitToBranch(app, alice, repo, {
94 const gate = await commitToBranch(f.app, f.alice, f.repo, {
55395 message: "The base's check",
55496 files: [{ path: "forge.json", content: checkJson("test -f NOPE.md") }],
55597 });
55698 if (!gate.ok) throw new Error(JSON.stringify(gate.error));
557 const branched = await commitToBranch(app, alice, repo, {
99 const branched = await commitToBranch(f.app, f.alice, f.repo, {
558100 branch: "feature",
559101 message: "Leave the check failing",
560102 files: [{ path: "a.txt", content: "a\n" }],
561103 });
562104 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
563 const opened = await openPull(app, alice, repo, {
105 const opened = await openPull(f.app, f.alice, f.repo, {
564106 title: "Feature",
565107 baseBranch: "main",
566108 headBranch: "feature",
567109 });
568110 if (!opened.ok) throw new Error(String(opened.error));
569111 const pull = opened.value;
570 const failed = await settled(
112 const failed = await f.settled(
571113 (r) => r.issueId === pull.id && r.commitOid === branched.value.commitOid,
572114 );
573115 expect(failed.state).toBe("failed");
@@ -575,7 +117,7 @@ describe("checks", () => {
575117 expect(failed.reason).toBe("pull");
576118 // A branch that heads no pull is nobody's to check: its push, handled
577119 // before the pull opened, left no run beside the pull's own.
578 expect(checksOf(branched.value.commitOid).map((r) => r.id)).toEqual([failed.id]);
120 expect(f.checksOf(branched.value.commitOid).map((r) => r.id)).toEqual([failed.id]);
579121 expect(failed.summary).toBe("check failed: check exited with code 1");
580122 // `test -f` is not `bun test`: its output is not this shape, so nothing
581123 // is guessed at it.
@@ -585,9 +127,9 @@ describe("checks", () => {
585127 // The check names the base tip it merged with; a default-branch push
586128 // merged nothing and records none (#1810).
587129 expect(failed.baseOid).toBe(gate.value.commitOid);
588 const onMain = await settled((r) => r.commitOid === gate.value.commitOid && r.issueId === null);
130 const onMain = await f.settled((r) => r.commitOid === gate.value.commitOid && r.issueId === null);
589131 expect(onMain.baseOid).toBe("");
590 const seen = checkSummary(app.db, pull.id, branched.value.commitOid);
132 const seen = checkSummary(f.app.db, pull.id, branched.value.commitOid);
591133 const line = checksLine(
592134 "/alice/lib",
593135 seen,
@@ -612,16 +154,16 @@ describe("checks", () => {
612154 expect(checksText(green, false, true, "main", false)).toContain("Check passed");
613155 expect(checksText(green, false, true, "main", false)).not.toContain("older");
614156 expect(
615 checksForHead(app.db, pull.id, branched.value.commitOid)[0]?.id,
157 checksForHead(f.app.db, pull.id, branched.value.commitOid)[0]?.id,
616158 ).toBe(failed.id);
617159
618 const fixed = await commitToBranch(app, alice, repo, {
160 const fixed = await commitToBranch(f.app, f.alice, f.repo, {
619161 branch: "feature",
620162 message: "Fix the check",
621163 files: [{ path: "NOPE.md", content: "there\n" }],
622164 });
623165 if (!fixed.ok) throw new Error(JSON.stringify(fixed.error));
624 const passed = await settled(
166 const passed = await f.settled(
625167 (r) => r.issueId === pull.id && r.commitOid === fixed.value.commitOid,
626168 );
627169 expect(passed.state).toBe("ok");
@@ -630,14 +172,14 @@ describe("checks", () => {
630172
631173 test("a bun test failure in the check step is summarised: how many passed, how many failed, and their names", async () => {
632174 // Invariant: a red `bun test` check's summary and detail carry the counts and the failing names.
633 const gate = await commitToBranch(app, alice, repo, {
175 const gate = await commitToBranch(f.app, f.alice, f.repo, {
634176 message: "The base's check runs bun test",
635177 files: [
636178 { path: "forge.json", content: checkJson("bun test check.test.ts") },
637179 ],
638180 });
639181 if (!gate.ok) throw new Error(JSON.stringify(gate.error));
640 const branched = await commitToBranch(app, alice, repo, {
182 const branched = await commitToBranch(f.app, f.alice, f.repo, {
641183 branch: "red-tests",
642184 message: "Two failing tests",
643185 files: [
@@ -654,14 +196,14 @@ describe("checks", () => {
654196 ],
655197 });
656198 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
657 const opened = await openPull(app, alice, repo, {
199 const opened = await openPull(f.app, f.alice, f.repo, {
658200 title: "Red tests",
659201 baseBranch: "main",
660202 headBranch: "red-tests",
661203 });
662204 if (!opened.ok) throw new Error(String(opened.error));
663205 const pull = opened.value;
664 const failed = await settled(
206 const failed = await f.settled(
665207 (r) => r.issueId === pull.id && r.commitOid === branched.value.commitOid,
666208 );
667209 expect(failed.state).toBe("failed");
@@ -677,21 +219,21 @@ describe("checks", () => {
677219
678220 test("a reader's fork pull is checked at the base's snapshot once a writer approves each head", async () => {
679221 // Invariant: a reader's fork pull runs nothing until a writer approves, and each new head waits for its own approval.
680 const gate = await commitToBranch(app, alice, repo, {
222 const gate = await commitToBranch(f.app, f.alice, f.repo, {
681223 message: "The base's check passes",
682224 files: [{ path: "forge.json", content: checkJson("true") }],
683225 });
684226 if (!gate.ok) throw new Error(JSON.stringify(gate.error));
685 const forked = await forkRepo(app, bob, repo);
227 const forked = await forkRepo(f.app, f.bob, f.repo);
686228 if (!forked.ok) throw new Error(String(forked.error));
687229 const fork = forked.value;
688 const committed = await commitToBranch(app, bob, fork, {
230 const committed = await commitToBranch(f.app, f.bob, fork, {
689231 branch: "from-fork",
690232 message: "From the fork",
691233 files: [{ path: "FORK.md", content: "hi\n" }],
692234 });
693235 if (!committed.ok) throw new Error(JSON.stringify(committed.error));
694 const opened = await openPull(app, bob, repo, {
236 const opened = await openPull(f.app, f.bob, f.repo, {
695237 title: "From the fork",
696238 baseBranch: "main",
697239 headBranch: "from-fork",
@@ -701,16 +243,16 @@ describe("checks", () => {
701243 const pull = opened.value;
702244 expect(pull.crossRepo).toBe(true);
703245 // Bob cannot write: nothing of his runs until Alice approves the head.
704 const approved = await submitReview(app, alice, repo, pull, {
246 const approved = await submitReview(f.app, f.alice, f.repo, pull, {
705247 state: "approved",
706248 });
707249 if (!approved.ok) throw new Error(String(approved.error));
708 const first = await settled(
250 const first = await f.settled(
709251 (r) => r.issueId === pull.id && r.commitOid === committed.value.commitOid,
710252 );
711253 expect(first.state).toBe("ok");
712254 // The fork pushes again: the base's snapshot moves, and so does the check.
713 const again = await commitToBranch(app, bob, fork, {
255 const again = await commitToBranch(f.app, f.bob, fork, {
714256 branch: "from-fork",
715257 message: "Again",
716258 files: [{ path: "FORK.md", content: "hi again\n" }],
@@ -719,40 +261,40 @@ describe("checks", () => {
719261 // The pull follows the fork's branch on its own clock: approve the new
720262 // head only once its move was announced and answered (the bus delivers
721263 // in step, so the push's handler has met the approval guard by then).
722 await headAnnounced(pull, again.value.commitOid);
723 const reapproved = await submitReview(app, alice, repo, pull, {
264 await f.headAnnounced(pull, again.value.commitOid);
265 const reapproved = await submitReview(f.app, f.alice, f.repo, pull, {
724266 state: "approved",
725267 });
726268 if (!reapproved.ok) throw new Error(String(reapproved.error));
727 const second = await settled(
269 const second = await f.settled(
728270 (r) => r.issueId === pull.id && r.commitOid === again.value.commitOid,
729271 );
730272 expect(second.state).toBe("ok");
731273 // A new head waits for its own approval: the push, handled before the
732274 // approval was given, ran nothing; the one check is the approval's.
733275 expect(second.reason).toBe("approved");
734 expect(checksOf(again.value.commitOid).map((r) => r.id)).toEqual([second.id]);
276 expect(f.checksOf(again.value.commitOid).map((r) => r.id)).toEqual([second.id]);
735277 });
736278
737279 test("approving a head whose check is already green queues no second check (#4391)", async () => {
738280 // Invariant: a second approval of a head already green on the same base queues nothing.
739 const based = await commitToBranch(app, alice, repo, {
281 const based = await commitToBranch(f.app, f.alice, f.repo, {
740282 message: "The base's check passes",
741283 files: [{ path: "forge.json", content: checkJson("true") }],
742284 });
743285 if (!based.ok) throw new Error(JSON.stringify(based.error));
744286 // Bob's fork, made by the fork test above when it ran first.
745287 const fork =
746 getRepo(app.db, "bob", repo.name) ??
747 (await forkRepo(app, bob, repo).then((f) => (f.ok ? f.value : undefined)));
288 getRepo(f.app.db, "bob", f.repo.name) ??
289 (await forkRepo(f.app, f.bob, f.repo).then((made) => (made.ok ? made.value : undefined)));
748290 if (fork === undefined) throw new Error("no fork");
749 const committed = await commitToBranch(app, bob, fork, {
291 const committed = await commitToBranch(f.app, f.bob, fork, {
750292 branch: "green-then-approved",
751293 message: "Green before the second look",
752294 files: [{ path: "GREEN.md", content: "x\n" }],
753295 });
754296 if (!committed.ok) throw new Error(JSON.stringify(committed.error));
755 const opened = await openPull(app, bob, repo, {
297 const opened = await openPull(f.app, f.bob, f.repo, {
756298 title: "Green before the second look",
757299 baseBranch: "main",
758300 headBranch: "green-then-approved",
@@ -761,16 +303,16 @@ describe("checks", () => {
761303 if (!opened.ok) throw new Error(String(opened.error));
762304 const pull = opened.value;
763305 const mine = (): Run[] =>
764 listRuns(app.db, repo.id, { kind: "check" }).filter(
306 listRuns(f.app.db, f.repo.id, { kind: "check" }).filter(
765307 (r) => r.issueId === pull.id && r.commitOid === committed.value.commitOid,
766308 );
767309 // The first approval is what lets a reader's head run at all.
768 const approved = await submitReview(app, alice, repo, pull, { state: "approved" });
310 const approved = await submitReview(f.app, f.alice, f.repo, pull, { state: "approved" });
769311 if (!approved.ok) throw new Error(String(approved.error));
770 const first = await settled((r) => r.issueId === pull.id);
312 const first = await f.settled((r) => r.issueId === pull.id);
771313 expect(first.state).toBe("ok");
772314 // A second approval of the same green head on the same base is nothing new.
773 const again = await submitReview(app, alice, repo, pull, {
315 const again = await submitReview(f.app, f.alice, f.repo, pull, {
774316 state: "approved",
775317 body: "Still good.",
776318 });
@@ -778,22 +320,22 @@ describe("checks", () => {
778320 // The fork moves on and its new head is approved: once that check
779321 // settles, the second approval's handler (begun earlier, doing no
780322 // more) has answered too.
781 const next = await commitToBranch(app, bob, fork, {
323 const next = await commitToBranch(f.app, f.bob, fork, {
782324 branch: "green-then-approved",
783325 message: "After the second look",
784326 files: [{ path: "GREEN.md", content: "y\n" }],
785327 });
786328 if (!next.ok) throw new Error(JSON.stringify(next.error));
787 await headAnnounced(pull, next.value.commitOid);
788 const third = await submitReview(app, alice, repo, pull, { state: "approved" });
329 await f.headAnnounced(pull, next.value.commitOid);
330 const third = await submitReview(f.app, f.alice, f.repo, pull, { state: "approved" });
789331 if (!third.ok) throw new Error(String(third.error));
790 await settled((r) => r.issueId === pull.id && r.commitOid === next.value.commitOid);
332 await f.settled((r) => r.issueId === pull.id && r.commitOid === next.value.commitOid);
791333 expect(mine().map((r) => r.id)).toEqual([first.id]);
792334 });
793335
794336 test("a commit already proven green is not run again; the re-request answers with the earlier verdict (#1795)", async () => {
795337 // Invariant: a re-requested check of a commit already green answers with that verdict without running the command.
796 const made = await commitToBranch(app, alice, repo, {
338 const made = await commitToBranch(f.app, f.alice, f.repo, {
797339 message: "Proven once",
798340 files: [
799341 { path: "forge.json", content: checkJson("echo ran-once; test -f OK.md # proven") },
@@ -801,626 +343,28 @@ describe("checks", () => {
801343 ],
802344 });
803345 if (!made.ok) throw new Error(JSON.stringify(made.error));
804 const first = await settled((r) => r.commitOid === made.value.commitOid);
346 const first = await f.settled((r) => r.commitOid === made.value.commitOid);
805347 expect(first.state).toBe("ok");
806 expect(readRunLog(dataDir, first.id).lines.join("\n")).toContain("ran-once");
807 const asked = await checks.request(repo, "main", made.value.commitOid, "again");
808 const second = await settled((r) => r.id === asked!.id);
348 expect(readRunLog(f.dataDir, first.id).lines.join("\n")).toContain("ran-once");
349 const asked = await f.checks.request(f.repo, "main", made.value.commitOid, "again");
350 const second = await f.settled((r) => r.id === asked!.id);
809351 expect(second.state).toBe("ok");
810352 expect(second.summary).toBe(`already proven green by ${first.id}`);
811 expect(readRunLog(dataDir, second.id).lines.join("\n")).not.toContain("ran-once");
353 expect(readRunLog(f.dataDir, second.id).lines.join("\n")).not.toContain("ran-once");
812354 });
813
814 test("a newer commit supersedes a check still waiting for the same environment", async () => {
815 // Invariant: a queued check is cancelled as superseded when a newer commit asks for the same environment.
816 const slow = await commitToBranch(app, alice, repo, {
817 message: "A slow check",
818 files: [{ path: "forge.json", content: checkJson(held("supersede")) }],
819 });
820 if (!slow.ok) throw new Error(JSON.stringify(slow.error));
821 // While that one holds the only slot, two more arrive: the first of
822 // them is superseded by the second before it ever runs.
823 const one = await checks.request(
824 repo,
825 "main",
826 slow.value.commitOid,
827 "push",
828 );
829 expect(one).toBeDefined();
830 const two = await commitToBranch(app, alice, repo, {
831 message: "Two",
832 files: [{ path: "TWO.md", content: "2\n" }],
833 });
834 if (!two.ok) throw new Error(JSON.stringify(two.error));
835 await until(() => checksOf(two.value.commitOid).length > 0, "two queued");
836 const three = await commitToBranch(app, alice, repo, {
837 message: "Three",
838 files: [{ path: "THREE.md", content: "3\n" }],
839 });
840 if (!three.ok) throw new Error(JSON.stringify(three.error));
841 await until(() => checksOf(three.value.commitOid).length > 0, "three queued");
842 release("supersede");
843 const last = await settled((r) => r.commitOid === three.value.commitOid);
844 expect(last.state).toBe("ok");
845 const middle = checksOf(two.value.commitOid)[0];
846 expect(middle?.state).toBe("cancelled");
847 expect(middle?.summary).toContain("superseded by");
848 });
849
850 test("a new head on a pull stops its running check for the old head, superseded not failed", async () => {
851 // Invariant: a pull's new head cuts off the running check of its old head, which ends cancelled as superseded.
852 const slow = await commitToBranch(app, alice, repo, {
853 branch: "supersede",
854 message: "A check that never ends on its own",
855 files: [
856 { path: "forge.json", content: checkJson(held("never")) },
857 { path: "S.md", content: "s\n" },
858 ],
859 });
860 if (!slow.ok) throw new Error(JSON.stringify(slow.error));
861 const opened = await openPull(app, alice, repo, {
862 title: "supersede",
863 baseBranch: "main",
864 headBranch: "supersede",
865 });
866 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
867 const env = `pull-${opened.value.number}`;
868 const old = await checks.request(
869 repo,
870 env,
871 slow.value.commitOid,
872 "push",
873 opened.value.id,
874 );
875 if (!old) throw new Error("no run");
876 await until(() => getRun(app.db, old.id)?.state === "running", "old head running");
877 const next = await commitToBranch(app, alice, repo, {
878 branch: "supersede",
879 message: "Fast",
880 files: [
881 { path: "forge.json", content: checkJson("test -f S.md") },
882 ],
883 });
884 if (!next.ok) throw new Error(JSON.stringify(next.error));
885 const fresh = await settled((r) => r.commitOid === next.value.commitOid);
886 expect(fresh.state).toBe("ok");
887 const stopped = getRun(app.db, old.id);
888 expect(stopped?.state).toBe("cancelled");
889 expect(stopped?.summary).toContain("superseded by");
890 // Stopped, not only marked: the old check, which never ends on its own,
891 // was cut off and wrote its end.
892 await until(
893 () => readRunLog(dataDir, old.id).lines.join("\n").includes("superseded"),
894 "the old head's check cut off",
895 );
896 });
897
898 for (const ending of ["merge", "close"] as const)
899 test(`${ending === "merge" ? "merging" : "closing"} a pull leaves no queued or running run for it`, async () => {
900 // Invariant: a pull merged or closed has its queued checks cancelled with that reason.
901 const blocker = await commitToBranch(app, alice, repo, {
902 message: "Hold the lane",
903 files: [{ path: "forge.json", content: checkJson(held(`lane-${ending}`)) }],
904 });
905 if (!blocker.ok) throw new Error(JSON.stringify(blocker.error));
906 await checks.request(repo, "lane", blocker.value.commitOid, "push");
907 const branched = await commitToBranch(app, alice, repo, {
908 branch: `lands-${ending}`,
909 message: "Lands before its check runs",
910 files: [{ path: "LANDS.md", content: "x\n" }],
911 });
912 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
913 const opened = await openPull(app, alice, repo, {
914 title: "Lands soon",
915 baseBranch: "main",
916 headBranch: `lands-${ending}`,
917 });
918 if (!opened.ok) throw new Error(String(opened.error));
919 const pull = opened.value;
920 const mine = (): Run[] =>
921 listRuns(app.db, repo.id, { kind: "check" }).filter(
922 (r) => r.issueId === pull.id,
923 );
924 await until(() => mine().length > 0, "the pull's check queued");
925 expect(mine().some((r) => r.state === "queued")).toBe(true);
926 if (ending === "merge") {
927 const landed = await mergePull(app, alice, repo, pull, "merge");
928 if (!landed.ok) throw new Error(String(landed.error));
929 } else {
930 const closed = setIssueState(app, alice, repo, pull, "closed");
931 if (!closed.ok) throw new Error(String(closed.error));
932 }
933 await until(() => mine().every((r) => r.state !== "queued"), "the pull's check gone");
934 expect(mine().filter((r) => r.state === "queued")).toEqual([]);
935 expect(
936 mine().some(
937 (r) =>
938 r.summary === (ending === "merge" ? "pull landed" : "pull closed"),
939 ),
940 ).toBe(true);
941 // The lane is let go, and left empty for the next test.
942 release(`lane-${ending}`);
943 await until(
944 () => checksOf(blocker.value.commitOid).every((r) => r.state !== "queued" && r.state !== "running"),
945 "the lane's holder done",
946 );
947 });
948
949 test("a queued gate keeps the last free slot from pull checks", () => {
950 // One slot free, a gate queued: the pull check passes, the gate starts.
951 expect(holdsLastSlot(1, true)).toBe(true);
952 // With no gate queued the pull check takes it, as every slot.
953 expect(holdsLastSlot(1, false)).toBe(false);
954 // More than one free: the pull check starts, one is still left.
955 expect(holdsLastSlot(2, true)).toBe(false);
956 });
957
958 test("pickNext takes the newest queued check, unless one has starved", () => {
959 const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "a" });
960 const b = createRun(app.db, { repoId: repo.id, kind: "check", env: "b" });
961 const c = createRun(app.db, { repoId: repo.id, kind: "check", env: "c" });
962 // ids are creation-ordered, so this is oldest-to-newest already.
963 expect(pickNext([a, b, c])?.id).toBe(c.id);
964 expect(pickNext([c, a, b])?.id).toBe(c.id);
965 expect(pickNext([a])?.id).toBe(a.id);
966 expect(pickNext([])).toBeUndefined();
967
968 // `a` has waited past the starvation bound: it outranks `c`, however new.
969 app.db
970 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
971 .run(a.id, new Date(Date.now() - STARVATION_MS - 1_000).toISOString());
972 const starved = getRun(app.db, a.id)!;
973 expect(pickNext([starved, b, c])?.id).toBe(starved.id);
974 // Just short of the bound, the newest still wins.
975 app.db
976 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
977 .run(a.id, new Date(Date.now() - STARVATION_MS + 1_000).toISOString());
978 expect(pickNext([getRun(app.db, a.id)!, b, c])?.id).toBe(c.id);
979
980 // These never named a real commit: cancel them rather than leave them
981 // for the live checks service running beside this test to trip over.
982 for (const run of [a, b, c]) cancelRun(app.db, run.id);
983 });
984
985 test("pickNext puts an approved pull's check first, newest among them, and the starved before all", () => {
986 const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "a" });
987 const b = createRun(app.db, { repoId: repo.id, kind: "check", env: "b" });
988 const c = createRun(app.db, { repoId: repo.id, kind: "check", env: "c" });
989 const now = Date.parse(c.createdAt);
990 const only = (...ids: string[]) => (run: { id: string }) => ids.includes(run.id);
991 // An older approved check beats a newer unapproved one.
992 expect(pickNext([a, b, c], now, only(a.id))?.id).toBe(a.id);
993 // Newest first among the approved.
994 expect(pickNext([a, b, c], now, only(a.id, b.id))?.id).toBe(b.id);
995 // With none approved, the order is unchanged.
996 expect(pickNext([a, b, c], now, () => false)?.id).toBe(c.id);
997 // A starved unapproved check still goes first.
998 const later = now + STARVATION_MS + 1_000;
999 const old = { ...a, createdAt: new Date(now - 1_000).toISOString() };
1000 const fresh = { ...b, createdAt: new Date(later).toISOString() };
1001 const approvedFresh = { ...c, createdAt: new Date(later + 1).toISOString() };
1002 expect(pickNext([fresh, approvedFresh, old], later, only(approvedFresh.id))?.id).toBe(old.id);
1003 for (const run of [a, b, c]) cancelRun(app.db, run.id);
1004 });
1005
1006 test("when a slot frees, the newest of several queued checks runs first", async () => {
1007 // Invariant: of several checks queued behind a held slot, the newest starts first.
1008 // Three checks, on three environments so none supersedes another. Their
1009 // commits are written before the slot is taken: written after, on a
1010 // loaded machine they can outlast the holder, and the oldest then finds
1011 // the slot free instead of queueing.
1012 const oldest = await commitToBranch(app, alice, repo, {
1013 branch: "order-a",
1014 message: "A",
1015 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
1016 });
1017 if (!oldest.ok) throw new Error(JSON.stringify(oldest.error));
1018 const middle = await commitToBranch(app, alice, repo, {
1019 branch: "order-b",
1020 message: "B",
1021 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
1022 });
1023 if (!middle.ok) throw new Error(JSON.stringify(middle.error));
1024 const newest = await commitToBranch(app, alice, repo, {
1025 branch: "order-c",
1026 message: "C",
1027 files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
1028 });
1029 if (!newest.ok) throw new Error(JSON.stringify(newest.error));
1030
1031 // A slow check on its own environment holds the only slot, and the
1032 // three queue behind it — oldest requested first.
1033 const holding = await commitToBranch(app, alice, repo, {
1034 branch: "order-hold",
1035 message: "Hold the slot",
1036 files: [
1037 { path: "forge.json", content: checkJson(held("order")) },
1038 ],
1039 });
1040 if (!holding.ok) throw new Error(JSON.stringify(holding.error));
1041 const holder = await checks.request(
1042 repo,
1043 "order-hold",
1044 holding.value.commitOid,
1045 "push",
1046 );
1047 expect(holder).toBeDefined();
1048
1049 const oldestRun = await checks.request(
1050 repo,
1051 "order-a",
1052 oldest.value.commitOid,
1053 "push",
1054 );
1055 const middleRun = await checks.request(
1056 repo,
1057 "order-b",
1058 middle.value.commitOid,
1059 "push",
1060 );
1061 const newestRun = await checks.request(
1062 repo,
1063 "order-c",
1064 newest.value.commitOid,
1065 "push",
1066 );
1067 expect(oldestRun && middleRun && newestRun).toBeTruthy();
1068 expect(oldestRun?.state).toBe("queued");
1069 expect(middleRun?.state).toBe("queued");
1070 expect(newestRun?.state).toBe("queued");
1071
1072 release("order");
1073 await settled((r) => r.id === newestRun!.id);
1074 await settled((r) => r.id === middleRun!.id);
1075 await settled((r) => r.id === oldestRun!.id);
1076
1077 const runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
1078 const startedAt = (id: string): number =>
1079 Date.parse(runs.find((r) => r.id === id)!.startedAt!);
1080 expect(startedAt(newestRun!.id)).toBeLessThan(startedAt(middleRun!.id));
1081 expect(startedAt(middleRun!.id)).toBeLessThan(startedAt(oldestRun!.id));
1082 });
1083
1084 test("a check in flight when the forge stops is queued again, and the next forge runs it", async () => {
1085 // Invariant: stopping the service cancels a running check as queued again, attempt + 1, and the next service runs it to the end.
1086 // Hangs the first time it runs, passes the second: no wait to tune.
1087 const id = await slowCheck(
1088 `if [ -f '${dataDir}/ran-once' ]; then test -f OK.md; else touch '${dataDir}/ran-once'; ${held("never")}; fi`,
1089 "A slower check",
1090 );
1091 // Running is said before the command begins: stop only once it has.
1092 await until(() => existsSync(join(dataDir, "ran-once")), "the command begun");
1093 const oid = getRun(app.db, id)!.commitOid;
1094 const cut = getRun(app.db, id);
1095
1096 // This forge stops — a release — with the check under way.
1097 await checks.stop();
1098 const runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
1099 const interrupted = runs.find((r) => r.id === cut!.id);
1100 expect(interrupted?.state).toBe("cancelled");
1101 expect(interrupted?.summary).toContain("queued again");
1102 const again = runs.find((r) => r.commitOid === oid && r.state === "queued");
1103 expect(again?.attempt).toBe(1);
1104
1105 // The forge that carries on finds it queued and runs it to the end.
1106 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
1107 const done = await settled((r) => r.id === again!.id);
1108 expect(done.state).toBe("ok");
1109 });
1110
1111 async function slowCheck(command: string, message: string): Promise<string> {
1112 const made = await commitToBranch(app, alice, repo, {
1113 message,
1114 files: [{ path: "forge.json", content: checkJson(command) }],
1115 });
1116 if (!made.ok) throw new Error(JSON.stringify(made.error));
1117 const oid = made.value.commitOid;
1118 await until(() => checksOf(oid)[0]?.state === "running", `${message} running`);
1119 return checksOf(oid)[0]!.id;
1120 }
1121
1122 test("a roll waits for a running check to finish, within its budget, instead of throwing it away", async () => {
1123 // Invariant: stop with a drain budget lets a running check finish to its verdict, leaving nothing queued.
1124 await checks.stop();
1125 checks = startChecks(app, {
1126 forgeUrl: servers.url,
1127 concurrency: 1,
1128 drainMs: 30_000,
1129 });
1130 const id = await slowCheck(held("drained"), "A drained check");
1131 // Let go only once the stop is under way: a stop that did not wait
1132 // would have cut it off already.
1133 const stopping = checks.stop();
1134 release("drained");
1135 await stopping;
1136 const run = getRun(app.db, id);
1137 expect(run?.state).toBe("ok");
1138 expect(
1139 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).some(
1140 (r) => r.state === "queued",
1141 ),
1142 ).toBe(false);
1143 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
1144 });
1145
1146 test("a person's stop cuts a running check off at once, and queues it again", async () => {
1147 // Invariant: stop without drain cancels a running check at once as queued again, and the next service runs it.
1148 await checks.stop();
1149 checks = startChecks(app, {
1150 forgeUrl: servers.url,
1151 concurrency: 1,
1152 drainMs: 30_000,
1153 });
1154 const id = await slowCheck(held("cut"), "A cut-off check");
1155 await checks.stop({ drain: false });
1156 const run = getRun(app.db, id);
1157 expect(run?.state).toBe("cancelled");
1158 expect(run?.summary).toContain("queued again");
1159 const again = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find(
1160 (r) => r.commitOid === run?.commitOid && r.state === "queued",
1161 );
1162 expect(again).toBeDefined();
1163 release("cut");
1164 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
1165 const done = await settled((r) => r.id === again?.id);
1166 expect(done.state).toBe("ok");
1167 });
1168
1169 test("a check whose pull lands while it runs ends as superseded, not failed", async () => {
1170 // Invariant: a running check whose row is cancelled under it logs superseded with the reason, never failed, even when its command fails.
1171 const id = await slowCheck(held("overtaken", "false"), "A check overtaken");
1172 cancelRun(app.db, id, "pull landed");
1173 release("overtaken");
1174 const log = (): string => readRunLog(dataDir, id).lines.join("\n");
1175 await until(() => log().includes("superseded"), "superseded in the log");
1176 const lines = log();
1177 expect(lines).toContain("superseded: pull landed");
1178 expect(lines).not.toContain("failed:");
1179 expect(getRun(app.db, id)?.state).toBe("cancelled");
1180 });
1181
1182355 test("a check that is declared wrong fails with the reason", async () => {
1183356 // Invariant: a malformed check declaration fails the run with the parse reason.
1184 const wrong = await commitToBranch(app, alice, repo, {
357 const wrong = await commitToBranch(f.app, f.alice, f.repo, {
1185358 message: "A wrong check",
1186359 files: [{ path: "forge.json", content: '{"check": "bun test"}\n' }],
1187360 });
1188361 if (!wrong.ok) throw new Error(JSON.stringify(wrong.error));
1189 const run = await settled((r) => r.commitOid === wrong.value.commitOid);
362 const run = await f.settled((r) => r.commitOid === wrong.value.commitOid);
1190363 expect(run.state).toBe("failed");
1191364 expect(run.summary).toBe(
1192365 "check failed: forge.json: check must be an object",
1193366 );
1194367 });
1195
1196 test("an open pull whose head was never checked is checked when the service boots", async () => {
1197 // Invariant: a booting service checks the head of an open pull that has no check, with reason boot.
1198 await checks.stop();
1199 // The pull is checked by the base's check, so the base must ask for it.
1200 const gate = await commitToBranch(app, alice, repo, {
1201 message: "The base's check wants QUIET.md",
1202 files: [{ path: "forge.json", content: checkJson("test -f QUIET.md") }],
1203 });
1204 if (!gate.ok) throw new Error(JSON.stringify(gate.error));
1205 const branch = await commitToBranch(app, alice, repo, {
1206 branch: "quiet",
1207 message: "Quiet",
1208 files: [{ path: "QUIET.md", content: "quiet\n" }],
1209 });
1210 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
1211 // The pull's check is the base's (#1275): give main a valid one.
1212 const base = await commitToBranch(app, alice, repo, {
1213 branch: "main",
1214 message: "The base's check",
1215 files: [{ path: "forge.json", content: checkJson("test -f QUIET.md") }],
1216 });
1217 if (!base.ok) throw new Error(JSON.stringify(base.error));
1218 // Opened while no checks service was listening: nothing asked for a check.
1219 const opened = await openPull(app, alice, repo, {
1220 title: "Quiet",
1221 baseBranch: "main",
1222 headBranch: "quiet",
1223 });
1224 if (!opened.ok) throw new Error(String(opened.error));
1225 expect(
1226 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
1227 (r) => r.commitOid === opened.value.headOid,
1228 ),
1229 ).toHaveLength(0);
1230 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 });
1231 const done = await settled((r) => r.commitOid === opened.value.headOid);
1232 expect(done.reason).toBe("boot");
1233 expect(done.state).toBe("ok");
1234 });
1235 });
1236
1237 describe("a check its forge died under", () => {
1238 let dataDir: string;
1239 let app: App;
1240 let alice: User;
1241 let repo: Repo;
1242 let pull: Pull;
1243 let stale: string;
1244
1245 beforeAll(async () => {
1246 dataDir = await mkdtemp(join(tmpdir(), "forge-checks-abandoned-"));
1247 app = createApp({ dataDir, port: 0 });
1248 const a = await createUser(app.db, {
1249 origin: "seed",
1250 trusted: true,
1251 username: "alice",
1252 email: "[email protected]",
1253 password: "a-good-passphrase",
1254 adminIfFirst: true,
1255 });
1256 if (!a.ok) throw new Error(a.error);
1257 alice = a.value;
1258 const made = await createRepo(app, alice, { name: "lib" });
1259 if (!made.ok) throw new Error(String(made.error));
1260 repo = made.value;
1261 const seeded = await commitToBranch(app, alice, repo, {
1262 message: "Start",
1263 files: [{ path: "README.md", content: "# lib\n" }],
1264 });
1265 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
1266 const first = await commitToBranch(app, alice, repo, {
1267 branch: "feature",
1268 message: "First",
1269 files: [{ path: "A.md", content: "a\n" }],
1270 });
1271 if (!first.ok) throw new Error(JSON.stringify(first.error));
1272 stale = first.value.commitOid;
1273 const second = await commitToBranch(app, alice, repo, {
1274 branch: "feature",
1275 message: "Second",
1276 files: [{ path: "B.md", content: "b\n" }],
1277 });
1278 if (!second.ok) throw new Error(JSON.stringify(second.error));
1279 const opened = await openPull(app, alice, repo, {
1280 title: "Feature",
1281 baseBranch: "main",
1282 headBranch: "feature",
1283 });
1284 if (!opened.ok) throw new Error(String(opened.error));
1285 pull = opened.value;
1286 });
1287
1288 afterAll(async () => {
1289 app.close();
1290 await rm(dataDir, { recursive: true, force: true });
1291 });
1292
1293 /** A check that was running when its forge died: claimed, lease long lapsed. */
1294 const abandoned = (commitOid: string, attempt = 0): Run => {
1295 const run = createRun(app.db, {
1296 repoId: repo.id,
1297 kind: "check",
1298 env: `pull-${pull.number}`,
1299 commitOid,
1300 reason: "push",
1301 issueId: pull.id,
1302 attempt,
1303 });
1304 claimRun(app.db, run.id);
1305 app.db
1306 .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1")
1307 .run(run.id, new Date(Date.now() - STALE_AFTER_MS - 1_000).toISOString());
1308 return getRun(app.db, run.id)!;
1309 };
1310
1311 const word = () =>
1312 pullWord(
1313 {
1314 db: app.db,
1315 dir: repoDir(app.reposRoot, repo.ownerName, repo.name),
1316 runtime: undefined,
1317 record: undefined,
1318 history: servingHistory(),
1319 viewer: undefined,
1320 },
1321 pull,
1322 );
1323
1324 const queuedFor = (commitOid: string): Run[] =>
1325 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter(
1326 (r) => r.commitOid === commitOid && r.state === "queued",
1327 );
1328
1329 test("a check for the head of an open pull is queued again, attempt + 1, and the change reads working", async () => {
1330 const run = abandoned(pull.headOid);
1331 expect((await word()).word).toBe("working");
1332 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
1333 // A second forge sweeping the same lapse finds nothing left to move.
1334 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(0);
1335 const cut = getRun(app.db, run.id)!;
1336 expect(cut.state).toBe("cancelled");
1337 expect(cut.summary).toBe(REQUEUED_SUMMARY);
1338 const again = queuedFor(pull.headOid);
1339 expect(again).toHaveLength(1);
1340 expect(again[0]!.attempt).toBe(1);
1341 expect(again[0]!.issueId).toBe(pull.id);
1342 expect(again[0]!.env).toBe(`pull-${pull.number}`);
1343 expect(await word()).toEqual({ word: "working", note: "queued to check" });
1344 cancelRun(app.db, again[0]!.id);
1345 });
1346
1347 test("a check whose lease is still held is left alone", () => {
1348 const run = createRun(app.db, {
1349 repoId: repo.id,
1350 kind: "check",
1351 env: `pull-${pull.number}`,
1352 commitOid: pull.headOid,
1353 issueId: pull.id,
1354 });
1355 claimRun(app.db, run.id);
1356 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(0);
1357 expect(getRun(app.db, run.id)?.state).toBe("running");
1358 cancelRun(app.db, run.id);
1359 });
1360
1361 test("at the ceiling the check fails once, saying the forge kept going away", async () => {
1362 const run = abandoned(pull.headOid, CHECK_RESTART_LIMIT);
1363 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
1364 const failed = getRun(app.db, run.id)!;
1365 expect(failed.state).toBe("failed");
1366 expect(failed.summary).toContain(
1367 "the forge kept going away under this check",
1368 );
1369 expect(queuedFor(pull.headOid)).toHaveLength(0);
1370 expect((await word()).word).toBe("stopped");
1371 });
1372
1373 test("a check for a commit that no longer heads an open pull is cancelled, not queued again", () => {
1374 const run = abandoned(stale);
1375 expect(requeueAbandonedChecks(app, STALE_AFTER_MS)).toBe(1);
1376 const cut = getRun(app.db, run.id)!;
1377 expect(cut.state).toBe("cancelled");
1378 expect(cut.summary).toContain("no longer heads an open pull");
1379 expect(queuedFor(stale)).toHaveLength(0);
1380 });
1381 });
1382
1383 describe("startChecks: a booting forge removes only checkouts nobody is running (#1884)", () => {
1384 test("a live run's checkout survives, a lapsed run's and a stranger's go", async () => {
1385 // Invariant: the boot sweep removes checkouts of lapsed or unknown runs and keeps a live run's.
1386 const dir = await mkdtemp(join(tmpdir(), "forge-sweep-"));
1387 const db = createApp({ dataDir: dir, port: 0 });
1388 try {
1389 const u = await createUser(db.db, {
1390 origin: "seed",
1391 trusted: true,
1392 username: "alice",
1393 email: "[email protected]",
1394 password: "a-good-passphrase",
1395 adminIfFirst: true,
1396 });
1397 if (!u.ok) throw new Error(u.error);
1398 const made = await createRepo(db, u.value, { name: "lib" });
1399 if (!made.ok) throw new Error(String(made.error));
1400 const mk = (): Run => {
1401 const run = createRun(db.db, { repoId: made.value.id, kind: "check", env: "x" });
1402 claimRun(db.db, run.id);
1403 return run;
1404 };
1405 const live = mk();
1406 const lapsed = mk();
1407 db.db
1408 .query("UPDATE run SET heartbeat_at = ?2 WHERE id = ?1")
1409 .run(lapsed.id, new Date(Date.now() - STALE_AFTER_MS - 5_000).toISOString());
1410 const where = (id: string): string => join(dir, "checks", id);
1411 for (const id of [live.id, lapsed.id, "no-such-run"])
1412 mkdirSync(where(id), { recursive: true });
1413 const handle = startChecks(db, { forgeUrl: "http://localhost" });
1414 await sweepSoon(db);
1415 await handle.stop();
1416 expect(existsSync(where(live.id))).toBe(true);
1417 expect(existsSync(where(lapsed.id))).toBe(false);
1418 expect(existsSync(where("no-such-run"))).toBe(false);
1419 } finally {
1420 db.close();
1421 await rm(dir, { recursive: true, force: true });
1422 }
1423 });
1424368 });
1425369
1426370 describe("checks: a finished shard is announced as the runner writes it (#2889)", () => {
test/helpers/checks-forge.ts added +152
@@ -0,0 +1,152 @@
1 /**
2 * A forge with its checks service running, for the tests of a repository's
3 * own check (`test/checks*.test.ts`): alice (admin) and bob (a member, a
4 * reader of alice's repository) and alice's public `lib`, one check slot.
5 * `lib`'s main starts with `OK.md` and no forge.json: no check is asked for
6 * it, and a check that passes by `test -f OK.md` (`held`'s default) passes.
7 *
8 * `checksForge` registers the beforeAll that boots it and the afterAll that
9 * stops it; the fields are set once beforeAll has run. `checks` is the live
10 * service and may be replaced by a test that stops and starts it again.
11 */
12
13 import { afterAll, beforeAll } from "bun:test";
14 import { writeFileSync } from "node:fs";
15 import { mkdtemp, rm } from "node:fs/promises";
16 import { tmpdir } from "node:os";
17 import { join } from "node:path";
18 import { createApp, type App } from "../../src/app.ts";
19 import { commitToBranch, createRepo } from "../../src/actions/repos.ts";
20 import { startChecks, type ChecksHandle } from "../../src/checks/service.ts";
21 import { listRuns, type Run } from "../../src/core/runs.ts";
22 import type { Pull } from "../../src/core/pulls.ts";
23 import { createUser, type User } from "../../src/core/users.ts";
24 import type { Repo } from "../../src/core/repos.ts";
25 import { serve, type Servers } from "../../src/http/server.ts";
26 import { listEvents } from "../../src/core/events.ts";
27 import { promote } from "./members.ts";
28
29 /** A forge.json declaring `run` as the check, with no install step. */
30 export const checkJson = (run: string, extra = ""): string =>
31 `{"check": {"run": ${JSON.stringify(run)}, "install": ""${extra}}}\n`;
32
33 export type ChecksForge = {
34 dataDir: string;
35 app: App;
36 servers: Servers;
37 checks: ChecksHandle;
38 alice: User;
39 bob: User;
40 repo: Repo;
41 /** Waits for the first check `where` names to end ok or failed. */
42 settled(where: (run: Run) => boolean, timeoutMs?: number): Promise<Run>;
43 /** Waits until `done` holds, polling the database; a check's own pace, not a timer's. */
44 until(done: () => boolean, what: string): Promise<void>;
45 /**
46 * A check command that holds its slot until the test lets it go
47 * (`release`), then runs `then`: the test is the clock, so no wait is
48 * guessed at. Never released, it runs until it is cut off.
49 */
50 held(name: string, then?: string): string;
51 release(name: string): void;
52 /** Waits until a fork pull's new head is announced on the bus as `pull.pushed`. */
53 headAnnounced(pull: Pull, headOid: string): Promise<void>;
54 checksOf(commitOid: string): Run[];
55 /** Commits `command` as main's check and waits for its run to be running; its id. */
56 slowCheck(command: string, message: string): Promise<string>;
57 };
58
59 export function checksForge(prefix = "forge-checks-"): ChecksForge {
60 const f = {} as ChecksForge;
61
62 f.settled = async (where, timeoutMs = 4_000) => {
63 const deadline = Date.now() + timeoutMs;
64 while (Date.now() < deadline) {
65 const run = listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).find(where);
66 if (run && (run.state === "ok" || run.state === "failed")) return run;
67 await Bun.sleep(10);
68 }
69 throw new Error("check did not settle");
70 };
71
72 f.until = async (done, what) => {
73 const deadline = Date.now() + 4_000;
74 while (!done()) {
75 if (Date.now() > deadline) throw new Error(`never came: ${what}`);
76 await Bun.sleep(10);
77 }
78 };
79
80 f.held = (name, then = "test -f OK.md") =>
81 `while [ ! -f '${f.dataDir}/go-${name}' ]; do sleep 0.01; done; ${then}`;
82 f.release = (name) => writeFileSync(join(f.dataDir, `go-${name}`), "");
83
84 f.headAnnounced = (pull, headOid) =>
85 f.until(
86 () =>
87 listEvents(f.app.db, { repoId: f.repo.id, kind: "pull.pushed" }).some(
88 (e) => e.payload["number"] === pull.number && e.payload["headOid"] === headOid,
89 ),
90 "the pull's new head announced",
91 );
92
93 f.checksOf = (commitOid) =>
94 listRuns(f.app.db, f.repo.id, { kind: "check", limit: 50 }).filter(
95 (r) => r.commitOid === commitOid,
96 );
97
98 f.slowCheck = async (command, message) => {
99 const made = await commitToBranch(f.app, f.alice, f.repo, {
100 message,
101 files: [{ path: "forge.json", content: checkJson(command) }],
102 });
103 if (!made.ok) throw new Error(JSON.stringify(made.error));
104 const oid = made.value.commitOid;
105 await f.until(() => f.checksOf(oid)[0]?.state === "running", `${message} running`);
106 return f.checksOf(oid)[0]!.id;
107 };
108
109 beforeAll(async () => {
110 f.dataDir = await mkdtemp(join(tmpdir(), prefix));
111 f.app = createApp({ dataDir: f.dataDir, port: 0 });
112 f.servers = await serve(f.app);
113 f.checks = startChecks(f.app, { forgeUrl: f.servers.url, concurrency: 1 });
114 const a = await createUser(f.app.db, {
115 origin: "seed",
116 trusted: true,
117 username: "alice",
118 email: "[email protected]",
119 password: "a-good-passphrase",
120 adminIfFirst: true,
121 });
122 if (!a.ok) throw new Error(a.error);
123 f.alice = a.value;
124 const b = await createUser(f.app.db, {
125 origin: "seed",
126 trusted: true,
127 username: "bob",
128 email: "[email protected]",
129 password: "a-good-passphrase",
130 });
131 if (!b.ok) throw new Error(b.error);
132 f.bob = b.value;
133 promote(f.app.db, f.bob);
134 const made = await createRepo(f.app, f.alice, { name: "lib", visibility: "public" });
135 if (!made.ok) throw new Error(String(made.error));
136 f.repo = made.value;
137 const seeded = await commitToBranch(f.app, f.alice, f.repo, {
138 message: "Seed OK.md",
139 files: [{ path: "OK.md", content: "ok\n" }],
140 });
141 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
142 });
143
144 afterAll(async () => {
145 await f.checks.stop();
146 f.servers.stop();
147 f.app.close();
148 await rm(f.dataDir, { recursive: true, force: true });
149 });
150
151 return f;
152 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs