trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns16g rEventsFlows

test/apps-adversarial-test-step.test.ts runs under 10 s #3581

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/3567-test-apps-adversarial-test-step-test-ts → main 2 commits fork 1 thread · 1 unresolved

  • Merged. mason/3567-test-apps-adversarial-test-step-test-ts was merged into main as 1369f0d.

Review

Sign in to review this change.
2 files changed +136 −133
scripts/test-times.json +1 −1
@@ -2,7 +2,7 @@
22 "test/apps-adversarial-grants.test.ts": 2.5,
33 "test/apps-adversarial-rules.test.ts": 29.7,
44 "test/apps-adversarial-teardown.test.ts": 3,
5 "test/apps-adversarial-test-step.test.ts": 33.5,
5 "test/apps-adversarial-test-step.test.ts": 1.8,
junipercrew

1.8 s is a local figure without cells. Replace it with the recorded cell time so the shards balance.
66 "test/apps-adversarial.test.ts": 18,
77 "test/apps-busy.test.ts": 26.4,
88 "test/apps-crew.test.ts": 3.8,
test/apps-adversarial-test-step.test.ts +135 −132
@@ -3,7 +3,9 @@
33 * runtime: what a failing install and a closed pull's test step leave
44 * behind. Split from apps-adversarial.test.ts so no one file sets the
55 * gate's floor (#2038); each file of the split boots its own forge and
6 * runtime.
6 * runtime. Here one app serves both tests, each through a pull of its own,
7 * and every wait is on the forge's event bus — nothing polls or sleeps
8 * (#3567).
79 */
810
911 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
@@ -17,6 +19,8 @@ import { newApp } from "../src/actions/apps.ts";
1719 import { mergePull, openPull } from "../src/actions/pulls.ts";
1820 import { commitToBranch } from "../src/actions/repos.ts";
1921 import { getPull } from "../src/core/pulls.ts";
22 import type { ForgeEvent } from "../src/core/events.ts";
23 import type { Repo } from "../src/core/repos.ts";
2024 import { listRuns, type Run } from "../src/core/runs.ts";
2125 import { createUser, type User } from "../src/core/users.ts";
2226 import { serve, type Servers } from "../src/http/server.ts";
@@ -30,6 +34,7 @@ describe("apps adversarial: the install and test steps", () => {
3034 let servers: Servers;
3135 let runtime: AppsRuntime;
3236 let alice: User;
37 let repo: Repo;
3338
3439 const viaHost = (host: string, path = "/", init: RequestInit = {}) =>
3540 fetch(`${servers.url}${path}`, {
@@ -38,26 +43,75 @@ describe("apps adversarial: the install and test steps", () => {
3843 redirect: "manual",
3944 });
4045
41 async function settled(
42 repoId: string,
43 where: (run: Run) => boolean,
44 timeoutMs = 60_000,
45 ): Promise<Run> {
46 const deadline = Date.now() + timeoutMs;
47 while (Date.now() < deadline) {
48 const run = listRuns(app.db, repoId, { kind: "deploy", limit: 50 }).find(
49 where,
50 );
51 if (
52 run &&
53 (run.state === "ok" ||
54 run.state === "failed" ||
55 run.state === "cancelled")
56 )
57 return run;
58 await Bun.sleep(50);
59 }
60 throw new Error("run did not settle in time");
46 /**
47 * The first value `look` gives, asked now and again after every event
48 * the forge appends — a run's progress and an app's serving are both
49 * announced on the bus, so nothing here polls on a timer.
50 */
51 function when<T>(
52 look: (event?: ForgeEvent) => T | undefined,
53 what: string,
54 timeoutMs = 20_000,
55 ): Promise<T> {
56 return new Promise((resolve, reject) => {
57 const done = (value: T): void => {
58 unsubscribe();
59 clearTimeout(timer);
60 resolve(value);
61 };
62 const unsubscribe = app.bus.subscribe((event) => {
63 const value = look(event);
64 if (value !== undefined) done(value);
65 });
66 const timer = setTimeout(() => {
67 unsubscribe();
68 reject(new Error(`${what} did not happen in time`));
69 }, timeoutMs);
70 const now = look();
71 if (now !== undefined) done(now);
72 });
73 }
74
75 const settled = (where: (run: Run) => boolean): Promise<Run> =>
76 when(
77 () =>
78 listRuns(app.db, repo.id, { kind: "deploy", limit: 50 }).find(
79 (run) =>
80 where(run) &&
81 (run.state === "ok" ||
82 run.state === "failed" ||
83 run.state === "cancelled"),
84 ),
85 "the deploy settling",
86 );
87
88 /** The pull branch carrying only this forge.json, opened against the default branch. */
89 async function pullWith(
90 branch: string,
91 manifest: Record<string, string>,
92 ): Promise<{ number: number; commitOid: string }> {
93 const committed = await commitToBranch(app, alice, repo, {
94 branch,
95 base: repo.defaultBranch,
96 message: branch,
97 files: [
98 {
99 path: "forge.json",
100 content: `${JSON.stringify({ app: manifest }, null, 2)}\n`,
101 },
102 ],
103 });
104 if (!committed.ok) throw new Error(JSON.stringify(committed.error));
105 const opened = await openPull(app, alice, repo, {
106 title: branch,
107 baseBranch: repo.defaultBranch,
108 headBranch: branch,
109 });
110 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
111 return {
112 number: opened.value.number,
113 commitOid: committed.value.commitOid,
114 };
61115 }
62116
63117 beforeAll(async () => {
@@ -81,6 +135,15 @@ describe("apps adversarial: the install and test steps", () => {
81135 });
82136 if (!created.ok) throw new Error(created.error);
83137 alice = created.value;
138 const made = await newApp(app, alice, {
139 name: "steps",
140 visibility: "public",
141 });
142 if (!made.ok) throw new Error(made.error);
143 repo = made.value.repo;
144 const production = await settled((r) => r.env === "production");
145 if (production.state !== "ok")
146 throw new Error(`production did not deploy: ${production.summary}`);
84147 }, 30_000);
85148
86149 afterAll(async () => {
@@ -92,139 +155,79 @@ describe("apps adversarial: the install and test steps", () => {
92155 });
93156
94157 test("install failing fails before test or start ever run — nothing from a half-installed tree", async () => {
95 const made = await newApp(app, alice, {
96 name: "badinstall",
97 visibility: "public",
158 const testMarker = join(work, "install-test-marker");
159 const runMarker = join(work, "install-run-marker");
160 const pull = await pullWith("bad-install", {
161 install: "exit 1",
162 run: `touch ${runMarker} && bun run src/server.ts`,
163 test: `touch ${testMarker} && bun test`,
98164 });
99 expect(made.ok).toBe(true);
100 if (!made.ok) return;
101 const repo = made.value.repo;
102 await settled(repo.id, (r) => r.env === "production");
103
104 const testMarker = join(work, `install-test-marker-${Date.now()}`);
105 const runMarker = join(work, `install-run-marker-${Date.now()}`);
106 const committed = await commitToBranch(app, alice, repo, {
107 branch: "bad-install",
108 base: repo.defaultBranch,
109 message: "install fails",
110 files: [
111 {
112 path: "forge.json",
113 content: `${JSON.stringify(
114 {
115 app: {
116 install: "exit 1",
117 run: `touch ${runMarker} && bun run src/server.ts`,
118 test: `touch ${testMarker} && bun test`,
119 },
120 },
121 null,
122 2,
123 )}\n`,
124 },
125 ],
126 });
127 expect(committed.ok).toBe(true);
128 if (!committed.ok) return;
129 const opened = await openPull(app, alice, repo, {
130 title: "Bad install",
131 baseBranch: repo.defaultBranch,
132 headBranch: "bad-install",
133 });
134 expect(opened.ok).toBe(true);
135 if (!opened.ok) return;
136165
137166 const failed = await settled(
138 repo.id,
139 (r) =>
140 r.env === `pull-${opened.value.number}` &&
141 r.commitOid === committed.value.commitOid,
167 (r) => r.env === `pull-${pull.number}` && r.commitOid === pull.commitOid,
142168 );
143169 expect(failed.state).toBe("failed");
144170 expect(failed.summary).toContain("install");
145171 // Neither `test` nor `run` ever executed.
146172 expect(existsSync(testMarker)).toBe(false);
147173 expect(existsSync(runMarker)).toBe(false);
148 expect(
149 (await viaHost(`badinstall--${opened.value.number}.${DOMAIN}`)).status,
150 ).toBe(502);
151 }, 60_000);
174 expect((await viaHost(`steps--${pull.number}.${DOMAIN}`)).status).toBe(
175 502,
176 );
177 }, 20_000);
152178
153179 test("closing a pull whose deploy already answered kills its still-running test step — no orphan `bun test`", async () => {
154 const made = await newApp(app, alice, {
155 name: "slowtest",
156 visibility: "public",
157 });
158 expect(made.ok).toBe(true);
159 if (!made.ok) return;
160 const repo = made.value.repo;
161 await settled(repo.id, (r) => r.env === "production");
162
163180 // A merge lands this same forge.json on the default branch and
164 // production redeploys it too, running this same test step to
165 // completion (nothing aborts a deploy that was never in flight when the
166 // pull closed) — a marker keyed by $FORGE_APP_ENV keeps that legitimate
167 // production run from being mistaken for the preview's orphan.
168 const marker = join(work, `slowtest-marker-${Date.now()}`);
169 const committed = await commitToBranch(app, alice, repo, {
170 branch: "slow-test",
171 base: repo.defaultBranch,
172 message: "slow test",
173 files: [
174 {
175 path: "forge.json",
176 content: `${JSON.stringify(
177 {
178 app: {
179 run: "bun run src/server.ts",
180 test: `sleep 5 && touch ${marker}-$FORGE_APP_ENV`,
181 },
182 },
183 null,
184 2,
185 )}\n`,
186 },
187 ],
181 // production redeploys it too; only the preview's test step sleeps, so
182 // production's legitimate run of it finishes at once and can never be
183 // mistaken for the preview's orphan. The sleep is far longer than this
184 // test may take: the preview's run can only settle in time if the close
185 // killed the step.
186 const marker = join(work, "slowtest-marker");
187 const serving = when(
188 (event) =>
189 event?.kind === "app.serving" &&
190 event.repoId === repo.id &&
191 event.payload.env !== "production"
192 ? String(event.payload.env)
193 : undefined,
194 "the preview serving",
195 );
196 const pull = await pullWith("slow-test", {
197 run: "bun run src/server.ts",
198 test: `if [ "$FORGE_APP_ENV" = preview ]; then sleep 30 && touch ${marker}; fi`,
188199 });
189 expect(committed.ok).toBe(true);
190 if (!committed.ok) return;
191 const opened = await openPull(app, alice, repo, {
192 title: "Slow test",
193 baseBranch: repo.defaultBranch,
194 headBranch: "slow-test",
195 });
196 expect(opened.ok).toBe(true);
197 if (!opened.ok) return;
200 const env = `pull-${pull.number}`;
201 expect(await serving).toBe(env);
202 expect((await viaHost(`steps--${pull.number}.${DOMAIN}`)).status).toBe(
203 200,
204 );
198205
199 // Wait for the app to answer — well before the test step's sleep ends.
200 const upDeadline = Date.now() + 10_000;
201 let up = false;
202 while (Date.now() < upDeadline) {
203 if (
204 (await viaHost(`slowtest--${opened.value.number}.${DOMAIN}`))
205 .status === 200
206 ) {
207 up = true;
208 break;
209 }
210 await Bun.sleep(50);
211 }
212 expect(up).toBe(true);
206 // The app answers while its test step is still sleeping.
207 const preview = (): Run | undefined =>
208 listRuns(app.db, repo.id, { kind: "deploy", limit: 50 }).find(
209 (r) => r.env === env && r.commitOid === pull.commitOid,
210 );
211 expect(preview()?.state).toBe("running");
213212
214213 // Close it while the test step is still sleeping.
215214 const closed = await mergePull(
216215 app,
217216 alice,
218217 repo,
219 getPull(app.db, repo.id, opened.value.number)!,
218 getPull(app.db, repo.id, pull.number)!,
220219 "merge",
221220 );
222221 expect(closed.ok).toBe(true);
223222
224 // Wait well past the sleep the test step was doing, and confirm it
225 // never finished: the preview's own marker is not there (production's,
226 // from its own unrelated redeploy of the same merged commit, may be).
227 await Bun.sleep(6_500);
228 expect(existsSync(`${marker}-preview`)).toBe(false);
229 }, 60_000);
223 // The step's outcome is only reported once its process has exited, so
224 // a run settled as cancelled is a step that is gone — the shell that
225 // would have touched the marker with it.
226 const ended = await settled(
227 (r) => r.env === env && r.commitOid === pull.commitOid,
228 );
229 expect(ended.state).toBe("failed");
230 expect(ended.summary).toBe("test: test cancelled");
231 expect(existsSync(marker)).toBe(false);
232 }, 20_000);
230233 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs