trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns12g rEventsFlows

The studio suites depend on test order: 7-12 tests fail under every --randomize seed #2069

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/2063-the-studio-suites-depend-on-test-order-7 → main 9 commits fork

  • Merged. mason/2063-the-studio-suites-depend-on-test-order-7 was merged into main as c08a5f8.

Review

Sign in to review this change.
4 files changed +152 −93
test/studio-door.test.ts +17 −19
@@ -5,7 +5,15 @@
55 * (#2038); the rest are test/studio*.test.ts.
66 */
77
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import {
9 afterAll,
10 afterEach,
11 beforeAll,
12 beforeEach,
13 describe,
14 expect,
15 test,
16 } from "bun:test";
917 import { mkdtemp, rm } from "node:fs/promises";
1018 import { tmpdir } from "node:os";
1119 import { join } from "node:path";
@@ -327,17 +335,6 @@ describe("every open pull's preview on the studio page", () => {
327335 .filter((r) => !isTerminal(r.state))
328336 .map((r) => `${r.env}:${r.state}`);
329337
330 /** Wait until the runtime has no deploy open or queued, so the next case
331 * does not queue behind this one's (concurrency is 1). */
332 async function deploysIdle(): Promise<void> {
333 const deadline = Date.now() + WAIT_MS;
334 while (unfinished().length > 0) {
335 if (Date.now() > deadline)
336 throw new Error(`deploys still open: ${unfinished().join(", ")}`);
337 await Bun.sleep(50);
338 }
339 }
340
341338 /** Wait for `env` to serve; fail naming the state it reached. */
342339 async function previewServes(record: AppRecord, env: EnvName): Promise<void> {
343340 const deadline = Date.now() + WAIT_MS;
@@ -354,7 +351,9 @@ describe("every open pull's preview on the studio page", () => {
354351 }
355352 }
356353
357 beforeAll(async () => {
354 // A fresh app for each test: one test's open pull would otherwise be the
355 // next's preview, and its deploys queue ahead of the next's (concurrency 1).
356 beforeEach(async () => {
358357 dataDir = await mkdtemp(join(tmpdir(), "forge-studio-previews-"));
359358 app = createApp({ dataDir, port: 0 });
360359 servers = await serve(app);
@@ -413,7 +412,7 @@ describe("every open pull's preview on the studio page", () => {
413412 if (!enabled.ok) throw new Error(String(enabled.error));
414413 }, 30_000);
415414
416 afterAll(async () => {
415 afterEach(async () => {
417416 await runtime.stop();
418417 servers.stop();
419418 app.close();
@@ -451,9 +450,6 @@ describe("every open pull's preview on the studio page", () => {
451450 const after = await (await get("/alice/widget")).text();
452451 expect(after).not.toContain("<h2>Previews</h2>");
453452 expect(after).not.toContain(`pull-${pull.number}/open`);
454 // The merge queues production's deploy and the pull's was cut short;
455 // leave the single deploy slot free for the next case.
456 await deploysIdle();
457453 }, 120_000);
458454
459455 test("a change's page names its preview's data as its own copy and gives the moment it was taken; production's card does not", async () => {
@@ -1181,7 +1177,9 @@ describe("the app's live fragment: /:owner/:name/app/state", () => {
11811177 redirect: "manual",
11821178 });
11831179
1184 beforeAll(async () => {
1180 // A fresh app for each test: one test's pull or preview slot would
1181 // otherwise show up in the next's head card and previews.
1182 beforeEach(async () => {
11851183 dataDir = await mkdtemp(join(tmpdir(), "forge-studio-state-"));
11861184 app = createApp({ dataDir, port: 0 });
11871185 servers = await serve(app);
@@ -1225,7 +1223,7 @@ describe("the app's live fragment: /:owner/:name/app/state", () => {
12251223 record = getAppByRepo(app.db, repo.id)!;
12261224 });
12271225
1228 afterAll(async () => {
1226 afterEach(async () => {
12291227 servers.stop();
12301228 app.close();
12311229 await rm(dataDir, { recursive: true, force: true });
test/studio-runs.test.ts +37 −21
@@ -5,7 +5,16 @@
55 * test/studio*.test.ts.
66 */
77
8 import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test";
8 import {
9 afterAll,
10 afterEach,
11 beforeAll,
12 beforeEach,
13 describe,
14 expect,
15 spyOn,
16 test,
17 } from "bun:test";
918 import { mkdtemp, rm } from "node:fs/promises";
1019 import { tmpdir } from "node:os";
1120 import { join } from "node:path";
@@ -877,7 +886,9 @@ describe("the app's page tails the deploy that is bringing it alive", () => {
877886 redirect: "manual",
878887 });
879888
880 beforeAll(async () => {
889 // A fresh app for each test: one test's tell or run would otherwise fill
890 // "Now" for the next, and its production slot stand in for the next's.
891 beforeEach(async () => {
881892 dataDir = await mkdtemp(join(tmpdir(), "forge-studio-deploy-tail-"));
882893 app = createApp({ dataDir, port: 0 });
883894 servers = await serve(app);
@@ -918,7 +929,7 @@ describe("the app's page tails the deploy that is bringing it alive", () => {
918929 expect(addToCrew(app, a.value, repo, "scout").ok).toBe(true);
919930 });
920931
921 afterAll(async () => {
932 afterEach(async () => {
922933 servers.stop();
923934 app.close();
924935 await rm(dataDir, { recursive: true, force: true });
@@ -1425,7 +1436,12 @@ describe('a tell nobody on the crew can take is not "quiet"', () => {
14251436 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
14261437 const enabled = await enableApp(app, alice, repo, { slug: "pad" });
14271438 if (!enabled.ok) throw new Error(String(enabled.error));
1428 // No crew at all — the exact state `tell` cannot route out of.
1439 // No crew at all — the exact state `tell` cannot route out of. Every
1440 // test below reads the one tell told into it here.
1441 const told = await post("/alice/pad/tell", {
1442 text: "Add a dark mode toggle.",
1443 });
1444 if (told.status !== 303) throw new Error(`tell: ${told.status}`);
14291445 });
14301446
14311447 afterAll(async () => {
@@ -1435,11 +1451,6 @@ describe('a tell nobody on the crew can take is not "quiet"', () => {
14351451 });
14361452
14371453 test('a crewless tell renders a "Now" row, not the empty state, and links to where a crew member is added', async () => {
1438 const told = await post("/alice/pad/tell", {
1439 text: "Add a dark mode toggle.",
1440 });
1441 expect(told.status).toBe(303);
1442
14431454 const page = await (await get("/alice/pad")).text();
14441455 expect(page).not.toContain("Nothing yet. Tell the crew something.");
14451456 expect(page).toContain("Add a dark mode toggle.");
@@ -1852,18 +1863,6 @@ describe("a deploy that just landed is news on the front door", () => {
18521863 deployRun =
18531864 finishRun(app.db, deployRun.id, { state: "ok", summary: "deploy" }) ??
18541865 deployRun;
1855 const servingAt = new Date().toISOString();
1856 putEnv(record, "production", {
1857 live: {
1858 commitOid: "6c99218",
1859 handle: { port: 4321 },
1860 runId: deployRun.id,
1861 since: servingAt,
1862 servingAt,
1863 createdAt: servingAt,
1864 manifest: { tty: null },
1865 },
1866 });
18671866
18681867 // A second app, otherwise identical, that never shipped — the control
18691868 // that must keep reading quiet.
@@ -1885,6 +1884,23 @@ describe("a deploy that just landed is news on the front door", () => {
18851884 quietRepo = madeQuiet.value;
18861885 });
18871886
1887 // Each test starts from the merge's deploy serving just now; a test that
1888 // wants another production slot puts its own.
1889 beforeEach(() => {
1890 const servingAt = new Date().toISOString();
1891 putEnv(record, "production", {
1892 live: {
1893 commitOid: "6c99218",
1894 handle: { port: 4321 },
1895 runId: deployRun.id,
1896 since: servingAt,
1897 servingAt,
1898 createdAt: servingAt,
1899 manifest: { tty: null },
1900 },
1901 });
1902 });
1903
18881904 afterAll(async () => {
18891905 servers.stop();
18901906 app.close();
test/studio-shipping.test.ts +55 −23
@@ -5,7 +5,14 @@
55 * test/studio*.test.ts.
66 */
77
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import {
9 afterAll,
10 beforeAll,
11 beforeEach,
12 describe,
13 expect,
14 test,
15 } from "bun:test";
916 import { mkdtemp, rm } from "node:fs/promises";
1017 import { tmpdir } from "node:os";
1118 import { join } from "node:path";
@@ -1179,6 +1186,12 @@ describe("previewLatency: the median seconds from push to serving", () => {
11791186 await rm(dataDir, { recursive: true, force: true });
11801187 });
11811188
1189 // Each test reads only the deploys it seeds: the window has no upper
1190 // bound, so another test's deploys would otherwise land in it.
1191 beforeEach(() => {
1192 app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repo.id);
1193 });
1194
11821195 /** A deploy run stamped with a known push instant and, unless null, known started/serving instants. */
11831196 const seedDeploy = (
11841197 createdAt: string,
@@ -1252,7 +1265,7 @@ describe("previewLatency: the median seconds from push to serving", () => {
12521265 });
12531266
12541267 test("the split between waiting to be picked up and coming up, seeded beside a total", () => {
1255 const now = Date.now() + 10_000_000; // clear of the previous test's window
1268 const now = Date.now();
12561269 const since = new Date(now - 60_000).toISOString();
12571270
12581271 // Three deploys with known queue and boot segments: (queue, boot) in seconds.
@@ -1293,7 +1306,7 @@ describe("previewLatency: the median seconds from push to serving", () => {
12931306 });
12941307
12951308 test("a deploy that finished without ever serving is counted apart, never in the median; one still running is counted as neither", () => {
1296 const now = Date.now() + 20_000_000; // clear of the earlier tests' windows
1309 const now = Date.now();
12971310 const since = new Date(now - 60_000).toISOString();
12981311
12991312 // Two served deploys, 4s and 6s.
@@ -1351,7 +1364,7 @@ describe("previewLatency: the median seconds from push to serving", () => {
13511364 });
13521365
13531366 test("a week with only never-served deploys still reports the failed count, with no median to give", () => {
1354 const now = Date.now() + 30_000_000;
1367 const now = Date.now();
13551368 const since = new Date(now - 60_000).toISOString();
13561369
13571370 const failed = createRun(app.db, { repoId: repo.id, kind: "deploy" });
@@ -1369,7 +1382,7 @@ describe("previewLatency: the median seconds from push to serving", () => {
13691382 });
13701383
13711384 test("a deploy the forge's own restart cancelled is never counted as failed, nor in the total it is 'of' (#255, #274)", () => {
1372 const now = Date.now() + 40_000_000;
1385 const now = Date.now();
13731386 const since = new Date(now - 60_000).toISOString();
13741387
13751388 // A real failure: counted.
@@ -1482,12 +1495,40 @@ describe("the studio's own words for how fast a change reaches its preview", ()
14821495 await rm(dataDir, { recursive: true, force: true });
14831496 });
14841497
1498 // Each test reads only the deploys it seeds; the agent run from
1499 // beforeAll stays, so the page always has history behind it.
1500 beforeEach(() => {
1501 app.db
1502 .query("DELETE FROM run WHERE repo_id = ?1 AND kind = 'deploy'")
1503 .run(repo.id);
1504 });
1505
14851506 const get = (path: string) =>
14861507 fetch(`${base}${path}`, {
14871508 headers: { cookie: `${ownerCookie}; forge_mode=studio` },
14881509 redirect: "manual",
14891510 });
14901511
1512 /** A deploy run stamped with a known push instant and serving instant, and unless null a known started instant. */
1513 const seedDeploy = (
1514 createdAt: string,
1515 startedAt: string | null,
1516 servingAt: string,
1517 ): void => {
1518 const deploy = createRun(app.db, { repoId: repo.id, kind: "deploy" });
1519 app.db
1520 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
1521 .run(deploy.id, createdAt);
1522 if (startedAt !== null) {
1523 app.db
1524 .query("UPDATE run SET started_at = ?2 WHERE id = ?1")
1525 .run(deploy.id, startedAt);
1526 }
1527 app.db
1528 .query("UPDATE run SET serving_at = ?2 WHERE id = ?1")
1529 .run(deploy.id, servingAt);
1530 };
1531
14911532 test("fewer than three served deploys this week says so plainly rather than a median of one", async () => {
14921533 const before = await (await get("/alice/trickle")).text();
14931534 expect(before).toContain("not enough deploys this week to say");
@@ -1506,24 +1547,6 @@ describe("the studio's own words for how fast a change reaches its preview", ()
15061547 });
15071548
15081549 test("the split reads as the total's explanation, and says when it is drawn from fewer deploys", async () => {
1509 const seedDeploy = (
1510 createdAt: string,
1511 startedAt: string | null,
1512 servingAt: string,
1513 ): void => {
1514 const deploy = createRun(app.db, { repoId: repo.id, kind: "deploy" });
1515 app.db
1516 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
1517 .run(deploy.id, createdAt);
1518 if (startedAt !== null) {
1519 app.db
1520 .query("UPDATE run SET started_at = ?2 WHERE id = ?1")
1521 .run(deploy.id, startedAt);
1522 }
1523 app.db
1524 .query("UPDATE run SET serving_at = ?2 WHERE id = ?1")
1525 .run(deploy.id, servingAt);
1526 };
15271550 // Three deploys with a known queue and boot each.
15281551 const now = Date.now();
15291552 const segments: readonly (readonly [number, number])[] = [
@@ -1561,6 +1584,15 @@ describe("the studio's own words for how fast a change reaches its preview", ()
15611584 });
15621585
15631586 test("a deploy that finished without ever serving is counted and worded, and links to the runs list", async () => {
1587 // Four served deploys first, so the failed one is the fifth.
1588 const now = Date.now();
1589 for (let i = 0; i < 4; i++) {
1590 seedDeploy(
1591 new Date(now + i * 1_000).toISOString(),
1592 null,
1593 new Date(now + i * 1_000 + 5_000).toISOString(),
1594 );
1595 }
15641596 const before = await (await get("/alice/trickle")).text();
15651597
15661598 const failed = createRun(app.db, {
test/studio.test.ts +43 −30
@@ -62,6 +62,35 @@ describe("the studio", () => {
6262 redirect: "manual",
6363 });
6464
65 /** An app of alice's with scout on its crew. */
66 const makeApp = async (name: string, description = ""): Promise<Repo> => {
67 const made = await createRepo(app, alice, {
68 name,
69 visibility: "private",
70 description,
71 });
72 if (!made.ok) throw new Error(String(made.error));
73 const seeded = await commitToBranch(app, alice, made.value, {
74 message: "Start",
75 files: [
76 {
77 path: "package.json",
78 content: `{"name":"${name}","scripts":{"start":"bun run server.ts"}}\n`,
79 },
80 {
81 path: "server.ts",
82 content:
83 "Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response('ok') });\n",
84 },
85 ],
86 });
87 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
88 const enabled = await enableApp(app, alice, made.value, { slug: name });
89 if (!enabled.ok) throw new Error(String(enabled.error));
90 expect(addToCrew(app, alice, made.value, "scout").ok).toBe(true);
91 return made.value;
92 };
93
6594 beforeAll(async () => {
6695 dataDir = await mkdtemp(join(tmpdir(), "forge-studio-"));
6796 app = createApp({ dataDir, port: 0 });
@@ -87,33 +116,9 @@ describe("the studio", () => {
87116 })
88117 ).headers.get("set-cookie") ?? ""
89118 ).split(";")[0]!;
90 const made = await createRepo(app, alice, {
91 name: "notes",
92 visibility: "private",
93 description: "a place for notes",
94 });
95 if (!made.ok) throw new Error(String(made.error));
96 repo = made.value;
97 const seeded = await commitToBranch(app, alice, repo, {
98 message: "Start",
99 files: [
100 {
101 path: "package.json",
102 content: '{"name":"notes","scripts":{"start":"bun run server.ts"}}\n',
103 },
104 {
105 path: "server.ts",
106 content:
107 "Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response('ok') });\n",
108 },
109 ],
110 });
111 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
112 const enabled = await enableApp(app, alice, repo, { slug: "notes" });
113 if (!enabled.ok) throw new Error(String(enabled.error));
114119 const scout = await createAgent(app, alice, { username: "scout" });
115120 if (!scout.ok) throw new Error(String(scout.error));
116 expect(addToCrew(app, alice, repo, "scout").ok).toBe(true);
121 repo = await makeApp("notes", "a place for notes");
117122 });
118123
119124 afterAll(async () => {
@@ -278,6 +283,14 @@ describe("the studio", () => {
278283 });
279284
280285 test("ship lands a change; say tells the crew", async () => {
286 // Its own app, so the tell it ships is the only one waiting there.
287 const repo = await makeApp("shipnotes");
288 const told = await post("/alice/shipnotes/tell", {
289 text: "Make the notes page faster.",
290 });
291 expect(told.status).toBe(303);
292 const tell = listIssues(app.db, repo.id, { kind: "issue", limit: 5 })[0]!;
293 expect(tell.assigneeName).toBe("scout");
281294 const branch = await commitToBranch(app, alice, repo, {
282295 branch: "feature/fast",
283296 base: "main",
@@ -287,13 +300,13 @@ describe("the studio", () => {
287300 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
288301 const opened = await openPull(app, alice, repo, {
289302 title: "Faster",
290 body: "Closes #1",
303 body: `Closes #${tell.number}`,
291304 baseBranch: "main",
292305 headBranch: "feature/fast",
293306 });
294307 if (!opened.ok) throw new Error(String(opened.error));
295308 const pull = opened.value;
296 const said = await post(`/alice/notes/pulls/${pull.number}/say`, {
309 const said = await post(`/alice/shipnotes/pulls/${pull.number}/say`, {
297310 text: "Keep the images lazy too.",
298311 });
299312 expect(said.status).toBe(303);
@@ -310,7 +323,7 @@ describe("the studio", () => {
310323 // A bare POST — a button with no fields, or a client that sent no body
311324 // at all — ships too: an empty form is not a bad request.
312325 const shipped = await fetch(
313 `${base}/alice/notes/pulls/${pull.number}/ship`,
326 `${base}/alice/shipnotes/pulls/${pull.number}/ship`,
314327 {
315328 method: "POST",
316329 headers: { cookie: `${cookie}; forge_mode=studio`, origin: base },
@@ -319,8 +332,8 @@ describe("the studio", () => {
319332 );
320333 expect(shipped.status).toBe(303);
321334 expect(getPull(app.db, repo.id, pull.number)?.mergedAt).not.toBeNull();
322 expect(getIssue(app.db, repo.id, 1)?.state).toBe("closed");
323 const page = await (await get("/alice/notes")).text();
335 expect(getIssue(app.db, repo.id, tell.number)?.state).toBe("closed");
336 const page = await (await get("/alice/shipnotes")).text();
324337 expect(page).toContain("shipped");
325338 // What was told and then shipped is not still "waiting" in Now.
326339 expect(page).not.toContain(">waiting<");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs