trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns15g rEventsFlows

A test that catches dev's words on a studio page, for every studio page there will be #84

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/62-a-test-that-catches-dev-s-words-on-a-stu → main 1 commit fork

  • Merged. scout/62-a-test-that-catches-dev-s-words-on-a-stu was merged into main as d627897.

Review

Sign in to review this change.
3 files changed +407 −19
forge.json +1 −1
@@ -6,7 +6,7 @@
66 "ready": 90
77 },
88 "check": {
9 "run": "bun x tsc --noEmit && bun test --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/repos.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/web.test.ts test/git-trust.test.ts",
9 "run": "bun x tsc --noEmit && bun test --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/studio-words.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/repos.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/web.test.ts test/git-trust.test.ts",
1010 "timeout": 1800
1111 }
1212 }
src/web/pages/studio.ts +2 −18
@@ -50,12 +50,7 @@ import {
5050 pullForBranch,
5151 type Pull,
5252 } from "../../core/pulls.ts";
53 import {
54 canWrite,
55 getRepoById,
56 listReposVisibleTo,
57 type Repo,
58 } from "../../core/repos.ts";
53 import { canWrite, listReposVisibleTo, type Repo } from "../../core/repos.ts";
5954 import { listReviews } from "../../core/reviews.ts";
6055 import {
6156 activeRunFor,
@@ -1507,13 +1502,6 @@ export async function studioPull(
15071502 (d) => d.pull?.id === pull.id,
15081503 );
15091504
1510 const headRepo = pull.crossRepo
1511 ? getRepoById(db, pull.headRepoId)
1512 : undefined;
1513 const headLabel = headRepo
1514 ? `${headRepo.ownerName}/${headRepo.name}:${pull.headBranch}`
1515 : pull.headBranch;
1516
15171505 const head = html`<div class="issue-head">
15181506 <h1>${pull.title} <span class="number">#${pull.number}</span></h1>
15191507 <div class="issue-meta">
@@ -1531,7 +1519,6 @@ export async function studioPull(
15311519 >${icon("external")}view on origin</a
15321520 >`,
15331521 )}
1534 <span class="mono">${headLabel} → ${pull.baseBranch}</span>
15351522 </div>
15361523 </div>`;
15371524
@@ -1560,10 +1547,7 @@ export async function studioPull(
15601547 ),
15611548 );
15621549
1563 const talk = conversation(ctx, repo, pull, {
1564 openVerb: html`wants to merge <span class="mono">${headLabel}</span> into
1565 <span class="mono">${pull.baseBranch}</span> ·`,
1566 });
1550 const talk = conversation(ctx, repo, pull, { openVerb: "opened this" });
15671551
15681552 const body = html`${repoHeader(ctx, repo, "pulls", {
15691553 compact: true,
test/studio-words.test.ts added +404
@@ -0,0 +1,404 @@
1 /**
2 * One mechanical check for every studio page: none of dev's vocabulary
3 * leaks onto it, the security floor's headers are still on it, the studio
4 * chrome is, and it stays small enough that a diff could never hide inside
5 * it. `docs/STUDIO.md`'s five words are each one thing in git; this is the
6 * test that nothing else says the git word instead.
7 *
8 * To register a new studio page for this check: add one row to `pages`
9 * (built in the test below, once the fixture that gives it something real
10 * to show exists) with the studio word it speaks and the path to fetch.
11 * Nothing else in this file changes — the same forbidden-vocabulary,
12 * chrome, header and size assertions walk it too. A route not yet wired
13 * into the router (`http/server.ts`) answers 404 and is skipped rather
14 * than failed, so a row for a page still being built does not block this
15 * file from landing first.
16 */
17 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
18 import { mkdtemp, rm } from "node:fs/promises";
19 import { tmpdir } from "node:os";
20 import { join } from "node:path";
21 import { createApp, type App } from "../src/app.ts";
22 import { enableApp } from "../src/actions/apps.ts";
23 import { previewEnv, type EnvName } from "../src/apps/data.ts";
24 import { AppsRuntime } from "../src/apps/runtime.ts";
25 import { getAppByRepo, type AppRecord } from "../src/core/apps.ts";
26 import { addToCrew, createAgent } from "../src/actions/crew.ts";
27 import { assignIssue, openIssue } from "../src/actions/issues.ts";
28 import { mergePull, openPull } from "../src/actions/pulls.ts";
29 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
30 import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
31 import { createUser, getUserByName } from "../src/core/users.ts";
32 import { serve, type Servers } from "../src/http/server.ts";
33
34 /**
35 * Dev's own words, each with the reason it must never reach a studio page.
36 * Checked with every `<script>` stripped first: the shared client script
37 * (`shellJs`, `src/web/client.ts`) ships on every page and classifies a raw
38 * agent log line by matching literal text like "branch " and "committed
39 * the", so its source would otherwise trip this file's own checks on pages
40 * that never rendered a word of it.
41 */
42 const DEV_WORDS: readonly { pattern: RegExp; because: string }[] = [
43 {
44 pattern: /diff-table/,
45 because: "a change is judged by what it does, never its diff",
46 },
47 {
48 pattern: /class="label lc-/,
49 because: "a label chip is dev's own sorting, not one of the five words",
50 },
51 {
52 pattern: /opened by/i,
53 because: "the studio says who made a change in its own words",
54 },
55 {
56 pattern: /checks (passed|failed)/i,
57 because: "a check reads as one of the five words, not dev's pass/fail",
58 },
59 {
60 pattern: /\brepository\b/i,
61 because: "the studio calls it an app",
62 },
63 {
64 pattern: /\bcommit\b/i,
65 because: "a short sha in a caption is fine; the word never is",
66 },
67 {
68 pattern: /\bbranch\b/i,
69 because: "a change is judged by what it does, not where it lives",
70 },
71 {
72 pattern: /\bmerge\b/i,
73 because: "the studio's own word for a merge is shipped",
74 },
75 ];
76
77 /**
78 * What a page itself says, in words a person reads — `<main id="main">`,
79 * `page()`'s own (`src/web/layout.ts`), not the one sitewide shell around
80 * every page in both faces on purpose: the topbar (mode switch, the "New
81 * repository" item beside "New app" in the "+" menu, search) and the
82 * footer. A hover title, an aria-label ("Where this repository is read
83 * from", on the mirror pin every repository carries) or an href (the deep
84 * link to dev's own commit view under a short sha, `?mode=dev` itself) is
85 * a destination or a hint for a screen reader, not a word the page said
86 * out loud — the risk this file's own header warns about, a short sha in
87 * a caption is fine and the link it sits on is how the studio keeps dev
88 * one click away.
89 */
90 const pageContent = (html: string): string =>
91 (html.match(/<main[^>]*id="main"[\s\S]*?<\/main>/)?.[0] ?? html)
92 .replace(/\s(?:aria-label|title|placeholder|href)="[^"]*"/g, "")
93 // The pull/issue state badge (`stateBadge`, "Merged") and the
94 // conversation's own "merged this" history line (`stateEvents`,
95 // `src/web/pages/thread.ts`) are the one thread dev's page reads too —
96 // docs/STUDIO.md says so explicitly ("the conversation: read it, add
97 // to it"). The studio's own word for the state is `wordBadge`
98 // elsewhere on the page (`data-word="shipped"`), checked below; these
99 // are the shared history underneath it, not a word the studio page
100 // itself chose to say.
101 .replace(/data-pr="merged"[\s\S]*?Merged<\/span/, 'data-pr="merged">…</span')
102 .replace(/<span>merged this<\/span>/, "<span>…</span>")
103 // Every icon (`icon()`, `src/web/icons.ts`) is a decorative, aria-hidden
104 // svg named after a Lucide glyph — `ico-merge` is a shape, not a word,
105 // and it rides on things dev and studio share alike (a timeline dot).
106 .replace(/<svg[\s\S]*?<\/svg>/g, "");
107
108 /** Generous next to a real page (tens of KB); nowhere near the 2.5 MB one
109 * file diff cost the studio pull page before it had its own render. */
110 const MAX_BODY_BYTES = 300_000;
111
112 describe("dev's vocabulary never reaches a studio page", () => {
113 let dataDir: string;
114 let app: App;
115 let servers: Servers;
116 let runtime: AppsRuntime;
117 let base: string;
118 let cookie: string;
119 let record: AppRecord;
120 let tellNumber: number;
121 let runId: string;
122 let openPullNumber: number;
123 let mergedPullNumber: number;
124 let openBranch: string;
125 let mergedBranch: string;
126
127 interface EnvSlot {
128 readonly app: AppRecord;
129 readonly env: EnvName;
130 readonly chain: Promise<void>;
131 readonly restarts: number[];
132 live?: unknown;
133 }
134 const putLive = (env: EnvName, live: unknown): void => {
135 (runtime as unknown as { envs: Map<string, EnvSlot> }).envs.set(
136 `${record.id}/${env}`,
137 { app: record, env, chain: Promise.resolve(), restarts: [], live },
138 );
139 };
140
141 beforeAll(async () => {
142 dataDir = await mkdtemp(join(tmpdir(), "forge-studio-words-"));
143 app = createApp({ dataDir, port: 0 });
144 servers = await serve(app);
145 base = servers.url;
146 // No `.start()`: this test boots one forge and drives it through the
147 // real actions, but the preview and production environments below are
148 // injected directly, the way `test/studio.test.ts`'s own fragment tests
149 // do, so nothing here waits on a real install or a real deploy.
150 runtime = new AppsRuntime(app, {
151 domain: "studio-words.test",
152 scheme: "http",
153 publicPort: servers.port,
154 forgeUrl: servers.url,
155 concurrency: 1,
156 });
157
158 const a = await createUser(app.db, {
159 username: "alice",
160 email: "[email protected]",
161 password: "a-good-passphrase",
162 adminIfFirst: true,
163 });
164 if (!a.ok) throw new Error(a.error);
165 cookie = (
166 (
167 await fetch(`${base}/signin`, {
168 method: "POST",
169 body: new URLSearchParams({
170 username: "alice",
171 password: "a-good-passphrase",
172 }),
173 redirect: "manual",
174 })
175 ).headers.get("set-cookie") ?? ""
176 ).split(";")[0]!;
177
178 const made = await createRepo(app, a.value, {
179 name: "vocab",
180 visibility: "private",
181 description: "a place to catch dev's words",
182 });
183 if (!made.ok) throw new Error(String(made.error));
184 const repo = made.value;
185 const seeded = await commitToBranch(app, a.value, repo, {
186 message: "Start",
187 files: [
188 {
189 path: "package.json",
190 content: '{"name":"vocab","scripts":{"start":"bun run server.ts"}}\n',
191 },
192 {
193 path: "server.ts",
194 content:
195 "Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response('ok') });\n",
196 },
197 ],
198 });
199 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
200 const enabled = await enableApp(app, a.value, repo, { slug: "vocab" });
201 if (!enabled.ok) throw new Error(String(enabled.error));
202 record = getAppByRepo(app.db, repo.id)!;
203
204 const scout = await createAgent(app, a.value, { username: "scout" });
205 if (!scout.ok) throw new Error(String(scout.error));
206 expect(addToCrew(app, a.value, repo, "scout").ok).toBe(true);
207 const scoutUser = getUserByName(app.db, "scout")!;
208
209 // A tell, assigned, with a run — the studio's *tell* and *working*.
210 const told = openIssue(app, a.value, repo, {
211 title: "Make the export button work",
212 body: "It does nothing right now.",
213 });
214 if (!told.ok) throw new Error(String(told.error));
215 tellNumber = told.value.number;
216 expect(assignIssue(app, a.value, repo, told.value, scoutUser).ok).toBe(
217 true,
218 );
219 const run = createRun(app.db, {
220 repoId: repo.id,
221 kind: "agent",
222 issueId: told.value.id,
223 actorId: scoutUser.id,
224 reason: "assigned",
225 });
226 claimRun(app.db, run.id);
227 finishRun(app.db, run.id, {
228 state: "ok",
229 summary: "Wired up the export handler.",
230 costUsd: 0.34,
231 turns: 6,
232 });
233 runId = run.id;
234
235 // An open pull with a green check and a preview — the studio's *look*.
236 openBranch = "add-export";
237 const openBranchMade = await commitToBranch(app, a.value, repo, {
238 branch: openBranch,
239 base: repo.defaultBranch,
240 message: "Add export",
241 files: [{ path: "export.ts", content: "export {};\n" }],
242 });
243 if (!openBranchMade.ok) throw new Error(JSON.stringify(openBranchMade.error));
244 const opened = await openPull(app, scoutUser, repo, {
245 title: "Add the export button",
246 body: `Closes #${tellNumber}.`,
247 baseBranch: repo.defaultBranch,
248 headBranch: openBranch,
249 });
250 if (!opened.ok) throw new Error(String(opened.error));
251 openPullNumber = opened.value.number;
252 const check = createRun(app.db, {
253 repoId: repo.id,
254 kind: "check",
255 issueId: opened.value.id,
256 env: `pull-${opened.value.number}`,
257 commitOid: opened.value.headOid,
258 reason: "pull",
259 });
260 claimRun(app.db, check.id);
261 finishRun(app.db, check.id, { state: "ok", summary: "check passed in 1s" });
262 putLive(previewEnv(opened.value.number), {
263 commitOid: opened.value.headOid,
264 handle: { port: 4321 },
265 runId: check.id,
266 since: new Date().toISOString(),
267 });
268
269 // A merged change, production caught up to it — the studio's *shipped*.
270 mergedBranch = "faster-notes";
271 const mergedBranchMade = await commitToBranch(app, a.value, repo, {
272 branch: mergedBranch,
273 base: repo.defaultBranch,
274 message: "Faster",
275 files: [{ path: "FAST.md", content: "fast\n" }],
276 });
277 if (!mergedBranchMade.ok)
278 throw new Error(JSON.stringify(mergedBranchMade.error));
279 const toMerge = await openPull(app, a.value, repo, {
280 title: "Make the notes page faster",
281 body: "",
282 baseBranch: repo.defaultBranch,
283 headBranch: mergedBranch,
284 });
285 if (!toMerge.ok) throw new Error(String(toMerge.error));
286 mergedPullNumber = toMerge.value.number;
287 const mergedCheck = createRun(app.db, {
288 repoId: repo.id,
289 kind: "check",
290 issueId: toMerge.value.id,
291 env: `pull-${toMerge.value.number}`,
292 commitOid: toMerge.value.headOid,
293 reason: "pull",
294 });
295 claimRun(app.db, mergedCheck.id);
296 finishRun(app.db, mergedCheck.id, {
297 state: "ok",
298 summary: "check passed in 1s",
299 });
300 const merged = await mergePull(app, a.value, repo, toMerge.value, "squash");
301 if (!merged.ok) throw new Error(String(merged.error));
302 const mergedOid = merged.value.mergeOid;
303 const deploy = createRun(app.db, {
304 repoId: repo.id,
305 kind: "deploy",
306 env: "production",
307 commitOid: mergedOid,
308 reason: "push",
309 });
310 claimRun(app.db, deploy.id);
311 finishRun(app.db, deploy.id, { state: "ok", summary: "deploy" });
312 putLive("production", {
313 commitOid: mergedOid,
314 handle: { port: 4321 },
315 runId: deploy.id,
316 since: new Date().toISOString(),
317 });
318 });
319
320 afterAll(async () => {
321 servers.stop();
322 app.close();
323 await rm(dataDir, { recursive: true, force: true });
324 });
325
326 test("home, app, tell, working, look and shipped: no dev word, the studio chrome, the security floor, and a page a diff would never fit in", async () => {
327 const pages: { word: string; path: string }[] = [
328 { word: "home", path: "/" },
329 { word: "app", path: "/alice/vocab" },
330 { word: "tell", path: `/alice/vocab/issues/${tellNumber}` },
331 { word: "working", path: `/alice/vocab/runs/${runId}` },
332 { word: "look", path: `/alice/vocab/pulls/${openPullNumber}` },
333 { word: "shipped", path: `/alice/vocab/pulls/${mergedPullNumber}` },
334 ];
335
336 for (const p of pages) {
337 const res = await fetch(`${base}${p.path}`, {
338 headers: { cookie: `${cookie}; forge_mode=studio` },
339 redirect: "manual",
340 });
341 // A route not yet wired into the router answers 404 — skip it rather
342 // than fail, so a row for a page still being built (docs/STUDIO.md's
343 // lists, as they land) does not block this file.
344 if (res.status === 404) continue;
345 expect(res.status).toBe(200);
346 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
347 expect(res.headers.get("referrer-policy")).toBe("same-origin");
348 expect(res.headers.get("content-security-policy")).toContain(
349 "default-src 'none'",
350 );
351
352 const html = await res.text();
353 expect(
354 html.length,
355 `${p.word} (${p.path}) is ${html.length} bytes — a diff would blow through ${MAX_BODY_BYTES}`,
356 ).toBeLessThan(MAX_BODY_BYTES);
357 // The studio chrome: the topbar's own tab reads "Studio", never
358 // "Code", the moment the mode and the repository are the studio's —
359 // and it is the only source of the literal word on any of these
360 // pages, so its presence here is the chrome, proven.
361 expect(html).toContain("Studio");
362
363 const visible = pageContent(html);
364 for (const { pattern, because } of DEV_WORDS) {
365 expect(
366 visible,
367 `${p.word} (${p.path}) said dev's word ${pattern} — ${because}`,
368 ).not.toMatch(pattern);
369 }
370 expect(
371 visible,
372 `${p.word} (${p.path}) named the open branch "${openBranch}"`,
373 ).not.toContain(openBranch);
374 expect(
375 visible,
376 `${p.word} (${p.path}) named the merged branch "${mergedBranch}"`,
377 ).not.toContain(mergedBranch);
378 }
379
380 // A run id is a key, not a headline: it rides hrefs (the dev link, the
381 // steer form) but never becomes the page's own title.
382 const transcript = await (
383 await fetch(`${base}/alice/vocab/runs/${runId}`, {
384 headers: { cookie: `${cookie}; forge_mode=studio` },
385 redirect: "manual",
386 })
387 ).text();
388 const title = transcript.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? "";
389 const h1 = transcript.match(/<h1>([\s\S]*?)<\/h1>/)?.[1] ?? "";
390 expect(title).not.toContain(runId);
391 expect(h1).not.toContain(runId);
392 });
393
394 test("?mode=dev on the same routes still carries dev's own words — the assertions above are real, not vacuous", async () => {
395 const dev = await (
396 await fetch(`${base}/alice/vocab/pulls/${openPullNumber}?mode=dev`, {
397 headers: { cookie },
398 redirect: "manual",
399 })
400 ).text();
401 expect(dev).toContain("diff-table");
402 expect(dev).toContain(openBranch);
403 });
404 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs