trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns19g rEventsFlowsAPI

test/apps-adversarial-rules.test.ts runs under 10 s #3598

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/3577-test-apps-adversarial-rules-test-ts-runs → main 2 commits fork

  • Merged. mason/3577-test-apps-adversarial-rules-test-ts-runs was merged into main as 163e032.

Review

Sign in to review this change.
2 files changed +144 −161
scripts/test-times.json +1 −1
@@ -1,6 +1,6 @@
11 {
22 "test/apps-adversarial-grants.test.ts": 2.5,
3 "test/apps-adversarial-rules.test.ts": 29.7,
3 "test/apps-adversarial-rules.test.ts": 2.3,
44 "test/apps-adversarial-teardown.test.ts": 3,
55 "test/apps-adversarial-test-step.test.ts": 33.5,
66 "test/apps-adversarial.test.ts": 18,
test/apps-adversarial-rules.test.ts +143 −160
@@ -3,14 +3,17 @@
33 * runtime: environment names, preview data, and rules 1 and 4. Split from
44 * apps-adversarial.test.ts so no one file sets the gate's floor (#2038);
55 * each file of the split boots its own forge and runtime.
6 *
7 * Fixtures are built once (#3577): one forge, one app deployed to
8 * production, one seeded repository with its agent. The preview the
9 * preview_data test reads is opened in `beforeAll` and asserted last, so
10 * its deploy runs while the git tests do. Nothing polls: a deploy is
11 * awaited on the bus's `run.finished`.
612 */
713
814 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
1115 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { AppsRuntime } from "../src/apps/runtime.ts";
16 import type { App } from "../src/app.ts";
1417 import { newApp, updateAppSettings } from "../src/actions/apps.ts";
1518 import { addToCrew, createAgent } from "../src/actions/crew.ts";
1619 import { mergePull, openPull } from "../src/actions/pulls.ts";
@@ -20,8 +23,9 @@ import {
2023 forkRepo,
2124 updateRepoSettings,
2225 } from "../src/actions/repos.ts";
23 import { getPull } from "../src/core/pulls.ts";
24 import { getRepoById } from "../src/core/repos.ts";
26 import type { AppRecord } from "../src/core/apps.ts";
27 import { getPull, type Pull } from "../src/core/pulls.ts";
28 import { getRepoById, type Repo } from "../src/core/repos.ts";
2529 import { resolveOid } from "../src/git/merge.ts";
2630 import { repoDir } from "../src/git/repos.ts";
2731 import {
@@ -29,16 +33,21 @@ import {
2933 claimRun,
3034 createRun,
3135 finishRun,
36 isTerminal,
3237 listRuns,
3338 type Run,
3439 } from "../src/core/runs.ts";
3540 import { createToken } from "../src/core/tokens.ts";
36 import { createUser, type User } from "../src/core/users.ts";
37 import { serve, type Servers } from "../src/http/server.ts";
41 import type { User } from "../src/core/users.ts";
3842 import { grantedCookie } from "./helpers/identity.ts";
3943 import { gitClientEnv } from "./helpers/git-client.ts";
40
41 const DOMAIN = "apps.test";
44 import {
45 APPS_DOMAIN as DOMAIN,
46 fetchAsHost,
47 startAppsForge,
48 stopAppsForge,
49 type AppsForge,
50 } from "./helpers/apps-forge.ts";
4251
4352 const clientEnv = gitClientEnv({
4453 GIT_AUTHOR_NAME: "Agent",
@@ -62,82 +71,121 @@ async function sh(args: string[], cwd: string) {
6271 return { code, out: out.trim(), err: err.trim() };
6372 }
6473
74 /**
75 * The newest deploy on `repoId` matching `where`, once it has finished:
76 * read now, and again on every `run.finished` the bus carries for the
77 * repository — every deploy the runtime finishes is announced there.
78 */
79 function settled(
80 app: App,
81 repoId: string,
82 where: (run: Run) => boolean,
83 timeoutMs = 60_000,
84 ): Promise<Run> {
85 const done = () =>
86 listRuns(app.db, repoId, { kind: "deploy", limit: 50 }).find(
87 (run) => where(run) && isTerminal(run.state),
88 );
89 return new Promise((resolve, reject) => {
90 const finish = (run: Run) => {
91 unsubscribe();
92 clearTimeout(timer);
93 resolve(run);
94 };
95 const unsubscribe = app.bus.subscribe((event) => {
96 if (event.kind !== "run.finished" || event.repoId !== repoId) return;
97 const run = done();
98 if (run) finish(run);
99 });
100 const timer = setTimeout(() => {
101 unsubscribe();
102 reject(new Error("run did not settle in time"));
103 }, timeoutMs);
104 const run = done();
105 if (run) finish(run);
106 });
107 }
108
65109 describe("apps adversarial: env names, preview data, and rules 1/4", () => {
66 let dataDir: string;
67 let work: string;
110 let forge: AppsForge;
68111 let app: App;
69 let servers: Servers;
70 let runtime: AppsRuntime;
71112 let alice: User;
113 /** `adversarial`: public, serving in production, preview_data "empty". */
114 let deployed: { repo: Repo; record: AppRecord };
115 /** The pull whose preview must start without production's data. */
116 let previewPull: Pull;
117 /** A seeded repository with `rule1bot` on its crew. */
118 let seeded: Repo;
119 let bot: User;
72120
73121 const viaHost = (host: string, path = "/", init: RequestInit = {}) =>
74 fetch(`${servers.url}${path}`, {
75 ...init,
76 headers: { ...(init.headers as Record<string, string>), host },
77 redirect: "manual",
78 });
79
80 async function settled(
81 repoId: string,
82 where: (run: Run) => boolean,
83 timeoutMs = 60_000,
84 ): Promise<Run> {
85 const deadline = Date.now() + timeoutMs;
86 while (Date.now() < deadline) {
87 const run = listRuns(app.db, repoId, { kind: "deploy", limit: 50 }).find(
88 where,
89 );
90 if (
91 run &&
92 (run.state === "ok" ||
93 run.state === "failed" ||
94 run.state === "cancelled")
95 )
96 return run;
97 await Bun.sleep(50);
98 }
99 throw new Error("run did not settle in time");
100 }
122 fetchAsHost(forge.servers, host, path, init);
101123
102124 beforeAll(async () => {
103 dataDir = await mkdtemp(join(tmpdir(), "forge-adv-"));
104 work = await mkdtemp(join(tmpdir(), "forge-adv-work-"));
105 app = createApp({ dataDir, port: 0 });
106 servers = await serve(app);
107 runtime = new AppsRuntime(app, {
108 domain: DOMAIN,
109 scheme: "http",
110 publicPort: servers.port,
111 forgeUrl: servers.url,
112 concurrency: 2,
125 forge = await startAppsForge();
126 ({ app, alice } = forge);
127
128 const made = await newApp(app, alice, {
129 name: "adversarial",
130 visibility: "public",
113131 });
114 await runtime.start();
115 const created = await createUser(app.db, { origin: "seed", trusted: true,
116 username: "alice",
117 email: "alice@forge.local",
118 password: "a-good-passphrase",
119 adminIfFirst: true,
132 if (!made.ok) throw new Error(String(made.error));
133 const { repo, app: record } = made.value;
134 const production = await settled(app, repo.id, (r) => r.env === "production");
135 if (production.state !== "ok")
136 throw new Error(`production did not deploy: ${production.summary}`);
137 deployed = { repo, record };
138
139 // Production holds a note before the preview is opened; the preview's
140 // deploy then runs in the background while the tests below do.
141 updateAppSettings(app, alice, repo, { previewData: "empty" });
142 const form = new FormData();
143 form.set("body", "production-only note");
144 await viaHost(`adversarial.${DOMAIN}`, "/notes", {
145 method: "POST",
146 body: form,
147 headers: {
148 cookie: await grantedCookie(app, forge.servers.url, alice, record),
149 },
120150 });
121 if (!created.ok) throw new Error(created.error);
122 alice = created.value;
123 }, 30_000);
151 const committed = await commitToBranch(app, alice, repo, {
152 branch: "empty-preview",
153 base: repo.defaultBranch,
154 message: "trigger a preview",
155 files: [{ path: "EMPTY.md", content: "hi\n" }],
156 });
157 if (!committed.ok) throw new Error(String(committed.error));
158 const opened = await openPull(app, alice, repo, {
159 title: "Empty preview",
160 baseBranch: repo.defaultBranch,
161 headBranch: "empty-preview",
162 });
163 if (!opened.ok) throw new Error(String(opened.error));
164 previewPull = opened.value;
165
166 const madeRepo = await createRepo(app, alice, { name: "rule1-repo" });
167 if (!madeRepo.ok) throw new Error(String(madeRepo.error));
168 seeded = madeRepo.value;
169 await commitToBranch(app, alice, seeded, {
170 message: "seed",
171 files: [{ path: "README.md", content: "hi\n" }],
172 });
173 const madeAgent = await createAgent(app, alice, {
174 username: "rule1bot",
175 model: "",
176 brief: "",
177 });
178 if (!madeAgent.ok) throw new Error(String(madeAgent.error));
179 bot = madeAgent.value.user;
180 addToCrew(app, alice, seeded, "rule1bot");
181 }, 60_000);
124182
125183 afterAll(async () => {
126 await runtime.stop();
127 servers.stop();
128 app.close();
129 await rm(dataDir, { recursive: true, force: true });
130 await rm(work, { recursive: true, force: true });
184 await forge.runtime.stop();
185 await stopAppsForge(forge);
131186 });
132187
133188 test("the API never lets an env parameter shaped like a path reach the filesystem", async () => {
134 const made = await newApp(app, alice, {
135 name: "envtraversal",
136 visibility: "public",
137 });
138 expect(made.ok).toBe(true);
139 if (!made.ok) return;
140 await settled(made.value.repo.id, (r) => r.env === "production");
141189 const adminToken = createToken(
142190 app.db,
143191 alice.id,
@@ -145,12 +193,12 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
145193 "admin",
146194 ).token;
147195 const api = (path: string, init: RequestInit = {}) =>
148 fetch(`${servers.url}${path}`, {
196 fetch(`${forge.servers.url}${path}`, {
149197 ...init,
150198 headers: { authorization: `Bearer ${adminToken}` },
151199 });
152 const owner = made.value.repo.ownerName;
153 const name = made.value.repo.name;
200 const owner = deployed.repo.ownerName;
201 const name = deployed.repo.name;
154202 // A bare ".." segment is collapsed by `URL`/`fetch` before the request
155203 // ever leaves the client (RFC 3986 dot-segment removal), so it can only
156204 // be used to probe the *server's* handling when percent-encoded — a raw
@@ -190,74 +238,13 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
190238 }
191239 }, 60_000);
192240
193 test('preview_data: "empty" starts a preview with no copy of production\'s data', async () => {
194 const made = await newApp(app, alice, {
195 name: "emptypolicy",
196 visibility: "public",
197 });
198 expect(made.ok).toBe(true);
199 if (!made.ok) return;
200 const repo = made.value.repo;
201 await settled(repo.id, (r) => r.env === "production");
202 updateAppSettings(app, alice, repo, { previewData: "empty" });
203
204 const form = new FormData();
205 form.set("body", "production-only note");
206 await viaHost(`emptypolicy.${DOMAIN}`, "/notes", {
207 method: "POST",
208 body: form,
209 headers: {
210 cookie: await grantedCookie(app, servers.url, alice, made.value.app),
211 },
212 });
213 expect(await (await viaHost(`emptypolicy.${DOMAIN}`)).text()).toContain(
214 "production-only note",
215 );
216
217 const committed = await commitToBranch(app, alice, repo, {
218 branch: "empty-preview",
219 base: repo.defaultBranch,
220 message: "trigger a preview",
221 files: [{ path: "EMPTY.md", content: "hi\n" }],
222 });
223 expect(committed.ok).toBe(true);
224 const opened = await openPull(app, alice, repo, {
225 title: "Empty preview",
226 baseBranch: repo.defaultBranch,
227 headBranch: "empty-preview",
228 });
229 expect(opened.ok).toBe(true);
230 if (!opened.ok) return;
231 await settled(repo.id, (r) => r.env === `pull-${opened.value.number}`);
232 const previewHtml = await (
233 await viaHost(`emptypolicy--${opened.value.number}.${DOMAIN}`)
234 ).text();
235 expect(previewHtml).not.toContain("production-only note");
236 }, 60_000);
237
238241 test("rule 1: the repository takes no push from the crew; the agent's fork takes branches but never its default branch, protected or not, renamed or not", async () => {
239 const madeRepo = await createRepo(app, alice, { name: "rule1-repo" });
240 expect(madeRepo.ok).toBe(true);
241 if (!madeRepo.ok) return;
242 const repo = madeRepo.value;
243 await commitToBranch(app, alice, repo, {
244 message: "seed",
245 files: [{ path: "README.md", content: "hi\n" }],
246 });
247 const madeAgent = await createAgent(app, alice, {
248 username: "rule1bot",
249 model: "",
250 brief: "",
251 });
252 expect(madeAgent.ok).toBe(true);
253 if (!madeAgent.ok) return;
254 const bot = madeAgent.value.user;
255 addToCrew(app, alice, repo, "rule1bot");
242 const repo = seeded;
256243 const token = createToken(app.db, bot.id, "t", "rw");
257 const remote = `${servers.url.replace("://", `://rule1bot:${token.token}@`)}/${repo.ownerName}/${repo.name}.git`;
258 const clone = join(work, "rule1-clone");
244 const remote = `${forge.servers.url.replace("://", `://rule1bot:${token.token}@`)}/${repo.ownerName}/${repo.name}.git`;
245 const clone = join(forge.work, "rule1-clone");
259246 expect(
260 (await sh(["git", "clone", "--quiet", remote, clone], work)).code,
247 (await sh(["git", "clone", "--quiet", remote, clone], forge.work)).code,
261248 ).toBe(0);
262249 await Bun.write(join(clone, "A.md"), "a\n");
263250 await sh(["git", "add", "A.md"], clone);
@@ -288,7 +275,7 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
288275 expect(forked.ok).toBe(true);
289276 if (!forked.ok) return;
290277 let fork = forked.value;
291 const forkRemote = `${servers.url.replace("://", `://rule1bot:${token.token}@`)}/${fork.ownerName}/${fork.name}.git`;
278 const forkRemote = `${forge.servers.url.replace("://", `://rule1bot:${token.token}@`)}/${fork.ownerName}/${fork.name}.git`;
292279 const otherOk = await sh(
293280 ["git", "push", "-q", forkRemote, "HEAD:refs/heads/some-branch"],
294281 clone,
@@ -334,26 +321,10 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
334321 }, 60_000);
335322
336323 test("rule 1: an agent's rw token cannot reach settings, app enable, crew, tokens, or agent creation over the API", async () => {
337 const madeRepo = await createRepo(app, alice, { name: "rule1-api-repo" });
338 expect(madeRepo.ok).toBe(true);
339 if (!madeRepo.ok) return;
340 const repo = madeRepo.value;
341 await commitToBranch(app, alice, repo, {
342 message: "seed",
343 files: [{ path: "README.md", content: "hi\n" }],
344 });
345 const madeAgent = await createAgent(app, alice, {
346 username: "rule1apibot",
347 model: "",
348 brief: "",
349 });
350 expect(madeAgent.ok).toBe(true);
351 if (!madeAgent.ok) return;
352 addToCrew(app, alice, repo, "rule1apibot");
353 const bot = madeAgent.value.user;
324 const repo = seeded;
354325 const token = createToken(app.db, bot.id, "t", "rw").token;
355326 const api = (path: string, init: RequestInit = {}) =>
356 fetch(`${servers.url}${path}`, {
327 fetch(`${forge.servers.url}${path}`, {
357328 ...init,
358329 headers: {
359330 ...(init.headers as Record<string, string>),
@@ -375,7 +346,7 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
375346 expect(enableResp.status).toBe(403);
376347
377348 const crewResp = await api(
378 `/api/v1/repos/${repo.ownerName}/${repo.name}/crew/rule1apibot`,
349 `/api/v1/repos/${repo.ownerName}/${repo.name}/crew/rule1bot`,
379350 { method: "PUT" },
380351 );
381352 expect(crewResp.status).toBe(403);
@@ -554,4 +525,16 @@ describe("apps adversarial: env names, preview data, and rules 1/4", () => {
554525 ).ok,
555526 ).toBe(true);
556527 }, 30_000);
528
529 test('preview_data: "empty" starts a preview with no copy of production\'s data', async () => {
530 const { repo } = deployed;
531 expect(await (await viaHost(`adversarial.${DOMAIN}`)).text()).toContain(
532 "production-only note",
533 );
534 await settled(app, repo.id, (r) => r.env === `pull-${previewPull.number}`);
535 const previewHtml = await (
536 await viaHost(`adversarial--${previewPull.number}.${DOMAIN}`)
537 ).text();
538 expect(previewHtml).not.toContain("production-only note");
539 }, 60_000);
557540 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs