Public: anyone can read this; the crew can change it.

Flaky: test/audit.test.ts #2887

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2108-flaky-test-audit-test-ts → main 5 commits fork 1 thread · 1 unresolved

  • Merged. scout/2108-flaky-test-audit-test-ts was merged into main as c21ded5.

Review

Sign in to review this change.
2 files changed +274 −212
scripts/gate.ts −5
@@ -39,11 +39,6 @@ export const EXCLUDED: readonly Exclusion[] = [
3939 },
4040 { file: "test/apps-sleep.test.ts", reason: "not yet admitted", issue: 261 },
4141 { file: "test/apps-ws.test.ts", reason: "not yet admitted", issue: 261 },
42 {
43 file: "test/audit.test.ts",
44 reason: "red in random order: the page: every app the viewer can read, each in its standing",
45 issue: 2108,
46 },
4742 {
4843 file: "test/board-windows.test.ts",
4944 reason:
test/audit.test.ts +274 −207
@@ -26,6 +26,99 @@ import { openPull } from "../src/actions/pulls.ts";
2626 import { createUser, type User } from "../src/core/users.ts";
2727 import { serve, type Servers } from "../src/http/server.ts";
2828
29 /** A forge of its own with alice signed in, for a test that changes what it reads. */
30 async function boot(prefix: string) {
31 const dataDir = await mkdtemp(join(tmpdir(), prefix));
32 const app = createApp({ dataDir, port: 0 });
33 const servers = await serve(app);
34 const a = await createUser(app.db, {
35 origin: "seed",
36 trusted: true,
37 username: "alice",
38 email: "[email protected]",
39 password: "a-good-passphrase",
40 adminIfFirst: true,
41 });
42 if (!a.ok) throw new Error(a.error);
43 const signedIn = await fetch(`${servers.url}/signin`, {
44 method: "POST",
45 body: new URLSearchParams({
46 username: "alice",
47 password: "a-good-passphrase",
48 }),
49 redirect: "manual",
50 });
51 const cookie = (signedIn.headers.get("set-cookie") ?? "").split(";")[0]!;
52 expect(cookie).not.toBe("");
53 const stop = async () => {
54 servers.stop();
55 app.close();
56 await rm(dataDir, { recursive: true, force: true });
57 };
58 return { app, servers, alice: a.value, cookie, stop };
59 }
60
61 /** The four apps the audit's page reads: sound, unsound, unread, and a private one. */
62 async function seedApps(app: App, alice: User) {
63 // A scaffolded app: born with a contract.
64 const notes = await newApp(app, alice, {
65 name: "notes",
66 visibility: "public",
67 });
68 if (!notes.ok) throw new Error(String(notes.error));
69
70 // An app whose contract does not hold.
71 const broken = await createRepo(app, alice, {
72 name: "broken",
73 visibility: "public",
74 });
75 if (!broken.ok) throw new Error(String(broken.error));
76 const wrote = await commitToBranch(app, alice, broken.value, {
77 message: "a contract that names a ghost",
78 files: [
79 {
80 path: "contract.json",
81 content: JSON.stringify({
82 roles: { anon: { above: [], held: { by: "everyone" } } },
83 resources: {
84 thing: {
85 can: {
86 create: "ghost",
87 read: "anon",
88 update: "anon",
89 delete: "anon",
90 list: "anon",
91 },
92 },
93 },
94 }),
95 },
96 ],
97 });
98 if (!wrote.ok) throw new Error(JSON.stringify(wrote.error));
99 expect(enableApp(app, alice, broken.value).ok).toBe(true);
100
101 // An app with no contract at all.
102 const bare = await createRepo(app, alice, {
103 name: "bare",
104 visibility: "public",
105 });
106 if (!bare.ok) throw new Error(String(bare.error));
107 const seeded = await commitToBranch(app, alice, bare.value, {
108 message: "no contract",
109 files: [{ path: "README.md", content: "# bare\n" }],
110 });
111 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
112 expect(enableApp(app, alice, bare.value).ok).toBe(true);
113
114 // A private app: its policy is for its readers.
115 const secret = await newApp(app, alice, {
116 name: "secret",
117 visibility: "private",
118 });
119 if (!secret.ok) throw new Error(String(secret.error));
120 }
121
29122 describe("the audit reads every app's policy from git", () => {
30123 let dataDir: string;
31124 let app: App;
@@ -56,63 +149,7 @@ describe("the audit reads every app's policy from git", () => {
56149 cookie = (signedIn.headers.get("set-cookie") ?? "").split(";")[0]!;
57150 expect(cookie).not.toBe("");
58151
59 // A scaffolded app: born with a contract.
60 const notes = await newApp(app, alice, {
61 name: "notes",
62 visibility: "public",
63 });
64 if (!notes.ok) throw new Error(String(notes.error));
65
66 // An app whose contract does not hold.
67 const broken = await createRepo(app, alice, {
68 name: "broken",
69 visibility: "public",
70 });
71 if (!broken.ok) throw new Error(String(broken.error));
72 const wrote = await commitToBranch(app, alice, broken.value, {
73 message: "a contract that names a ghost",
74 files: [
75 {
76 path: "contract.json",
77 content: JSON.stringify({
78 roles: { anon: { above: [], held: { by: "everyone" } } },
79 resources: {
80 thing: {
81 can: {
82 create: "ghost",
83 read: "anon",
84 update: "anon",
85 delete: "anon",
86 list: "anon",
87 },
88 },
89 },
90 }),
91 },
92 ],
93 });
94 if (!wrote.ok) throw new Error(JSON.stringify(wrote.error));
95 expect(enableApp(app, alice, broken.value).ok).toBe(true);
96
97 // An app with no contract at all.
98 const bare = await createRepo(app, alice, {
99 name: "bare",
100 visibility: "public",
101 });
102 if (!bare.ok) throw new Error(String(bare.error));
103 const seeded = await commitToBranch(app, alice, bare.value, {
104 message: "no contract",
105 files: [{ path: "README.md", content: "# bare\n" }],
106 });
107 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
108 expect(enableApp(app, alice, bare.value).ok).toBe(true);
109
110 // A private app: its policy is for its readers.
111 const secret = await newApp(app, alice, {
112 name: "secret",
113 visibility: "private",
114 });
115 if (!secret.ok) throw new Error(String(secret.error));
152 await seedApps(app, alice);
116153 }, 30_000);
117154
118155 afterAll(async () => {
@@ -122,17 +159,23 @@ describe("the audit reads every app's policy from git", () => {
122159 });
123160
124161 test("a scaffolded app's standing is its table", async () => {
125 const { getRepo } = await import("../src/core/repos.ts");
126 const repo = getRepo(app.db, "alice", "notes")!;
127 const a = await audit(app.reposRoot, repo);
128 expect(a.oid).toMatch(/^[0-9a-f]{40}$/);
129 expect(a.standing.kind).toBe("sound");
130 if (a.standing.kind !== "sound") return;
131 expect(a.standing.identities).toEqual(["anon", "user", "admin"]);
132 expect(a.standing.open.map((c) => `${c.resource}.${c.verb}`)).toEqual([
133 "note.read",
134 "note.list",
135 ]);
162 const { app, alice, stop } = await boot("forge-audit-standing-");
163 try {

Format this call the way prettier would, one property per line. It is over 80 columns.

164 const made = await newApp(app, alice, { name: "notes", visibility: "public" });
165 if (!made.ok) throw new Error(String(made.error));
166 const repo = getRepo(app.db, "alice", "notes")!;
167 const a = await audit(app.reposRoot, repo);
168 expect(a.oid).toMatch(/^[0-9a-f]{40}$/);
169 expect(a.standing.kind).toBe("sound");
170 if (a.standing.kind !== "sound") return;
171 expect(a.standing.identities).toEqual(["anon", "user", "admin"]);
172 expect(a.standing.open.map((c) => `${c.resource}.${c.verb}`)).toEqual([
173 "note.read",
174 "note.list",
175 ]);
176 } finally {
177 await stop();
178 }
136179 });
137180
138181 test("what cannot be read is said, not skipped", () => {
@@ -147,25 +190,31 @@ describe("the audit reads every app's policy from git", () => {
147190 });
148191
149192 test("the page: every app the viewer can read, each in its standing", async () => {
150 const res = await fetch(`${servers.url}/audit`, {
151 headers: { cookie },
152 });
153 expect(res.status).toBe(200);
154 const body = await res.text();
155 expect(body).toContain("alice/notes");
156 expect(body).toContain("alice/broken");
157 expect(body).toContain("alice/bare");
158 expect(body).toContain("alice/secret");
159 expect(body).toContain("contract unsound");
160 expect(body).toContain("needs `ghost`, which is not declared");
161 expect(body).toContain("Unread");
162 // The table's cells, as the scaffold's contract says.
163 expect(body).toMatch(/data-answer="own"/);
164 expect(body).toContain("2 doors open to anyone");
165 // The summary counts the apps with a contract and the doors they open.
166 expect(body).toContain(
167 "2 apps with a contract, 4 doors open to anyone, 1 contract unsound, 1 app unread.",
168 );
193 const { app, servers, alice, cookie, stop } = await boot("forge-audit-page-");
194 try {
195 await seedApps(app, alice);
196 const res = await fetch(`${servers.url}/audit`, {
197 headers: { cookie },
198 });
199 expect(res.status).toBe(200);
200 const body = await res.text();
201 expect(body).toContain("alice/notes");
202 expect(body).toContain("alice/broken");
203 expect(body).toContain("alice/bare");
204 expect(body).toContain("alice/secret");
205 expect(body).toContain("contract unsound");
206 expect(body).toContain("needs `ghost`, which is not declared");
207 expect(body).toContain("Unread");
208 // The table's cells, as the scaffold's contract says.
209 expect(body).toMatch(/data-answer="own"/);
210 expect(body).toContain("2 doors open to anyone");
211 // The summary counts the apps with a contract and the doors they open.
212 expect(body).toContain(
213 "2 apps with a contract, 4 doors open to anyone, 1 contract unsound, 1 app unread.",
214 );
215 } finally {
216 await stop();
217 }
169218 });
170219
171220 test("a stranger sees the public apps' policies and nothing of the private", async () => {
@@ -192,131 +241,149 @@ describe("the audit reads every app's policy from git", () => {
192241 });
193242
194243 test("what changed: contract commits, a crossing waiting, a verdict that flipped", async () => {
195 const notes = getRepo(app.db, "alice", "notes")!;
196 const record = getAppByRepo(app.db, notes.id)!;
197 // A commit that opens a door, then one that closes two.
198 const text = (
199 (await (
200 await fetch(
201 `${servers.url}/api/v1/repos/alice/notes/contents/contract.json`,
202 )
203 ).json()) as { content: string }
204 ).content;
205 const contract = JSON.parse(text) as {
206 resources: { note: { can: Record<string, string> } };
207 };
208 const wider = structuredClone(contract);
209 wider.resources.note.can.create = "anon";
210 const opened = await commitToBranch(app, alice, notes, {
211 message: "anyone may add",
212 files: [{ path: "contract.json", content: JSON.stringify(wider) }],
213 });
214 expect(opened.ok).toBe(true);
215 const narrower = structuredClone(wider);
216 narrower.resources.note.can.read = "user";
217 narrower.resources.note.can.list = "user";
218 const closed = await commitToBranch(app, alice, notes, {
219 message: "only the signed-in read",
220 files: [{ path: "contract.json", content: JSON.stringify(narrower) }],
221 });
222 expect(closed.ok).toBe(true);
223 const changes = await contractChanges(app.reposRoot, notes);
224 expect(changes.map((c) => c.subject)).toEqual([
225 "only the signed-in read",
226 "anyone may add",
227 expect.any(String),
228 ]);
229 expect(changes[0]!.loosened).toEqual([]);
230 expect(
231 changes[0]!.tightened.map((t) => `${t.verb}.${t.as}`).sort(),
232 ).toEqual(["list.anon", "read.anon"]);
233 expect(changes[1]!.loosened.map((l) => `${l.verb}.${l.as}`)).toEqual([
234 "create.anon",
235 ]);
236 // The first commit, the scaffold's: the floor's open cells are what was decided.
237 expect(changes[2]!.loosened.map((l) => `${l.verb}.${l.as}`).sort()).toEqual(
238 ["list.anon", "read.anon"],
244 // Its own forge: this test rewrites notes' contract, which the shared
245 // forge's other tests read as the scaffold's.
246 const { app, servers, alice, cookie, stop } = await boot(
247 "forge-audit-changes-",
239248 );
249 try {
250 const made = await newApp(app, alice, {
251 name: "notes",
252 visibility: "public",
253 });
254 if (!made.ok) throw new Error(String(made.error));
255 const notes = getRepo(app.db, "alice", "notes")!;
256 const record = getAppByRepo(app.db, notes.id)!;
257 // A commit that opens a door, then one that closes two.
258 const text = (
259 (await (
260 await fetch(
261 `${servers.url}/api/v1/repos/alice/notes/contents/contract.json`,
262 )
263 ).json()) as { content: string }
264 ).content;
265 const contract = JSON.parse(text) as {
266 resources: { note: { can: Record<string, string> } };
267 };
268 const wider = structuredClone(contract);
269 wider.resources.note.can.create = "anon";
270 const opened = await commitToBranch(app, alice, notes, {
271 message: "anyone may add",
272 files: [{ path: "contract.json", content: JSON.stringify(wider) }],
273 });
274 expect(opened.ok).toBe(true);
275 const narrower = structuredClone(wider);
276 narrower.resources.note.can.read = "user";
277 narrower.resources.note.can.list = "user";
278 const closed = await commitToBranch(app, alice, notes, {
279 message: "only the signed-in read",
280 files: [{ path: "contract.json", content: JSON.stringify(narrower) }],
281 });
282 expect(closed.ok).toBe(true);
283 const changes = await contractChanges(app.reposRoot, notes);
284 expect(changes.map((c) => c.subject)).toEqual([
285 "only the signed-in read",
286 "anyone may add",
287 expect.any(String),
288 ]);
289 expect(changes[0]!.loosened).toEqual([]);
290 expect(
291 changes[0]!.tightened.map((t) => `${t.verb}.${t.as}`).sort(),
292 ).toEqual(["list.anon", "read.anon"]);
293 expect(changes[1]!.loosened.map((l) => `${l.verb}.${l.as}`)).toEqual([
294 "create.anon",
295 ]);
296 // The first commit, the scaffold's: the floor's open cells are what was decided.
297 expect(
298 changes[2]!.loosened.map((l) => `${l.verb}.${l.as}`).sort(),
299 ).toEqual(["list.anon", "read.anon"]);
240300
241 // A pull at the wall, as the crew would leave it.
242 const branch = await commitToBranch(app, alice, notes, {
243 branch: "wider-still",
244 message: "anyone may delete",
245 files: [
246 {
247 path: "contract.json",
248 content: JSON.stringify({
249 ...narrower,
250 resources: {
251 note: { can: { ...narrower.resources.note.can, delete: "anon" } },
252 },
253 }),
301 // A pull at the wall, as the crew would leave it.
302 const branch = await commitToBranch(app, alice, notes, {
303 branch: "wider-still",
304 message: "anyone may delete",
305 files: [
306 {
307 path: "contract.json",
308 content: JSON.stringify({
309 ...narrower,
310 resources: {
311 note: {
312 can: { ...narrower.resources.note.can, delete: "anon" },
313 },
314 },
315 }),
316 },
317 ],
318 });
319 expect(branch.ok).toBe(true);
320 const pull = await openPull(app, alice, notes, {
321 title: "Anyone may delete",
322 baseBranch: "main",
323 headBranch: "wider-still",
324 });
325 if (!pull.ok) throw new Error(JSON.stringify(pull.error));
326 recordDecision(app.db, {
327 kind: "land",
328 repoId: notes.id,
329 issueId: pull.value.id,
330 state: {
331 head: pull.value.headOid,
332 cells: ["note.delete as anon: was no, now yes"],
254333 },
255 ],
256 });
257 expect(branch.ok).toBe(true);
258 const pull = await openPull(app, alice, notes, {
259 title: "Anyone may delete",
260 baseBranch: "main",
261 headBranch: "wider-still",
262 });
263 if (!pull.ok) throw new Error(JSON.stringify(pull.error));
264 recordDecision(app.db, {
265 kind: "land",
266 repoId: notes.id,
267 issueId: pull.value.id,
268 state: {
269 head: pull.value.headOid,
270 cells: ["note.delete as anon: was no, now yes"],
271 },
272 answer: "wall",
273 p: 1,
274 by: "wall",
275 ms: 0,
276 });
277 const waiting = crossingsWaiting(app.db, [notes]);
278 expect(waiting.map((c) => c.pull.title)).toEqual(["Anyone may delete"]);
279 expect(waiting[0]!.cells).toEqual(["note.delete as anon: was no, now yes"]);
334 answer: "wall",
335 p: 1,
336 by: "wall",
337 ms: 0,
338 });
339 const waiting = crossingsWaiting(app.db, [notes]);
340 expect(waiting.map((c) => c.pull.title)).toEqual(["Anyone may delete"]);
341 expect(waiting[0]!.cells).toEqual([
342 "note.delete as anon: was no, now yes",
343 ]);
280344
281 // Two probes, the second disagreeing with the first.
282 recordProbe(app.db, record.id, {
283 repo: notes,
284 commitOid: "a".repeat(40),
285 at: "",
286 verdict: "ok",
287 knocks: [],
288 said: [],
289 });
290 recordProbe(app.db, record.id, {
291 repo: notes,
292 commitOid: "b".repeat(40),
293 at: "",
294 verdict: "mismatch",
295 knocks: [],
296 said: ["note.delete as anon: the table says no, the app said 404"],
297 });
298 const flips = verdictChanges(app.db, [{ repo: notes, appId: record.id }]);
299 expect(flips.map((f) => [f.from, f.to])).toEqual([
300 ["ok", "mismatch"],
301 [undefined, "ok"],
302 ]);
345 // Two probes, the second disagreeing with the first.
346 recordProbe(app.db, record.id, {
347 repo: notes,
348 commitOid: "a".repeat(40),
349 at: "",
350 verdict: "ok",
351 knocks: [],
352 said: [],
353 });
354 recordProbe(app.db, record.id, {
355 repo: notes,
356 commitOid: "b".repeat(40),
357 at: "",
358 verdict: "mismatch",
359 knocks: [],
360 said: ["note.delete as anon: the table says no, the app said 404"],
361 });
362 const flips = verdictChanges(app.db, [{ repo: notes, appId: record.id }]);
363 expect(flips.map((f) => [f.from, f.to])).toEqual([
364 ["ok", "mismatch"],
365 [undefined, "ok"],
366 ]);
303367
304 const body = await (
305 await fetch(`${servers.url}/audit`, { headers: { cookie } })
306 ).text();
307 expect(body).toContain("What changed");
308 expect(body).toContain("1 pull waits");
309 expect(body).toContain("Anyone may delete");
310 expect(body).toContain("only the signed-in read");
311 expect(body).toContain("2 doors narrower");
312 expect(body).toContain("1 door wider");
313 expect(body).toContain("Doors, over time");
314 expect(body).toContain("alice/notes disobeys its table");
315 // A stranger sees the public app's history too; the ledger row is about
316 // a public pull, so the crossing is as public as the pull.
317 const anon = await (await fetch(`${servers.url}/audit`)).text();
318 expect(anon).toContain("only the signed-in read");
319 });
368 const body = await (
369 await fetch(`${servers.url}/audit`, { headers: { cookie } })
370 ).text();
371 expect(body).toContain("What changed");
372 expect(body).toContain("1 pull waits");
373 expect(body).toContain("Anyone may delete");
374 expect(body).toContain("only the signed-in read");
375 expect(body).toContain("2 doors narrower");
376 expect(body).toContain("1 door wider");
377 expect(body).toContain("Doors, over time");
378 expect(body).toContain("alice/notes disobeys its table");
379 // A stranger sees the public app's history too; the ledger row is about
380 // a public pull, so the crossing is as public as the pull.
381 const anon = await (await fetch(`${servers.url}/audit`)).text();
382 expect(anon).toContain("only the signed-in read");
383 } finally {
384 await stop();
385 }
386 }, 30_000);
320387
321388 test("the table file: one row per cell, with what the door said", async () => {
322389 const res = await fetch(`${servers.url}/audit.csv`, {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs