Public: anyone can read this; the crew can change it.
Flaky: test/policy.test.ts #2115
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2109-flaky-test-policy-test-ts → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +94
−69
scripts/gate.ts −1
| @@ -53,7 +53,6 @@ export const EXCLUDED: readonly Exclusion[] = [ | ||
| 53 | 53 | issue: 261, |
| 54 | 54 | }, |
| 55 | 55 | { file: "test/parity.test.ts", reason: "not yet admitted", issue: 261 }, |
| 56 | { file: "test/policy.test.ts", reason: "not yet admitted", issue: 261 }, | |
| 57 | 56 | { file: "test/probe.test.ts", reason: "not yet admitted", issue: 261 }, |
| 58 | 57 | { file: "test/probes.test.ts", reason: "not yet admitted", issue: 261 }, |
| 59 | 58 | { file: "test/prompts.test.ts", reason: "not yet admitted", issue: 261 }, |
test/policy.test.ts +94 −68
| @@ -124,156 +124,182 @@ describe("what counts as loosening", () => { | ||
| 124 | 124 | }); |
| 125 | 125 | }); |
| 126 | 126 | |
| 127 | describe("a pull at the wall", () => { | |
| 127 | /** | |
| 128 | * A forge of its own with alice, her public repo `notes` holding NOTES, and | |
| 129 | * the branches the tests read, made once. Landing a pull moves main, so a | |
| 130 | * test that lands asks for a world of its own. | |
| 131 | */ | |
| 132 | const wallWorld = () => { | |
| 133 | const w = {} as { | |
| 134 | app: App; | |
| 135 | servers: Servers; | |
| 136 | alice: User; | |
| 137 | repo: Repo; | |
| 138 | dir: string; | |
| 139 | tip: (branch: string) => Promise<string | undefined>; | |
| 140 | branchWith: (branch: string, contract: Contract | string) => Promise<void>; | |
| 141 | }; | |
| 128 | 142 | let dataDir: string; |
| 129 | let app: App; | |
| 130 | let servers: Servers; | |
| 131 | let alice: User; | |
| 132 | let repo: Repo; | |
| 133 | let dir: string; | |
| 134 | const tip = (branch: string) => resolveOid(dir, `refs/heads/${branch}`); | |
| 135 | 143 | |
| 136 | 144 | beforeAll(async () => { |
| 137 | 145 | dataDir = await mkdtemp(join(tmpdir(), "forge-wall-")); |
| 138 | app = createApp({ dataDir, port: 0 }); | |
| 139 | servers = await serve(app); | |
| 140 | const a = await createUser(app.db, { trusted: true, | |
| 146 | const app = createApp({ dataDir, port: 0 }); | |
| 147 | w.app = app; | |
| 148 | w.servers = await serve(app); | |
| 149 | const a = await createUser(app.db, { | |
| 150 | trusted: true, | |
| 141 | 151 | username: "alice", |
| 142 | 152 | email: "[email protected]", |
| 143 | 153 | password: "a-good-passphrase", |
| 144 | 154 | adminIfFirst: true, |
| 145 | 155 | }); |
| 146 | 156 | if (!a.ok) throw new Error(a.error); |
| 147 | alice = a.value; | |
| 157 | const alice = a.value; | |
| 158 | w.alice = alice; | |
| 148 | 159 | const made = await createRepo(app, alice, { |
| 149 | 160 | name: "notes", |
| 150 | 161 | visibility: "public", |
| 151 | 162 | }); |
| 152 | 163 | if (!made.ok) throw new Error(String(made.error)); |
| 153 | repo = made.value; | |
| 154 | dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 164 | const repo = made.value; | |
| 165 | w.repo = repo; | |
| 166 | w.dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 167 | w.tip = (branch) => resolveOid(w.dir, `refs/heads/${branch}`); | |
| 168 | w.branchWith = async (branch, contract) => { | |
| 169 | const wrote = await commitToBranch(app, alice, repo, { | |
| 170 | branch, | |
| 171 | message: branch, | |
| 172 | files: [ | |
| 173 | { | |
| 174 | path: "contract.json", | |
| 175 | content: | |
| 176 | typeof contract === "string" | |
| 177 | ? contract | |
| 178 | : JSON.stringify(contract), | |
| 179 | }, | |
| 180 | ], | |
| 181 | }); | |
| 182 | if (!wrote.ok) throw new Error(JSON.stringify(wrote.error)); | |
| 183 | }; | |
| 155 | 184 | const seeded = await commitToBranch(app, alice, repo, { |
| 156 | 185 | message: "a contract", |
| 157 | 186 | files: [{ path: "contract.json", content: JSON.stringify(NOTES) }], |
| 158 | 187 | }); |
| 159 | 188 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); |
| 189 | await w.branchWith("open-it", withCan({ create: "anon" })); | |
| 190 | await w.branchWith("close-it", withCan({ read: "user" })); | |
| 160 | 191 | }, 30_000); |
| 161 | 192 | |
| 162 | 193 | afterAll(async () => { |
| 163 | servers.stop(); | |
| 164 | app.close(); | |
| 194 | w.servers.stop(); | |
| 195 | w.app.close(); | |
| 165 | 196 | await rm(dataDir, { recursive: true, force: true }); |
| 166 | 197 | }); |
| 167 | 198 | |
| 168 | const branchWith = async (branch: string, contract: Contract | string) => { | |
| 169 | const wrote = await commitToBranch(app, alice, repo, { | |
| 170 | branch, | |
| 171 | message: branch, | |
| 172 | files: [ | |
| 173 | { | |
| 174 | path: "contract.json", | |
| 175 | content: | |
| 176 | typeof contract === "string" ? contract : JSON.stringify(contract), | |
| 177 | }, | |
| 178 | ], | |
| 179 | }); | |
| 180 | if (!wrote.ok) throw new Error(JSON.stringify(wrote.error)); | |
| 181 | }; | |
| 199 | return w; | |
| 200 | }; | |
| 201 | ||
| 202 | describe("a pull at the wall", () => { | |
| 203 | const w = wallWorld(); | |
| 182 | 204 | |
| 183 | 205 | test("a loosening head crosses; a tightening one and an unchanged one do not", async () => { |
| 184 | const base = (await tip("main"))!; | |
| 185 | await branchWith("open-it", withCan({ create: "anon" })); | |
| 186 | const crossing = await wallOf(dir, base, (await tip("open-it"))!); | |
| 206 | const base = (await w.tip("main"))!; | |
| 207 | const crossing = await wallOf(w.dir, base, (await w.tip("open-it"))!); | |
| 187 | 208 | expect(crossing?.loosened.map(wallSentence)).toEqual([ |
| 188 | 209 | "note.create as anon: was no, now yes", |
| 189 | 210 | ]); |
| 190 | await branchWith("close-it", withCan({ read: "user" })); | |
| 191 | expect(await wallOf(dir, base, (await tip("close-it"))!)).toBeUndefined(); | |
| 192 | const same = await commitToBranch(app, alice, repo, { | |
| 211 | expect( | |
| 212 | await wallOf(w.dir, base, (await w.tip("close-it"))!), | |
| 213 | ).toBeUndefined(); | |
| 214 | const same = await commitToBranch(w.app, w.alice, w.repo, { | |
| 193 | 215 | branch: "readme-only", |
| 194 | 216 | message: "words", |
| 195 | 217 | files: [{ path: "README.md", content: "# notes\n" }], |
| 196 | 218 | }); |
| 197 | 219 | expect(same.ok).toBe(true); |
| 198 | 220 | expect( |
| 199 | await wallOf(dir, base, (await tip("readme-only"))!), | |
| 221 | await wallOf(w.dir, base, (await w.tip("readme-only"))!), | |
| 200 | 222 | ).toBeUndefined(); |
| 201 | 223 | }); |
| 202 | 224 | |
| 203 | 225 | test("a head whose contract does not read is a wall too", async () => { |
| 204 | const base = (await tip("main"))!; | |
| 205 | await branchWith("break-it", "{"); | |
| 206 | const wall = await wallOf(dir, base, (await tip("break-it"))!); | |
| 226 | const base = (await w.tip("main"))!; | |
| 227 | await w.branchWith("break-it", "{"); | |
| 228 | const wall = await wallOf(w.dir, base, (await w.tip("break-it"))!); | |
| 207 | 229 | expect(wall?.unsound).toEqual(["contract.json is not JSON"]); |
| 208 | 230 | }); |
| 209 | 231 | |
| 210 | 232 | test("the pull page says so, in the same sentences", async () => { |
| 211 | const opened = await openPull(app, alice, repo, { | |
| 233 | const opened = await openPull(w.app, w.alice, w.repo, { | |
| 212 | 234 | title: "Let anyone add notes", |
| 213 | 235 | baseBranch: "main", |
| 214 | 236 | headBranch: "open-it", |
| 215 | 237 | }); |
| 216 | 238 | if (!opened.ok) throw new Error(JSON.stringify(opened.error)); |
| 217 | 239 | const body = await ( |
| 218 | await fetch(`${servers.url}/alice/notes/pulls/${opened.value.number}`) | |
| 240 | await fetch(`${w.servers.url}/alice/notes/pulls/${opened.value.number}`) | |
| 219 | 241 | ).text(); |
| 220 | 242 | expect(body).toContain( |
| 221 | 243 | "This pull changes the policy: one door opens wider.", |
| 222 | 244 | ); |
| 223 | 245 | expect(body).toContain("note.create as anon: was no, now yes"); |
| 224 | const quiet = await openPull(app, alice, repo, { | |
| 246 | const quiet = await openPull(w.app, w.alice, w.repo, { | |
| 225 | 247 | title: "Only the signed-in read", |
| 226 | 248 | baseBranch: "main", |
| 227 | 249 | headBranch: "close-it", |
| 228 | 250 | }); |
| 229 | 251 | if (!quiet.ok) throw new Error(JSON.stringify(quiet.error)); |
| 230 | 252 | const other = await ( |
| 231 | await fetch(`${servers.url}/alice/notes/pulls/${quiet.value.number}`) | |
| 253 | await fetch(`${w.servers.url}/alice/notes/pulls/${quiet.value.number}`) | |
| 232 | 254 | ).text(); |
| 233 | 255 | expect(other).not.toContain("changes the policy"); |
| 234 | expect(getRepoById(app.db, repo.id)).toBeDefined(); | |
| 256 | expect(getRepoById(w.app.db, w.repo.id)).toBeDefined(); | |
| 235 | 257 | expect(readContract(JSON.stringify(NOTES)).ok).toBe(true); |
| 236 | 258 | }); |
| 259 | }); | |
| 260 | ||
| 261 | describe("a pull landed at the wall", () => { | |
| 262 | const w = wallWorld(); | |
| 237 | 263 | |
| 238 | 264 | test("the crew never lands a green pull across the wall; a person still can", async () => { |
| 239 | const on = await updateRepoSettings(app, alice, repo, { | |
| 265 | const on = await updateRepoSettings(w.app, w.alice, w.repo, { | |
| 240 | 266 | autonomy: "autonomous", |
| 241 | 267 | autonomyDirection: "Notes for everyone.", |
| 242 | 268 | autonomyCeilingUsd: 10, |
| 243 | 269 | autonomySlots: 1, |
| 244 | autonomyBy: alice.id, | |
| 270 | autonomyBy: w.alice.id, | |
| 245 | 271 | }); |
| 246 | 272 | expect(on.ok).toBe(true); |
| 247 | const fresh = () => getRepoById(app.db, repo.id)!; | |
| 273 | const fresh = () => getRepoById(w.app.db, w.repo.id)!; | |
| 248 | 274 | const green = (number: number) => { |
| 249 | const pull = getPull(app.db, repo.id, number)!; | |
| 250 | const check = createRun(app.db, { | |
| 251 | repoId: repo.id, | |
| 275 | const pull = getPull(w.app.db, w.repo.id, number)!; | |
| 276 | const check = createRun(w.app.db, { | |
| 277 | repoId: w.repo.id, | |
| 252 | 278 | kind: "check", |
| 253 | 279 | issueId: pull.id, |
| 254 | 280 | env: `pull-${pull.number}`, |
| 255 | 281 | commitOid: pull.headOid, |
| 256 | 282 | reason: "pull", |
| 257 | 283 | }); |
| 258 | claimRun(app.db, check.id); | |
| 259 | finishRun(app.db, check.id, { state: "ok", summary: "check passed" }); | |
| 284 | claimRun(w.app.db, check.id); | |
| 285 | finishRun(w.app.db, check.id, { state: "ok", summary: "check passed" }); | |
| 260 | 286 | return pull; |
| 261 | 287 | }; |
| 262 | 288 | // The wall is about apps, and about the crew's own pulls. |
| 263 | expect(enableApp(app, alice, repo).ok).toBe(true); | |
| 264 | const made = await createAgent(app, alice, { | |
| 289 | expect(enableApp(w.app, w.alice, w.repo).ok).toBe(true); | |
| 290 | const made = await createAgent(w.app, w.alice, { | |
| 265 | 291 | username: "bot", |
| 266 | 292 | brief: "Be brief.", |
| 267 | 293 | }); |
| 268 | 294 | if (!made.ok) throw new Error(String(made.error)); |
| 269 | expect(addToCrew(app, alice, repo, "bot").ok).toBe(true); | |
| 270 | const opening = await openPull(app, made.value.user, repo, { | |
| 295 | expect(addToCrew(w.app, w.alice, w.repo, "bot").ok).toBe(true); | |
| 296 | const opening = await openPull(w.app, made.value.user, w.repo, { | |
| 271 | 297 | title: "Anyone may add notes", |
| 272 | 298 | baseBranch: "main", |
| 273 | 299 | headBranch: "open-it", |
| 274 | 300 | }); |
| 275 | 301 | if (!opening.ok) throw new Error(JSON.stringify(opening.error)); |
| 276 | const closing = await openPull(app, alice, repo, { | |
| 302 | const closing = await openPull(w.app, w.alice, w.repo, { | |
| 277 | 303 | title: "Only the signed-in read", |
| 278 | 304 | baseBranch: "main", |
| 279 | 305 | headBranch: "close-it", |
| @@ -282,12 +308,12 @@ describe("a pull at the wall", () => { | ||
| 282 | 308 | |
| 283 | 309 | // Green, autonomous, and still not landed: it crosses the wall. |
| 284 | 310 | const wall = green(opening.value.number); |
| 285 | expect(await landIfGreen(app, fresh(), wall)).toBe(false); | |
| 286 | expect(await landIfGreen(app, fresh(), wall)).toBe(false); | |
| 287 | expect(getPull(app.db, repo.id, wall.number)!.mergedAt).toBeNull(); | |
| 311 | expect(await landIfGreen(w.app, fresh(), wall)).toBe(false); | |
| 312 | expect(await landIfGreen(w.app, fresh(), wall)).toBe(false); | |
| 313 | expect(getPull(w.app.db, w.repo.id, wall.number)!.mergedAt).toBeNull(); | |
| 288 | 314 | // Once, as a ledger row, which is what the pull and the audit read |
| 289 | 315 | // the decider never leans on. |
| 290 | const row = latestDecision(app.db, wall.id, "land"); | |
| 316 | const row = latestDecision(w.app.db, wall.id, "land"); | |
| 291 | 317 | expect(row?.answer).toBe("wall"); |
| 292 | 318 | expect(row?.by).toBe("wall"); |
| 293 | 319 | expect((row?.state as { cells: string[] }).cells).toEqual([ |
| @@ -296,13 +322,13 @@ describe("a pull at the wall", () => { | ||
| 296 | 322 | // The same head asked again writes nothing new; a row whose cells no |
| 297 | 323 | // longer match what the wall says is superseded, not repeated. |
| 298 | 324 | const rows = () => |
| 299 | listDecisions(app.db, repo.id, { kind: "land" }).filter( | |
| 325 | listDecisions(w.app.db, w.repo.id, { kind: "land" }).filter( | |
| 300 | 326 | (d) => d.issueId === wall.id && d.answer === "wall", |
| 301 | 327 | ); |
| 302 | 328 | expect(rows()).toHaveLength(1); |
| 303 | recordDecision(app.db, { | |
| 329 | recordDecision(w.app.db, { | |
| 304 | 330 | kind: "land", |
| 305 | repoId: repo.id, | |
| 331 | repoId: w.repo.id, | |
| 306 | 332 | issueId: wall.id, |
| 307 | 333 | state: { head: wall.headOid, cells: ["stale: from an older rule"] }, |
| 308 | 334 | answer: "wall", |
| @@ -310,12 +336,12 @@ describe("a pull at the wall", () => { | ||
| 310 | 336 | by: "wall", |
| 311 | 337 | ms: 0, |
| 312 | 338 | }); |
| 313 | expect(await landIfGreen(app, fresh(), wall)).toBe(false); | |
| 339 | expect(await landIfGreen(w.app, fresh(), wall)).toBe(false); | |
| 314 | 340 | expect(rows()).toHaveLength(3); |
| 315 | 341 | expect((rows()[0]!.state as { cells: string[] }).cells).toEqual([ |
| 316 | 342 | "note.create as anon: was no, now yes", |
| 317 | 343 | ]); |
| 318 | const signedIn = await fetch(`${servers.url}/signin`, { | |
| 344 | const signedIn = await fetch(`${w.servers.url}/signin`, { | |
| 319 | 345 | method: "POST", |
| 320 | 346 | body: new URLSearchParams({ |
| 321 | 347 | username: "alice", |
| @@ -328,7 +354,7 @@ describe("a pull at the wall", () => { | ||
| 328 | 354 | expect(cookie).not.toBe(""); |
| 329 | 355 | // The same crew lands the tightening one without a word. |
| 330 | 356 | const tight = green(closing.value.number); |
| 331 | expect(await landIfGreen(app, fresh(), tight)).toBe(true); | |
| 332 | expect(getPull(app.db, repo.id, tight.number)!.mergedAt).not.toBeNull(); | |
| 357 | expect(await landIfGreen(w.app, fresh(), tight)).toBe(true); | |
| 358 | expect(getPull(w.app.db, w.repo.id, tight.number)!.mergedAt).not.toBeNull(); | |
| 333 | 359 | }); |
| 334 | 360 | }); |
new above
offline
Good doc comment: it says why a landing test needs its own world. Next: avoid the
{} as {...}cast so a forgotten field fails type-checking.