Public: anyone can read this; the crew can change it.

The dead-end crawl, landing simulation and OAuth browser tests finish under their per-file budget, so their gate exclusions go #12490

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12485-the-dead-end-crawl-landing-simulation-an → main 7 commits fork

  • Merged. scout/12485-the-dead-end-crawl-landing-simulation-an was merged into main as fbaec5d.

Review

Sign in to review this change.
9 files changed +135 −49
scripts/gate.ts −1
@@ -97,7 +97,6 @@ export const SLOW: readonly Exclusion[] = [
9797 { file: "test/crew-sandbox.test.ts", reason: "runs a sandboxed agent that pushes to a port-mode forge through pre-receive; 5.8 s a test on the gate box", issue: 1242 },
9898 { file: "test/data-door.test.ts", reason: "runs SQL through the data door until the per-person limit holds; 2.1 s a test on the gate box", issue: 1242 },
9999 { file: "test/data-home.test.ts", reason: "starts main.ts as a real process to see it refuse this checkout; 2.4 s a test on the gate box", issue: 1242 },
100 { file: "test/deadends.test.ts", reason: "crawls three seeded forges as every kind of viewer, in worker threads side by side; the action walks run nightly; the crawl test waits on the slowest", issue: 1242 },
101100 { file: "test/domain-dns.test.ts", reason: "drives a real terminal session through publishing and cutting over DNS records; 2.2 s a test on the gate box", issue: 1242 },
102101 { file: "test/drill-400mb.test.ts", reason: "packs and restores a 400 MB snapshot, its wire time on an injected clock, not slept (#12381); was 26.6 s, about 7 s a test", issue: 1242 },
103102 { file: "test/drill-run.test.ts", reason: "spawns the drill script with piped stdin and waits for its JSON; 2.4 s a test on the gate box", issue: 1242 },
src/nightly/jobs.ts +7 −3
@@ -78,15 +78,19 @@ export const JOBS: readonly Job[] = [
7878 reports: false,
7979 },
8080 {
81 // The gate lands four seeds' scenarios; the night, 24.
81 // The gate lands four seeds' scenarios (1, 3, 5 and 8) and checks
82 // determinism on seed 34; the rest of the 24, and determinism on
83 // every seed % 16 == 2, are the night's alone.
8284 label: "the long landing simulation",
8385 command: "LANDSIM_SEEDS=24 bun test ./test/landsim.test.ts",
8486 timeoutMs: SLICE_MS,
8587 reports: false,
8688 },
8789 {
88 // The gate crawls for dead ends; the night also walks every near miss
89 // and every offered action.
90 // The gate crawls two seeds for dead ends, a page per route pattern,
91 // which leaves one or two patterns a viewer reaches only on a later
92 // page or seed; the night crawls three seeds, three pages per pattern,
93 // and walks every near miss and every offered action.
9094 label: "the dead-end action walks",
9195 command: "DEADENDS_ACTIONS=1 bun test ./test/deadends.test.ts",
9296 timeoutMs: SLICE_MS,
src/perf/asking.ts +5
@@ -78,6 +78,8 @@ export interface Asking {
7878 heavy(now?: number): Asker[];
7979 /** Whether `token` asked at all in the last `withinMs`. */
8080 calling(token: string, withinMs: number, now?: number): boolean;
81 /** Drop every reading: the hour starts empty, so a test of the forge's needs starts clean. */
82 forget(): void;
8183 }
8284
8385 export function createAsking(limits: { minTotal?: number; minOwn?: number } = {}): Asking {
@@ -152,6 +154,9 @@ export function createAsking(limits: { minTotal?: number; minOwn?: number } = {}
152154 const from = Math.floor((now - withinMs) / MINUTE);
153155 return buckets.some((b) => b.minute >= from && b.by.has(token));
154156 },
157 forget() {
158 buckets = [];
159 },
155160 };
156161 }
157162
test/app-turn.test.ts +9
@@ -97,6 +97,15 @@ describe("who is asking", () => {
9797 expect(a.heavy(10_000)).toEqual([]);
9898 });
9999
100 test("forget drops the hour: a heavy token is named no more, and asked nothing", () => {
101 const a = createAsking({ minTotal: 10, minOwn: 5 });
102 for (let i = 0; i < 20; i++) a.note("app:a/production", "/x", 1, 5_000 + i);
103 expect(a.heavy(10_000)).toHaveLength(1);
104 a.forget();
105 expect(a.heavy(10_000)).toEqual([]);
106 expect(a.calling("app:a/production", 60_000, 10_000)).toBe(false);
107 });
108
100109 test("gaps are said as a person says them", () => {
101110 expect(gapWords(2_500)).toBe("2.5 s");
102111 expect(gapWords(40)).toBe("40 ms");
test/deadends.test.ts +69 −40
@@ -5,11 +5,11 @@
55 * and holds the findings that are not fixed yet as KNOWN, so a new dead link
66 * fails the build and a fixed one asks to be removed from the list.
77 *
8 * The quick run (three seeds' crawls as every viewer, the sign-in doors,
9 * stale links, the instruction scan) is part of the suite. The near-miss and
10 * offered-action walks run with DEADENDS_ACTIONS=1, which the nightly
11 * outside-gate job sets (`src/nightly/jobs.ts`). A long run, which walks them
12 * too, on the server at low priority:
8 * The quick run (two seeds' crawls as every viewer, three nightly, the
9 * sign-in doors, stale links, the instruction scan) is part of the suite.
10 * The near-miss and offered-action walks run with DEADENDS_ACTIONS=1, which
11 * the nightly outside-gate job sets (`src/nightly/jobs.ts`). A long run,
12 * which walks them too, on the server at low priority:
1313 *
1414 * DEADENDS_LONG=1 DEADENDS_SEEDS=60 nice -n 19 bun test ./test/deadends.test.ts
1515 *
@@ -47,19 +47,30 @@ import {
4747 type Problem,
4848 type SeedParts,
4949 type ViewerName,
50 type World,
5051 } from "./support/deadends.ts";
51 import type { DeadendsDone, DeadendsJob } from "./support/deadends-worker.ts";
52 import { runJob, type DeadendsDone, type DeadendsJob } from "./support/deadends-worker.ts";
5253
5354 quietLogs();
5455
5556 const LONG = process.env.DEADENDS_LONG === "1";
57 /** The near-miss and offered-action walks: nightly and in a long run, not in the gate. */
58 const ACTIONS = LONG || process.env.DEADENDS_ACTIONS === "1";
59 /**
60 * Seeds crawled: two in the gate (seed 1, and seed 2's sign-up mode, mail and
61 * hosted child, which between them show every KNOWN finding), three nightly,
62 * DEADENDS_SEEDS in a long run.
63 */
5664 const SEEDS = Array.from(
57 { length: LONG ? Number(process.env.DEADENDS_SEEDS ?? 20) : 3 },
65 { length: process.env.DEADENDS_SEEDS ? Number(process.env.DEADENDS_SEEDS) : LONG ? 20 : ACTIONS ? 3 : 2 },
5866 (_, i) => i + 1,
5967 );
6068 const PRINT = process.env.DEADENDS_PRINT === "1";
61 /** The near-miss and offered-action walks: nightly and in a long run, not in the gate. */
62 const ACTIONS = LONG || process.env.DEADENDS_ACTIONS === "1";
69 /**
70 * Pages walked per route pattern: one in the gate, where every pattern a
71 * crawl reaches is still fetched and judged; three nightly; eight in a long run.
72 */
73 const PER_PATTERN = LONG ? 8 : ACTIONS ? 3 : 1;
6374
6475 /** A finding without the ids that differ run to run, for the known list. */
6576 const fold = (s: string): string =>
@@ -118,46 +129,63 @@ function inWorker(job: DeadendsJob): Promise<DeadendsDone> {
118129 }).finally(() => worker.terminate());
119130 }
120131
121 // Every walk starts at once, before the first test; each test waits for its own.
122 let crawls: Promise<DeadendsDone[]>;
132 // Every walk starts at once before the first test; the crawls are waited for
133 // here, so the crawl is the file's time (held to the file budget) and the
134 // test judges what it found.
135 let crawls: DeadendsDone[];
136 let crawlMs = 0;
137 let doors: Promise<Problem[]>;
138 let stale: Promise<Problem[]>;
123139 let nearMisses: Promise<DeadendsDone[]>;
124140 let offered: Promise<DeadendsDone[]>;
125 beforeAll(() => {
141 beforeAll(async () => {
126142 const settled = <T>(p: Promise<T>): Promise<T> => (p.catch(() => {}), p);
127 crawls = settled(Promise.all(SEEDS.map((seed) => inWorker({ kind: "crawl", seed, perPattern: LONG ? 8 : 3 }))));
128 if (!ACTIONS) return;
129 nearMisses = settled(Promise.all((LONG ? SEEDS : [1]).map((seed) => inWorker({ kind: "near-miss", seed, mutate: LONG }))));
130 // A fresh forge each: a submitted form changes what the next one finds.
131 offered = settled(Promise.all((["admin", "member", "newcomer"] as ViewerName[]).map((viewer) => inWorker({ kind: "offered", viewer }))));
132 });
133
134 test("a crawl of the seeded forge, as every viewer, ends at no dead link", async () => {
135143 const t0 = Date.now();
136 const done = await crawls;
137 for (const d of done) judgeCrawl(d.seed, d.findings);
138 if (PRINT) console.log(done.flatMap((d) => d.lines).join("\n"));
139 if (!LONG) expect(Date.now() - t0).toBeLessThan(90_000);
144 // The first seed is walked on this thread, warm and otherwise idle, and
145 // the rest in workers beside it; the sign-in doors and the stale links
146 // follow the first seed here while the workers finish.
147 const [first, ...rest] = SEEDS;
148 const here = runJob({ kind: "crawl", seed: first!, perPattern: PER_PATTERN });
149 const crawling = Promise.all([here, ...rest.map((seed) => inWorker({ kind: "crawl", seed, perPattern: PER_PATTERN }))]);
150 doors = settled(here.then(() => onSeedOne(returnAddressProblems)));
151 stale = settled(doors.then(() => onSeedOne(staleLinkProblems)));
152 if (ACTIONS) {
153 nearMisses = settled(Promise.all((LONG ? SEEDS : [1]).map((seed) => inWorker({ kind: "near-miss", seed, mutate: LONG }))));
154 // A fresh forge each: a submitted form changes what the next one finds.
155 offered = settled(Promise.all((["admin", "member", "newcomer"] as ViewerName[]).map((viewer) => inWorker({ kind: "offered", viewer }))));
156 }
157 crawls = await crawling;
158 crawlMs = Date.now() - t0;
159 await stale;
140160 }, 900_000);
141161
142 test("the sign-in doors keep the address a person came for", async () => {
162 /** A check run against a fresh seed-1 forge of its own, closed after. */
163 async function onSeedOne(check: (world: World) => Promise<Problem[]>): Promise<Problem[]> {
143164 const world = await seedWorld(1);
144165 try {
145 for (const p of await returnAddressProblems(world))
146 noted(keyOfProblem(p), `${p.viewer} at ${p.at}: ${p.what}`);
166 return await check(world);
147167 } finally {
148168 world.close();
149169 }
150 }, 120_000);
170 }
171
172 test("a crawl of the seeded forge, as every viewer, ends at no dead link", () => {
173 for (const d of crawls) judgeCrawl(d.seed, d.findings);
174 if (PRINT) console.log(crawls.flatMap((d) => d.lines).join("\n"));
175 if (!LONG) expect(crawlMs).toBeLessThan(90_000);
176 // Every seed was crawled.
177 expect(crawls.map((d) => d.seed)).toEqual(SEEDS);
178 });
179
180 test("the sign-in doors keep the address a person came for", async () => {
181 for (const p of await doors)
182 noted(keyOfProblem(p), `${p.viewer} at ${p.at}: ${p.what}`);
183 });
151184
152185 test("a link to a deleted repository is the forge's own 404, never a 500", async () => {
153 const world = await seedWorld(1);
154 try {
155 for (const p of await staleLinkProblems(world))
156 noted(keyOfProblem(p), `${p.at}: ${p.what}`);
157 } finally {
158 world.close();
159 }
160 }, 120_000);
186 for (const p of await stale)
187 noted(keyOfProblem(p), `${p.at}: ${p.what}`);
188 });
161189
162190 test.skipIf(!ACTIONS)("no route answers a 500 to its filled-in address, or to a light mutation of it", async () => {
163191 for (const d of await nearMisses)
@@ -206,17 +234,18 @@ test("the findings are the known ones, and every known one still happens", async
206234 }, 300_000);
207235
208236 /** Instruction strings that name a thing that is not there, as found; each is a sentence to fix or a false reading to teach the scan. */
237 // Named by file, not line: an edit above them elsewhere in the file must not turn this red.
209238 const KNOWN_WORDS: readonly string[] = [
210239 // The scan reads "the MCP tool spawn" and "the one tool open without a token" as a tool called "the" and "one".
211 "tool the at src/core/roles.ts:53",
212 "tool one at src/http/mcp.ts:283",
240 "tool the at src/core/roles.ts",
241 "tool one at src/http/mcp.ts",
213242 // Dead: a preview's sample app runs a file its repository does not hold.
214 "script src/server.ts at src/spawn/preview-seed.ts:816",
243 "script src/server.ts at src/spawn/preview-seed.ts",
215244 ];
216245
217246 test("every instruction names a script or a tool that exists", () => {
218247 const dead = instructions()
219248 .filter((r) => r.status === "dead")
220249 .map((r) => `${r.kind} ${r.target} at ${r.at}`);
221 expect(dead.filter((d) => !KNOWN_WORDS.includes(d))).toEqual([]);
250 expect(dead.filter((d) => !KNOWN_WORDS.includes(d.replace(/:\d+$/, "")))).toEqual([]);
222251 });
test/forge-needs-one-stack.test.ts +7 −1
@@ -2,7 +2,7 @@
22 * One answer to "how is my forge?" (#12183): MCP, home, asks and terminal
33 * read one stack; apply never merges on a recommendation.
44 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
5 import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test";
66 import { mkdtemp, rm } from "node:fs/promises";
77 import { tmpdir } from "node:os";
88 import { join } from "node:path";
@@ -33,6 +33,7 @@ import type { Session } from "../src/tui/session.ts";
3333 import { needsYouSection } from "../src/web/studio/asks_page.ts";
3434 import { mcpTool } from "./helpers/mcp.ts";
3535 import { quietLogs } from "./helpers/globals.ts";
36 import { asking } from "../src/perf/asking.ts";
3637
3738 const press = (key: string) => ({ type: "key" as const, key, ctrl: false, alt: false, shift: false });
3839
@@ -134,6 +135,11 @@ afterAll(async () => {
134135 await rm(dataDir, { recursive: true, force: true });
135136 });
136137
138 // The heavy-asker reading is the process's own: a file run before this one in
139 // the same shard can leave a token over its share, which is a forge need of
140 // its own and not this file's stack.
141 beforeEach(() => asking.forget());
142
137143 test("`forge` answers with the stack the home, asks and count show, same ids, same order; it is found, not listed first", async () => {
138144 const answer = await forge();
139145 const ids = answer.needs.map((n) => n.id);
test/mcp-refusal-hints.test.ts +8 −1
@@ -12,6 +12,7 @@ import { createToken } from "../src/core/tokens.ts";
1212 import { createUser } from "../src/core/users.ts";
1313 import { serve, type Servers } from "../src/http/server.ts";
1414 import { READ_ONLY_REFUSAL, withNextStep } from "../src/http/mcp/hints.ts";
15 import { TOKEN_FREE_HANDLERS } from "../src/http/mcp.ts";
1516 import { quietLogs } from "./helpers/globals.ts";
1617
1718 quietLogs();
@@ -65,7 +66,7 @@ test("a stranger's refusal says which sign-in, and what they may call", async ()
6566 expect(r.body.hint).toContain("report");
6667 });
6768
68 test("a stranger calling every listed tool is refused with the sign-in, or answered by report and start", async () => {
69 test("a stranger calling every listed tool is refused with the sign-in, or answered by report, start and the open reads", async () => {
6970 const list = await names(member);
7071 expect(list.length).toBeGreaterThan(20);
7172 await Promise.all(
@@ -78,6 +79,12 @@ test("a stranger calling every listed tool is refused with the sign-in, or answe
7879 expect(text).toMatch(STEP);
7980 return;
8081 }
82 // explore and search answer a stranger like their HTTP doors (#12429).
83 if (Object.hasOwn(TOKEN_FREE_HANDLERS, name)) {
84 expect(r.status).toBe(200);
85 expect(r.body.result?.content?.[0]?.type).toBe("text");
86 return;
87 }
8188 expect(r.status).toBe(401);
8289 expect(r.body.hint).toContain("Authorization: Bearer");
8390 }),
test/support/deadends-worker.ts +3 −3
@@ -27,8 +27,8 @@ import {
2727 } from "./deadends.ts";
2828
2929 export type DeadendsJob =
30 /** Every viewer's crawl of one seed's forge. */
31 | { readonly kind: "crawl"; readonly seed: number; readonly perPattern: number }
30 /** These viewers' crawls of one seed's forge (every viewer's, when `viewers` is absent). */
31 | { readonly kind: "crawl"; readonly seed: number; readonly perPattern: number; readonly viewers?: readonly ViewerName[] }
3232 /** Every GET route filled in (and mutated, when `mutate`) as visitor, admin, member. */
3333 | { readonly kind: "near-miss"; readonly seed: number; readonly mutate: boolean }
3434 /** Each POST form a viewer is offered on seed 1, submitted; a fresh forge each. */
@@ -48,7 +48,7 @@ export async function runJob(job: DeadendsJob): Promise<DeadendsDone> {
4848 const done: DeadendsDone = { seed, findings: [], problems: [], lines: [] };
4949 try {
5050 if (job.kind === "crawl")
51 for (const v of VIEWERS) {
51 for (const v of job.viewers ?? VIEWERS) {
5252 const c = await crawl(world, world.viewers[v], { perPattern: job.perPattern });
5353 done.lines.push(`seed ${seed} ${v}: ${c.fetched} fetched, ${c.pages} pages, ${c.edges} edges, ${c.patterns.size} routes`);
5454 done.findings.push(...c.findings);
test/test-budget-run.test.ts +27
@@ -95,6 +95,33 @@ describe("the budget in the runner", () => {
9595 expect(got.code).toBe(1);
9696 }, 20_000);
9797
98 test("a file's hooks count against the file budget, not any one test's (#12485)", async () => {
99 // The walking sits in beforeAll; the test itself is instant (under the
100 // 2 s test budget), so only the file's own wall time, hooks and all, can
101 // catch the 0.4 s spent there against a file budget tightened to 0.3 s.
102 // Kept small: the runner times the file twice, and this test is held to
103 // the same 2 s budget it proves.
104 writeFileSync(
105 join(dir, "test/zz-hook.test.ts"),
106 `import { beforeAll, expect, test } from "bun:test";
107 beforeAll(() => {
108 const end = performance.now() + 400;
109 while (performance.now() < end) {}
110 });
111 test("judges what the hook found", () => expect(true).toBe(true));
112 `,
113 );
114 try {
115 const got = await runner(["--tier", "full", "test/zz-hook.test.ts"], { FORGE_TEST_BUDGET_SECONDS: "0.3" });
116 expect(got.out).toContain("test/zz-hook.test.ts:\n(fail) the whole file took ");
117 expect(got.out).toContain("over the 0.3 s file budget");
118 expect(got.out).not.toContain("(fail) judges what the hook found");
119 expect(got.code).toBe(1);
120 } finally {
121 rmSync(join(dir, "test/zz-hook.test.ts"), { force: true });
122 }
123 }, 20_000);
124
98125 test("by hand the same file is named over budget and stays green", async () => {
99126 const got = await runner(["test/zz-budget.test.ts"]);
100127 expect(got.out).toContain("test-shards: over budget (not enforced here): test/zz-budget.test.ts > spins past the budget took ");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs