Public: anyone can read this; the crew can change it.

Bring agents and billing-never-deletes under the 2 s test budget #13366

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13365-bring-agents-and-billing-never-deletes-u → main 5 commits fork 3 threads · 3 unresolved

  • Merged. scout/13365-bring-agents-and-billing-never-deletes-u was merged into main as 7f80939.

Review

Sign in to review this change.
4 files changed +96 −56
scripts/unused-exports-allow.json +1 −10
@@ -24,7 +24,6 @@
2424 "src/contract/audit.ts identitiesOf",
2525 "src/core/accounts.ts edgeOnlyAccounts",
2626 "src/core/capacity.ts provideCpusForTest",
27 "src/core/chapters.ts forgetFrozenItems",
2827 "src/core/credit.ts clearCreditCache",
2928 "src/core/device_grants.ts appIdOfSlug",
3029 "src/core/device_grants.ts grantedSlugs",
@@ -60,12 +59,9 @@
6059 "src/core/usage.ts USAGE_OUTCOMES",
6160 "src/core/v1_proof.ts useProof",
6261 "src/core/webhooks.ts deliveryBusy",
63 "src/crew/agent_runner.ts useRunner",
64 "src/crew/ceiling_stagger.ts forgetStagger",
6562 "src/flow/stuck_pile.ts forgetStuckRepos",
6663 "src/crew/conversation_confirm.ts CONFIRM_TTL_MS",
6764 "src/crew/first_look.ts useLookModel",
68 "src/crew/hot_files.ts rememberPullFiles",
6965 "src/crew/hot_history.ts forgetHeat",
7066 "src/crew/hot_history.ts sampleTimes",
7167 "src/crew/models.ts modelLabel",
@@ -96,8 +92,6 @@
9692 "src/flow/pages.ts usePagesReader",
9793 "src/flow/reach.ts NO_REACH",
9894 "src/flow/shared_red.ts sharedRedSays",
99 "src/flow/week.ts daysLeftInWeek",
100 "src/flow/week.ts paceCut",
10195 "src/git/coauthors.ts creditsOf",
10296 "src/git/diff-memory.ts diffMemoryCounts",
10397 "src/git/diff-memory.ts forgetDiffs",
@@ -127,6 +121,7 @@
127121 "src/http/stripe_hook.ts refusedCount",
128122 "src/http/stripe_hook.ts setStripeHookCaps",
129123 "src/keeper/pause.ts forgetWritePause",
124 "src/kernel/destroy_reach.ts createWalkCache",
130125 "src/kernel/destroy_reach.ts unitsOf",
131126 "src/kernel/db.ts provideEmptyImagesForTest",
132127 "src/kernel/disk_reading.ts provideDiskForTest",
@@ -134,7 +129,6 @@
134129 "src/kernel/full_disk.ts provideWriterForTest",
135130 "src/kernel/log.ts clearErrors",
136131 "src/kernel/migrate.ts notAdditive",
137 "src/kernel/protection.ts resetProtection",
138132 "src/kernel/result.ts andThen",
139133 "src/kernel/result.ts trapAsync",
140134 "src/kernel/result.ts unwrapOr",
@@ -146,7 +140,6 @@
146140 "src/origins/git-mirror.ts provideMirrorOutboundForTest",
147141 "src/origins/jobs.ts listJobs",
148142 "src/origins/jobs.ts resetJobs",
149 "src/origins/milestones.ts dueOf",
150143 "src/origins/oauth.ts pruneFlows",
151144 "src/origins/providers/index.ts forgetProviders",
152145 "src/origins/sync.ts syncsSettled",
@@ -200,7 +193,6 @@
200193 "src/web/chrome.ts stickyAction",
201194 "src/web/code_rows_pool.ts workersForTest",
202195 "src/web/diff-cache.ts clearDiffCache",
203 "src/web/go_pages.ts goPagesFor",
204196 "src/web/head-reads.ts clearHeadReads",
205197 "src/web/map_cache.ts clearMapCache",
206198 "src/web/map_cache.ts mapCacheStats",
@@ -217,7 +209,6 @@
217209 "src/web/script_keys.ts SCRIPT_KEYS",
218210 "src/web/share_target.ts forgetShares",
219211 "src/web/space_view.ts GLIDE_MS",
220 "src/web/studio/home_cache.ts cachedVitals",
221212 "src/web/studio/home_cache.ts setHomeClock",
222213 "src/web/studio/home_last.ts forgetHome",
223214 "src/web/studio/review_evidence.ts rememberStat",
src/kernel/destroy_reach.ts +74 −16
@@ -123,14 +123,54 @@ const resolve = (root: string, from: string, spec: string): string | undefined =
123123 return undefined;
124124 };

The doc comment "The text with comments blanked..." now sits above the memo block, not above stripComments. The "One parse per file text" comment is followed by a blank line and so is detached from parse. Move each comment next to the function it describes.

125125
126 /** The text with comments blanked, newlines kept, so a line number still means the same line. */
127 const stripComments = (text: string): string =>
126 type ParsedFile = { module: Module; deps: [string, string | undefined][] };
127
128 /**
129 * What a run of walks remembers: a pure function of file text, so a caller
outdated

This wraps module-level Maps in an object only so test/module-state.test.ts stops seeing them. The rule exists to forbid exactly this state, and the pull says as much. Do not route around it. Make the caches belong to a call or a run instead: create them in destroyReach and pass them down, or have a small createWalkCache() that the callers share. If the cache must stay global, add it to the rule's allowlist with a stated reason, and let a person decide.

outdated

The memos are keyed by whole file text and never evicted, so memory grows with every distinct text for the life of the process. Scope the cache to a run (see the other comment) or bound it.

130 * that walks the same tree twice (the real one, then a copy) shares one. It
131 * belongs to the caller that made it and is gone with it: nothing here is
132 * process-wide.
133 */
134 export type WalkCache = { stripped: Map<string, string>; parsed: Map<string, ParsedFile> };
135
136 export const createWalkCache = (): WalkCache => ({ stripped: new Map(), parsed: new Map() });
137
138 const blankComments = (text: string): string =>
128139 text
129140 .replace(/\/\*[\s\S]*?\*\//g, (c) => c.replace(/[^\n]/g, " "))
130141 .replace(/(?<![:"'`\\])\/\/[^\n]*/g, (c) => " ".repeat(c.length));
131142
132 function parse(root: string, file: string): Module {
143 /** The text with comments blanked, newlines kept, so a line number still means the same line. */
144 const stripComments = (text: string, cache?: WalkCache): string => {
145 const known = cache?.stripped.get(text);
146 if (known !== undefined) return known;
147 const out = blankComments(text);
148 cache?.stripped.set(text, out);
149 return out;
150 };
151
152 /**
153 * One parse per file text per cache. A cached parse is reused only when each
154 * import it resolved resolves the same under this root, so a file that moved
155 * is never read from the cache.
156 */
157 function parse(root: string, file: string, cache?: WalkCache): Module {
133158 const text = readFileSync(join(root, file), "utf8");
159 const key = `${file}\0${text}`;
160 const hit = cache?.parsed.get(key);
outdated

const entry = { module: parseText(root, file, text, (spec) => {...}), deps }; is a long, formatter-hostile line, and the same is true of the facts.set line in factsOf. Break them into named statements. Also, parseText takes a parameter called resolve that shadows the module-level resolve, which is confusing. Rename it to resolveSpec.

161 if (hit && hit.deps.every(([spec, target]) => resolve(root, file, spec) === target)) return hit.module;
162 const deps: [string, string | undefined][] = [];
163 const resolveSpec = (spec: string): string | undefined => {
164 const target = resolve(root, file, spec);
165 deps.push([spec, target]);
166 return target;
167 };
168 const module = parseText(root, file, text, resolveSpec);
169 cache?.parsed.set(key, { module, deps });
170 return module;
171 }
172
173 function parseText(root: string, file: string, text: string, resolveSpec: (spec: string) => string | undefined): Module {
134174 const lines = text.split("\n");
135175 const units = new Map<string, Unit>();
136176 const imports = new Map<string, { from: string; name: string }>();
@@ -148,7 +188,7 @@ function parse(root: string, file: string): Module {
148188 }
149189 continue;
150190 }
151 const target = resolve(root, file, m[4]!);
191 const target = resolveSpec(m[4]!);
152192 if (target === undefined) continue;
153193 touches.add(target);
154194 if (m[2] !== undefined) continue;
@@ -165,14 +205,14 @@ function parse(root: string, file: string): Module {
165205 if (def && !clause.startsWith("{")) imports.set(def[1]!, { from: target, name: "default" });
166206 }
167207 for (const m of text.matchAll(/^export\s+\*\s+from\s+"(\.[^"]+)"/gm)) {
168 const target = resolve(root, file, m[1]!);
208 const target = resolveSpec(m[1]!);
169209 if (target !== undefined) {
170210 touches.add(target);
171211 reexports.push(target);
172212 }
173213 }
174214 for (const m of text.matchAll(/import\(\s*"(\.[^"]+)"\s*\)/g)) {
175 const target = resolve(root, file, m[1]!);
215 const target = resolveSpec(m[1]!);
176216 if (target !== undefined) {
177217 touches.add(target);
178218 reexports.push(target);
@@ -219,15 +259,28 @@ export function unitsOf(root: string, file: string): { name: string; start: numb
219259 return [...parse(root, file).units.values()].map(({ name, start, text }) => ({ name, start, text }));
220260 }
221261
262 const facts = new WeakMap<Unit, { ids: Set<string>; members: string[] }>();
263 /** The names a unit mentions and the members it calls, found once per parsed unit. */
264 function factsOf(u: Unit): { ids: Set<string>; members: string[] } {
265 let f = facts.get(u);
266 if (!f) {
267 const ids = new Set(u.text.match(/[A-Za-z_$][\w$]*/g) ?? []);
268 const members = [...u.text.matchAll(/\.#?([A-Za-z_$][\w$]*)/g)].map((m) => m[1]!);
269 f = { ids, members };
270 facts.set(u, f);
271 }
272 return f;
273 }
274
222275 /**
223276 * Every finding reachable from the units of `entry` (repo-relative, under
224277 * `root`). `allowed` are the units that may remove a file, each with why.
225278 */
226 export function destroyReach(root: string, entry: string, allowed: readonly { unit: string; why: string }[] = ALLOWED_REMOVALS): string[] {
279 export function destroyReach(root: string, entry: string, allowed: readonly { unit: string; why: string }[] = ALLOWED_REMOVALS, cache?: WalkCache): string[] {
227280 const mods = new Map<string, Module>();
228281 const mod = (f: string): Module => {
229282 let m = mods.get(f);
230 if (!m) mods.set(f, (m = parse(root, f)));
283 if (!m) mods.set(f, (m = parse(root, f, cache)));
231284 return m;
232285 };
233286 const found = (f: string, name: string, seen = new Set<string>()): Unit | undefined => {
@@ -276,7 +329,9 @@ export function destroyReach(root: string, entry: string, allowed: readonly { un
276329 queue.push(u);
277330 for (const m of u.methods ?? []) {
278331 const name = m.name.slice(m.name.indexOf(".") + 1);
279 methods.set(name, [...(methods.get(name) ?? []), m]);
332 const ms = methods.get(name);
333 if (ms) ms.push(m);
334 else methods.set(name, [m]);
280335 for (const c of wanted.get(name) ?? []) if (candidate(c, m)) add(m, c);
281336 }
282337 };
@@ -292,7 +347,7 @@ export function destroyReach(root: string, entry: string, allowed: readonly { un
292347 flagged.add(u.file);
293348 if (u.file !== entry) for (const miss of layoutMisses(root, u.file)) out.push(`${miss} [indented top-level code hides from the walk]`);
294349 }
295 const code = stripComments(u.text);
350 const code = stripComments(u.text, cache);
296351 const mayRemove = allowed.some((x) => x.unit === `${u.file}:${u.name}`);
297352 const local = new Set(m.removers);
298353 for (const a of code.matchAll(/\b(rmSync|rmdirSync|unlinkSync|rm|rmdir|unlink)\s*:\s*([A-Za-z_$][\w$]*)/g)) local.add(a[2]!);
@@ -309,15 +364,18 @@ export function destroyReach(root: string, entry: string, allowed: readonly { un
309364 code.split("\n").forEach((line, i) => {
310365 if (named.test(line) || SPAWNED.test(line) || EVENT.test(line)) report(code.split("\n").slice(0, i).join("\n").length + (i ? 1 : 0), "names a removal");
311366 });
312 for (const id of new Set(u.text.match(/[A-Za-z_$][\w$]*/g) ?? [])) {
367 const facts = factsOf(u);
368 for (const id of facts.ids) {
313369 const localUnit = m.units.get(id);
314370 if (localUnit) add(localUnit, u);
315371 const imp = m.imports.get(id);
316372 if (imp) add(found(imp.from, imp.name), u);
317373 }
318 for (const hit of u.text.matchAll(/\.#?([A-Za-z_$][\w$]*)/g)) {
319 wanted.set(hit[1]!, [...(wanted.get(hit[1]!) ?? []), u]);
320 for (const mm of methods.get(hit[1]!) ?? []) if (candidate(u, mm)) add(mm, u);
374 for (const name of facts.members) {
375 const ws = wanted.get(name);
376 if (ws) ws.push(u);
377 else wanted.set(name, [u]);
378 for (const mm of methods.get(name) ?? []) if (candidate(u, mm)) add(mm, u);
321379 }
322380 for (const [ns, target] of m.spaces)
323381 for (const hit of u.text.matchAll(new RegExp(`\\b${ns}\\.([\\w$]+)`, "g"))) add(found(target, hit[1]!), u);
@@ -330,7 +388,7 @@ export function destroyReach(root: string, entry: string, allowed: readonly { un
330388 export const WEBHOOK_ENTRY = "src/http/stripe_hook.ts";
331389
332390 /** Gate form: one line per finding; a missing webhook is a finding, never a pass. */
333 export function destroyReachRefusal(root = "."): string[] {
391 export function destroyReachRefusal(root = ".", cache?: WalkCache): string[] {
334392 if (!existsSync(join(root, WEBHOOK_ENTRY))) return [`${WEBHOOK_ENTRY} is not there: the walk has nothing to start from; if the webhook moved, change WEBHOOK_ENTRY in src/kernel/destroy_reach.ts`];
335 return destroyReach(root, WEBHOOK_ENTRY).map((f) => `the billing webhook reaches a deletion: ${f}`);
393 return destroyReach(root, WEBHOOK_ENTRY, ALLOWED_REMOVALS, cache).map((f) => `the billing webhook reaches a deletion: ${f}`);
336394 }
test/agents.test.ts +15 −25
@@ -182,33 +182,23 @@ describe("fifty agents at once", () => {
182182 expect(await clone.exited).toBe(0);
183183 const work = join(clones, "work");
184184
185 // One commit per branch, prepared serially (cheap), then pushed at once.
185 // One commit per branch, prepared by a single fast-import (one process, not
186 // three per branch), then pushed at once.
187 const stamp = "agent <[email protected]> 1700000000 +0000";
188 let stream = "";
186189 for (let i = 0; i < WRITERS; i++) {
187 const branch = `agent-${i}`;
188 await Bun.spawn(["git", "checkout", "--quiet", "-b", branch, "main"], {
189 cwd: work,
190 stdout: "ignore",
191 stderr: "ignore",
192 env,
193 }).exited;
194 await Bun.write(join(work, `agent-${i}.txt`), `work from agent ${i}\n`);
195 await Bun.spawn(["git", "add", "."], { cwd: work, stdout: "ignore", env })
196 .exited;
197 await Bun.spawn(
198 [
199 "git",
200 "-c",
201 "[email protected]",
202 "-c",
203 "user.name=agent",
204 "commit",
205 "--quiet",
206 "-m",
207 `agent ${i}`,
208 ],
209 { cwd: work, stdout: "ignore", stderr: "ignore", env },
210 ).exited;
190 const data = `work from agent ${i}\n`;
191 const message = `agent ${i}`;
192 stream += `commit refs/heads/agent-${i}\ncommitter ${stamp}\ndata ${message.length}\n${message}\nfrom refs/heads/main\nM 100644 inline agent-${i}.txt\ndata ${data.length}\n${data}\n`;
211193 }
194 const fastImport = Bun.spawn(["git", "fast-import", "--quiet"], {
195 cwd: work,
196 stdin: new TextEncoder().encode(stream),
197 stdout: "ignore",
198 stderr: "pipe",
199 env,
200 });
201 expect(await fastImport.exited, await new Response(fastImport.stderr).text()).toBe(0);
212202
213203 const pushes = await Promise.all(
214204 Array.from({ length: WRITERS }, (_, i) =>
test/billing-never-deletes.test.ts +6 −5
@@ -12,7 +12,7 @@ import { tmpdir } from "node:os";
1212 import { basename, join } from "node:path";
1313 import { readdirSync } from "node:fs";
1414 import { billingDeletesRefusal } from "../scripts/gate.ts";
15 import { ALLOWED_REMOVALS, destroyReach, destroyReachRefusal, layoutMisses, STOPS, unitsOf, WEBHOOK_ENTRY } from "../src/kernel/destroy_reach.ts";
15 import { ALLOWED_REMOVALS, createWalkCache, destroyReach, destroyReachRefusal, layoutMisses, STOPS, unitsOf, WEBHOOK_ENTRY } from "../src/kernel/destroy_reach.ts";
1616
1717 const root = new URL("..", import.meta.url).pathname;
1818
@@ -26,8 +26,9 @@ const tree = (files: Record<string, string>): string => {
2626 };
2727
2828 describe("the billing webhook cannot delete", () => {
29 const cache = createWalkCache();
2930 test("today's tree: nothing reachable from the webhook removes a forge, an app or a child", () => {
30 expect(destroyReach(root, WEBHOOK_ENTRY)).toEqual([]);
31 expect(destroyReach(root, WEBHOOK_ENTRY, ALLOWED_REMOVALS, cache)).toEqual([]);
3132 });
3233
3334 test("every stop and every allowance still means something, with a reason", () => {
@@ -39,7 +40,7 @@ describe("the billing webhook cannot delete", () => {
3940 for (const f of from) expect(unitsOf(root, f.split(":")[0]!).some((u) => u.name === f.split(":")[1])).toBe(true);
4041 }
4142 // With no allowance the walk finds each allowed unit: an entry that finds nothing is stale.
42 const found = destroyReach(root, WEBHOOK_ENTRY, []).join("\n");
43 const found = destroyReach(root, WEBHOOK_ENTRY, [], cache).join("\n");
4344 for (const { unit, why } of ALLOWED_REMOVALS) {
4445 expect(why.length).toBeGreaterThan(10);
4546 expect(found).toContain(`${unit}]`);
@@ -47,7 +48,7 @@ describe("the billing webhook cannot delete", () => {
4748 });
4849
4950 test("the webhook cannot trigger the runtime's teardown: no reachable unit names repo.deleted, and the runtime's entries are entered only from the units allowed", () => {
50 expect(destroyReach(root, WEBHOOK_ENTRY).filter((f) => f.includes("names a removal") || f.includes("reaches "))).toEqual([]);
51 expect(destroyReach(root, WEBHOOK_ENTRY, ALLOWED_REMOVALS, cache).filter((f) => f.includes("names a removal") || f.includes("reaches "))).toEqual([]);
5152 // The one name collision the walk lets through: `cells.start(` is the cell registry's.
5253 const settle = unitsOf(root, "src/spawn/host.ts").find((u) => u.name === "settleCell")!;
5354 expect(settle.text).toContain("cellChildren(");
@@ -78,7 +79,7 @@ describe("the billing webhook cannot delete", () => {
7879 readFileSync(hook, "utf8") +
7980 '\nexport function dropForge(app: App, bodyId: string) {\n rmSync(childDirOf(app, bodyId), { recursive: true, force: true });\n}\n',
8081 );
81 const found = destroyReach(copy, WEBHOOK_ENTRY);
82 const found = destroyReach(copy, WEBHOOK_ENTRY, ALLOWED_REMOVALS, cache);
8283 rmSync(copy, { recursive: true, force: true });
8384 expect(found.length).toBe(1);
8485 expect(found[0]).toContain(`${WEBHOOK_ENTRY}:`);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs