Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7vxdkmn19wjbtnbvbxvk3cc is past its budget: A run was killed twice: budget: a review run took 20 min, limit 10. #14622

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14609-sentinel-budget-01m7vxdkmn19wjbtnbvbxvk3 → main 1 commit fork

  • Merged. mason/14609-sentinel-budget-01m7vxdkmn19wjbtnbvbxvk3 was merged into main as 2795d45.

Review

Sign in to review this change.
2 files changed +62 −4
src/crew/mirror.ts +41 −4
@@ -21,6 +21,7 @@
2121 import { existsSync, mkdirSync, rmSync } from "node:fs";
2222 import { join } from "node:path";
2323 import { runGit } from "../git/env.ts";
24 import { err } from "../kernel/result.ts";
2425 import { createLog } from "../kernel/log.ts";
2526 import { crewRoot } from "./workdir.ts";
2627
@@ -32,14 +33,38 @@ export const mirrorDir = (dataDir: string, repoId: string): string =>
3233 // One chain per repository: two fetches into one mirror never overlap.
3334 const chains = new Map<string, Promise<unknown>>();
3435
35 function exclusive<T>(key: string, job: () => Promise<T>): Promise<T> {
36 const run = (chains.get(key) ?? Promise.resolve()).then(job, job);
36 /**
37 * `job`, once every earlier job on `key` has finished. A caller whose `signal`
38 * aborts while it still waits its turn gets `stopped()` at once, and its job
39 * never runs: a run stopped at its budget does not sit out other runs' clones
40 * first (#14609). Once the job has started, the signal is the job's to heed.
41 */
42 function exclusive<T>(
43 key: string,
44 job: () => Promise<T>,
45 leave?: { signal: AbortSignal; stopped: () => T },
46 ): Promise<T> {
47 let started = false;
48 const turn = async (): Promise<T> => {
49 if (leave?.signal.aborted) return leave.stopped();
50 started = true;
51 return job();
52 };
53 const run = (chains.get(key) ?? Promise.resolve()).then(turn, turn);
3754 const tail = run.catch(() => undefined);
3855 chains.set(key, tail);
3956 void tail.then(() => {
4057 if (chains.get(key) === tail) chains.delete(key);
4158 });
42 return run;
59 if (leave === undefined) return run;
60 return new Promise<T>((resolve, reject) => {
61 const stop = (): void => {
62 if (!started) resolve(leave.stopped());
63 };
64 if (leave.signal.aborted) stop();
65 leave.signal.addEventListener("abort", stop, { once: true });
66 run.then(resolve, reject).finally(() => leave.signal.removeEventListener("abort", stop));
67 });
4368 }
4469
4570 /** What a mirror holds: branches and tags, never pull refs. */
@@ -74,6 +99,9 @@ async function refresh(
7499 return undefined;
75100 }
76101
102 /** The code of a clone whose run was stopped before its turn in the queue came. */
103 const STOPPED = -4;
104
77105 /** Git's wording for a damaged object store; a timeout kill or a dead origin says none of it. */
78106 const damaged = (stderr: string): boolean =>
79107 /bad object|corrupt|inflate|did not receive expected object|missing object|unable to parse|invalid object|checkout failed/i.test(
@@ -127,6 +155,15 @@ export async function cloneWithMirror(input: {
127155 const plain = () =>
128156 runGit(["clone", "--quiet", "--", input.url, input.dir], opts);
129157 if (!validId(input.repoId)) return plain();
158 const signal = input.signal;
159 const leave =
160 signal === undefined
161 ? undefined
162 : {
163 signal,
164 stopped: () =>
165 err({ code: STOPPED, stderr: "stopped while waiting for the repository's clone queue", args: ["clone", input.url] }),
166 };
130167 return exclusive(input.repoId, async () => {
131168 const mirror = await refresh(mirrorDir(input.dataDir, input.repoId), input.url, opts);
132169 if (mirror === undefined) return plain();
@@ -145,5 +182,5 @@ export async function cloneWithMirror(input: {
145182 rmSync(input.dir, { recursive: true, force: true });
146183 rmSync(mirror, { recursive: true, force: true });
147184 return plain();
148 });
185 }, leave);
149186 }
test/crew-mirror.test.ts +21
@@ -116,6 +116,27 @@ test("the borrowing clone runs inside the repository's lock", async () => {
116116 expect(s.log()).toEqual(["start", "end", "start", "end", "start", "end", "start", "end"]);
117117 });
118118
119 test("a clone queued behind another's leaves the queue the moment its run is stopped", async () => {
120 // Run 01m7vxdkmn19wjbtnbvbxvk3cc (#14609): a review stopped at its 10 min
121 // limit sat in the queue until 20 min, behind other runs' clones.
122 const s = slow(1.5);
123 const holder = cloneWithMirror({ dataDir: data(), repoId: "r9", url: s.url, dir: join(root, "queue-a"), env: s.env });
124 const ctl = new AbortController();
125 const dir = join(root, "queue-b");
126 // Both joined the repository's queue as they were called: the second waits.
127 const queued = cloneWithMirror({ dataDir: data(), repoId: "r9", url: s.url, dir, env: s.env, signal: ctl.signal });
128 const stopped = Date.now();
129 ctl.abort();
130 const result = await queued;
131 expect(Date.now() - stopped).toBeLessThan(500);
132 expect(result.ok).toBe(false);
133 if (!result.ok) expect(result.error.stderr).toContain("stopped");
134 expect((await holder).ok).toBe(true);
135 // It never ran: the holder's mirror build and clone are the only conversations.
136 expect(s.log()).toEqual(["start", "end", "start", "end"]);
137 expect(existsSync(join(dir, "a.txt"))).toBe(false);
138 });
139
119140 test("an agent's sandbox denies the mirrors", () => {
120141 const policy = sandboxPolicy({ dataDir: data(), agentName: "scout", urls: [] });
121142 expect(policy.denyRead).toContain(join(data(), "crew", ".mirrors"));

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs