Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7vxdkmn19wjbtnbvbxvk3cc is past its budget: A run was killed twice: budget: a review run took 20 min, limit 10. #14622
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14609-sentinel-budget-01m7vxdkmn19wjbtnbvbxvk3 → main 1 commit fork
2 files
changed +62
−4
src/crew/mirror.ts +41 −4
| @@ -21,6 +21,7 @@ | ||
| 21 | 21 | import { existsSync, mkdirSync, rmSync } from "node:fs"; |
| 22 | 22 | import { join } from "node:path"; |
| 23 | 23 | import { runGit } from "../git/env.ts"; |
| 24 | import { err } from "../kernel/result.ts"; | |
| 24 | 25 | import { createLog } from "../kernel/log.ts"; |
| 25 | 26 | import { crewRoot } from "./workdir.ts"; |
| 26 | 27 | |
| @@ -32,14 +33,38 @@ export const mirrorDir = (dataDir: string, repoId: string): string => | ||
| 32 | 33 | // One chain per repository: two fetches into one mirror never overlap. |
| 33 | 34 | const chains = new Map<string, Promise<unknown>>(); |
| 34 | 35 | |
| 35 | function exclusive<T>(key: string, job: () => Promise<T>): Promise<T> { | |
| 36 | const run = (chains.get(key) ?? Promise.resolve()).then(job, job); | |
| 36 | /** | |
| 37 | * `job`, once every earlier job on `key` has finished. A caller whose `signal` | |
| 38 | * aborts while it still waits its turn gets `stopped()` at once, and its job | |
| 39 | * never runs: a run stopped at its budget does not sit out other runs' clones | |
| 40 | * first (#14609). Once the job has started, the signal is the job's to heed. | |
| 41 | */ | |
| 42 | function exclusive<T>( | |
| 43 | key: string, | |
| 44 | job: () => Promise<T>, | |
| 45 | leave?: { signal: AbortSignal; stopped: () => T }, | |
| 46 | ): Promise<T> { | |
| 47 | let started = false; | |
| 48 | const turn = async (): Promise<T> => { | |
| 49 | if (leave?.signal.aborted) return leave.stopped(); | |
| 50 | started = true; | |
| 51 | return job(); | |
| 52 | }; | |
| 53 | const run = (chains.get(key) ?? Promise.resolve()).then(turn, turn); | |
| 37 | 54 | const tail = run.catch(() => undefined); |
| 38 | 55 | chains.set(key, tail); |
| 39 | 56 | void tail.then(() => { |
| 40 | 57 | if (chains.get(key) === tail) chains.delete(key); |
| 41 | 58 | }); |
| 42 | return run; | |
| 59 | if (leave === undefined) return run; | |
| 60 | return new Promise<T>((resolve, reject) => { | |
| 61 | const stop = (): void => { | |
| 62 | if (!started) resolve(leave.stopped()); | |
| 63 | }; | |
| 64 | if (leave.signal.aborted) stop(); | |
| 65 | leave.signal.addEventListener("abort", stop, { once: true }); | |
| 66 | run.then(resolve, reject).finally(() => leave.signal.removeEventListener("abort", stop)); | |
| 67 | }); | |
| 43 | 68 | } |
| 44 | 69 | |
| 45 | 70 | /** What a mirror holds: branches and tags, never pull refs. */ |
| @@ -74,6 +99,9 @@ async function refresh( | ||
| 74 | 99 | return undefined; |
| 75 | 100 | } |
| 76 | 101 | |
| 102 | /** The code of a clone whose run was stopped before its turn in the queue came. */ | |
| 103 | const STOPPED = -4; | |
| 104 | ||
| 77 | 105 | /** Git's wording for a damaged object store; a timeout kill or a dead origin says none of it. */ |
| 78 | 106 | const damaged = (stderr: string): boolean => |
| 79 | 107 | /bad object|corrupt|inflate|did not receive expected object|missing object|unable to parse|invalid object|checkout failed/i.test( |
| @@ -127,6 +155,15 @@ export async function cloneWithMirror(input: { | ||
| 127 | 155 | const plain = () => |
| 128 | 156 | runGit(["clone", "--quiet", "--", input.url, input.dir], opts); |
| 129 | 157 | if (!validId(input.repoId)) return plain(); |
| 158 | const signal = input.signal; | |
| 159 | const leave = | |
| 160 | signal === undefined | |
| 161 | ? undefined | |
| 162 | : { | |
| 163 | signal, | |
| 164 | stopped: () => | |
| 165 | err({ code: STOPPED, stderr: "stopped while waiting for the repository's clone queue", args: ["clone", input.url] }), | |
| 166 | }; | |
| 130 | 167 | return exclusive(input.repoId, async () => { |
| 131 | 168 | const mirror = await refresh(mirrorDir(input.dataDir, input.repoId), input.url, opts); |
| 132 | 169 | if (mirror === undefined) return plain(); |
| @@ -145,5 +182,5 @@ export async function cloneWithMirror(input: { | ||
| 145 | 182 | rmSync(input.dir, { recursive: true, force: true }); |
| 146 | 183 | rmSync(mirror, { recursive: true, force: true }); |
| 147 | 184 | return plain(); |
| 148 | }); | |
| 185 | }, leave); | |
| 149 | 186 | } |
test/crew-mirror.test.ts +21
| @@ -116,6 +116,27 @@ test("the borrowing clone runs inside the repository's lock", async () => { | ||
| 116 | 116 | expect(s.log()).toEqual(["start", "end", "start", "end", "start", "end", "start", "end"]); |
| 117 | 117 | }); |
| 118 | 118 | |
| 119 | test("a clone queued behind another's leaves the queue the moment its run is stopped", async () => { | |
| 120 | // Run 01m7vxdkmn19wjbtnbvbxvk3cc (#14609): a review stopped at its 10 min | |
| 121 | // limit sat in the queue until 20 min, behind other runs' clones. | |
| 122 | const s = slow(1.5); | |
| 123 | const holder = cloneWithMirror({ dataDir: data(), repoId: "r9", url: s.url, dir: join(root, "queue-a"), env: s.env }); | |
| 124 | const ctl = new AbortController(); | |
| 125 | const dir = join(root, "queue-b"); | |
| 126 | // Both joined the repository's queue as they were called: the second waits. | |
| 127 | const queued = cloneWithMirror({ dataDir: data(), repoId: "r9", url: s.url, dir, env: s.env, signal: ctl.signal }); | |
| 128 | const stopped = Date.now(); | |
| 129 | ctl.abort(); | |
| 130 | const result = await queued; | |
| 131 | expect(Date.now() - stopped).toBeLessThan(500); | |
| 132 | expect(result.ok).toBe(false); | |
| 133 | if (!result.ok) expect(result.error.stderr).toContain("stopped"); | |
| 134 | expect((await holder).ok).toBe(true); | |
| 135 | // It never ran: the holder's mirror build and clone are the only conversations. | |
| 136 | expect(s.log()).toEqual(["start", "end", "start", "end"]); | |
| 137 | expect(existsSync(join(dir, "a.txt"))).toBe(false); | |
| 138 | }); | |
| 139 | ||
| 119 | 140 | test("an agent's sandbox denies the mirrors", () => { |
| 120 | 141 | const policy = sandboxPolicy({ dataDir: data(), agentName: "scout", urls: [] }); |
| 121 | 142 | expect(policy.denyRead).toContain(join(data(), "crew", ".mirrors")); |
new above
offline