Public: anyone can read this; the crew can change it.
Sentinel: budget:01m7vny26brgeygne5gcc2z0wb is past its budget: A run was killed twice: budget: a review run took 11 min, limit 10. #14624
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14601-sentinel-budget-01m7vny26brgeygne5gcc2z0 → main 4 commits fork 3 threads · 3 unresolved
3 files
changed +152
−21
src/crew/mirror.ts +1 −4
| @@ -20,7 +20,7 @@ | ||
| 20 | 20 | |
| 21 | 21 | import { existsSync, mkdirSync, rmSync } from "node:fs"; |
| 22 | 22 | import { join } from "node:path"; |
| 23 | import { runGit } from "../git/env.ts"; | |
| 23 | import { STOPPED, runGit } from "../git/env.ts"; | |
| 24 | 24 | import { err } from "../kernel/result.ts"; |
| 25 | 25 | import { createLog } from "../kernel/log.ts"; |
| 26 | 26 | import { crewRoot } from "./workdir.ts"; |
| @@ -99,9 +99,6 @@ async function refresh( | ||
| 99 | 99 | return undefined; |
| 100 | 100 | } |
| 101 | 101 | |
| 102 | /** The code of a clone whose run was stopped before its turn in the queue came. */ | |
| 103 | const STOPPED = -4; | |
| 104 | ||
| 105 | 102 | /** Git's wording for a damaged object store; a timeout kill or a dead origin says none of it. */ |
| 106 | 103 | const damaged = (stderr: string): boolean => |
| 107 | 104 | /bad object|corrupt|inflate|did not receive expected object|missing object|unable to parse|invalid object|checkout failed/i.test( |
src/git/env.ts +89 −17
| @@ -146,21 +146,70 @@ export class Gate { | ||
| 146 | 146 | * deploy gets the next permit while previews are waiting. |
| 147 | 147 | */ |
| 148 | 148 | async acquire(options: { first?: boolean } = {}): Promise<() => void> { |
| 149 | const taken = this.take(options.first === true); | |
| 150 | // Free, the permit is taken without an await: the same ticks as ever. | |
| 151 | return taken instanceof Promise ? await taken : taken; | |
| 152 | } | |
| 153 | ||
| 154 | /** | |
| 155 | * `acquire`, or undefined when `signal` aborts first: an aborted waiter | |
| 156 | * leaves the queue at once and never holds a permit. | |
| 157 | */ | |
| 158 | async acquireUnless(signal: AbortSignal | undefined): Promise<(() => void) | undefined> { | |
| 159 | const taken = this.take(false, signal); | |
| 160 | return taken instanceof Promise ? await taken : taken; | |
| 161 | } | |
| 162 | ||
| 163 | /** | |
| 164 | * A permit at once when one is free (or undefined when `signal` has | |
| 165 | * already aborted), else a promise of one. Synchronous on the free path | |
| 166 | * so a permit costs no microtask more than it did before #14601: the | |
| 167 | * checks scheduler's ordering (`quotas-running.test.ts`) leans on it. | |
| 168 | * Without a signal nothing can abort, so the first overload is never | |
| 169 | * undefined. | |
| 170 | */ | |
| 171 | private take(first: boolean): (() => void) | Promise<() => void>; | |
| 172 | private take( | |
| 173 | first: boolean, | |
| 174 | signal: AbortSignal | undefined, | |
| 175 | ): (() => void) | undefined | Promise<(() => void) | undefined>; | |
| 176 | private take( | |
| 177 | first: boolean, | |
| 178 | signal?: AbortSignal, | |
| 179 | ): (() => void) | undefined | Promise<(() => void) | undefined> { | |
| 180 | if (signal?.aborted === true) return undefined; | |
| 149 | 181 | if (this.active < this.limit) { |
| 150 | 182 | this.active++; |
| 151 | } else { | |
| 152 | // Wait for a permit to be *handed* to us. The releaser does not | |
| 153 | // decrement when it wakes somebody: the count stays taken and simply | |
| 154 | // changes owner. Decrementing first would open a window in which a | |
| 155 | // caller arriving synchronously slips in ahead of the queue and the | |
| 156 | // limit is briefly exceeded — a semaphore that is only usually one. | |
| 157 | const first = options.first === true; | |
| 158 | await new Promise<void>((wake) => { | |
| 159 | const behind = first ? this.waiting.findIndex((w) => !w.first) : -1; | |
| 160 | if (behind === -1) this.waiting.push({ wake, first }); | |
| 161 | else this.waiting.splice(behind, 0, { wake, first }); | |
| 162 | }); | |
| 183 | return this.permit(); | |
| 163 | 184 | } |
| 185 | // Wait for a permit to be *handed* to us. The releaser does not | |
| 186 | // decrement when it wakes somebody: the count stays taken and simply | |
| 187 | // changes owner. Decrementing first would open a window in which a | |
| 188 | // caller arriving synchronously slips in ahead of the queue and the | |
| 189 | // limit is briefly exceeded — a semaphore that is only usually one. | |
| 190 | return new Promise<(() => void) | undefined>((resolve) => { | |
| 191 | const leave = (): void => { | |
| 192 | const at = this.waiting.indexOf(waiter); | |
| 193 | if (at === -1) return; | |
| 194 | this.waiting.splice(at, 1); | |
| 195 | resolve(undefined); | |
| 196 | }; | |
| 197 | const waiter = { | |
| 198 | wake: () => { | |
| 199 | signal?.removeEventListener("abort", leave); | |
| 200 | resolve(this.permit()); | |
| 201 | }, | |
| 202 | first, | |
| 203 | }; | |
| 204 | const behind = first ? this.waiting.findIndex((w) => !w.first) : -1; | |
| 205 | if (behind === -1) this.waiting.push(waiter); | |
| 206 | else this.waiting.splice(behind, 0, waiter); | |
| 207 | signal?.addEventListener("abort", leave, { once: true }); | |
| 208 | }); | |
| 209 | } | |
| 210 | ||
| 211 | /** The release of a permit already counted in `active`. */ | |
| 212 | private permit(): () => void { | |
| 164 | 213 | if (this.active > this.peak) this.peak = this.active; |
| 165 | 214 | let released = false; |
| 166 | 215 | return () => { |
| @@ -188,6 +237,25 @@ export class Gate { | ||
| 188 | 237 | } |
| 189 | 238 | } |
| 190 | 239 | |
| 240 | /** | |
| 241 | * `run`, unless `signal` aborts while it waits for a permit: then `left()` | |
| 242 | * at once, and `work` never runs. A run stopped at its budget does not sit | |
| 243 | * out every other git process before it can stop (#14601). | |
| 244 | */ | |
| 245 | async runUnless<T>( | |
| 246 | signal: AbortSignal | undefined, | |
| 247 | work: () => Promise<T>, | |
| 248 | left: () => T, | |
| 249 | ): Promise<T> { | |
| 250 | const release = await this.acquireUnless(signal); | |
| 251 | if (release === undefined) return left(); | |
| 252 | try { | |
| 253 | return await work(); | |
| 254 | } finally { | |
| 255 | release(); | |
| 256 | } | |
| 257 | } | |
| 258 | ||
| 191 | 259 | stats(): { active: number; waiting: number; limit: number; peak: number } { |
| 192 | 260 | return { |
| 193 | 261 | active: this.active, |
| @@ -284,6 +352,9 @@ export interface RunOptions { | ||
| 284 | 352 | /** The code of a failure over `maxBytes`. */ |
| 285 | 353 | export const OVER_CAP = -2; |
| 286 | 354 | |
| 355 | /** The code of a git call whose caller was stopped before it ran. */ | |
| 356 | export const STOPPED = -4; | |
| 357 | ||
| 287 | 358 | /** Run git to completion, capturing output as bytes. Non-zero exit is an Err. */ |
| 288 | 359 | export async function runGitRaw( |
| 289 | 360 | args: readonly string[], |
| @@ -296,7 +367,7 @@ export async function runGitRaw( | ||
| 296 | 367 | return err({ code: 128, stderr: "fatal: bad object", args }); |
| 297 | 368 | // Timed from before the gate: a wait for a permit is the queue #8158 names. |
| 298 | 369 | const start = performance.now(); |
| 299 | return gitGate.run(async () => { | |
| 370 | return gitGate.runUnless(options.signal, async () => { | |
| 300 | 371 | let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">; |
| 301 | 372 | try { |
| 302 | 373 | proc = Bun.spawn(["git", ...args], { |
| @@ -336,7 +407,7 @@ export async function runGitRaw( | ||
| 336 | 407 | options.signal?.removeEventListener("abort", stop); |
| 337 | 408 | noteGit(gitRecord, args, start); |
| 338 | 409 | } |
| 339 | }); | |
| 410 | }, () => err({ code: STOPPED, stderr: "stopped while waiting for a git permit", args })); | |
| 340 | 411 | } |
| 341 | 412 | |
| 342 | 413 | /** Run git to completion, capturing stdout as trimmed text. */ |
| @@ -424,8 +495,9 @@ export async function runGitCapture( | ||
| 424 | 495 | if (hold !== undefined && !(await namesOnlyOwn(hold, args, options.stdin))) |
| 425 | 496 | return { code: 128, stdout: "", stderr: "fatal: bad object" }; |
| 426 | 497 | const start = performance.now(); |
| 427 | return gitGate.run(async () => { | |
| 428 | if (options.signal?.aborted === true) return { code: 1, stdout: "", stderr: "", timedOut: true }; | |
| 498 | const stopped = (): GitCapture => ({ code: 1, stdout: "", stderr: "", timedOut: true }); | |
| 499 | return gitGate.runUnless(options.signal, async () => { | |
| 500 | if (options.signal?.aborted === true) return stopped(); | |
| 429 | 501 | let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">; |
| 430 | 502 | try { |
| 431 | 503 | proc = Bun.spawn(["git", ...args], { |
| @@ -470,7 +542,7 @@ export async function runGitCapture( | ||
| 470 | 542 | options.signal?.removeEventListener("abort", abort); |
| 471 | 543 | noteGit(gitRecord, args, start); |
| 472 | 544 | } |
| 473 | }); | |
| 545 | }, stopped); | |
| 474 | 546 | } |
| 475 | 547 | |
| 476 | 548 | /** |
test/git-gate-leave.test.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | // A git call whose run was stopped leaves the permit queue at once (#14601). | |
| 2 | // Run 01m7vny26brgeygne5gcc2z0wb: a review stopped at its 10 min limit spent | |
| 3 | // 681.5 s in `clone`, all of it before the agent began. | |
| 4 | import { expect, test } from "bun:test"; | |
| 5 | import { Gate, gitGate, runGit, runGitCapture, STOPPED } from "../src/git/env.ts"; | |
| 6 | ||
| 7 | test("an aborted waiter leaves the queue, never runs, and hands nothing on", async () => { | |
| 8 | const gate = new Gate(1); | |
| 9 | const release = await gate.acquire(); | |
| 10 | const ctl = new AbortController(); | |
| 11 | let ran = false; | |
| 12 | const waiting = gate.runUnless(ctl.signal, async () => { | |
| 13 | ran = true; | |
| 14 | return "ran"; | |
| 15 | }, () => "left"); | |
| 16 | const after = gate.run(async () => "after"); | |
| 17 | expect(gate.stats().waiting).toBe(2); | |
| 18 | ctl.abort(); | |
| 19 | expect(await waiting).toBe("left"); | |
| 20 | expect(gate.stats().waiting).toBe(1); | |
| 21 | release(); | |
| 22 | // The permit went to the next waiter, not to the one that left. | |
| 23 | expect(await after).toBe("after"); | |
| 24 | expect(ran).toBe(false); | |
| 25 | expect(gate.stats().active).toBe(0); | |
| 26 | }); | |
| 27 | ||
| 28 | test("a signal aborted after the permit was handed on does not take it back", async () => { | |
| 29 | const gate = new Gate(1); | |
| 30 | const release = await gate.acquire(); | |
| 31 | const ctl = new AbortController(); | |
| 32 | const { promise: started, resolve: start } = Promise.withResolvers<void>(); | |
| 33 | const waiting = gate.runUnless(ctl.signal, async () => { | |
| 34 | start(); | |
| 35 | return "ran"; | |
| 36 | }, () => "left"); | |
| 37 | release(); | |
| 38 | ctl.abort(); | |
| 39 | await started; | |
| 40 | expect(await waiting).toBe("ran"); | |
| 41 | expect(gate.stats()).toMatchObject({ active: 0, waiting: 0 }); | |
| 42 | }); | |
| 43 | ||
| 44 | test("a git call stopped while every permit is held answers at once", async () => { | |
| 45 | const held: (() => void)[] = []; | |
| 46 | for (let i = 0; i < gitGate.limit; i++) held.push(await gitGate.acquire()); | |
| 47 | try { | |
| 48 | const ctl = new AbortController(); | |
| 49 | const raw = runGit(["--version"], { signal: ctl.signal }); | |
| 50 | const capture = runGitCapture(["--version"], { signal: ctl.signal }); | |
| 51 | const t0 = Date.now(); | |
| 52 | ctl.abort(); | |
| 53 | const [r, c] = await Promise.all([raw, capture]); | |
| 54 | expect(Date.now() - t0).toBeLessThan(200); | |
| 55 | expect(r.ok).toBe(false); | |
| 56 | if (!r.ok) expect(r.error.code).toBe(STOPPED); | |
| 57 | expect(c.timedOut).toBe(true); | |
| 58 | expect(gitGate.stats().waiting).toBe(0); | |
| 59 | } finally { | |
| 60 | for (const release of held) release(); | |
| 61 | } | |
| 62 | }); |
new above
offline
The
as () => voidcast hides thattakemay return undefined. Narrow the types so no cast is needed.