Public: anyone can read this; the crew can change it.

Sentinel: budget:01m7vny26brgeygne5gcc2z0wb is past its budget: A run was killed twice: budget: a review run took 11 min, limit 10. #14624

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/14601-sentinel-budget-01m7vny26brgeygne5gcc2z0 → main 4 commits fork 3 threads · 3 unresolved

  • Merged. torque/14601-sentinel-budget-01m7vny26brgeygne5gcc2z0 was merged into main as ad31a4c.

Review

Sign in to review this change.
3 files changed +152 −21
src/crew/mirror.ts +1 −4
@@ -20,7 +20,7 @@
2020
2121 import { existsSync, mkdirSync, rmSync } from "node:fs";
2222 import { join } from "node:path";
23 import { runGit } from "../git/env.ts";
23 import { STOPPED, runGit } from "../git/env.ts";
2424 import { err } from "../kernel/result.ts";
2525 import { createLog } from "../kernel/log.ts";
2626 import { crewRoot } from "./workdir.ts";
@@ -99,9 +99,6 @@ async function refresh(
9999 return undefined;
100100 }
101101
102 /** The code of a clone whose run was stopped before its turn in the queue came. */
103 const STOPPED = -4;
104
105102 /** Git's wording for a damaged object store; a timeout kill or a dead origin says none of it. */
106103 const damaged = (stderr: string): boolean =>
107104 /bad object|corrupt|inflate|did not receive expected object|missing object|unable to parse|invalid object|checkout failed/i.test(
src/git/env.ts +89 −17
@@ -146,21 +146,70 @@ export class Gate {
146146 * deploy gets the next permit while previews are waiting.
147147 */
148148 async acquire(options: { first?: boolean } = {}): Promise<() => void> {
149 const taken = this.take(options.first === true);
150 // Free, the permit is taken without an await: the same ticks as ever.
151 return taken instanceof Promise ? await taken : taken;
outdated

The as () => void cast hides that take may return undefined. Narrow the types so no cast is needed.

152 }
153
154 /**
155 * `acquire`, or undefined when `signal` aborts first: an aborted waiter
156 * leaves the queue at once and never holds a permit.
157 */
158 async acquireUnless(signal: AbortSignal | undefined): Promise<(() => void) | undefined> {
159 const taken = this.take(false, signal);
160 return taken instanceof Promise ? await taken : taken;
161 }
162
163 /**
164 * A permit at once when one is free (or undefined when `signal` has
165 * already aborted), else a promise of one. Synchronous on the free path
166 * so a permit costs no microtask more than it did before #14601: the
167 * checks scheduler's ordering (`quotas-running.test.ts`) leans on it.
168 * Without a signal nothing can abort, so the first overload is never
169 * undefined.
170 */
171 private take(first: boolean): (() => void) | Promise<() => void>;
172 private take(
173 first: boolean,
174 signal: AbortSignal | undefined,
175 ): (() => void) | undefined | Promise<(() => void) | undefined>;
176 private take(
177 first: boolean,
178 signal?: AbortSignal,
179 ): (() => void) | undefined | Promise<(() => void) | undefined> {
180 if (signal?.aborted === true) return undefined;
149181 if (this.active < this.limit) {
150182 this.active++;
151 } else {
152 // Wait for a permit to be *handed* to us. The releaser does not
153 // decrement when it wakes somebody: the count stays taken and simply
154 // changes owner. Decrementing first would open a window in which a
155 // caller arriving synchronously slips in ahead of the queue and the
156 // limit is briefly exceeded — a semaphore that is only usually one.
157 const first = options.first === true;
158 await new Promise<void>((wake) => {
159 const behind = first ? this.waiting.findIndex((w) => !w.first) : -1;
160 if (behind === -1) this.waiting.push({ wake, first });
161 else this.waiting.splice(behind, 0, { wake, first });
162 });
183 return this.permit();
163184 }
185 // Wait for a permit to be *handed* to us. The releaser does not
186 // decrement when it wakes somebody: the count stays taken and simply
187 // changes owner. Decrementing first would open a window in which a
188 // caller arriving synchronously slips in ahead of the queue and the
189 // limit is briefly exceeded — a semaphore that is only usually one.
190 return new Promise<(() => void) | undefined>((resolve) => {
191 const leave = (): void => {
192 const at = this.waiting.indexOf(waiter);
193 if (at === -1) return;
194 this.waiting.splice(at, 1);
195 resolve(undefined);
196 };
197 const waiter = {
198 wake: () => {
199 signal?.removeEventListener("abort", leave);
200 resolve(this.permit());
201 },
202 first,
203 };
204 const behind = first ? this.waiting.findIndex((w) => !w.first) : -1;
205 if (behind === -1) this.waiting.push(waiter);
206 else this.waiting.splice(behind, 0, waiter);
207 signal?.addEventListener("abort", leave, { once: true });
208 });
209 }
210
211 /** The release of a permit already counted in `active`. */
212 private permit(): () => void {
164213 if (this.active > this.peak) this.peak = this.active;
165214 let released = false;
166215 return () => {
@@ -188,6 +237,25 @@ export class Gate {
188237 }
189238 }
190239
240 /**
241 * `run`, unless `signal` aborts while it waits for a permit: then `left()`
242 * at once, and `work` never runs. A run stopped at its budget does not sit
243 * out every other git process before it can stop (#14601).
244 */
245 async runUnless<T>(
246 signal: AbortSignal | undefined,
247 work: () => Promise<T>,
248 left: () => T,
249 ): Promise<T> {
250 const release = await this.acquireUnless(signal);
251 if (release === undefined) return left();
252 try {
253 return await work();
254 } finally {
255 release();
256 }
257 }
258
191259 stats(): { active: number; waiting: number; limit: number; peak: number } {
192260 return {
193261 active: this.active,
@@ -284,6 +352,9 @@ export interface RunOptions {
284352 /** The code of a failure over `maxBytes`. */
285353 export const OVER_CAP = -2;
286354
355 /** The code of a git call whose caller was stopped before it ran. */
356 export const STOPPED = -4;
357
287358 /** Run git to completion, capturing output as bytes. Non-zero exit is an Err. */
288359 export async function runGitRaw(
289360 args: readonly string[],
@@ -296,7 +367,7 @@ export async function runGitRaw(
296367 return err({ code: 128, stderr: "fatal: bad object", args });
297368 // Timed from before the gate: a wait for a permit is the queue #8158 names.
298369 const start = performance.now();
299 return gitGate.run(async () => {
370 return gitGate.runUnless(options.signal, async () => {
300371 let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">;
301372 try {
302373 proc = Bun.spawn(["git", ...args], {
@@ -336,7 +407,7 @@ export async function runGitRaw(
336407 options.signal?.removeEventListener("abort", stop);
337408 noteGit(gitRecord, args, start);
338409 }
339 });
410 }, () => err({ code: STOPPED, stderr: "stopped while waiting for a git permit", args }));
340411 }
341412
342413 /** Run git to completion, capturing stdout as trimmed text. */
@@ -424,8 +495,9 @@ export async function runGitCapture(
424495 if (hold !== undefined && !(await namesOnlyOwn(hold, args, options.stdin)))
425496 return { code: 128, stdout: "", stderr: "fatal: bad object" };
426497 const start = performance.now();
427 return gitGate.run(async () => {
428 if (options.signal?.aborted === true) return { code: 1, stdout: "", stderr: "", timedOut: true };
498 const stopped = (): GitCapture => ({ code: 1, stdout: "", stderr: "", timedOut: true });
499 return gitGate.runUnless(options.signal, async () => {
500 if (options.signal?.aborted === true) return stopped();
429501 let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">;
430502 try {
431503 proc = Bun.spawn(["git", ...args], {
@@ -470,7 +542,7 @@ export async function runGitCapture(
470542 options.signal?.removeEventListener("abort", abort);
471543 noteGit(gitRecord, args, start);
472544 }
473 });
545 }, stopped);
474546 }
475547
476548 /**
test/git-gate-leave.test.ts added +62
@@ -0,0 +1,62 @@
1 // A git call whose run was stopped leaves the permit queue at once (#14601).
2 // Run 01m7vny26brgeygne5gcc2z0wb: a review stopped at its 10 min limit spent
3 // 681.5 s in `clone`, all of it before the agent began.
4 import { expect, test } from "bun:test";
5 import { Gate, gitGate, runGit, runGitCapture, STOPPED } from "../src/git/env.ts";
6
7 test("an aborted waiter leaves the queue, never runs, and hands nothing on", async () => {
8 const gate = new Gate(1);
9 const release = await gate.acquire();
10 const ctl = new AbortController();
11 let ran = false;
12 const waiting = gate.runUnless(ctl.signal, async () => {
13 ran = true;
14 return "ran";
15 }, () => "left");
16 const after = gate.run(async () => "after");
17 expect(gate.stats().waiting).toBe(2);
18 ctl.abort();
19 expect(await waiting).toBe("left");
20 expect(gate.stats().waiting).toBe(1);
21 release();
22 // The permit went to the next waiter, not to the one that left.
23 expect(await after).toBe("after");
24 expect(ran).toBe(false);
25 expect(gate.stats().active).toBe(0);
26 });
27
28 test("a signal aborted after the permit was handed on does not take it back", async () => {
29 const gate = new Gate(1);
30 const release = await gate.acquire();
31 const ctl = new AbortController();
32 const { promise: started, resolve: start } = Promise.withResolvers<void>();
33 const waiting = gate.runUnless(ctl.signal, async () => {
34 start();
35 return "ran";
36 }, () => "left");
37 release();
38 ctl.abort();
39 await started;
40 expect(await waiting).toBe("ran");
41 expect(gate.stats()).toMatchObject({ active: 0, waiting: 0 });
42 });
43
44 test("a git call stopped while every permit is held answers at once", async () => {
45 const held: (() => void)[] = [];
46 for (let i = 0; i < gitGate.limit; i++) held.push(await gitGate.acquire());
47 try {
48 const ctl = new AbortController();
49 const raw = runGit(["--version"], { signal: ctl.signal });
50 const capture = runGitCapture(["--version"], { signal: ctl.signal });
51 const t0 = Date.now();
52 ctl.abort();
53 const [r, c] = await Promise.all([raw, capture]);
54 expect(Date.now() - t0).toBeLessThan(200);
55 expect(r.ok).toBe(false);
56 if (!r.ok) expect(r.error.code).toBe(STOPPED);
57 expect(c.timedOut).toBe(true);
58 expect(gitGate.stats().waiting).toBe(0);
59 } finally {
60 for (const release of held) release();
61 }
62 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs