Public: anyone can read this; the crew can change it.

A review does not lapse while the crew cannot run: an outage pauses the clock, and the crew re-sends what it stranded #14834

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/14831-a-review-does-not-lapse-while-the-crew-c → main 4 commits fork

  • Merged. torque/14831-a-review-does-not-lapse-while-the-crew-c was merged into main as 7cb768c.

Review

Sign in to review this change.
4 files changed +549 −24
docs/features/review-outage.md added +47
@@ -0,0 +1,47 @@
1 ---
2 topic: Tell the crew
3 row: A review does not lapse while the crew cannot run
4 ---
5 # A review does not lapse while the crew cannot run (#14831)
6
7 A green agent pull goes to the crew's reviewer. If the reviewer has not been
8 asked within `REVIEW_LAPSE_MS` (60 min, `src/crew/review.ts`), the pull lapses
9 and waits for you: home shows it as "waiting for your review · the reviewer did
10 not answer". A lapse means the reviewer was asked and did not answer, so the
11 clock counts only the minutes when a review could have run.
12
13 ## What pauses the clock
14
15 The forge reads the run table (`src/crew/review_pause.ts`). The crew counts as
16 down at a given moment when an agent run died before its agent started (at the
17 clone, fetch or checkout) in the 15 minutes before it, and either:
18
19 - no agent run got as far as its agent in those 15 minutes, or
20 - the clone breaker stands: three such deaths came after the last run that got
21 through.
22
23 The lapse clock stands still for every down stretch, the same way a hold stops
24 it. A forge where nothing ran at all is never down, so a quiet forge's clock
25 runs as before. Stretches are read back at most 24 hours: a head that has been
26 green longer than that lapsed long ago. The read comes from the run table, so a
27 restart forgets nothing.
28
29 ## What is re-sent
30
31 - While the crew is down, no review is sent, whether to the crew's reviewer, a
32 named team or for a stranger's pull: it would die at its clone too.
33 - A review run that died before its agent started is not an attempt. It does
34 not use up one of the two tries.
35 - Once the crew is back, the review sweep sends every green pull the outage
36 stranded, oldest first, within the reviewer slots and at most `SWEEP_SENDS`
37 (10) a pass. The server log says `review: paused …` once when the outage
38 starts, and `review: re-sent N green pulls the crew's outage stranded` once
39 when they are all sent.
40
41 ## What you see
42
43 There is nothing new on a page. During an outage a green pull does not appear
44 under "waiting for your review" until it has been waiting for 60 minutes of
45 time when the crew could run. After the outage it gets its review like any
46 newly green pull. `test/review-outage.test.ts` replays a two-hour clone storm
47 and shows each step.
src/crew/review.ts +73 −24
@@ -34,6 +34,7 @@ import { reviewerPaused } from "./reviewer_paused.ts";
3434 import { crewFilters, isStrangerPull } from "./stranger.ts";
3535 import { queueAgentRun } from "./dispatch.ts";
3636 import { type Handover, handoverOf, handoversOf } from "./guard.ts";
37 import { crewDownNow, crewDownStretches, diedBeforeAgent, liveMs, type Stretch } from "./review_pause.ts";
3738
3839 const log = createLog("crew");
3940
@@ -194,9 +195,11 @@ export function reviewStage(
194195 greenAt: string,
195196 attempts: readonly ReviewAttempt[],
196197 now: Date,
198 /** Stretches the crew could not run (`review_pause.ts`): the lapse clock stands still in them. */
199 paused: readonly Stretch[] = [],
197200 ): "send" | "wait" | "lapsed" {
198201 const at = now.getTime();
199 if (at - Date.parse(greenAt) >= REVIEW_LAPSE_MS) return "lapsed";
202 if (liveMs(Date.parse(greenAt), at, paused) >= REVIEW_LAPSE_MS) return "lapsed";
200203 const last = attempts.at(-1);
201204 if (last === undefined) return "send";
202205 const fresh = at - Date.parse(last.createdAt) < REVIEW_WAIT_MS;
@@ -212,12 +215,17 @@ export function reviewStage(
212215 * from when the pull was last let go, whichever is later, and only runs
213216 * sent since then count: a pull held past the lapse and then released is
214217 * sent to review like any newly green one, never landed as "lapsed".
218 *
219 * An outage stops it too (#14831): the stretches the crew could not run are
220 * carried as `paused`, and a review run that died before its agent started
221 * (clone, fetch, checkout) is no attempt — the reviewer was never asked.
215222 */
216223 export function reviewAttempts(
217224 db: Database,
218225 pull: Pull,
226 now = new Date(),
219227 ): ReviewHistory | undefined {
220 return reviewAttemptsFor(db, [pull]).get(pull.id);
228 return reviewAttemptsFor(db, [pull], now).get(pull.id);
221229 }
222230
223231 /**
@@ -228,7 +236,9 @@ export function reviewAttempts(
228236 export function reviewAttemptsFor(
229237 db: Database,
230238 pulls: readonly Pull[],
239 now = new Date(),
231240 ): Map<string, ReviewHistory> {
241 const found: { pull: Pull; since: string; attempts: ReviewAttempt[]; checkMinutes: number }[] = [];
232242 const out = new Map<string, ReviewHistory>();
233243 const CHUNK = 400;
234244 for (let i = 0; i < pulls.length; i += CHUNK) {
@@ -248,13 +258,14 @@ export function reviewAttemptsFor(
248258 const released = lastReleasesFor(db, ids);
249259 const reviews = new Map<string, { createdAt: string; active: boolean; reviewerId: string | null }[]>();
250260 for (const r of db
251 .query<{ issue_id: string; created_at: string; state: string; actor_id: string | null }, string[]>(
252 `SELECT issue_id, created_at, state, actor_id FROM run INDEXED BY run_review_attempts
261 .query<{ issue_id: string; created_at: string; state: string; actor_id: string | null; summary: string | null }, string[]>(
262 `SELECT issue_id, created_at, state, actor_id, summary FROM run INDEXED BY run_review_attempts
253263 WHERE kind = 'agent' AND reason = 'review'
254264 AND issue_id IN (${ids.map((_, n) => `?${n + 1}`).join(",")})
255265 ORDER BY created_at, id`,
256266 )
257267 .all(...ids)) {
268 if (diedBeforeAgent(r.state, r.summary)) continue;
258269 const attempt = {
259270 createdAt: r.created_at,
260271 active: r.state === "queued" || r.state === "running",
@@ -282,14 +293,23 @@ export function reviewAttemptsFor(
282293 const release = released.get(pull.id);
283294 const since = release !== undefined && release > green ? release : green;
284295 const attempts = (reviews.get(pull.id) ?? []).filter((a) => a.createdAt >= since);
285 const first = attempts[0];
286 const lateAt =
287 first !== undefined && Date.parse(first.createdAt) - Date.parse(since) >= REVIEW_LAPSE_MS
288 ? first.createdAt
289 : undefined;
290 out.set(pull.id, { greenAt: since, attempts, lateAt, checkMinutes: Math.round(checkMs / 60_000) });
296 found.push({ pull, since, attempts, checkMinutes: Math.round(checkMs / 60_000) });
291297 }
292298 }
299 if (found.length === 0) return out;
300 // One read of the crew's outages for the set, from the oldest head's green on.
301 const at = now.getTime();
302 const all = crewDownStretches(db, Math.min(...found.map((f) => Date.parse(f.since))), at);
303 for (const { pull, since, attempts, checkMinutes } of found) {
304 const from = Date.parse(since);
305 const paused = all.filter(([, b]) => b > from);
306 const first = attempts[0];
307 const lateAt =
308 first !== undefined && liveMs(from, Date.parse(first.createdAt), paused) >= REVIEW_LAPSE_MS
309 ? first.createdAt
310 : undefined;
311 out.set(pull.id, { greenAt: since, attempts, lateAt, checkMinutes, paused });
312 }
293313 return out;
294314 }
295315
@@ -301,6 +321,8 @@ export interface ReviewHistory {
301321 readonly lateAt: string | undefined;
302322 /** How long the check took, from queued to green. */
303323 readonly checkMinutes: number;
324 /** Stretches since `greenAt` the crew could not run: the lapse clock stood still (#14831). */
325 readonly paused: readonly Stretch[];
304326 }
305327
306328 /**
@@ -312,10 +334,10 @@ export function stageOf(
312334 history: ReviewHistory,
313335 now: Date,
314336 ): "send" | "wait" | "lapsed" | "late" {
315 const age = now.getTime() - Date.parse(history.greenAt);
337 const age = liveMs(Date.parse(history.greenAt), now.getTime(), history.paused);
316338 if (history.attempts.length === 0 && age >= REVIEW_LAPSE_MS)
317339 return age < REVIEW_LAPSE_MS + LATE_SEND_MS ? "late" : "lapsed";
318 return reviewStage(history.lateAt ?? history.greenAt, history.attempts, now);
340 return reviewStage(history.lateAt ?? history.greenAt, history.attempts, now, history.paused);
319341 }
320342
321343 /** Why an agent's green pull waits for a person, and since when. */
@@ -412,7 +434,7 @@ export function reviewWaitsFor(
412434 // (`guard.ts`), so the card, the thread note and the held state cannot
413435 // answer differently.
414436 const over = handovers ?? handoversOf(db, open.map((p) => ({ id: p.pull.id, repo: p.repo })));
415 const histories = reviewAttemptsFor(db, open.map((p) => p.pull));
437 const histories = reviewAttemptsFor(db, open.map((p) => p.pull), now);
416438 const reviewers = seated;
417439 for (const { repo, pull } of open) {
418440 const handover = over.get(pull.id);
@@ -485,6 +507,8 @@ export async function sendToReview(
485507 if (!filters && guestWaits(app.db, pull)) return "held";
486508 // The reviewer's slots bound every route, the teams' too. The most (PILLAR_MAX) is no cap: the sweep already bounds what it sends.
487509 if (repo.reviewerSlots < PILLAR_MAX && reviewsInFlight(app.db, repo.id) >= repo.reviewerSlots) return "wait";
510 // A review sent while runs die at their clone would die too: it waits for the crew (#14831).
511 if (crewDownNow(app.db, now.getTime())) return "wait";
488512 if (filters) return sendStrangerToReview(app, repo, reviewers, pull);
489513 if (teamsOfPull(app.db, pull).length > 0) {
490514 const sent = await sendToTeams(app, repo, reviewers, pull, now);
@@ -493,7 +517,7 @@ export async function sendToReview(
493517 const seen = reviewSummary(app.db, pull.id, pull.headOid);
494518 if (seen.approvals > 0 || seen.changesRequested > 0) return "done";
495519 if (handedOver(app.db, repo.id, pull) !== undefined) return "held";
496 const history = reviewAttempts(app.db, pull);
520 const history = reviewAttempts(app.db, pull, now);
497521 if (history === undefined) return "wait";
498522 const stage = stageOf(history, now);
499523 // An autonomous repository sends a pull nobody reviewed whatever its age,
@@ -606,10 +630,10 @@ async function sendToTeams(
606630 const pending = named.filter((v) => v.state === undefined || v.stale);
607631 if (pending.length === 0) return "done";
608632 if (handedOver(app.db, repo.id, pull) !== undefined) return "held";
609 const history = reviewAttempts(app.db, pull);
633 const history = reviewAttempts(app.db, pull, now);
610634 const issue = getIssueById(app.db, pull.id);
611635 if (history === undefined || !issue) return "wait";
612 if (now.getTime() - Date.parse(history.greenAt) >= REVIEW_LAPSE_MS) return "lapsed";
636 if (liveMs(Date.parse(history.greenAt), now.getTime(), history.paused) >= REVIEW_LAPSE_MS) return "lapsed";
613637 let sent = false;
614638 // A reviewer on two pending teams is asked once: their verdict answers both.
615639 const asked = new Set<string>();
@@ -644,6 +668,13 @@ export const SWEEP_SENDS = 10;
644668 /** The last skip line said for a pull, so a stall is said once, not every tick. */
645669 const said = new Map<string, string>();
646670
671 /**
672 * The crew's outage, as the sweep last saw it (#14831): whether it said the crew was down,
673 * and how many pulls the outage stranded it has sent again since, said once when a pass
674 * sends none of them. Two numbers, gone on restart: the run table holds the outage itself.
675 */
676 const outage = { down: false, resent: 0 };
677
647678 /** Every open pull of a repository, oldest first, paged past the list cap. */
648679 function openPullsOldestFirst(db: Database, repoId: string): Pull[] {
649680 const all: Pull[] = [];
@@ -670,7 +701,15 @@ function openPullsOldestFirst(db: Database, repoId: string): Pull[] {
670701 * pull passed over for a reason gets one line saying why. The tick's net
671702 * under the check event; returns how many went.
672703 */
673 export async function sweepReviews(app: App): Promise<number> {
704 export async function sweepReviews(app: App, now = new Date()): Promise<number> {
705 // While runs die at their clone nothing is sent: the lapse clock stands still meanwhile.
706 if (crewDownNow(app.db, now.getTime())) {
707 if (!outage.down) log.info("review: paused, crew runs die before their agent starts");
708 outage.down = true;
709 return 0;
710 }
711 outage.down = false;
712 let resent = 0;
674713 const repoIds = app.db
675714 .query<{ repo_id: string }, []>(
676715 "SELECT DISTINCT repo_id FROM repo_crew WHERE role = 'reviewer'",
@@ -683,15 +722,17 @@ export async function sweepReviews(app: App): Promise<number> {
683722 const repo = getRepoById(app.db, id);
684723 if (!repo) continue;
685724 for (const pull of openPullsOldestFirst(app.db, repo.id)) {
686 if (queued >= SWEEP_SENDS) return forgetSettled(stalled, queued);
725 if (queued >= SWEEP_SENDS) return forgetSettled(stalled, queued, resent);
687726 // An agent's green pull, or a stranger's the crew filters (#9575).
688727 const owed =
689728 (pull.authorIsAgent && greenHead(app.db, pull)) ||
690729 (!pull.authorIsAgent && crewFilters(repo) && isStrangerPull(app.db, repo, pull));
691730 if (!owed) continue;
692 const result = await sendToReview(app, repo, pull);
693 if (result === "sent") queued++;
694 else if (result === "lapsed" || result === "wait") {
731 const result = await sendToReview(app, repo, pull, now);
732 if (result === "sent") {
733 queued++;
734 if ((reviewAttempts(app.db, pull, now)?.paused.length ?? 0) > 0) resent++;
735 } else if (result === "lapsed" || result === "wait") {
695736 stalled.add(pull.id);
696737 const key = `${pull.headOid}:${result}`;
697738 if (said.get(pull.id) === key) continue;
@@ -706,11 +747,19 @@ export async function sweepReviews(app: App): Promise<number> {
706747 }
707748 }
708749 }
709 return forgetSettled(stalled, queued);
750 return forgetSettled(stalled, queued, resent);
710751 }
711752
712 /** Drop the said-lines of pulls that no longer stall, so the map stays the size of the stall. */
713 function forgetSettled(stalled: Set<string>, queued: number): number {
753 /**
754 * Drop the said-lines of pulls that no longer stall, so the map stays the size of the stall,
755 * and say how many pulls an outage stranded were sent again, once they all are.
756 */
757 function forgetSettled(stalled: Set<string>, queued: number, resent: number): number {
714758 for (const id of said.keys()) if (!stalled.has(id)) said.delete(id);
759 outage.resent += resent;
760 if (resent === 0 && outage.resent > 0) {
761 log.info(`review: re-sent ${outage.resent} green pulls the crew's outage stranded`);
762 outage.resent = 0;
763 }
715764 return queued;
716765 }
src/crew/review_pause.ts added +126
@@ -0,0 +1,126 @@
1 /**
2 * When the crew could not run (#14831). On 2026-11-20 every crew run died at its clone for nine
3 * hours; green pulls sat past `REVIEW_LAPSE_MS` and lapsed to a person though no reviewer was ever
4 * asked. A lapse means "asked and did not answer", so the lapse clock counts only time a review
5 * could have run.
6 *
7 * The crew is down at instant t when a run died before its agent started (clone, fetch,
8 * checkout: `interruptionOf` = "clone") within `DOWN_WINDOW_MS` before t, and either no run
9 * reached its agent in that window, or the clone breaker stands: `CLONE_TRIP` such deaths came
10 * after the newest run that did (`clone_breaker.ts`). A quiet forge, where nothing ended at all,
11 * is never down: there the clock runs as it always did. Read from the run table alone, so a
12 * restart forgets nothing; pure over the rows and the instants.
13 */
14
15 import type { Database } from "bun:sqlite";
16 import { CLONE_TRIP } from "./clone_breaker.ts";
17 import { interruptionOf } from "./interrupted.ts";
18
19 /** How long one run's death at its clone, or one run that reached its agent, speaks for. */
20 export const DOWN_WINDOW_MS = 15 * 60_000;
21 /** How far back the stretches are read: a head green longer than this lapsed long ago. */
22 export const PAUSE_LOOKBACK_MS = 24 * 60 * 60_000;
23
24 /** One ended agent run: when, and whether it reached its agent. */
25 export interface Ended {
26 readonly at: number;
27 readonly reached: boolean;
28 }
29
30 /** A stretch the crew was down, as epoch ms, `[from, to)`. */
31 export type Stretch = readonly [number, number];
32
33 /** The last index in sorted `xs` with a value at or under `t`, or -1. */
34 function lastAtOrBefore(xs: readonly number[], t: number): number {
35 let lo = 0;
36 let hi = xs.length;
37 while (lo < hi) {
38 const mid = (lo + hi) >> 1;
39 if (xs[mid]! <= t) lo = mid + 1;
40 else hi = mid;
41 }
42 return lo - 1;
43 }
44
45 interface Index {
46 readonly reached: number[];
47 readonly prestart: number[];
48 }
49
50 function indexOf(ended: readonly Ended[]): Index {
51 const reached: number[] = [];
52 const prestart: number[] = [];
53 for (const e of [...ended].sort((a, b) => a.at - b.at)) (e.reached ? reached : prestart).push(e.at);
54 return { reached, prestart };
55 }
56
57 function downIn(ix: Index, t: number): boolean {
58 const p = lastAtOrBefore(ix.prestart, t);
59 // No run died at its clone in the window: nothing says the crew is down.
60 if (p < 0 || ix.prestart[p]! <= t - DOWN_WINDOW_MS) return false;
61 const r = lastAtOrBefore(ix.reached, t);
62 if (r < 0 || ix.reached[r]! <= t - DOWN_WINDOW_MS) return true;
63 // A run reached its agent in the window; the breaker reopens after CLONE_TRIP deaths since.
64 const after = p - lastAtOrBefore(ix.prestart, ix.reached[r]!);
65 return after >= CLONE_TRIP;
66 }
67
68 /** Whether the crew is down at `t` (epoch ms). */
69 export const downAt = (ended: readonly Ended[], t: number): boolean => downIn(indexOf(ended), t);
70
71 /**
72 * The stretches within `[from, to)` the crew was down, merged and in order. Down is constant
73 * between consecutive edges (each end, and each end plus the window), so it is read once per edge.
74 */
75 export function downStretches(ended: readonly Ended[], from: number, to: number): Stretch[] {
76 if (to <= from) return [];
77 const ix = indexOf(ended);
78 const edges = new Set<number>([from]);
79 for (const e of ended)
80 for (const x of [e.at, e.at + DOWN_WINDOW_MS]) if (x > from && x < to) edges.add(x);
81 const sorted = [...edges].sort((a, b) => a - b);
82 const out: [number, number][] = [];
83 sorted.forEach((start, i) => {
84 if (!downIn(ix, start)) return;
85 const end = sorted[i + 1] ?? to;
86 const last = out.at(-1);
87 if (last !== undefined && last[1] === start) last[1] = end;
88 else out.push([start, end]);
89 });
90 return out;
91 }
92
93 /** Milliseconds of `[from, to)` outside every stretch: the time a review could have run. */
94 export function liveMs(from: number, to: number, paused: readonly Stretch[]): number {
95 let ms = Math.max(0, to - from);
96 for (const [a, b] of paused) ms -= Math.max(0, Math.min(b, to) - Math.max(a, from));
97 return ms;
98 }
99
100 /** The agent runs that ended from `from - DOWN_WINDOW_MS` on (`run_kind_finished`). */
101 export function endedSince(db: Database, from: number): Ended[] {
102 return db
103 .query<{ finished_at: string; state: string; summary: string | null }, [string]>(
104 `SELECT finished_at, state, summary FROM run INDEXED BY run_kind_finished
105 WHERE kind = 'agent' AND finished_at IS NOT NULL AND finished_at >= ?1`,
106 )
107 .all(new Date(from - DOWN_WINDOW_MS).toISOString())
108 .map((r) => ({
109 at: Date.parse(r.finished_at),
110 reached: !diedBeforeAgent(r.state, r.summary),
111 }));
112 }
113
114 /** Whether a review run that ended so died before its agent started: no attempt. */
115 export const diedBeforeAgent = (state: string, summary: string | null): boolean =>
116 state === "failed" && interruptionOf(summary ?? "") === "clone";
117
118 /** The stretches the crew was down from `from` (no further back than the lookback) to `now`. */
119 export function crewDownStretches(db: Database, from: number, now: number): Stretch[] {
120 const start = Math.max(from, now - PAUSE_LOOKBACK_MS);
121 return downStretches(endedSince(db, start), start, now);
122 }
123
124 /** Whether the crew is down now: a review sent now would die at its clone, so none is sent. */
125 export const crewDownNow = (db: Database, now: number): boolean =>
126 downAt(endedSince(db, now), now);
test/review-outage.test.ts added +303
@@ -0,0 +1,303 @@
1 /**
2 * A review does not lapse while the crew cannot run (#14831). On 2026-11-20 every crew run died
3 * at its clone for nine hours; green pulls lapsed to a person though no reviewer was ever asked.
4 * The lapse clock stands still while runs die before their agent starts, a review that died at
5 * its clone is no attempt, and once the crew is back every stranded green pull is sent again.
6 */
7 import { seatAs } from "./helpers/seat.ts";
8 import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { addToCrew, createAgent } from "../src/actions/crew.ts";
14 import { nameTeams } from "../src/actions/issue_teams.ts";
15 import { openIssue } from "../src/actions/issues.ts";
16 import { openPull } from "../src/actions/pulls.ts";
17 import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
18 import { getPull, type Pull } from "../src/core/pulls.ts";
19 import { getRepoById, type Repo } from "../src/core/repos.ts";
20 import { saveTeam } from "../src/core/teams.ts";
21 import { markVerified } from "../src/core/verify.ts";
22 import { logLevel, setLogLevel } from "../src/kernel/log.ts";
23 import { setLimit } from "../src/settings/limits.ts";
24 import { createRun, listRuns } from "../src/core/runs.ts";
25 import { createUser, getUserByName, type User } from "../src/core/users.ts";
26 import {
27 REVIEW_LAPSE_MS,
28 reviewAttempts,
29 sendToReview,
30 sweepReviews,
31 waitsForPerson,
32 } from "../src/crew/review.ts";
33 import {
34 DOWN_WINDOW_MS,
35 downAt,
36 downStretches,
37 endedSince,
38 liveMs,
39 type Ended,
40 } from "../src/crew/review_pause.ts";
41
42 const MIN = 60_000;
43
44 describe("when the crew is down", () => {
45 const died = (m: number): Ended => ({ at: m * MIN, reached: false });
46 const ran = (m: number): Ended => ({ at: m * MIN, reached: true });
47
48 test("a quiet forge, where nothing ended, is never down: its clock runs as before", () => {
49 expect(downStretches([], 0, 600 * MIN)).toEqual([]);
50 expect(liveMs(0, 90 * MIN, [])).toBe(90 * MIN);
51 });
52
53 test("runs dying at their clone every 5 min for 2 h are one stretch, ending a window after the last", () => {
54 const storm = Array.from({ length: 25 }, (_, i) => died(10 + 5 * i)); // 10 … 130 min
55 expect(downStretches(storm, 0, 300 * MIN)).toEqual([[10 * MIN, 130 * MIN + DOWN_WINDOW_MS]]);
56 // 300 min less the 135 min down: 165 min live.
57 expect(liveMs(0, 300 * MIN, downStretches(storm, 0, 300 * MIN))).toBe(165 * MIN);
58 });
59
60 test("a run that reached its agent in the window is the crew up, until the breaker trips again", () => {
61 expect(downAt([died(0), ran(2)], 5 * MIN)).toBe(false);
62 expect(downAt([ran(0), died(1), died(2)], 3 * MIN)).toBe(false);
63 // CLONE_TRIP (3) deaths after the last run that got through: the breaker stands.
64 expect(downAt([ran(0), died(1), died(2), died(3)], 4 * MIN)).toBe(true);
65 // A death older than the window says nothing.
66 expect(downAt([died(0)], 16 * MIN)).toBe(false);
67 });
68
69 test("the breaker's stretch: down from the CLONE_TRIP-th death after a run got through, a window past the last", () => {
70 // Without the breaker branch the run at 0 holds the crew up until 15 min, and the stretch
71 // starts there.
72 expect(downStretches([ran(0), died(1), died(2), died(3)], 0, 60 * MIN)).toEqual([[3 * MIN, 3 * MIN + DOWN_WINDOW_MS]]);
73 });
74 });
75
76 describe("a clone storm strands no green pull", () => {
77 let dataDir: string;
78 let app: App;
79 let alice: User;
80 let repo: Repo;
81 let open: Repo;
82 let teamPull: Pull;
83 let strangerPull: Pull;
84 // The storm's shape: green at T0, every agent run dies at its clone from T0+5 to T0+125 min.
85 const T0 = Date.now() - 4 * 60 * MIN;
86 const iso = (m: number) => new Date(T0 + m * MIN).toISOString();
87 const DURING = new Date(T0 + 60 * MIN);
88 // 160 min after the green, 25 min of it live (5 before the storm, 20 since the window closed).
89 const AFTER = new Date(T0 + 160 * MIN);
90 const STORM_DEATHS = 25;
91
92 beforeAll(async () => {
93 dataDir = await mkdtemp(join(tmpdir(), "forge-review-outage-"));
94 app = createApp({ dataDir, port: 0 });
95 const a = await createUser(app.db, {
96 origin: "seed",
97 trusted: true,
98 username: "alice",
99 email: "[email protected]",
100 password: "a-good-passphrase",
101 adminIfFirst: true,
102 });
103 if (!a.ok) throw new Error(a.error);
104 alice = a.value;
105 markVerified(app.db, alice.id, new Date().toISOString());
106 for (const [key, value] of [
107 ["signup.per_source", 1000],
108 ["guest.forks_per_day", 1000],
109 ["guest.forks_forge_per_day", 1000],
110 ["guest.pulls_per_day", 1000],
111 ] as const)
112 setLimit(app.db, key, value, "setup");
113 const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
114 if (!made.ok) throw new Error(String(made.error));
115 repo = made.value;
116 await commitToBranch(app, alice, repo, {
117 message: "Start",
118 files: [{ path: "README.md", content: "# engine\n" }],
119 });
120 // A public autonomous repository: the crew's reviewer reads a stranger's pull first (#9575).
121 const pub = await createRepo(app, alice, { name: "open", visibility: "public" });
122 if (!pub.ok) throw new Error(String(pub.error));
123 await commitToBranch(app, alice, pub.value, {
124 message: "Start",
125 files: [{ path: "README.md", content: "# open\n" }],
126 });
127 app.db.query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2").run(alice.id, pub.value.id);
128 open = getRepoById(app.db, pub.value.id)!;
129 for (const name of ["scout", "juniper"]) {
130 const agent = await createAgent(app, alice, { username: name, brief: `I am ${name}.` });
131 if (!agent.ok) throw new Error(String(agent.error));
132 expect(addToCrew(app, alice, repo, name).ok).toBe(true);
133 }
134 expect(addToCrew(app, alice, open, "juniper").ok).toBe(true);
135 const juniper = getUserByName(app.db, "juniper")!.id;
136 seatAs(app.db, repo.id, juniper, "reviewer");
137 seatAs(app.db, open.id, juniper, "reviewer");
138 saveTeam(app.db, "design", { charter: "", dial: 0, members: [juniper] });
139
140 // The storm: every agent run dies at its clone, one every 5 minutes for two hours.
141 for (let m = 5; m <= 125; m += 5) {
142 const run = createRun(app.db, { repoId: repo.id, kind: "agent", reason: "pick" });
143 ended(run.id, "failed", "clone failed: fatal: early EOF", m - 1, m);
144 }
145 }, 60_000);
146
147 afterAll(async () => {
148 await app.close?.();
149 await rm(dataDir, { recursive: true, force: true });
150 });
151
152 /** An agent's pull on `engine`, its check green at T0; `body` may close an issue. */
153 async function greenPull(name: string, body = ""): Promise<Pull> {
154 await commitToBranch(app, alice, repo, {
155 branch: `feature/${name}`,
156 base: "main",
157 message: name,
158 files: [{ path: `docs/${name}.md`, content: `${name}\n` }],
159 });
160 const opened = await openPull(app, alice, repo, {
161 title: name,
162 body,
163 baseBranch: "main",
164 headBranch: `feature/${name}`,
165 });
166 if (!opened.ok) throw new Error(String(opened.error));
167 const pull = opened.value;
168 app.db.query("UPDATE issue SET author_id = ?2 WHERE id = ?1").run(pull.id, getUserByName(app.db, "scout")!.id);
169 const check = createRun(app.db, {
170 repoId: repo.id,
171 kind: "check",
172 issueId: pull.id,
173 env: `pull-${pull.number}`,
174 commitOid: pull.headOid,
175 reason: "pull",
176 });
177 ended(check.id, "ok", "check passed", -10, 0);
178 return { ...pull, authorIsAgent: true };
179 }
180
181 function ended(id: string, state: string, summary: string, createdMin: number, finishedMin: number): void {
182 app.db
183 .query(
184 "UPDATE run SET state = ?2, summary = ?3, created_at = ?4, started_at = ?4, finished_at = ?5 WHERE id = ?1",
185 )
186 .run(id, state, summary, iso(createdMin), iso(finishedMin));
187 }
188
189 const reviews = (pull: Pull) =>
190 listRuns(app.db, pull.repoId, { issueId: pull.id }).filter((r) => r.reason === "review");
191
192 test("the storm's rows are what the outage read finds", () => {
193 expect(endedSince(app.db, T0).length).toBe(STORM_DEATHS);
194 expect(endedSince(app.db, T0).every((e) => !e.reached)).toBe(true);
195 });
196
197 test("while the crew is down a team pull and a stranger's pull are not sent; once it is back, both are", async () => {
198 // A pull closing an issue that names the design team, green at T0.
199 const issue = openIssue(app, alice, repo, { title: "A chart" });
200 if (!issue.ok) throw new Error(issue.error);
201 const named = nameTeams(app, alice, repo, issue.value, ["design"]);
202 if (!named.ok) throw new Error(named.error);
203 teamPull = await greenPull("team", `Closes #${issue.value.number}`);
204 // A stranger's pull from a fork of the autonomous repository.
205 const made = await createUser(app.db, {
206 origin: "seed",
207 trusted: true,
208 username: "stranger",
209 email: "[email protected]",
210 password: "correct-horse-battery",
211 });
212 if (!made.ok) throw new Error(made.error);
213 markVerified(app.db, made.value.id, new Date().toISOString());
214 app.db.query("UPDATE user SET created_at = ?1 WHERE id = ?2").run("2026-01-01T00:00:00.000Z", made.value.id);
215 const stranger = getUserByName(app.db, "stranger")!;
216 const fork = await forkRepo(app, stranger, open);
217 if (!fork.ok) throw new Error(String(fork.error));
218 const change = await commitToBranch(app, stranger, fork.value, {
219 branch: "change",
220 base: "main",
221 message: "Change",
222 files: [{ path: "notes.md", content: "hi\n" }],
223 });
224 if (!change.ok) throw new Error(JSON.stringify(change.error));
225 const opened = await openPull(app, stranger, open, {
226 title: "A change",
227 body: "",
228 baseBranch: "main",
229 headBranch: "change",
230 headRepo: fork.value,
231 });
232 if (!opened.ok) throw new Error(String(opened.error));
233 strangerPull = getPull(app.db, open.id, opened.value.number)!;
234
235 // Without the outage check each would be sent here, and its review would die at its clone.
236 expect(await sendToReview(app, repo, teamPull, DURING)).toBe("wait");
237 expect(await sendToReview(app, open, strangerPull, DURING)).toBe("wait");
238 expect(reviews(teamPull)).toHaveLength(0);
239 expect(reviews(strangerPull)).toHaveLength(0);
240
241 // 160 min after the green with 25 of them live: the team route does not lapse.
242 expect(await sendToReview(app, repo, teamPull, AFTER)).toBe("sent");
243 expect(reviews(teamPull).map((r) => r.actorId)).toEqual([getUserByName(app.db, "juniper")!.id]);
244 expect(await sendToReview(app, open, strangerPull, AFTER)).toBe("sent");
245 expect(reviews(strangerPull)).toHaveLength(1);
246 }, 60_000);
247
248 test("two hours of runs dying at the clone: afterwards every green pull is sent, none waits for a person", async () => {
249 const pulls = [await greenPull("one"), await greenPull("two"), await greenPull("three")];
250 // The first pull was sent once during the storm, and its review died at its clone.
251 const juniper = getUserByName(app.db, "juniper")!.id;
252 const died = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: pulls[0]!.id, reason: "review", actorId: juniper });
253 ended(died.id, "failed", "clone failed: fatal: early EOF", 10, 11);
254
255 const found = logLevel();
256 setLogLevel("info");
257 const info = spyOn(console, "info").mockImplementation(() => {});
258 try {
259 // During the storm nothing is sent, and the clock stands still.
260 expect(await sweepReviews(app, DURING)).toBe(0);
261 expect(info.mock.calls.filter(([l]) => String(l).includes("review: paused"))).toHaveLength(1);
262
263 for (const pull of pulls) {
264 const history = reviewAttempts(app.db, pull, AFTER)!;
265 expect(history.attempts).toHaveLength(0); // the clone death used up no try
266 expect(liveMs(T0, AFTER.getTime(), history.paused)).toBe(25 * MIN);
267 expect(liveMs(T0, AFTER.getTime(), history.paused)).toBeLessThan(REVIEW_LAPSE_MS);
268 expect(waitsForPerson(app.db, repo.id, pull, AFTER)).toBeUndefined();
269 }
270
271 expect(await sweepReviews(app, AFTER)).toBe(3);
272 for (const pull of pulls) {
273 expect(reviews(pull).filter((r) => r.state === "queued")).toHaveLength(1);
274 expect(reviewAttempts(app.db, pull, AFTER)!.attempts).toHaveLength(1);
275 expect(waitsForPerson(app.db, repo.id, pull, AFTER)).toBeUndefined();
276 }
277 const resentLines = () => info.mock.calls.map(([l]) => String(l)).filter((l) => l.includes("review: re-sent"));
278 // Said once the pass after the last re-send, with the count of this pass's three.
279 expect(resentLines()).toHaveLength(0);
280 expect(await sweepReviews(app, AFTER)).toBe(0);
281 expect(resentLines()).toHaveLength(1);
282 expect(resentLines()[0]).toContain("re-sent 3 green pulls the crew's outage stranded");
283 expect(await sweepReviews(app, AFTER)).toBe(0);
284 expect(resentLines()).toHaveLength(1);
285 } finally {
286 info.mockRestore();
287 setLogLevel(found);
288 }
289 }, 60_000);
290
291 test("the outage read walks the finished-runs index", () => {
292 const plan = app.db
293 .query<{ detail: string }, [string]>(
294 `EXPLAIN QUERY PLAN SELECT finished_at, state, summary FROM run INDEXED BY run_kind_finished
295 WHERE kind = 'agent' AND finished_at IS NOT NULL AND finished_at >= ?1`,
296 )
297 .all("x")
298 .map((r) => r.detail)
299 .join("\n");
300 expect(plan).toContain("run_kind_finished");
301 expect(plan).not.toContain("TEMP B-TREE");
302 });
303 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs