Public: anyone can read this; the crew can change it.
A review does not lapse while the crew cannot run: an outage pauses the clock, and the crew re-sends what it stranded #14834
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14831-a-review-does-not-lapse-while-the-crew-c → main 4 commits fork
4 files
changed +549
−24
docs/features/review-outage.md added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | --- | |
| 2 | topic: Tell the crew | |
| 3 | row: A review does not lapse while the crew cannot run | |
| 4 | --- | |
| 5 | # A review does not lapse while the crew cannot run (#14831) | |
| 6 | ||
| 7 | A green agent pull goes to the crew's reviewer. If the reviewer has not been | |
| 8 | asked within `REVIEW_LAPSE_MS` (60 min, `src/crew/review.ts`), the pull lapses | |
| 9 | and waits for you: home shows it as "waiting for your review · the reviewer did | |
| 10 | not answer". A lapse means the reviewer was asked and did not answer, so the | |
| 11 | clock counts only the minutes when a review could have run. | |
| 12 | ||
| 13 | ## What pauses the clock | |
| 14 | ||
| 15 | The forge reads the run table (`src/crew/review_pause.ts`). The crew counts as | |
| 16 | down at a given moment when an agent run died before its agent started (at the | |
| 17 | clone, fetch or checkout) in the 15 minutes before it, and either: | |
| 18 | ||
| 19 | - no agent run got as far as its agent in those 15 minutes, or | |
| 20 | - the clone breaker stands: three such deaths came after the last run that got | |
| 21 | through. | |
| 22 | ||
| 23 | The lapse clock stands still for every down stretch, the same way a hold stops | |
| 24 | it. A forge where nothing ran at all is never down, so a quiet forge's clock | |
| 25 | runs as before. Stretches are read back at most 24 hours: a head that has been | |
| 26 | green longer than that lapsed long ago. The read comes from the run table, so a | |
| 27 | restart forgets nothing. | |
| 28 | ||
| 29 | ## What is re-sent | |
| 30 | ||
| 31 | - While the crew is down, no review is sent, whether to the crew's reviewer, a | |
| 32 | named team or for a stranger's pull: it would die at its clone too. | |
| 33 | - A review run that died before its agent started is not an attempt. It does | |
| 34 | not use up one of the two tries. | |
| 35 | - Once the crew is back, the review sweep sends every green pull the outage | |
| 36 | stranded, oldest first, within the reviewer slots and at most `SWEEP_SENDS` | |
| 37 | (10) a pass. The server log says `review: paused …` once when the outage | |
| 38 | starts, and `review: re-sent N green pulls the crew's outage stranded` once | |
| 39 | when they are all sent. | |
| 40 | ||
| 41 | ## What you see | |
| 42 | ||
| 43 | There is nothing new on a page. During an outage a green pull does not appear | |
| 44 | under "waiting for your review" until it has been waiting for 60 minutes of | |
| 45 | time when the crew could run. After the outage it gets its review like any | |
| 46 | newly green pull. `test/review-outage.test.ts` replays a two-hour clone storm | |
| 47 | and shows each step. |
src/crew/review.ts +73 −24
| @@ -34,6 +34,7 @@ import { reviewerPaused } from "./reviewer_paused.ts"; | ||
| 34 | 34 | import { crewFilters, isStrangerPull } from "./stranger.ts"; |
| 35 | 35 | import { queueAgentRun } from "./dispatch.ts"; |
| 36 | 36 | import { type Handover, handoverOf, handoversOf } from "./guard.ts"; |
| 37 | import { crewDownNow, crewDownStretches, diedBeforeAgent, liveMs, type Stretch } from "./review_pause.ts"; | |
| 37 | 38 | |
| 38 | 39 | const log = createLog("crew"); |
| 39 | 40 | |
| @@ -194,9 +195,11 @@ export function reviewStage( | ||
| 194 | 195 | greenAt: string, |
| 195 | 196 | attempts: readonly ReviewAttempt[], |
| 196 | 197 | now: Date, |
| 198 | /** Stretches the crew could not run (`review_pause.ts`): the lapse clock stands still in them. */ | |
| 199 | paused: readonly Stretch[] = [], | |
| 197 | 200 | ): "send" | "wait" | "lapsed" { |
| 198 | 201 | const at = now.getTime(); |
| 199 | if (at - Date.parse(greenAt) >= REVIEW_LAPSE_MS) return "lapsed"; | |
| 202 | if (liveMs(Date.parse(greenAt), at, paused) >= REVIEW_LAPSE_MS) return "lapsed"; | |
| 200 | 203 | const last = attempts.at(-1); |
| 201 | 204 | if (last === undefined) return "send"; |
| 202 | 205 | const fresh = at - Date.parse(last.createdAt) < REVIEW_WAIT_MS; |
| @@ -212,12 +215,17 @@ export function reviewStage( | ||
| 212 | 215 | * from when the pull was last let go, whichever is later, and only runs |
| 213 | 216 | * sent since then count: a pull held past the lapse and then released is |
| 214 | 217 | * sent to review like any newly green one, never landed as "lapsed". |
| 218 | * | |
| 219 | * An outage stops it too (#14831): the stretches the crew could not run are | |
| 220 | * carried as `paused`, and a review run that died before its agent started | |
| 221 | * (clone, fetch, checkout) is no attempt — the reviewer was never asked. | |
| 215 | 222 | */ |
| 216 | 223 | export function reviewAttempts( |
| 217 | 224 | db: Database, |
| 218 | 225 | pull: Pull, |
| 226 | now = new Date(), | |
| 219 | 227 | ): ReviewHistory | undefined { |
| 220 | return reviewAttemptsFor(db, [pull]).get(pull.id); | |
| 228 | return reviewAttemptsFor(db, [pull], now).get(pull.id); | |
| 221 | 229 | } |
| 222 | 230 | |
| 223 | 231 | /** |
| @@ -228,7 +236,9 @@ export function reviewAttempts( | ||
| 228 | 236 | export function reviewAttemptsFor( |
| 229 | 237 | db: Database, |
| 230 | 238 | pulls: readonly Pull[], |
| 239 | now = new Date(), | |
| 231 | 240 | ): Map<string, ReviewHistory> { |
| 241 | const found: { pull: Pull; since: string; attempts: ReviewAttempt[]; checkMinutes: number }[] = []; | |
| 232 | 242 | const out = new Map<string, ReviewHistory>(); |
| 233 | 243 | const CHUNK = 400; |
| 234 | 244 | for (let i = 0; i < pulls.length; i += CHUNK) { |
| @@ -248,13 +258,14 @@ export function reviewAttemptsFor( | ||
| 248 | 258 | const released = lastReleasesFor(db, ids); |
| 249 | 259 | const reviews = new Map<string, { createdAt: string; active: boolean; reviewerId: string | null }[]>(); |
| 250 | 260 | for (const r of db |
| 251 | .query<{ issue_id: string; created_at: string; state: string; actor_id: string | null }, string[]>( | |
| 252 | `SELECT issue_id, created_at, state, actor_id FROM run INDEXED BY run_review_attempts | |
| 261 | .query<{ issue_id: string; created_at: string; state: string; actor_id: string | null; summary: string | null }, string[]>( | |
| 262 | `SELECT issue_id, created_at, state, actor_id, summary FROM run INDEXED BY run_review_attempts | |
| 253 | 263 | WHERE kind = 'agent' AND reason = 'review' |
| 254 | 264 | AND issue_id IN (${ids.map((_, n) => `?${n + 1}`).join(",")}) |
| 255 | 265 | ORDER BY created_at, id`, |
| 256 | 266 | ) |
| 257 | 267 | .all(...ids)) { |
| 268 | if (diedBeforeAgent(r.state, r.summary)) continue; | |
| 258 | 269 | const attempt = { |
| 259 | 270 | createdAt: r.created_at, |
| 260 | 271 | active: r.state === "queued" || r.state === "running", |
| @@ -282,14 +293,23 @@ export function reviewAttemptsFor( | ||
| 282 | 293 | const release = released.get(pull.id); |
| 283 | 294 | const since = release !== undefined && release > green ? release : green; |
| 284 | 295 | const attempts = (reviews.get(pull.id) ?? []).filter((a) => a.createdAt >= since); |
| 285 | const first = attempts[0]; | |
| 286 | const lateAt = | |
| 287 | first !== undefined && Date.parse(first.createdAt) - Date.parse(since) >= REVIEW_LAPSE_MS | |
| 288 | ? first.createdAt | |
| 289 | : undefined; | |
| 290 | out.set(pull.id, { greenAt: since, attempts, lateAt, checkMinutes: Math.round(checkMs / 60_000) }); | |
| 296 | found.push({ pull, since, attempts, checkMinutes: Math.round(checkMs / 60_000) }); | |
| 291 | 297 | } |
| 292 | 298 | } |
| 299 | if (found.length === 0) return out; | |
| 300 | // One read of the crew's outages for the set, from the oldest head's green on. | |
| 301 | const at = now.getTime(); | |
| 302 | const all = crewDownStretches(db, Math.min(...found.map((f) => Date.parse(f.since))), at); | |
| 303 | for (const { pull, since, attempts, checkMinutes } of found) { | |
| 304 | const from = Date.parse(since); | |
| 305 | const paused = all.filter(([, b]) => b > from); | |
| 306 | const first = attempts[0]; | |
| 307 | const lateAt = | |
| 308 | first !== undefined && liveMs(from, Date.parse(first.createdAt), paused) >= REVIEW_LAPSE_MS | |
| 309 | ? first.createdAt | |
| 310 | : undefined; | |
| 311 | out.set(pull.id, { greenAt: since, attempts, lateAt, checkMinutes, paused }); | |
| 312 | } | |
| 293 | 313 | return out; |
| 294 | 314 | } |
| 295 | 315 | |
| @@ -301,6 +321,8 @@ export interface ReviewHistory { | ||
| 301 | 321 | readonly lateAt: string | undefined; |
| 302 | 322 | /** How long the check took, from queued to green. */ |
| 303 | 323 | readonly checkMinutes: number; |
| 324 | /** Stretches since `greenAt` the crew could not run: the lapse clock stood still (#14831). */ | |
| 325 | readonly paused: readonly Stretch[]; | |
| 304 | 326 | } |
| 305 | 327 | |
| 306 | 328 | /** |
| @@ -312,10 +334,10 @@ export function stageOf( | ||
| 312 | 334 | history: ReviewHistory, |
| 313 | 335 | now: Date, |
| 314 | 336 | ): "send" | "wait" | "lapsed" | "late" { |
| 315 | const age = now.getTime() - Date.parse(history.greenAt); | |
| 337 | const age = liveMs(Date.parse(history.greenAt), now.getTime(), history.paused); | |
| 316 | 338 | if (history.attempts.length === 0 && age >= REVIEW_LAPSE_MS) |
| 317 | 339 | return age < REVIEW_LAPSE_MS + LATE_SEND_MS ? "late" : "lapsed"; |
| 318 | return reviewStage(history.lateAt ?? history.greenAt, history.attempts, now); | |
| 340 | return reviewStage(history.lateAt ?? history.greenAt, history.attempts, now, history.paused); | |
| 319 | 341 | } |
| 320 | 342 | |
| 321 | 343 | /** Why an agent's green pull waits for a person, and since when. */ |
| @@ -412,7 +434,7 @@ export function reviewWaitsFor( | ||
| 412 | 434 | // (`guard.ts`), so the card, the thread note and the held state cannot |
| 413 | 435 | // answer differently. |
| 414 | 436 | const over = handovers ?? handoversOf(db, open.map((p) => ({ id: p.pull.id, repo: p.repo }))); |
| 415 | const histories = reviewAttemptsFor(db, open.map((p) => p.pull)); | |
| 437 | const histories = reviewAttemptsFor(db, open.map((p) => p.pull), now); | |
| 416 | 438 | const reviewers = seated; |
| 417 | 439 | for (const { repo, pull } of open) { |
| 418 | 440 | const handover = over.get(pull.id); |
| @@ -485,6 +507,8 @@ export async function sendToReview( | ||
| 485 | 507 | if (!filters && guestWaits(app.db, pull)) return "held"; |
| 486 | 508 | // The reviewer's slots bound every route, the teams' too. The most (PILLAR_MAX) is no cap: the sweep already bounds what it sends. |
| 487 | 509 | if (repo.reviewerSlots < PILLAR_MAX && reviewsInFlight(app.db, repo.id) >= repo.reviewerSlots) return "wait"; |
| 510 | // A review sent while runs die at their clone would die too: it waits for the crew (#14831). | |
| 511 | if (crewDownNow(app.db, now.getTime())) return "wait"; | |
| 488 | 512 | if (filters) return sendStrangerToReview(app, repo, reviewers, pull); |
| 489 | 513 | if (teamsOfPull(app.db, pull).length > 0) { |
| 490 | 514 | const sent = await sendToTeams(app, repo, reviewers, pull, now); |
| @@ -493,7 +517,7 @@ export async function sendToReview( | ||
| 493 | 517 | const seen = reviewSummary(app.db, pull.id, pull.headOid); |
| 494 | 518 | if (seen.approvals > 0 || seen.changesRequested > 0) return "done"; |
| 495 | 519 | if (handedOver(app.db, repo.id, pull) !== undefined) return "held"; |
| 496 | const history = reviewAttempts(app.db, pull); | |
| 520 | const history = reviewAttempts(app.db, pull, now); | |
| 497 | 521 | if (history === undefined) return "wait"; |
| 498 | 522 | const stage = stageOf(history, now); |
| 499 | 523 | // An autonomous repository sends a pull nobody reviewed whatever its age, |
| @@ -606,10 +630,10 @@ async function sendToTeams( | ||
| 606 | 630 | const pending = named.filter((v) => v.state === undefined || v.stale); |
| 607 | 631 | if (pending.length === 0) return "done"; |
| 608 | 632 | if (handedOver(app.db, repo.id, pull) !== undefined) return "held"; |
| 609 | const history = reviewAttempts(app.db, pull); | |
| 633 | const history = reviewAttempts(app.db, pull, now); | |
| 610 | 634 | const issue = getIssueById(app.db, pull.id); |
| 611 | 635 | if (history === undefined || !issue) return "wait"; |
| 612 | if (now.getTime() - Date.parse(history.greenAt) >= REVIEW_LAPSE_MS) return "lapsed"; | |
| 636 | if (liveMs(Date.parse(history.greenAt), now.getTime(), history.paused) >= REVIEW_LAPSE_MS) return "lapsed"; | |
| 613 | 637 | let sent = false; |
| 614 | 638 | // A reviewer on two pending teams is asked once: their verdict answers both. |
| 615 | 639 | const asked = new Set<string>(); |
| @@ -644,6 +668,13 @@ export const SWEEP_SENDS = 10; | ||
| 644 | 668 | /** The last skip line said for a pull, so a stall is said once, not every tick. */ |
| 645 | 669 | const said = new Map<string, string>(); |
| 646 | 670 | |
| 671 | /** | |
| 672 | * The crew's outage, as the sweep last saw it (#14831): whether it said the crew was down, | |
| 673 | * and how many pulls the outage stranded it has sent again since, said once when a pass | |
| 674 | * sends none of them. Two numbers, gone on restart: the run table holds the outage itself. | |
| 675 | */ | |
| 676 | const outage = { down: false, resent: 0 }; | |
| 677 | ||
| 647 | 678 | /** Every open pull of a repository, oldest first, paged past the list cap. */ |
| 648 | 679 | function openPullsOldestFirst(db: Database, repoId: string): Pull[] { |
| 649 | 680 | const all: Pull[] = []; |
| @@ -670,7 +701,15 @@ function openPullsOldestFirst(db: Database, repoId: string): Pull[] { | ||
| 670 | 701 | * pull passed over for a reason gets one line saying why. The tick's net |
| 671 | 702 | * under the check event; returns how many went. |
| 672 | 703 | */ |
| 673 | export async function sweepReviews(app: App): Promise<number> { | |
| 704 | export async function sweepReviews(app: App, now = new Date()): Promise<number> { | |
| 705 | // While runs die at their clone nothing is sent: the lapse clock stands still meanwhile. | |
| 706 | if (crewDownNow(app.db, now.getTime())) { | |
| 707 | if (!outage.down) log.info("review: paused, crew runs die before their agent starts"); | |
| 708 | outage.down = true; | |
| 709 | return 0; | |
| 710 | } | |
| 711 | outage.down = false; | |
| 712 | let resent = 0; | |
| 674 | 713 | const repoIds = app.db |
| 675 | 714 | .query<{ repo_id: string }, []>( |
| 676 | 715 | "SELECT DISTINCT repo_id FROM repo_crew WHERE role = 'reviewer'", |
| @@ -683,15 +722,17 @@ export async function sweepReviews(app: App): Promise<number> { | ||
| 683 | 722 | const repo = getRepoById(app.db, id); |
| 684 | 723 | if (!repo) continue; |
| 685 | 724 | for (const pull of openPullsOldestFirst(app.db, repo.id)) { |
| 686 | if (queued >= SWEEP_SENDS) return forgetSettled(stalled, queued); | |
| 725 | if (queued >= SWEEP_SENDS) return forgetSettled(stalled, queued, resent); | |
| 687 | 726 | // An agent's green pull, or a stranger's the crew filters (#9575). |
| 688 | 727 | const owed = |
| 689 | 728 | (pull.authorIsAgent && greenHead(app.db, pull)) || |
| 690 | 729 | (!pull.authorIsAgent && crewFilters(repo) && isStrangerPull(app.db, repo, pull)); |
| 691 | 730 | if (!owed) continue; |
| 692 | const result = await sendToReview(app, repo, pull); | |
| 693 | if (result === "sent") queued++; | |
| 694 | else if (result === "lapsed" || result === "wait") { | |
| 731 | const result = await sendToReview(app, repo, pull, now); | |
| 732 | if (result === "sent") { | |
| 733 | queued++; | |
| 734 | if ((reviewAttempts(app.db, pull, now)?.paused.length ?? 0) > 0) resent++; | |
| 735 | } else if (result === "lapsed" || result === "wait") { | |
| 695 | 736 | stalled.add(pull.id); |
| 696 | 737 | const key = `${pull.headOid}:${result}`; |
| 697 | 738 | if (said.get(pull.id) === key) continue; |
| @@ -706,11 +747,19 @@ export async function sweepReviews(app: App): Promise<number> { | ||
| 706 | 747 | } |
| 707 | 748 | } |
| 708 | 749 | } |
| 709 | return forgetSettled(stalled, queued); | |
| 750 | return forgetSettled(stalled, queued, resent); | |
| 710 | 751 | } |
| 711 | 752 | |
| 712 | /** Drop the said-lines of pulls that no longer stall, so the map stays the size of the stall. */ | |
| 713 | function forgetSettled(stalled: Set<string>, queued: number): number { | |
| 753 | /** | |
| 754 | * Drop the said-lines of pulls that no longer stall, so the map stays the size of the stall, | |
| 755 | * and say how many pulls an outage stranded were sent again, once they all are. | |
| 756 | */ | |
| 757 | function forgetSettled(stalled: Set<string>, queued: number, resent: number): number { | |
| 714 | 758 | for (const id of said.keys()) if (!stalled.has(id)) said.delete(id); |
| 759 | outage.resent += resent; | |
| 760 | if (resent === 0 && outage.resent > 0) { | |
| 761 | log.info(`review: re-sent ${outage.resent} green pulls the crew's outage stranded`); | |
| 762 | outage.resent = 0; | |
| 763 | } | |
| 715 | 764 | return queued; |
| 716 | 765 | } |
src/crew/review_pause.ts added +126
| @@ -0,0 +1,126 @@ | ||
| 1 | /** | |
| 2 | * When the crew could not run (#14831). On 2026-11-20 every crew run died at its clone for nine | |
| 3 | * hours; green pulls sat past `REVIEW_LAPSE_MS` and lapsed to a person though no reviewer was ever | |
| 4 | * asked. A lapse means "asked and did not answer", so the lapse clock counts only time a review | |
| 5 | * could have run. | |
| 6 | * | |
| 7 | * The crew is down at instant t when a run died before its agent started (clone, fetch, | |
| 8 | * checkout: `interruptionOf` = "clone") within `DOWN_WINDOW_MS` before t, and either no run | |
| 9 | * reached its agent in that window, or the clone breaker stands: `CLONE_TRIP` such deaths came | |
| 10 | * after the newest run that did (`clone_breaker.ts`). A quiet forge, where nothing ended at all, | |
| 11 | * is never down: there the clock runs as it always did. Read from the run table alone, so a | |
| 12 | * restart forgets nothing; pure over the rows and the instants. | |
| 13 | */ | |
| 14 | ||
| 15 | import type { Database } from "bun:sqlite"; | |
| 16 | import { CLONE_TRIP } from "./clone_breaker.ts"; | |
| 17 | import { interruptionOf } from "./interrupted.ts"; | |
| 18 | ||
| 19 | /** How long one run's death at its clone, or one run that reached its agent, speaks for. */ | |
| 20 | export const DOWN_WINDOW_MS = 15 * 60_000; | |
| 21 | /** How far back the stretches are read: a head green longer than this lapsed long ago. */ | |
| 22 | export const PAUSE_LOOKBACK_MS = 24 * 60 * 60_000; | |
| 23 | ||
| 24 | /** One ended agent run: when, and whether it reached its agent. */ | |
| 25 | export interface Ended { | |
| 26 | readonly at: number; | |
| 27 | readonly reached: boolean; | |
| 28 | } | |
| 29 | ||
| 30 | /** A stretch the crew was down, as epoch ms, `[from, to)`. */ | |
| 31 | export type Stretch = readonly [number, number]; | |
| 32 | ||
| 33 | /** The last index in sorted `xs` with a value at or under `t`, or -1. */ | |
| 34 | function lastAtOrBefore(xs: readonly number[], t: number): number { | |
| 35 | let lo = 0; | |
| 36 | let hi = xs.length; | |
| 37 | while (lo < hi) { | |
| 38 | const mid = (lo + hi) >> 1; | |
| 39 | if (xs[mid]! <= t) lo = mid + 1; | |
| 40 | else hi = mid; | |
| 41 | } | |
| 42 | return lo - 1; | |
| 43 | } | |
| 44 | ||
| 45 | interface Index { | |
| 46 | readonly reached: number[]; | |
| 47 | readonly prestart: number[]; | |
| 48 | } | |
| 49 | ||
| 50 | function indexOf(ended: readonly Ended[]): Index { | |
| 51 | const reached: number[] = []; | |
| 52 | const prestart: number[] = []; | |
| 53 | for (const e of [...ended].sort((a, b) => a.at - b.at)) (e.reached ? reached : prestart).push(e.at); | |
| 54 | return { reached, prestart }; | |
| 55 | } | |
| 56 | ||
| 57 | function downIn(ix: Index, t: number): boolean { | |
| 58 | const p = lastAtOrBefore(ix.prestart, t); | |
| 59 | // No run died at its clone in the window: nothing says the crew is down. | |
| 60 | if (p < 0 || ix.prestart[p]! <= t - DOWN_WINDOW_MS) return false; | |
| 61 | const r = lastAtOrBefore(ix.reached, t); | |
| 62 | if (r < 0 || ix.reached[r]! <= t - DOWN_WINDOW_MS) return true; | |
| 63 | // A run reached its agent in the window; the breaker reopens after CLONE_TRIP deaths since. | |
| 64 | const after = p - lastAtOrBefore(ix.prestart, ix.reached[r]!); | |
| 65 | return after >= CLONE_TRIP; | |
| 66 | } | |
| 67 | ||
| 68 | /** Whether the crew is down at `t` (epoch ms). */ | |
| 69 | export const downAt = (ended: readonly Ended[], t: number): boolean => downIn(indexOf(ended), t); | |
| 70 | ||
| 71 | /** | |
| 72 | * The stretches within `[from, to)` the crew was down, merged and in order. Down is constant | |
| 73 | * between consecutive edges (each end, and each end plus the window), so it is read once per edge. | |
| 74 | */ | |
| 75 | export function downStretches(ended: readonly Ended[], from: number, to: number): Stretch[] { | |
| 76 | if (to <= from) return []; | |
| 77 | const ix = indexOf(ended); | |
| 78 | const edges = new Set<number>([from]); | |
| 79 | for (const e of ended) | |
| 80 | for (const x of [e.at, e.at + DOWN_WINDOW_MS]) if (x > from && x < to) edges.add(x); | |
| 81 | const sorted = [...edges].sort((a, b) => a - b); | |
| 82 | const out: [number, number][] = []; | |
| 83 | sorted.forEach((start, i) => { | |
| 84 | if (!downIn(ix, start)) return; | |
| 85 | const end = sorted[i + 1] ?? to; | |
| 86 | const last = out.at(-1); | |
| 87 | if (last !== undefined && last[1] === start) last[1] = end; | |
| 88 | else out.push([start, end]); | |
| 89 | }); | |
| 90 | return out; | |
| 91 | } | |
| 92 | ||
| 93 | /** Milliseconds of `[from, to)` outside every stretch: the time a review could have run. */ | |
| 94 | export function liveMs(from: number, to: number, paused: readonly Stretch[]): number { | |
| 95 | let ms = Math.max(0, to - from); | |
| 96 | for (const [a, b] of paused) ms -= Math.max(0, Math.min(b, to) - Math.max(a, from)); | |
| 97 | return ms; | |
| 98 | } | |
| 99 | ||
| 100 | /** The agent runs that ended from `from - DOWN_WINDOW_MS` on (`run_kind_finished`). */ | |
| 101 | export function endedSince(db: Database, from: number): Ended[] { | |
| 102 | return db | |
| 103 | .query<{ finished_at: string; state: string; summary: string | null }, [string]>( | |
| 104 | `SELECT finished_at, state, summary FROM run INDEXED BY run_kind_finished | |
| 105 | WHERE kind = 'agent' AND finished_at IS NOT NULL AND finished_at >= ?1`, | |
| 106 | ) | |
| 107 | .all(new Date(from - DOWN_WINDOW_MS).toISOString()) | |
| 108 | .map((r) => ({ | |
| 109 | at: Date.parse(r.finished_at), | |
| 110 | reached: !diedBeforeAgent(r.state, r.summary), | |
| 111 | })); | |
| 112 | } | |
| 113 | ||
| 114 | /** Whether a review run that ended so died before its agent started: no attempt. */ | |
| 115 | export const diedBeforeAgent = (state: string, summary: string | null): boolean => | |
| 116 | state === "failed" && interruptionOf(summary ?? "") === "clone"; | |
| 117 | ||
| 118 | /** The stretches the crew was down from `from` (no further back than the lookback) to `now`. */ | |
| 119 | export function crewDownStretches(db: Database, from: number, now: number): Stretch[] { | |
| 120 | const start = Math.max(from, now - PAUSE_LOOKBACK_MS); | |
| 121 | return downStretches(endedSince(db, start), start, now); | |
| 122 | } | |
| 123 | ||
| 124 | /** Whether the crew is down now: a review sent now would die at its clone, so none is sent. */ | |
| 125 | export const crewDownNow = (db: Database, now: number): boolean => | |
| 126 | downAt(endedSince(db, now), now); |
test/review-outage.test.ts added +303
| @@ -0,0 +1,303 @@ | ||
| 1 | /** | |
| 2 | * A review does not lapse while the crew cannot run (#14831). On 2026-11-20 every crew run died | |
| 3 | * at its clone for nine hours; green pulls lapsed to a person though no reviewer was ever asked. | |
| 4 | * The lapse clock stands still while runs die before their agent starts, a review that died at | |
| 5 | * its clone is no attempt, and once the crew is back every stranded green pull is sent again. | |
| 6 | */ | |
| 7 | import { seatAs } from "./helpers/seat.ts"; | |
| 8 | import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 14 | import { nameTeams } from "../src/actions/issue_teams.ts"; | |
| 15 | import { openIssue } from "../src/actions/issues.ts"; | |
| 16 | import { openPull } from "../src/actions/pulls.ts"; | |
| 17 | import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts"; | |
| 18 | import { getPull, type Pull } from "../src/core/pulls.ts"; | |
| 19 | import { getRepoById, type Repo } from "../src/core/repos.ts"; | |
| 20 | import { saveTeam } from "../src/core/teams.ts"; | |
| 21 | import { markVerified } from "../src/core/verify.ts"; | |
| 22 | import { logLevel, setLogLevel } from "../src/kernel/log.ts"; | |
| 23 | import { setLimit } from "../src/settings/limits.ts"; | |
| 24 | import { createRun, listRuns } from "../src/core/runs.ts"; | |
| 25 | import { createUser, getUserByName, type User } from "../src/core/users.ts"; | |
| 26 | import { | |
| 27 | REVIEW_LAPSE_MS, | |
| 28 | reviewAttempts, | |
| 29 | sendToReview, | |
| 30 | sweepReviews, | |
| 31 | waitsForPerson, | |
| 32 | } from "../src/crew/review.ts"; | |
| 33 | import { | |
| 34 | DOWN_WINDOW_MS, | |
| 35 | downAt, | |
| 36 | downStretches, | |
| 37 | endedSince, | |
| 38 | liveMs, | |
| 39 | type Ended, | |
| 40 | } from "../src/crew/review_pause.ts"; | |
| 41 | ||
| 42 | const MIN = 60_000; | |
| 43 | ||
| 44 | describe("when the crew is down", () => { | |
| 45 | const died = (m: number): Ended => ({ at: m * MIN, reached: false }); | |
| 46 | const ran = (m: number): Ended => ({ at: m * MIN, reached: true }); | |
| 47 | ||
| 48 | test("a quiet forge, where nothing ended, is never down: its clock runs as before", () => { | |
| 49 | expect(downStretches([], 0, 600 * MIN)).toEqual([]); | |
| 50 | expect(liveMs(0, 90 * MIN, [])).toBe(90 * MIN); | |
| 51 | }); | |
| 52 | ||
| 53 | test("runs dying at their clone every 5 min for 2 h are one stretch, ending a window after the last", () => { | |
| 54 | const storm = Array.from({ length: 25 }, (_, i) => died(10 + 5 * i)); // 10 … 130 min | |
| 55 | expect(downStretches(storm, 0, 300 * MIN)).toEqual([[10 * MIN, 130 * MIN + DOWN_WINDOW_MS]]); | |
| 56 | // 300 min less the 135 min down: 165 min live. | |
| 57 | expect(liveMs(0, 300 * MIN, downStretches(storm, 0, 300 * MIN))).toBe(165 * MIN); | |
| 58 | }); | |
| 59 | ||
| 60 | test("a run that reached its agent in the window is the crew up, until the breaker trips again", () => { | |
| 61 | expect(downAt([died(0), ran(2)], 5 * MIN)).toBe(false); | |
| 62 | expect(downAt([ran(0), died(1), died(2)], 3 * MIN)).toBe(false); | |
| 63 | // CLONE_TRIP (3) deaths after the last run that got through: the breaker stands. | |
| 64 | expect(downAt([ran(0), died(1), died(2), died(3)], 4 * MIN)).toBe(true); | |
| 65 | // A death older than the window says nothing. | |
| 66 | expect(downAt([died(0)], 16 * MIN)).toBe(false); | |
| 67 | }); | |
| 68 | ||
| 69 | test("the breaker's stretch: down from the CLONE_TRIP-th death after a run got through, a window past the last", () => { | |
| 70 | // Without the breaker branch the run at 0 holds the crew up until 15 min, and the stretch | |
| 71 | // starts there. | |
| 72 | expect(downStretches([ran(0), died(1), died(2), died(3)], 0, 60 * MIN)).toEqual([[3 * MIN, 3 * MIN + DOWN_WINDOW_MS]]); | |
| 73 | }); | |
| 74 | }); | |
| 75 | ||
| 76 | describe("a clone storm strands no green pull", () => { | |
| 77 | let dataDir: string; | |
| 78 | let app: App; | |
| 79 | let alice: User; | |
| 80 | let repo: Repo; | |
| 81 | let open: Repo; | |
| 82 | let teamPull: Pull; | |
| 83 | let strangerPull: Pull; | |
| 84 | // The storm's shape: green at T0, every agent run dies at its clone from T0+5 to T0+125 min. | |
| 85 | const T0 = Date.now() - 4 * 60 * MIN; | |
| 86 | const iso = (m: number) => new Date(T0 + m * MIN).toISOString(); | |
| 87 | const DURING = new Date(T0 + 60 * MIN); | |
| 88 | // 160 min after the green, 25 min of it live (5 before the storm, 20 since the window closed). | |
| 89 | const AFTER = new Date(T0 + 160 * MIN); | |
| 90 | const STORM_DEATHS = 25; | |
| 91 | ||
| 92 | beforeAll(async () => { | |
| 93 | dataDir = await mkdtemp(join(tmpdir(), "forge-review-outage-")); | |
| 94 | app = createApp({ dataDir, port: 0 }); | |
| 95 | const a = await createUser(app.db, { | |
| 96 | origin: "seed", | |
| 97 | trusted: true, | |
| 98 | username: "alice", | |
| 99 | email: "[email protected]", | |
| 100 | password: "a-good-passphrase", | |
| 101 | adminIfFirst: true, | |
| 102 | }); | |
| 103 | if (!a.ok) throw new Error(a.error); | |
| 104 | alice = a.value; | |
| 105 | markVerified(app.db, alice.id, new Date().toISOString()); | |
| 106 | for (const [key, value] of [ | |
| 107 | ["signup.per_source", 1000], | |
| 108 | ["guest.forks_per_day", 1000], | |
| 109 | ["guest.forks_forge_per_day", 1000], | |
| 110 | ["guest.pulls_per_day", 1000], | |
| 111 | ] as const) | |
| 112 | setLimit(app.db, key, value, "setup"); | |
| 113 | const made = await createRepo(app, alice, { name: "engine", visibility: "private" }); | |
| 114 | if (!made.ok) throw new Error(String(made.error)); | |
| 115 | repo = made.value; | |
| 116 | await commitToBranch(app, alice, repo, { | |
| 117 | message: "Start", | |
| 118 | files: [{ path: "README.md", content: "# engine\n" }], | |
| 119 | }); | |
| 120 | // A public autonomous repository: the crew's reviewer reads a stranger's pull first (#9575). | |
| 121 | const pub = await createRepo(app, alice, { name: "open", visibility: "public" }); | |
| 122 | if (!pub.ok) throw new Error(String(pub.error)); | |
| 123 | await commitToBranch(app, alice, pub.value, { | |
| 124 | message: "Start", | |
| 125 | files: [{ path: "README.md", content: "# open\n" }], | |
| 126 | }); | |
| 127 | app.db.query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?1 WHERE id = ?2").run(alice.id, pub.value.id); | |
| 128 | open = getRepoById(app.db, pub.value.id)!; | |
| 129 | for (const name of ["scout", "juniper"]) { | |
| 130 | const agent = await createAgent(app, alice, { username: name, brief: `I am ${name}.` }); | |
| 131 | if (!agent.ok) throw new Error(String(agent.error)); | |
| 132 | expect(addToCrew(app, alice, repo, name).ok).toBe(true); | |
| 133 | } | |
| 134 | expect(addToCrew(app, alice, open, "juniper").ok).toBe(true); | |
| 135 | const juniper = getUserByName(app.db, "juniper")!.id; | |
| 136 | seatAs(app.db, repo.id, juniper, "reviewer"); | |
| 137 | seatAs(app.db, open.id, juniper, "reviewer"); | |
| 138 | saveTeam(app.db, "design", { charter: "", dial: 0, members: [juniper] }); | |
| 139 | ||
| 140 | // The storm: every agent run dies at its clone, one every 5 minutes for two hours. | |
| 141 | for (let m = 5; m <= 125; m += 5) { | |
| 142 | const run = createRun(app.db, { repoId: repo.id, kind: "agent", reason: "pick" }); | |
| 143 | ended(run.id, "failed", "clone failed: fatal: early EOF", m - 1, m); | |
| 144 | } | |
| 145 | }, 60_000); | |
| 146 | ||
| 147 | afterAll(async () => { | |
| 148 | await app.close?.(); | |
| 149 | await rm(dataDir, { recursive: true, force: true }); | |
| 150 | }); | |
| 151 | ||
| 152 | /** An agent's pull on `engine`, its check green at T0; `body` may close an issue. */ | |
| 153 | async function greenPull(name: string, body = ""): Promise<Pull> { | |
| 154 | await commitToBranch(app, alice, repo, { | |
| 155 | branch: `feature/${name}`, | |
| 156 | base: "main", | |
| 157 | message: name, | |
| 158 | files: [{ path: `docs/${name}.md`, content: `${name}\n` }], | |
| 159 | }); | |
| 160 | const opened = await openPull(app, alice, repo, { | |
| 161 | title: name, | |
| 162 | body, | |
| 163 | baseBranch: "main", | |
| 164 | headBranch: `feature/${name}`, | |
| 165 | }); | |
| 166 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 167 | const pull = opened.value; | |
| 168 | app.db.query("UPDATE issue SET author_id = ?2 WHERE id = ?1").run(pull.id, getUserByName(app.db, "scout")!.id); | |
| 169 | const check = createRun(app.db, { | |
| 170 | repoId: repo.id, | |
| 171 | kind: "check", | |
| 172 | issueId: pull.id, | |
| 173 | env: `pull-${pull.number}`, | |
| 174 | commitOid: pull.headOid, | |
| 175 | reason: "pull", | |
| 176 | }); | |
| 177 | ended(check.id, "ok", "check passed", -10, 0); | |
| 178 | return { ...pull, authorIsAgent: true }; | |
| 179 | } | |
| 180 | ||
| 181 | function ended(id: string, state: string, summary: string, createdMin: number, finishedMin: number): void { | |
| 182 | app.db | |
| 183 | .query( | |
| 184 | "UPDATE run SET state = ?2, summary = ?3, created_at = ?4, started_at = ?4, finished_at = ?5 WHERE id = ?1", | |
| 185 | ) | |
| 186 | .run(id, state, summary, iso(createdMin), iso(finishedMin)); | |
| 187 | } | |
| 188 | ||
| 189 | const reviews = (pull: Pull) => | |
| 190 | listRuns(app.db, pull.repoId, { issueId: pull.id }).filter((r) => r.reason === "review"); | |
| 191 | ||
| 192 | test("the storm's rows are what the outage read finds", () => { | |
| 193 | expect(endedSince(app.db, T0).length).toBe(STORM_DEATHS); | |
| 194 | expect(endedSince(app.db, T0).every((e) => !e.reached)).toBe(true); | |
| 195 | }); | |
| 196 | ||
| 197 | test("while the crew is down a team pull and a stranger's pull are not sent; once it is back, both are", async () => { | |
| 198 | // A pull closing an issue that names the design team, green at T0. | |
| 199 | const issue = openIssue(app, alice, repo, { title: "A chart" }); | |
| 200 | if (!issue.ok) throw new Error(issue.error); | |
| 201 | const named = nameTeams(app, alice, repo, issue.value, ["design"]); | |
| 202 | if (!named.ok) throw new Error(named.error); | |
| 203 | teamPull = await greenPull("team", `Closes #${issue.value.number}`); | |
| 204 | // A stranger's pull from a fork of the autonomous repository. | |
| 205 | const made = await createUser(app.db, { | |
| 206 | origin: "seed", | |
| 207 | trusted: true, | |
| 208 | username: "stranger", | |
| 209 | email: "[email protected]", | |
| 210 | password: "correct-horse-battery", | |
| 211 | }); | |
| 212 | if (!made.ok) throw new Error(made.error); | |
| 213 | markVerified(app.db, made.value.id, new Date().toISOString()); | |
| 214 | app.db.query("UPDATE user SET created_at = ?1 WHERE id = ?2").run("2026-01-01T00:00:00.000Z", made.value.id); | |
| 215 | const stranger = getUserByName(app.db, "stranger")!; | |
| 216 | const fork = await forkRepo(app, stranger, open); | |
| 217 | if (!fork.ok) throw new Error(String(fork.error)); | |
| 218 | const change = await commitToBranch(app, stranger, fork.value, { | |
| 219 | branch: "change", | |
| 220 | base: "main", | |
| 221 | message: "Change", | |
| 222 | files: [{ path: "notes.md", content: "hi\n" }], | |
| 223 | }); | |
| 224 | if (!change.ok) throw new Error(JSON.stringify(change.error)); | |
| 225 | const opened = await openPull(app, stranger, open, { | |
| 226 | title: "A change", | |
| 227 | body: "", | |
| 228 | baseBranch: "main", | |
| 229 | headBranch: "change", | |
| 230 | headRepo: fork.value, | |
| 231 | }); | |
| 232 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 233 | strangerPull = getPull(app.db, open.id, opened.value.number)!; | |
| 234 | ||
| 235 | // Without the outage check each would be sent here, and its review would die at its clone. | |
| 236 | expect(await sendToReview(app, repo, teamPull, DURING)).toBe("wait"); | |
| 237 | expect(await sendToReview(app, open, strangerPull, DURING)).toBe("wait"); | |
| 238 | expect(reviews(teamPull)).toHaveLength(0); | |
| 239 | expect(reviews(strangerPull)).toHaveLength(0); | |
| 240 | ||
| 241 | // 160 min after the green with 25 of them live: the team route does not lapse. | |
| 242 | expect(await sendToReview(app, repo, teamPull, AFTER)).toBe("sent"); | |
| 243 | expect(reviews(teamPull).map((r) => r.actorId)).toEqual([getUserByName(app.db, "juniper")!.id]); | |
| 244 | expect(await sendToReview(app, open, strangerPull, AFTER)).toBe("sent"); | |
| 245 | expect(reviews(strangerPull)).toHaveLength(1); | |
| 246 | }, 60_000); | |
| 247 | ||
| 248 | test("two hours of runs dying at the clone: afterwards every green pull is sent, none waits for a person", async () => { | |
| 249 | const pulls = [await greenPull("one"), await greenPull("two"), await greenPull("three")]; | |
| 250 | // The first pull was sent once during the storm, and its review died at its clone. | |
| 251 | const juniper = getUserByName(app.db, "juniper")!.id; | |
| 252 | const died = createRun(app.db, { repoId: repo.id, kind: "agent", issueId: pulls[0]!.id, reason: "review", actorId: juniper }); | |
| 253 | ended(died.id, "failed", "clone failed: fatal: early EOF", 10, 11); | |
| 254 | ||
| 255 | const found = logLevel(); | |
| 256 | setLogLevel("info"); | |
| 257 | const info = spyOn(console, "info").mockImplementation(() => {}); | |
| 258 | try { | |
| 259 | // During the storm nothing is sent, and the clock stands still. | |
| 260 | expect(await sweepReviews(app, DURING)).toBe(0); | |
| 261 | expect(info.mock.calls.filter(([l]) => String(l).includes("review: paused"))).toHaveLength(1); | |
| 262 | ||
| 263 | for (const pull of pulls) { | |
| 264 | const history = reviewAttempts(app.db, pull, AFTER)!; | |
| 265 | expect(history.attempts).toHaveLength(0); // the clone death used up no try | |
| 266 | expect(liveMs(T0, AFTER.getTime(), history.paused)).toBe(25 * MIN); | |
| 267 | expect(liveMs(T0, AFTER.getTime(), history.paused)).toBeLessThan(REVIEW_LAPSE_MS); | |
| 268 | expect(waitsForPerson(app.db, repo.id, pull, AFTER)).toBeUndefined(); | |
| 269 | } | |
| 270 | ||
| 271 | expect(await sweepReviews(app, AFTER)).toBe(3); | |
| 272 | for (const pull of pulls) { | |
| 273 | expect(reviews(pull).filter((r) => r.state === "queued")).toHaveLength(1); | |
| 274 | expect(reviewAttempts(app.db, pull, AFTER)!.attempts).toHaveLength(1); | |
| 275 | expect(waitsForPerson(app.db, repo.id, pull, AFTER)).toBeUndefined(); | |
| 276 | } | |
| 277 | const resentLines = () => info.mock.calls.map(([l]) => String(l)).filter((l) => l.includes("review: re-sent")); | |
| 278 | // Said once the pass after the last re-send, with the count of this pass's three. | |
| 279 | expect(resentLines()).toHaveLength(0); | |
| 280 | expect(await sweepReviews(app, AFTER)).toBe(0); | |
| 281 | expect(resentLines()).toHaveLength(1); | |
| 282 | expect(resentLines()[0]).toContain("re-sent 3 green pulls the crew's outage stranded"); | |
| 283 | expect(await sweepReviews(app, AFTER)).toBe(0); | |
| 284 | expect(resentLines()).toHaveLength(1); | |
| 285 | } finally { | |
| 286 | info.mockRestore(); | |
| 287 | setLogLevel(found); | |
| 288 | } | |
| 289 | }, 60_000); | |
| 290 | ||
| 291 | test("the outage read walks the finished-runs index", () => { | |
| 292 | const plan = app.db | |
| 293 | .query<{ detail: string }, [string]>( | |
| 294 | `EXPLAIN QUERY PLAN SELECT finished_at, state, summary FROM run INDEXED BY run_kind_finished | |
| 295 | WHERE kind = 'agent' AND finished_at IS NOT NULL AND finished_at >= ?1`, | |
| 296 | ) | |
| 297 | .all("x") | |
| 298 | .map((r) => r.detail) | |
| 299 | .join("\n"); | |
| 300 | expect(plan).toContain("run_kind_finished"); | |
| 301 | expect(plan).not.toContain("TEMP B-TREE"); | |
| 302 | }); | |
| 303 | }); |
new above
offline