Public: anyone can read this; the crew can change it.
When checks are scarce, they go to the work that matters most: the check queu… #12231
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12230-when-checks-are-scarce-they-go-to-the-wo → main 5 commits fork 2 threads · 2 unresolved
9 files
changed +346
−25
docs/features/ranked-checks.md added +21
| @@ -0,0 +1,21 @@ | ||
| 1 | # Ranked checks | |
| 2 | ||
| 3 | The owner's ranked list (Next up) decides which waiting work a worker takes | |
| 4 | first, not only which issue the crew starts. | |
| 5 | ||
| 6 | - **Checks.** `pickNext` (`checks/service.ts`) takes, in order: a check that | |
| 7 | has waited `STARVATION_MS` (half the lane budget), then an approved pull's | |
| 8 | check, then the rest. Inside each tier the check whose issue stands highest | |
| 9 | on the list goes first, the unranked last, newest first as the tie-break. | |
| 10 | - **Rework and re-checks** (`crew/requested.ts`, `crew/stale_reds.ts`) are | |
| 11 | taken in the same rank order, the unranked after, each group as it came. | |
| 12 | - **Home lane.** The checks lane lists waiting checks in the order they will | |
| 13 | start, by the same three tiers, so the tile saying "next in line" names the | |
| 14 | check that starts next. Approval is read in one batch per lane read; the | |
| 15 | database orders by rank before the 40-row bound, so a ranked check is never | |
| 16 | cut off behind newer unranked ones. | |
| 17 | - **Across repositories.** Ranks are per repository. Equal places in two | |
| 18 | repositories fall to the same tie-break (newest check first); nothing | |
| 19 | interleaves them by any other rule. | |
| 20 | ||
| 21 | Tests: `test/checks-ranked.test.ts`. |
src/checks/ranked_first.ts added +21
| @@ -0,0 +1,21 @@ | ||
| 1 | /** | |
| 2 | * The place of a queued check's issue on the owner's ranked list (#12230), | |
| 3 | * for `pickNext`: 1 at the top, undefined for a check whose issue is not | |
| 4 | * ranked (or that has none). Asked once per run for as long as it lives. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import type { Run } from "../core/runs.ts"; | |
| 9 | ||
| 10 | export function rankOnce(db: Database): (run: Run) => number | undefined { | |
| 11 | const known = new Map<string, number | null>(); | |
| 12 | return (run) => { | |
| 13 | if (run.issueId === null) return undefined; | |
| 14 | let pos = known.get(run.issueId); | |
| 15 | if (pos === undefined) { | |
| 16 | pos = db.query<{ pos: number }, [string]>("SELECT pos FROM issue_rank WHERE issue_id = ?1").get(run.issueId)?.pos ?? null; | |
| 17 | known.set(run.issueId, pos); | |
| 18 | } | |
| 19 | return pos ?? undefined; | |
| 20 | }; | |
| 21 | } |
src/checks/service.ts +18 −8
| @@ -89,7 +89,7 @@ import { | ||
| 89 | 89 | verifyDue, |
| 90 | 90 | } from "./verify.ts"; |
| 91 | 91 | import { getIssueById } from "../core/issues.ts"; |
| 92 | import { BUDGETS, budgetSummary } from "../kernel/budgets.ts"; | |
| 92 | import { BUDGETS, budgetSummary, STARVATION_MS } from "../kernel/budgets.ts"; | |
| 93 | 93 | import { transact } from "../kernel/db.ts"; |
| 94 | 94 | import { createLog } from "../kernel/log.ts"; |
| 95 | 95 | import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts"; |
| @@ -135,6 +135,7 @@ import { | ||
| 135 | 135 | } from "./install.ts"; |
| 136 | 136 | import { evictSoon, lend } from "./install_evict.ts"; |
| 137 | 137 | import { checkStage, type GateStage } from "./gate_reason.ts"; |
| 138 | import { rankOnce } from "./ranked_first.ts"; | |
| 138 | 139 | import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts"; |
| 139 | 140 | import type { JobOptions } from "../cluster/jobs.ts"; |
| 140 | 141 | import type { CheckDetail } from "../core/runs.ts"; |
| @@ -285,9 +286,11 @@ const TICK_MS = 5_000; | ||
| 285 | 286 | * fresh: the bound that keeps `pickNext` from starving a pull nobody has |
| 286 | 287 | * pushed to in a while. Half the lane's budget (#4061): at the budget |
| 287 | 288 | * itself, every check the newest-first order passed over waited exactly |
| 288 | * long enough to make the lane a cause, even when the lane kept up. | |
| 289 | * long enough to make the lane a cause, even when the lane kept up. Kept | |
| 290 | * in `kernel/budgets.ts`, so the home lane that shows this order reads | |
| 291 | * the same number. | |
| 289 | 292 | */ |
| 290 | export const STARVATION_MS = BUDGETS.laneWait / 2; | |
| 293 | export { STARVATION_MS } from "../kernel/budgets.ts"; | |
| 291 | 294 | |
| 292 | 295 | /** |
| 293 | 296 | * Whether a pull's check must pass over the free slots left (`free`): while |
| @@ -307,23 +310,29 @@ export function holdsLastSlot(free: number, gateWaits: boolean): boolean { | ||
| 307 | 310 | * the rest, newest first among themselves (#6396): a check that is the last |
| 308 | 311 | * thing between a pull and landing is not held up by a new pull's. Ties |
| 309 | 312 | * (down to the same instant) fall back to id order, which is itself |
| 310 | * creation order (`kernel/id.ts`). | |
| 313 | * creation order (`kernel/id.ts`). Among the rest, and among the approved, | |
| 314 | * the pull whose issue stands highest on the owner's ranked list (`rank`, | |
| 315 | * 1 at the top) goes first, the unranked last, newest first only as the | |
| 316 | * tie-break (#12230). | |
| 311 | 317 | */ |
| 312 | 318 | export function pickNext( |
| 313 | 319 | queued: readonly Run[], |
| 314 | 320 | now: number = Date.now(), |
| 315 | 321 | approved: (run: Run) => boolean = () => false, |
| 322 | rank: (run: Run) => number | undefined = () => undefined, | |
| 316 | 323 | ): Run | undefined { |
| 317 | 324 | if (queued.length === 0) return undefined; |
| 318 | 325 | const newer = (a: Run, b: Run): boolean => |
| 319 | 326 | a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id); |
| 320 | 327 | let oldest = queued[0]!; |
| 328 | const place = (run: Run): number => rank(run) ?? Number.MAX_SAFE_INTEGER; | |
| 329 | const better = (a: Run, b: Run): boolean => place(a) < place(b) || (place(a) === place(b) && newer(a, b)); | |
| 321 | 330 | let newest: Run | undefined; |
| 322 | 331 | let newestApproved: Run | undefined; |
| 323 | 332 | for (const run of queued) { |
| 324 | 333 | if (newer(oldest, run)) oldest = run; |
| 325 | if (newest === undefined || newer(run, newest)) newest = run; | |
| 326 | if (approved(run) && (newestApproved === undefined || newer(run, newestApproved))) | |
| 334 | if (newest === undefined || better(run, newest)) newest = run; | |
| 335 | if (approved(run) && (newestApproved === undefined || better(run, newestApproved))) | |
| 327 | 336 | newestApproved = run; |
| 328 | 337 | } |
| 329 | 338 | if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest; |
| @@ -662,11 +671,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 662 | 671 | } |
| 663 | 672 | let rest = queued; |
| 664 | 673 | const isApproved = approvedOnce(app.db); |
| 665 | let next = pickNext(rest, Date.now(), isApproved); | |
| 674 | const rankOf = rankOnce(app.db); | |
| 675 | let next = pickNext(rest, Date.now(), isApproved, rankOf); | |
| 666 | 676 | while (next !== undefined && atLimit(next)) { |
| 667 | 677 | const held = next; |
| 668 | 678 | rest = rest.filter((run) => run !== held); |
| 669 | next = pickNext(rest, Date.now(), isApproved); | |
| 679 | next = pickNext(rest, Date.now(), isApproved, rankOf); | |
| 670 | 680 | } |
| 671 | 681 | return next; |
| 672 | 682 | } |
src/core/rank.ts +11
| @@ -99,3 +99,14 @@ export function lift(db: Database, repo: Repo, issueId: string): boolean { | ||
| 99 | 99 | if (set.later !== undefined) lifted = removeDependency(db, set.later.id, issueId) || lifted; |
| 100 | 100 | return lifted; |
| 101 | 101 | } |
| 102 | ||
| 103 | /** | |
| 104 | * `items` in the order of the owner's ranked list (#12230): top place first, | |
| 105 | * the unranked after, each group keeping the order it came in. A stable sort, | |
| 106 | * so what the caller already ordered (oldest verdict first) is the tie-break. | |
| 107 | */ | |
| 108 | export function inRankOrder<T>(db: Database, repoId: string, items: readonly T[], idOf: (t: T) => string): T[] { | |
| 109 | const places = ranksInRepo(db, repoId).places; | |
| 110 | const of = (t: T) => places.get(idOf(t)) ?? Number.MAX_SAFE_INTEGER; | |
| 111 | return items.map((t, i) => ({ t, i })).sort((a, b) => of(a.t) - of(b.t) || a.i - b.i).map((x) => x.t); | |
| 112 | } |
src/crew/requested.ts +2 −1
| @@ -20,6 +20,7 @@ | ||
| 20 | 20 | * reason changes, so a stall is never silent and never a flood. |
| 21 | 21 | */ |
| 22 | 22 | |
| 23 | import { inRankOrder } from "../core/rank.ts"; | |
| 23 | 24 | import type { App } from "../app.ts"; |
| 24 | 25 | import { addComment, assignIssue } from "../actions/issues.ts"; |
| 25 | 26 | import { getAgent, listCrewRoles, type Agent } from "../core/agents.ts"; |
| @@ -116,7 +117,7 @@ export async function moveRequested( | ||
| 116 | 117 | const builders = crew.filter((c) => c.role === "builder").map((c) => c.agent); |
| 117 | 118 | let slots = free; |
| 118 | 119 | const slice = sliced(); |
| 119 | for (const requested of requestedPulls(app.db, repo.id)) { | |
| 120 | for (const requested of inRankOrder(app.db, repo.id, requestedPulls(app.db, repo.id), (r) => r.pullId)) { | |
| 120 | 121 | if (slice.due()) await slice.breathe(); |
| 121 | 122 | let move: RequestedMove; |
| 122 | 123 | if (verdictIsStale(requested)) move = await askAgain(app, repo, requested, now); |
src/crew/stale_reds.ts +2 −1
| @@ -18,6 +18,7 @@ | ||
| 18 | 18 | * a red main shares (`dispatch.ts` waits those out) and a machine failure. |
| 19 | 19 | */ |
| 20 | 20 | |
| 21 | import { inRankOrder } from "../core/rank.ts"; | |
| 21 | 22 | import type { App } from "../app.ts"; |
| 22 | 23 | import { addComment, assignIssue, setIssueState } from "../actions/issues.ts"; |
| 23 | 24 | import { checksFor } from "../checks/service.ts"; |
| @@ -130,7 +131,7 @@ export async function sweepStaleReds( | ||
| 130 | 131 | let budget = Math.min(PER_TICK, Math.max(room, 0)); |
| 131 | 132 | const main = redMain(app.db, repo.id); |
| 132 | 133 | const forgeLines: ForgeLines = new Map(); |
| 133 | for (const pull of listOpenPulls(app.db, repo.id)) { | |
| 134 | for (const pull of inRankOrder(app.db, repo.id, listOpenPulls(app.db, repo.id), (p) => p.id)) { | |
| 134 | 135 | if (!pull.authorIsAgent || pull.mergedAt !== null || pull.headOid === "") continue; |
| 135 | 136 | if (getHold(app.db, pull.id) !== undefined || authorsInRepoPull(pull)) continue; |
| 136 | 137 | const issue = getIssueById(app.db, pull.id); |
src/flow/lanes_read.ts +101 −15
| @@ -20,6 +20,8 @@ import type { App } from "../app.ts"; | ||
| 20 | 20 | import { nextUp, type NextRow } from "../core/next_up.ts"; |
| 21 | 21 | import { idleReason, type IdleReason } from "../crew/idle_reason.ts"; |
| 22 | 22 | import { listAutonomousRepos } from "../core/repos.ts"; |
| 23 | import { STARVATION_MS } from "../kernel/budgets.ts"; | |
| 24 | import { reviewSummariesFor } from "../core/reviews.ts"; | |
| 23 | 25 | import { weekShare } from "../crew/week_read.ts"; |
| 24 | 26 | import { firedAlarms } from "./alarms.ts"; |
| 25 | 27 | import { readBuilders, readKpis } from "./kpis.ts"; |
| @@ -138,6 +140,13 @@ interface Raw { | ||
| 138 | 140 | beat: string | null; |
| 139 | 141 | issue_id: string | null; |
| 140 | 142 | born: string | null; |
| 143 | /** Place of the row's issue on the owner's ranked list, 1 at the top; null when unranked. */ | |
| 144 | rk?: number | null; | |
| 145 | run_id?: string; | |
| 146 | /** Checks lane: an approved pull's check, which `pickNext` takes before the rest. */ | |
| 147 | approved?: boolean; | |
| 148 | /** Checks lane: the check that has waited so long `pickNext` takes it before all. */ | |
| 149 | starved?: boolean; | |
| 141 | 150 | total: number; |
| 142 | 151 | } |
| 143 | 152 | |
| @@ -147,17 +156,36 @@ const BUILD_WORDS: Record<string, string> = { | ||
| 147 | 156 | feedback: "Answering review notes.", |
| 148 | 157 | }; |
| 149 | 158 | |
| 150 | /** The queue position of each waiting row, from 1, oldest first: the order the next worker takes them. */ | |
| 151 | function positioned(rows: readonly Raw[]): Map<string, number> { | |
| 159 | /** | |
| 160 | * The waiting rows in the order the next worker takes them (#12230): for | |
| 161 | * checks, as `pickNext` does, a starved one first, then an approved pull's, | |
| 162 | * then the rest; within each, the rank of the row's issue on the owner's list | |
| 163 | * first, the unranked last; then `newestFirst` (checks: the order `pickNext` | |
| 164 | * breaks ties in) or oldest first. Ranks are per repository, so equal places | |
| 165 | * in two repositories fall to the same tie-break: the newest (or oldest) first. | |
| 166 | */ | |
| 167 | function takenOrder(rows: readonly Raw[], newestFirst: boolean): Raw[] { | |
| 168 | const place = (r: Raw) => r.rk ?? Number.MAX_SAFE_INTEGER; | |
| 169 | const tier = (r: Raw) => (r.starved ? 0 : r.approved ? 1 : 2); | |
| 170 | const by = (a: Raw, b: Raw) => (a.since < b.since ? -1 : a.since > b.since ? 1 : 0) * (newestFirst ? -1 : 1); | |
| 171 | return rows | |
| 172 | .filter((x) => x.state === "queued") | |
| 173 | .sort((a, b) => tier(a) - tier(b) || place(a) - place(b) || by(a, b)); | |
| 174 | } | |
| 175 | ||
| 176 | /** The queue position of each waiting row, from 1: the order the next worker takes them. */ | |
| 177 | function positioned(rows: readonly Raw[], newestFirst = false): Map<string, number> { | |
| 152 | 178 | const out = new Map<string, number>(); |
| 153 | 179 | let n = 0; |
| 154 | for (const r of rows | |
| 155 | .filter((x) => x.state === "queued") | |
| 156 | .sort((a, b) => (a.since < b.since ? -1 : a.since > b.since ? 1 : 0))) | |
| 157 | out.set(r.id, ++n); | |
| 180 | for (const r of takenOrder(rows, newestFirst)) out.set(r.id, ++n); | |
| 158 | 181 | return out; |
| 159 | 182 | } |
| 160 | 183 | |
| 184 | /** The rows with the waiting ones in the order they will be taken, after the running. */ | |
| 185 | function inTakenOrder(rows: readonly Raw[], newestFirst: boolean): Raw[] { | |
| 186 | return [...rows.filter((x) => x.state !== "queued"), ...takenOrder(rows, newestFirst)]; | |
| 187 | } | |
| 188 | ||
| 161 | 189 | const ordinal = (n: number): string => |
| 162 | 190 | n === 1 |
| 163 | 191 | ? "next in line" |
| @@ -198,18 +226,74 @@ function runRows( | ||
| 198 | 226 | kind: "agent" | "check", |
| 199 | 227 | where: string, |
| 200 | 228 | now: string, |
| 229 | newestFirst = false, | |
| 201 | 230 | ): Raw[] { |
| 202 | return db | |
| 203 | .query<Raw, [string, string, number]>( | |
| 231 | // For checks, the order `pickNext` takes them is settled before the scan | |
| 232 | // bound, so the check that starts next is never cut off the window: the | |
| 233 | // starved one, then approved pulls', then rank, then age. | |
| 234 | const tiers = kind === "check" ? checkTiers(db, now) : { starved: null, approved: [] as string[] }; | |
| 235 | const rows = db | |
| 236 | .query<Raw, [string, string, number, string | null, string]>( | |
| 204 | 237 | `SELECT COALESCE(i.id, x.id) AS id, x.repo_id, i.number, CASE WHEN i.id IS NULL THEN 'main' ELSE i.kind END AS kind, i.title, i.sealed, |
| 205 | 238 | u.username AS who, CASE WHEN x.state = 'running' THEN COALESCE(x.started_at, x.created_at) ELSE x.created_at END AS since, |
| 206 | x.state, x.reason, x.attempt, x.env, x.heartbeat_at AS beat, i.id AS issue_id, i.created_at AS born, COUNT(*) OVER () AS total | |
| 239 | x.state, x.reason, x.attempt, x.env, x.heartbeat_at AS beat, i.id AS issue_id, i.created_at AS born, k.pos AS rk, x.id AS run_id, COUNT(*) OVER () AS total | |
| 207 | 240 | FROM run x LEFT JOIN issue i ON i.id = x.issue_id LEFT JOIN user u ON u.id = x.actor_id |
| 241 | LEFT JOIN issue_rank k ON k.issue_id = x.issue_id | |
| 208 | 242 | WHERE x.kind = ?1 AND x.state IN ('queued','running') AND ${where} |
| 209 | 243 | AND (x.state = 'running' OR x.not_before IS NULL OR x.not_before <= ?2) |
| 210 | ORDER BY x.state DESC, x.created_at ASC LIMIT ?3`, | |
| 244 | ORDER BY x.state DESC, (x.id = ?4) DESC, (x.id IN (SELECT value FROM json_each(?5))) DESC, (k.pos IS NULL), k.pos, | |
| 245 | x.created_at ${newestFirst ? "DESC" : "ASC"}, x.id ${newestFirst ? "DESC" : "ASC"} LIMIT ?3`, | |
| 211 | 246 | ) |
| 212 | .all(kind, now, SCAN); | |
| 247 | .all(kind, now, SCAN, tiers.starved, JSON.stringify(tiers.approved)); | |
| 248 | if (kind === "check") { | |
| 249 | const approved = new Set(tiers.approved); | |
| 250 | for (const r of rows) | |
| 251 | if (r.state === "queued") { | |
| 252 | r.starved = r.run_id === tiers.starved; | |
| 253 | r.approved = approved.has(r.run_id!); | |
| 254 | } | |
| 255 | } | |
| 256 | return rows; | |
| 257 | } | |
| 258 | ||
| 259 | /** | |
| 260 | * Which queued checks `pickNext` takes first, read over the whole queued set | |
| 261 | * (small: checks waiting for a slot) rather than the scanned window: the | |
| 262 | * starved one — the oldest, once it has waited `STARVATION_MS` — and the | |
| 263 | * approved pulls' (an approval on the check's own commit, no change | |
| 264 | * requested, the rule of `approvedForCheck`), in one batch per round, never | |
| 265 | * one query per row. The starved one is approximate: `pickNext` passes over a | |
| 266 | * check whose repository is at its concurrency limit and takes the next, a | |
| 267 | * live state the lane does not read. | |
| 268 | */ | |
| 269 | function checkTiers(db: Database, now: string): { starved: string | null; approved: string[] } { | |
| 270 | const queued = db | |
| 271 | .query<{ id: string; at: string; issue_id: string | null; oid: string | null }, [string]>( | |
| 272 | `SELECT id, created_at AS at, issue_id, commit_oid AS oid FROM run | |
| 273 | WHERE kind = 'check' AND state = 'queued' AND (not_before IS NULL OR not_before <= ?1) | |
| 274 | ORDER BY created_at ASC, id ASC`, | |
| 275 | ) | |
| 276 | .all(now); | |
| 277 | const o = queued[0]; | |
| 278 | const starved = o !== undefined && Date.parse(now) - Date.parse(o.at) >= STARVATION_MS ? o.id : null; | |
| 279 | const approved: string[] = []; | |
| 280 | const left = queued.filter((r) => r.issue_id !== null && r.oid !== null); | |
| 281 | // One summary per issue and commit; a second commit of the same issue waits for the next round. | |
| 282 | while (left.length > 0) { | |
| 283 | const round = new Map<string, (typeof left)[number]>(); | |
| 284 | for (let i = left.length - 1; i >= 0; i--) { | |
| 285 | const r = left[i]!; | |
| 286 | if (round.has(r.issue_id!)) continue; | |
| 287 | round.set(r.issue_id!, r); | |
| 288 | left.splice(i, 1); | |
| 289 | } | |
| 290 | const sums = reviewSummariesFor(db, [...round].map(([id, r]) => ({ id, headOid: r.oid! }))); | |
| 291 | for (const [id, r] of round) { | |
| 292 | const seen = sums.get(id); | |
| 293 | if (seen !== undefined && seen.approvals > 0 && seen.changesRequested === 0) approved.push(r.id); | |
| 294 | } | |
| 295 | } | |
| 296 | return { starved, approved }; | |
| 213 | 297 | } |
| 214 | 298 | |
| 215 | 299 | /** Approved pulls waiting for a person, oldest approval first. */ |
| @@ -310,7 +394,8 @@ function carried( | ||
| 310 | 394 | ); |
| 311 | 395 | } |
| 312 | 396 | |
| 313 | function building(rows: readonly Raw[]): LaneRead { | |
| 397 | function building(raw: readonly Raw[]): LaneRead { | |
| 398 | const rows = inTakenOrder(raw, false); | |
| 314 | 399 | const line = positioned(rows); |
| 315 | 400 | const items = rows.map((r) => |
| 316 | 401 | r.state === "running" |
| @@ -330,8 +415,9 @@ function building(rows: readonly Raw[]): LaneRead { | ||
| 330 | 415 | return { items, total: rows[0]?.total ?? 0 }; |
| 331 | 416 | } |
| 332 | 417 | |
| 333 | function checks(rows: readonly Raw[]): LaneRead { | |
| 334 | const line = positioned(rows); | |
| 418 | function checks(raw: readonly Raw[]): LaneRead { | |
| 419 | const rows = inTakenOrder(raw, true); | |
| 420 | const line = positioned(rows, true); | |
| 335 | 421 | const items = rows.map((r) => { |
| 336 | 422 | const what = |
| 337 | 423 | r.kind === "main" |
| @@ -449,7 +535,7 @@ export function readFast(db: Database, now: Date): Fast { | ||
| 449 | 535 | return { |
| 450 | 536 | lanes: { |
| 451 | 537 | building: building(runRows(db, "agent", "x.reason != 'review'", iso)), |
| 452 | checks: checks(runRows(db, "check", "1 = 1", iso)), | |
| 538 | checks: checks(runRows(db, "check", "1 = 1", iso, true)), | |
| 453 | 539 | review: review(runRows(db, "agent", "x.reason = 'review'", iso)), |
| 454 | 540 | landing: landing(approvedRows(db), mergedRows(db, week)), |
| 455 | 541 | live: live(db, day), |
src/kernel/budgets.ts +7
| @@ -21,6 +21,13 @@ export const BUDGETS = { | ||
| 21 | 21 | heartbeat: 3 * 60_000, |
| 22 | 22 | } as const; |
| 23 | 23 | |
| 24 | /** | |
| 25 | * How long a queued check may wait before it outranks every other however | |
| 26 | * fresh or approved: half the lane's budget (#4061). One number, read by | |
| 27 | * `pickNext` and by the home lane that shows its order (#12230). | |
| 28 | */ | |
| 29 | export const STARVATION_MS = BUDGETS.laneWait / 2; | |
| 30 | ||
| 24 | 31 | /** The summary prefix every budget kill carries; vitals and retries read it. */ |
| 25 | 32 | export const BUDGET_PREFIX = "budget:"; |
| 26 | 33 |
test/checks-ranked.test.ts added +163
| @@ -0,0 +1,163 @@ | ||
| 1 | /** | |
| 2 | * Which queued check starts next follows the owner's ranked list (#12230): | |
| 3 | * top first, unranked last, newest only as the tie-break; an approved pull's | |
| 4 | * check still goes first and a starved one still outranks both. | |
| 5 | */ | |
| 6 | import { expect, test } from "bun:test"; | |
| 7 | import { mkdtemp } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { openPull } from "../src/actions/pulls.ts"; | |
| 11 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 12 | import { createApp } from "../src/app.ts"; | |
| 13 | import { rankOnce } from "../src/checks/ranked_first.ts"; | |
| 14 | import { pickNext, STARVATION_MS } from "../src/checks/service.ts"; | |
| 15 | import { BUDGETS, STARVATION_MS as BOUND } from "../src/kernel/budgets.ts"; | |
| 16 | import { inRankOrder, writeOrder } from "../src/core/rank.ts"; | |
| 17 | import { createRun, type Run } from "../src/core/runs.ts"; | |
| 18 | import { createUser } from "../src/core/users.ts"; | |
| 19 | import { readFast } from "../src/flow/lanes_read.ts"; | |
| 20 | ||
| 21 | async function setup() { | |
| 22 | const app = createApp({ dataDir: await mkdtemp(join(tmpdir(), "forge-ranked-")), port: 0 }); | |
| 23 | const u = await createUser(app.db, { | |
| 24 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 25 | password: "a-good-passphrase", adminIfFirst: true, | |
| 26 | }); | |
| 27 | if (!u.ok) throw new Error(u.error); | |
| 28 | const made = await createRepo(app, u.value, { name: "lib" }); | |
| 29 | if (!made.ok) throw new Error(String(made.error)); | |
| 30 | const first = await commitToBranch(app, u.value, made.value, { message: "Start", files: [{ path: "README.md", content: "x\n" }] }); | |
| 31 | if (!first.ok) throw new Error(JSON.stringify(first.error)); | |
| 32 | const pulls: string[] = []; | |
| 33 | for (const n of [1, 2, 3, 4, 5]) { | |
| 34 | const branch = `b${n}`; | |
| 35 | const c = await commitToBranch(app, u.value, made.value, { branch, message: branch, files: [{ path: `${branch}.md`, content: "x\n" }] }); | |
| 36 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 37 | const o = await openPull(app, u.value, made.value, { title: branch, baseBranch: "main", headBranch: branch }); | |
| 38 | if (!o.ok) throw new Error(String(o.error)); | |
| 39 | pulls.push(o.value.id); | |
| 40 | } | |
| 41 | // Checks queued in order 1..5, so pull 5's is the newest. | |
| 42 | const checks = pulls.map((issueId, i) => | |
| 43 | createRun(app.db, { repoId: made.value.id, kind: "check", env: `pull-${i + 1}`, issueId }), | |
| 44 | ); | |
| 45 | // Rank order: pull 2 first, then 4, 1, 3; pull 5 is unranked. | |
| 46 | writeOrder(app.db, made.value, [pulls[1]!, pulls[3]!, pulls[0]!, pulls[2]!], u.value.id); | |
| 47 | return { app, repo: made.value, pulls, checks }; | |
| 48 | } | |
| 49 | ||
| 50 | /** Make a queued check look as if it had waited `ms`, in the database and in the Run held. */ | |
| 51 | function age(app: { db: import("bun:sqlite").Database }, run: Run, ms: number): Run { | |
| 52 | const createdAt = new Date(Date.now() - ms).toISOString(); | |
| 53 | app.db.query("UPDATE run SET created_at = ?2 WHERE id = ?1").run(run.id, createdAt); | |
| 54 | return { ...run, createdAt }; | |
| 55 | } | |
| 56 | ||
| 57 | /** Approve the pull on its check's commit, as a review would. */ | |
| 58 | function approve(app: { db: import("bun:sqlite").Database }, issueId: string, run: Run): void { | |
| 59 | const reviewer = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!; | |
| 60 | app.db | |
| 61 | .query("INSERT INTO review (id, issue_id, author_id, state, commit_oid, body, created_at) VALUES (?1, ?2, ?3, 'approved', ?4, '', ?5)") | |
| 62 | .run(`rv-${run.id}`, issueId, reviewer.id, run.commitOid, new Date().toISOString()); | |
| 63 | } | |
| 64 | ||
| 65 | test("five checks start in the order of their issues' ranks, the unranked last", async () => { | |
| 66 | const { app, checks } = await setup(); | |
| 67 | const rank = rankOnce(app.db); | |
| 68 | const order: string[] = []; | |
| 69 | let rest: Run[] = [...checks]; | |
| 70 | for (let next = pickNext(rest, Date.now(), () => false, rank); next; next = pickNext(rest, Date.now(), () => false, rank)) { | |
| 71 | order.push(next.env); | |
| 72 | const took = next; | |
| 73 | rest = rest.filter((r) => r !== took); | |
| 74 | } | |
| 75 | expect(order).toEqual(["pull-2", "pull-4", "pull-1", "pull-3", "pull-5"]); | |
| 76 | }); | |
| 77 | ||
| 78 | test("approved goes before rank, a starved check before both, and unranked checks keep newest first", async () => { | |
| 79 | const { app, checks } = await setup(); | |
| 80 | const rank = rankOnce(app.db); | |
| 81 | expect(pickNext(checks, Date.now(), (r) => r.env === "pull-5", rank)?.env).toBe("pull-5"); | |
| 82 | expect(pickNext(checks, Date.now(), () => false)?.env).toBe("pull-5"); | |
| 83 | const starved = age(app, checks[2]!, STARVATION_MS + 1000); | |
| 84 | const fresh = checks.map((r) => (r.id === starved.id ? starved : r)); | |
| 85 | expect(pickNext(fresh, Date.now(), () => false, rank)?.env).toBe("pull-3"); | |
| 86 | }); | |
| 87 | ||
| 88 | test("the home lane lists waiting checks in the order they will start, the first as next", async () => { | |
| 89 | const { app } = await setup(); | |
| 90 | const lane = readFast(app.db, new Date(Date.now() + 1000)).lanes.checks; | |
| 91 | expect(lane.items.map((i) => i.title)).toEqual(["b2", "b4", "b1", "b3", "b5"]); | |
| 92 | expect(lane.items[0]!.hint).toContain("next in line"); | |
| 93 | expect(lane.items[0]!.position).toBe(1); | |
| 94 | }); | |
| 95 | ||
| 96 | test("rework and re-checks are taken in rank order, the unranked after, each group as it came", async () => { | |
| 97 | const { app, repo, pulls } = await setup(); | |
| 98 | expect(inRankOrder(app.db, repo.id, pulls, (p) => p)).toEqual([pulls[1]!, pulls[3]!, pulls[0]!, pulls[2]!, pulls[4]!]); | |
| 99 | }); | |
| 100 | ||
| 101 | test("the lane names the check pickNext takes first: an approved pull's, a starved one before it", async () => { | |
| 102 | const { app, pulls, checks } = await setup(); | |
| 103 | const at = () => readFast(app.db, new Date(Date.now() + 1000)).lanes.checks.items.map((i) => i.title); | |
| 104 | // An approval on pull 5's check commit puts it ahead of the ranked ones. | |
| 105 | approve(app, pulls[4]!, checks[4]!); | |
| 106 | expect(at()).toEqual(["b5", "b2", "b4", "b1", "b3"]); | |
| 107 | const first = readFast(app.db, new Date(Date.now() + 1000)).lanes.checks.items[0]!; | |
| 108 | expect(first.hint).toContain("next in line"); | |
| 109 | // A starved check (the oldest, past the bound) goes before the approved one. | |
| 110 | age(app, checks[2]!, STARVATION_MS + 5000); | |
| 111 | expect(at()).toEqual(["b3", "b5", "b2", "b4", "b1"]); | |
| 112 | }); | |
| 113 | ||
| 114 | test("a ranked check is not cut off the lane by newer unranked ones past the scan bound", async () => { | |
| 115 | const { app, repo } = await setup(); | |
| 116 | for (let i = 0; i < 60; i++) createRun(app.db, { repoId: repo.id, kind: "check", env: `extra-${i}` }); | |
| 117 | const lane = readFast(app.db, new Date(Date.now() + 1000)).lanes.checks; | |
| 118 | expect(lane.items.map((i) => i.title).slice(0, 4)).toEqual(["b2", "b4", "b1", "b3"]); | |
| 119 | expect(lane.items.length).toBeLessThanOrEqual(40); | |
| 120 | }); | |
| 121 | ||
| 122 | test("a ranked check that is the oldest of 60 newer unranked ones is still in the lane, first", async () => { | |
| 123 | const { app, repo, pulls, checks } = await setup(); | |
| 124 | // Only pull 2 ranked, and its check the oldest; the other four pulls' checks stay unranked. | |
| 125 | writeOrder(app.db, repo, [pulls[1]!], (app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!).id); | |
| 126 | age(app, checks[1]!, 60_000); | |
| 127 | for (let i = 0; i < 60; i++) createRun(app.db, { repoId: repo.id, kind: "check", env: `extra-${i}` }); | |
| 128 | const lane = readFast(app.db, new Date(Date.now() + 1000)).lanes.checks; | |
| 129 | expect(lane.items[0]!.title).toBe("b2"); | |
| 130 | expect(lane.items[0]!.hint).toContain("next in line"); | |
| 131 | }); | |
| 132 | ||
| 133 | test("an approved unranked check older than 40 newer ones is still in the lane, first", async () => { | |
| 134 | const { app, repo, pulls, checks } = await setup(); | |
| 135 | age(app, checks[4]!, 60_000); | |
| 136 | approve(app, pulls[4]!, checks[4]!); | |
| 137 | for (let i = 0; i < 45; i++) createRun(app.db, { repoId: repo.id, kind: "check", env: `extra-${i}` }); | |
| 138 | const lane = readFast(app.db, new Date(Date.now() + 1000)).lanes.checks; | |
| 139 | expect(lane.items[0]!.title).toBe("b5"); | |
| 140 | expect(lane.items[0]!.hint).toContain("next in line"); | |
| 141 | }); | |
| 142 | ||
| 143 | test("the lane and pickNext read one starvation bound", async () => { | |
| 144 | expect(STARVATION_MS).toBe(BOUND); | |
| 145 | expect(BOUND).toBe(BUDGETS.laneWait / 2); | |
| 146 | const { app, checks } = await setup(); | |
| 147 | const first = () => readFast(app.db, new Date()).lanes.checks.items[0]!.title; | |
| 148 | // Just inside the bound pull 3's check is not starved: rank decides. Just past it, it is. | |
| 149 | age(app, checks[2]!, BOUND - 5_000); | |
| 150 | expect(first()).toBe("b2"); | |
| 151 | expect(pickNext(checks.map((r, i) => (i === 2 ? { ...r, createdAt: new Date(Date.now() - BOUND + 5_000).toISOString() } : r)), Date.now(), () => false, rankOnce(app.db))?.env).toBe("pull-2"); | |
| 152 | age(app, checks[2]!, BOUND + 5_000); | |
| 153 | expect(first()).toBe("b3"); | |
| 154 | }); | |
| 155 | ||
| 156 | test("ranks are per repository: equal places in two repositories fall to the newest check", async () => { | |
| 157 | const mk = (id: string, repoId: string, createdAt: string) => ({ id, repoId, createdAt, env: id }) as unknown as Run; | |
| 158 | const a = mk("a", "repo-a", "2026-01-01T00:00:00.000Z"); | |
| 159 | const b = mk("b", "repo-b", "2026-01-01T00:00:01.000Z"); | |
| 160 | const rank = () => 1; | |
| 161 | expect(pickNext([a, b], Date.parse(b.createdAt), () => false, rank)?.id).toBe("b"); | |
| 162 | expect(pickNext([b, a], Date.parse(b.createdAt), () => false, rank)?.id).toBe("b"); | |
| 163 | }); |
new above
offline