Public: anyone can read this; the crew can change it.
Weekly pacing limits the whole crew, not each repository: the week fell 2 points in 40 minutes at midnight while it slowed only one #8992
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8989-weekly-pacing-limits-the-whole-crew-not → main 6 commits fork 2 threads · 2 unresolved
5 files
changed +338
−15
docs/features/crew-week.md added +11
| @@ -0,0 +1,11 @@ | ||
| 1 | # The week is one budget for the whole crew (#8989) | |
| 2 | ||
| 3 | `crewWeekFor` (`src/crew/week_read.ts`) reads the week once per set of accounts and decides in slots for the whole crew: its ceiling is `crew.concurrency`, its floor (`WEEK.floorSlots`) applies to the crew, not to each repository. `repoPace` gives a repository `budget - slots running in the other repositories on the same accounts`, held under the repository's own slots, which stay limits inside the budget. | |
| 4 | ||
| 5 | - Only new picks wait on it. Finishing, rework and reviews use `repoPace(..., false)` and run ahead. | |
| 6 | - `autonomyTick` drives repositories with a backed bet first, so a short budget goes to what the owner backed, then the rest in order. | |
| 7 | - A repository that hit its daily ceiling yesterday restarts at its own time in the first six hours of the UTC day (`src/crew/ceiling_stagger.ts`), so the midnight reset is not a burst. | |
| 8 | - Home and the terminal print the crew's number: "the week allows the whole crew N of M slots". | |
| 9 | - Picks made earlier in the same tick in other repositories count against the budget before their runs show, so a burst across repositories cannot pass the crew's floor. | |
| 10 | - A ceiling of 0 allows no spend and is held by the ceiling itself; it is never also held until an offset. The "hit yesterday" read is remembered per repository for one instant, so `repoPace` runs it once a tick. | |
| 11 | - Work a person assigns starts through the dispatcher, which does not read `repoPace`, so it is not held by the crew's budget. |
src/crew/autonomy.ts +48 −12
| @@ -13,6 +13,7 @@ | ||
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | 15 | import { tickBets } from "../actions/bets.ts"; |
| 16 | import { betsInUse } from "../core/bets.ts"; | |
| 16 | 17 | import { onlyListDocs, resolveListConflicts } from "../actions/list-conflicts.ts"; |
| 17 | 18 | import { closedToday, closeStale } from "./stale_pulls.ts"; |
| 18 | 19 | import type { Database } from "bun:sqlite"; |
| @@ -106,7 +107,8 @@ import { reviewSummary } from "../core/reviews.ts"; | ||
| 106 | 107 | import { standing } from "../core/standing.ts"; |
| 107 | 108 | import { pace, type Pace } from "./pacing.ts"; |
| 108 | 109 | import { heldByGateAlone, releaseFromGate } from "./release_later.ts"; |
| 109 | import { weekFor } from "./week_read.ts"; | |
| 110 | import { staggeredUntil } from "./ceiling_stagger.ts"; | |
| 111 | import { crewWeekFor, weekFor, weekScope } from "./week_read.ts"; | |
| 110 | 112 | import { announceGovernor, governorFor } from "./governed.ts"; |
| 111 | 113 | import { BREAKER_NOTE, breakerTripped } from "./breaker.ts"; |
| 112 | 114 | import { REPEAT_NOTE } from "./repeat_note.ts"; |
| @@ -364,8 +366,11 @@ function meanRunUsd(db: App["db"], repoId: string): number { | ||
| 364 | 366 | ); |
| 365 | 367 | } |
| 366 | 368 | |
| 369 | /** The issues each repository picked in this tick, by repository id: their runs may not show yet. */ | |
| 370 | export type TickPicks = ReadonlyMap<string, ReadonlySet<string>>; | |
| 371 | ||
| 367 | 372 | /** The crew's pace here: the operator's slots, held to what the windows have left and to the governor's. */ |
| 368 | export function repoPace(app: App, repo: Repo, now = new Date()): Pace { | |
| 373 | export function repoPace(app: App, repo: Repo, now = new Date(), crew = true, picked: TickPicks = new Map()): Pace { | |
| 369 | 374 | const accounts = grantedAccounts(app.db, repo.id, repo.ownerId).flatMap( |
| 370 | 375 | (a) => { |
| 371 | 376 | const s = standing(app.db, a.id, now); |
| @@ -375,10 +380,28 @@ export function repoPace(app: App, repo: Repo, now = new Date()): Pace { | ||
| 375 | 380 | const paced = pace(accounts, repo.autonomySlots, meanRunUsd(app.db, repo.id), now); |
| 376 | 381 | const governed = governorFor(app, repo, now).slots; |
| 377 | 382 | const limited = governed < paced.slots ? { ...paced, slots: governed } : paced; |
| 378 | // The week's own pace and the reserve for people (#8695). | |
| 379 | const week = weekFor(app, repo, now, { idleSlots: 0, startable: 0, gated: 0 }); | |
| 380 | return week !== null && week.slots < limited.slots | |
| 381 | ? { ...limited, slots: week.slots, reason: `${week.slots} of ${repo.autonomySlots} slots: ${week.sentence}` } | |
| 383 | // Finishing and rework run ahead of new picks: only a new pick waits on the crew's week and the midnight spread. | |
| 384 | if (!crew) return limited; | |
| 385 | const until = staggeredUntil(app.db, repo, now); | |
| 386 | if (until !== null) | |
| 387 | return { ...limited, slots: 0, reason: `0 of ${repo.autonomySlots} slots until ${until.toISOString().slice(11, 16)} UTC: the daily ceilings reset together and each repository restarts at its own time` }; | |
| 388 | // The week is one budget for the whole crew on these accounts, and the repository's own slots are limits inside it (#8989). | |
| 389 | const week = crewWeekFor(app, repo, now); | |
| 390 | if (week === null) return limited; | |
| 391 | const scope = weekScope(app, repo, now); | |
| 392 | let others = 0; | |
| 393 | for (const other of listAutonomousRepos(app.db)) { | |
| 394 | if (other.id === repo.id || other.autonomy !== "autonomous" || other.archivedAt !== null) continue; | |
| 395 | if (weekScope(app, other, now) !== scope) continue; | |
| 396 | const active = activeRuns(app, other.id); | |
| 397 | others += slottedRuns(active, listCrewRoles(app.db, other.id)).length; | |
| 398 | // Picked earlier in this tick and not yet running: a slot of the crew's week already spent. | |
| 399 | const running = new Set(active.map((r) => r.issueId)); | |
| 400 | others += [...(picked.get(other.id) ?? [])].filter((id) => !running.has(id)).length; | |
| 401 | } | |
| 402 | const here = Math.max(0, week.slots - others); | |
| 403 | return here < limited.slots | |
| 404 | ? { ...limited, slots: here, reason: `${here} of ${repo.autonomySlots} slots: the whole crew has ${week.slots} for the week, ${others} running elsewhere. ${week.sentence}` } | |
| 382 | 405 | : limited; |
| 383 | 406 | } |
| 384 | 407 | |
| @@ -405,7 +428,7 @@ function releaseForPace(app: App, repo: Repo, by: User, slotted: number, startab | ||
| 405 | 428 | function freeSlots(app: App, repo: Repo, now: Date): number { |
| 406 | 429 | if (autonomyStatus(app, repo, now).paused !== null) return 0; |
| 407 | 430 | const slotted = slottedRuns(activeRuns(app, repo.id), listCrewRoles(app.db, repo.id)); |
| 408 | return Math.max(0, repoPace(app, repo, now).slots - slotted.length); | |
| 431 | return Math.max(0, repoPace(app, repo, now, false).slots - slotted.length); | |
| 409 | 432 | } |
| 410 | 433 | |
| 411 | 434 | /** The pickable issues in slot order: the order Next up shows, small work held to its share (#7571). */ |
| @@ -453,6 +476,7 @@ export async function driveRepo( | ||
| 453 | 476 | app: App, |
| 454 | 477 | repo: Repo, |
| 455 | 478 | now = new Date(), |
| 479 | picked: Map<string, Set<string>> = new Map(), | |
| 456 | 480 | ): Promise<Drove> { |
| 457 | 481 | // One start per free builder slot (#8729). Each pass re-reads the ceiling, |
| 458 | 482 | // the paced slots and the held filters, so a start the dispatcher refused |
| @@ -460,10 +484,11 @@ export async function driveRepo( | ||
| 460 | 484 | // An issue picked this tick may not have its run yet: it is kept out of the |
| 461 | 485 | // next pass and counted against the slots until the run shows. |
| 462 | 486 | const taken = new Set<string>(); |
| 463 | const did = await driveOnce(app, repo, now, taken); | |
| 487 | picked.set(repo.id, taken); | |
| 488 | const did = await driveOnce(app, repo, now, taken, picked); | |
| 464 | 489 | if (did !== "picked") return did; |
| 465 | 490 | for (let n = 1; n < Math.max(1, repo.autonomySlots); n++) { |
| 466 | const next = await driveOnce(app, getRepoById(app.db, repo.id) ?? repo, now, taken); | |
| 491 | const next = await driveOnce(app, getRepoById(app.db, repo.id) ?? repo, now, taken, picked); | |
| 467 | 492 | if (next !== "picked") break; |
| 468 | 493 | } |
| 469 | 494 | return did; |
| @@ -474,6 +499,7 @@ async function driveOnce( | ||
| 474 | 499 | repo: Repo, |
| 475 | 500 | now: Date, |
| 476 | 501 | taken: Set<string>, |
| 502 | picked: TickPicks, | |
| 477 | 503 | ): Promise<Drove> { |
| 478 | 504 | if (repo.autonomy === "off") return "off"; |
| 479 | 505 | tickBets(app, repo, now); |
| @@ -490,7 +516,7 @@ async function driveOnce( | ||
| 490 | 516 | const pending = [...taken].filter((id) => !running.has(id)).length; |
| 491 | 517 | // Slots follow the subscriptions' remaining windows, never past the |
| 492 | 518 | // operator's; at a wall nothing new starts until the reset. |
| 493 | const paced = repoPace(app, repo, now); | |
| 519 | const paced = repoPace(app, repo, now, true, picked); | |
| 494 | 520 | if (paced.slots === 0 && slotted.length === 0) return "waiting"; |
| 495 | 521 | // The files the open pulls touch, read before the sort: a build that |
| 496 | 522 | // names a hot one waits for its pile to drain (#5934). |
| @@ -1271,6 +1297,15 @@ export function ensureDirectionIssue( | ||
| 1271 | 1297 | return opened.value; |
| 1272 | 1298 | } |
| 1273 | 1299 | |
| 1300 | /** | |
| 1301 | * Repositories with a backed bet go first, so a short week budget (#8989) is | |
| 1302 | * spent on what the owner backed before the rest; the order is otherwise kept. | |
| 1303 | */ | |
| 1304 | export function byBackedBets(db: App["db"], repos: Repo[]): Repo[] { | |
| 1305 | const backed = (r: Repo): number => (betsInUse(db, r.id) ? 0 : 1); | |
| 1306 | return [...repos].sort((a, b) => backed(a) - backed(b)); | |
| 1307 | } | |
| 1308 | ||
| 1274 | 1309 | /** Every repository that runs on its own, one turn each. */ |
| 1275 | 1310 | export async function autonomyTick( |
| 1276 | 1311 | app: App, |
| @@ -1290,7 +1325,8 @@ export async function autonomyTick( | ||
| 1290 | 1325 | } catch (error) { |
| 1291 | 1326 | log.error("trust note failed", { error: String(error) }); |
| 1292 | 1327 | } |
| 1293 | for (const repo of listAutonomousRepos(app.db)) { | |
| 1328 | const picked = new Map<string, Set<string>>(); | |
| 1329 | for (const repo of byBackedBets(app.db, listAutonomousRepos(app.db))) { | |
| 1294 | 1330 | // An archived repository is put away: the crew starts nothing in it. |
| 1295 | 1331 | if (repo.archivedAt !== null) continue; |
| 1296 | 1332 | try { |
| @@ -1341,7 +1377,7 @@ export async function autonomyTick( | ||
| 1341 | 1377 | const requested = await moveRequested(app, current, freeSlots(app, current, now), now); |
| 1342 | 1378 | await breathe(); |
| 1343 | 1379 | const unstuck = unstickNow(app, current, now); |
| 1344 | const did = await driveRepo(app, current, now); | |
| 1380 | const did = await driveRepo(app, current, now, picked); | |
| 1345 | 1381 | await breathe(); |
| 1346 | 1382 | // Rules only, no run: an issue noted already done on the base closes. |
| 1347 | 1383 | const triaged = await liveTriage(app, current, now).catch((error) => { |
src/crew/ceiling_stagger.ts added +56
| @@ -0,0 +1,56 @@ | ||
| 1 | /** | |
| 2 | * The daily ceilings reset together at 00:00 UTC. A repository that hit its | |
| 3 | * ceiling yesterday starts again at its own offset into the day, spread over | |
| 4 | * `SPREAD_MS` by a hash of its id, so the reset is not a burst (#8989). | |
| 5 | * Pure but for one read of yesterday's spend. | |
| 6 | */ | |
| 7 | ||
| 8 | import type { Database } from "bun:sqlite"; | |
| 9 | ||
| 10 | /** How far into the day the offsets reach. */ | |
| 11 | export const SPREAD_MS = 6 * 3_600_000; | |
| 12 | ||
| 13 | const DAY_MS = 86_400_000; | |
| 14 | ||
| 15 | /** A stable offset in [0, SPREAD_MS) from the repository's id. */ | |
| 16 | export function offsetOf(repoId: string): number { | |
| 17 | let h = 2166136261; | |
| 18 | for (let i = 0; i < repoId.length; i++) h = Math.imul(h ^ repoId.charCodeAt(i), 16777619) >>> 0; | |
| 19 | return h % SPREAD_MS; | |
| 20 | } | |
| 21 | ||
| 22 | interface Memo { | |
| 23 | readonly nowMs: number; | |
| 24 | readonly ceiling: number; | |
| 25 | readonly until: Date | null; | |
| 26 | } | |
| 27 | ||
| 28 | /** The answer per repository for the instant it was asked: `repoPace` asks many times a tick, with one `now`. */ | |
| 29 | const memo = new Map<string, Memo>(); | |
| 30 | ||
| 31 | /** Forget what was read; for tests that change yesterday's spend under one `now`. */ | |
| 32 | export function forgetStagger(): void { | |
| 33 | memo.clear(); | |
| 34 | } | |
| 35 | ||
| 36 | /** | |
| 37 | * When this repository may start again today, or null: it did not hit its | |
| 38 | * ceiling yesterday, or its time has come. A ceiling of 0 allows no spend at | |
| 39 | * all and is held by the ceiling itself, so it is never "hit yesterday" here. | |
| 40 | */ | |
| 41 | export function staggeredUntil(db: Database, repo: { id: string; autonomyCeilingUsd: number }, now: Date): Date | null { | |
| 42 | const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); | |
| 43 | const at = today + offsetOf(repo.id); | |
| 44 | if (now.getTime() >= at || !(repo.autonomyCeilingUsd > 0)) return null; | |
| 45 | const seen = memo.get(repo.id); | |
| 46 | if (seen !== undefined && seen.nowMs === now.getTime() && seen.ceiling === repo.autonomyCeilingUsd) return seen.until; | |
| 47 | const spent = | |
| 48 | db | |
| 49 | .query<{ s: number | null }, [string, string, string]>( | |
| 50 | `SELECT SUM(cost_usd) AS s FROM run WHERE repo_id = ?1 AND kind = 'agent' AND created_at >= ?2 AND created_at < ?3`, | |
| 51 | ) | |
| 52 | .get(repo.id, new Date(today - DAY_MS).toISOString(), new Date(today).toISOString())?.s ?? 0; | |
| 53 | const until = spent >= repo.autonomyCeilingUsd ? new Date(at) : null; | |
| 54 | memo.set(repo.id, { nowMs: now.getTime(), ceiling: repo.autonomyCeilingUsd, until }); | |
| 55 | return until; | |
| 56 | } |
src/crew/week_read.ts +27 −3
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | import type { App } from "../app.ts"; |
| 7 | 7 | import { listAutonomousRepos, type Repo } from "../core/repos.ts"; |
| 8 | 8 | import { standings, WINDOW_MS } from "../core/standing.ts"; |
| 9 | import { capacityCount } from "../core/capacity.ts"; | |
| 9 | 10 | import { grantedAccounts, PLATFORM } from "../core/subscriptions.ts"; |
| 10 | 11 | import { burnFrom, noteReading, readingsSince } from "./week_burn.ts"; |
| 11 | 12 | import { paceFloor } from "../core/capacity.ts"; |
| @@ -14,6 +15,8 @@ import { efficiencyWords, WEEK, weekDecision, type Efficiency, type WeekDecision | ||
| 14 | 15 | /** The week as the provider's meter reads it: a meter, or null while no account's limit is known. */ |
| 15 | 16 | interface Week { |
| 16 | 17 | readonly meter: WeekMeter | null; |
| 18 | /** The accounts the meter reads, as one key: repositories drawing on the same accounts share one week. */ | |
| 19 | readonly scope: string; | |
| 17 | 20 | } |
| 18 | 21 | |
| 19 | 22 | /** How long a repository's week is remembered: `repoPace` is asked many times a tick. */ |
| @@ -57,7 +60,7 @@ function readWeek(app: App, repo: Repo, now: Date): Week | null { | ||
| 57 | 60 | } |
| 58 | 61 | if (resetsAt === null || !(capUsd > 0)) { |
| 59 | 62 | // Granted accounts with no known limit are an unknown week; the forge's own token with none is not metered at all. |
| 60 | return granted.length > 0 ? { meter: null } : null; | |
| 63 | return granted.length > 0 ? { meter: null, scope: [...granted].sort().join(",") } : null; | |
| 61 | 64 | } |
| 62 | 65 | // The burn is read off the readings of the meter, not off dollars: the |
| 63 | 66 | // week's free share is stored as it is read and the burn is how fast it fell. |
| @@ -67,6 +70,7 @@ function readWeek(app: App, repo: Repo, now: Date): Week | null { | ||
| 67 | 70 | noteReading(app.db, scope, now.getTime(), freePct, WINDOW_MS.week); |
| 68 | 71 | const readings = readingsSince(app.db, scope, opened); |
| 69 | 72 | return { |
| 73 | scope, | |
| 70 | 74 | meter: { |
| 71 | 75 | freePct, |
| 72 | 76 | keptPct: (keptUsd / capUsd) * 100, |
| @@ -118,6 +122,26 @@ export function weekFor( | ||
| 118 | 122 | return week === null ? null : weekDecision({ ...week, now, ceiling: repo.autonomySlots, floorSlots: paceFloor(app.db), ...seen }); |
| 119 | 123 | } |
| 120 | 124 | |
| 125 | /** The key of the accounts a repository draws its week from; null while no account is metered. */ | |
| 126 | export function weekScope(app: App, repo: Repo, now: Date): string | null { | |
| 127 | return weekOf(app, repo, now)?.scope ?? null; | |
| 128 | } | |
| 129 | ||
| 130 | /** The slots the whole crew may run at once: what `crew.concurrency` says, else the repository's own. */ | |
| 131 | export function crewCeiling(app: App, repo: Repo): number { | |
| 132 | return capacityCount(app.db, "crew.concurrency", repo.autonomySlots); | |
| 133 | } | |
| 134 | ||
| 135 | /** | |
| 136 | * The week's decision for the whole crew drawing on this repository's | |
| 137 | * accounts: one budget in slots across every repository, its ceiling the | |
| 138 | * crew's (`crew.concurrency`), its floor the crew's too (#8989). | |
| 139 | */ | |
| 140 | export function crewWeekFor(app: App, repo: Repo, now: Date): WeekDecision | null { | |
| 141 | const week = weekOf(app, repo, now); | |
| 142 | return week === null ? null : weekDecision({ ...week, now, ceiling: crewCeiling(app, repo), floorSlots: paceFloor(app.db), idleSlots: 0, startable: 0, gated: 0 }); | |
| 143 | } | |
| 144 | ||
| 121 | 145 | /** |
| 122 | 146 | * The pace sentence and the efficiency beside it for the autonomous |
| 123 | 147 | * repository the crew works most this week (the most agent runs; the first by |
| @@ -129,10 +153,10 @@ export function weekWords(app: App, now: Date = new Date()): string | null { | ||
| 129 | 153 | let best: { words: string; runs: number } | null = null; |
| 130 | 154 | for (const repo of listAutonomousRepos(app.db)) { |
| 131 | 155 | if (repo.autonomy !== "autonomous" || repo.archivedAt !== null) continue; |
| 132 | const week = weekFor(app, repo, now); | |
| 156 | const week = crewWeekFor(app, repo, now); | |
| 133 | 157 | if (week === null) continue; |
| 134 | 158 | const efficiency = weekEfficiency(app, repo, now); |
| 135 | if (best === null || efficiency.runs > best.runs) best = { words: `${week.sentence} ${efficiencyWords(efficiency)}`, runs: efficiency.runs }; | |
| 159 | if (best === null || efficiency.runs > best.runs) best = { words: `${week.sentence} The week allows the whole crew ${week.slots} of ${crewCeiling(app, repo)} slots across every repository. ${efficiencyWords(efficiency)}`, runs: efficiency.runs }; | |
| 136 | 160 | } |
| 137 | 161 | return best?.words ?? null; |
| 138 | 162 | } |
test/crew-week.test.ts added +196
| @@ -0,0 +1,196 @@ | ||
| 1 | /** The week is one budget for the whole crew, not one per repository (#8989). */ | |
| 2 | import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 7 | import { createRepo, updateRepoSettings } from "../src/actions/repos.ts"; | |
| 8 | import { createApp, type App } from "../src/app.ts"; | |
| 9 | import { setCapacity } from "../src/core/capacity.ts"; | |
| 10 | import { getRepoById, type Repo } from "../src/core/repos.ts"; | |
| 11 | import { createRun } from "../src/core/runs.ts"; | |
| 12 | import { noteLimit, setHeadroom } from "../src/core/standing.ts"; | |
| 13 | import { addSubscription } from "../src/core/subscriptions.ts"; | |
| 14 | import { recordUsage } from "../src/core/usage.ts"; | |
| 15 | import { getUserByName, type User } from "../src/core/users.ts"; | |
| 16 | import { insertBet, markBacked } from "../src/core/bets.ts"; | |
| 17 | import { byBackedBets, repoPace } from "../src/crew/autonomy.ts"; | |
| 18 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 19 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 20 | import { settleApp } from "./helpers/tui.ts"; | |
| 21 | import { forgetStagger, offsetOf, SPREAD_MS, staggeredUntil } from "../src/crew/ceiling_stagger.ts"; | |
| 22 | import { forgetWeek, weekWords } from "../src/crew/week_read.ts"; | |
| 23 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 24 | import { quietLogs } from "./helpers/globals.ts"; | |
| 25 | ||
| 26 | quietLogs(); | |
| 27 | ||
| 28 | const DAY = 86_400_000; | |
| 29 | const NOW = new Date(Date.UTC(2026, 10, 3, 12, 0, 0)); | |
| 30 | let dir: string; | |
| 31 | let app: App; | |
| 32 | let alice: User; | |
| 33 | let repos: Repo[] = []; | |
| 34 | let sub: string; | |
| 35 | ||
| 36 | function spend(repo: Repo, usd: number, at: Date): void { | |
| 37 | const run = createRun(app.db, { repoId: repo.id, kind: "agent" }); | |
| 38 | app.db.query("UPDATE run SET state = 'ok' WHERE id = ?1").run(run.id); | |
| 39 | recordUsage(app.db, { runId: run.id, repoId: repo.id, issueId: null, agentId: null, subscriptionId: sub, tokensIn: 1, tokensOut: 1, costUsd: usd, wallMs: 1000, outcome: "ok", at: at.toISOString() }); | |
| 40 | } | |
| 41 | ||
| 42 | /** The measured readings: 44% free an hour ago, 42% now (about 2 points in the hour, against about 0.4 allowed). */ | |
| 43 | function measured(): void { | |
| 44 | app.db.query("DELETE FROM usage").run(); | |
| 45 | app.db.query("DELETE FROM week_reading").run(); | |
| 46 | spend(repos[0]!, 100, new Date(NOW.getTime() - 4 * DAY)); | |
| 47 | noteLimit(app.db, sub, { window: "week", resetsAt: new Date(NOW.getTime() - 3.5 * DAY).toISOString(), observedAt: new Date(NOW.getTime() - 3.5 * DAY - 3_600_000).toISOString(), said: "weekly limit" }); | |
| 48 | spend(repos[0]!, 56, new Date(NOW.getTime() - DAY)); | |
| 49 | spend(repos[0]!, 2, new Date(NOW.getTime() - 3_600_000)); | |
| 50 | app.db.query("INSERT INTO week_reading (scope, at, free_pct) VALUES (?1, ?2, 44)").run(sub, new Date(NOW.getTime() - 3_600_000).toISOString()); | |
| 51 | forgetWeek(); | |
| 52 | } | |
| 53 | ||
| 54 | function running(repo: Repo, n: number): void { | |
| 55 | const bot = getUserByName(app.db, "builderbot")!; | |
| 56 | for (let i = 0; i < n; i++) { | |
| 57 | const r = createRun(app.db, { repoId: repo.id, kind: "agent", actorId: bot.id }); | |
| 58 | app.db.query("UPDATE run SET state = 'running' WHERE id = ?1").run(r.id); | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | beforeAll(async () => { | |
| 63 | dir = await mkdtemp(join(tmpdir(), "forge-crew-week-")); | |
| 64 | app = createApp({ dataDir: dir, port: 0 }); | |
| 65 | const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-passphrase" }); | |
| 66 | if (!a.ok) throw new Error("signup failed"); | |
| 67 | alice = a.value; | |
| 68 | const bot = await createAgent(app, alice, { username: "builderbot" }); | |
| 69 | if (!bot.ok) throw new Error(String(bot.error)); | |
| 70 | const added = addSubscription(app.db, dir, { name: "main", token: "sk-ant-oat01-crew-week-1" }); | |
| 71 | if (!added.ok) throw new Error(added.error); | |
| 72 | sub = added.value.id; | |
| 73 | setHeadroom(app.db, sub, 0); | |
| 74 | for (const name of ["a", "b", "c", "d"]) { | |
| 75 | const made = await createRepo(app, alice, { name, visibility: "private" }); | |
| 76 | if (!made.ok) throw new Error(String(made.error)); | |
| 77 | expect(addToCrew(app, alice, made.value, "builderbot").ok).toBe(true); | |
| 78 | const on = await updateRepoSettings(app, alice, made.value, { autonomy: "autonomous", autonomyDirection: "Ship it.", autonomyCeilingUsd: 1000, autonomySlots: 5, autonomyBy: alice.id }); | |
| 79 | expect(on.ok).toBe(true); | |
| 80 | app.db.query("INSERT INTO subscription_grant (scope_id, agent_id, subscription_id) VALUES (?1, '', ?2)").run(made.value.id, sub); | |
| 81 | repos.push(getRepoById(app.db, made.value.id)!); | |
| 82 | } | |
| 83 | setCapacity(app.db, "crew.concurrency", "10", alice.id); | |
| 84 | }); | |
| 85 | ||
| 86 | beforeEach(() => { | |
| 87 | app.db.query("DELETE FROM run WHERE state IN ('queued','running')").run(); | |
| 88 | }); | |
| 89 | ||
| 90 | afterAll(async () => { | |
| 91 | app.close(); | |
| 92 | await rm(dir, { recursive: true, force: true }); | |
| 93 | }); | |
| 94 | ||
| 95 | test("the readings of 2026-11-03 slow the total: the whole crew holds at the floor, not each repository", () => { | |
| 96 | measured(); | |
| 97 | const slots = repos.map((r) => repoPace(app, r, NOW).slots); | |
| 98 | // Nothing running: each repository may start, but only to the crew's budget between them. | |
| 99 | expect(Math.max(...slots)).toBe(2); | |
| 100 | running(repos[0]!, 2); | |
| 101 | const after = repos.map((r) => repoPace(app, r, NOW).slots); | |
| 102 | expect(after).toEqual([2, 0, 0, 0]); | |
| 103 | expect(repoPace(app, repos[1]!, NOW).reason).toContain("whole crew has 2"); | |
| 104 | }); | |
| 105 | ||
| 106 | test("a repository's own slots stay limits inside the crew's budget", () => { | |
| 107 | measured(); | |
| 108 | app.db.query("UPDATE repo SET autonomy_slots = 1 WHERE id = ?1").run(repos[2]!.id); | |
| 109 | expect(repoPace(app, getRepoById(app.db, repos[2]!.id)!, NOW).slots).toBe(1); | |
| 110 | app.db.query("UPDATE repo SET autonomy_slots = 5 WHERE id = ?1").run(repos[2]!.id); | |
| 111 | }); | |
| 112 | ||
| 113 | test("finishing work is not held by the crew's budget, only new picks are", () => { | |
| 114 | measured(); | |
| 115 | running(repos[0]!, 2); | |
| 116 | expect(repoPace(app, repos[1]!, NOW).slots).toBe(0); | |
| 117 | expect(repoPace(app, repos[1]!, NOW, false).slots).toBeGreaterThan(0); | |
| 118 | }); | |
| 119 | ||
| 120 | test("home says the whole crew's number", () => { | |
| 121 | measured(); | |
| 122 | expect(weekWords(app, NOW)).toContain("The week allows the whole crew 2 of 10 slots across every repository."); | |
| 123 | }); | |
| 124 | ||
| 125 | test("a repository that hit its ceiling yesterday restarts at its own time, spread over the morning", () => { | |
| 126 | const midnight = new Date(Date.UTC(2026, 10, 3, 0, 5, 0)); | |
| 127 | for (const r of repos) { | |
| 128 | const run = createRun(app.db, { repoId: r.id, kind: "agent" }); | |
| 129 | app.db.query("UPDATE run SET cost_usd = 1000, created_at = ?2, state = 'ok' WHERE id = ?1").run(run.id, new Date(Date.UTC(2026, 10, 2, 20, 0, 0)).toISOString()); | |
| 130 | } | |
| 131 | const ready = repos.map((r) => staggeredUntil(app.db, r, midnight)?.getTime() ?? 0); | |
| 132 | expect(new Set(ready).size).toBeGreaterThan(1); | |
| 133 | for (const at of ready) expect(at).toBeLessThan(Date.UTC(2026, 10, 3) + SPREAD_MS); | |
| 134 | for (const r of repos) { | |
| 135 | expect(offsetOf(r.id)).toBeGreaterThanOrEqual(0); | |
| 136 | expect(offsetOf(r.id)).toBeLessThan(SPREAD_MS); | |
| 137 | } | |
| 138 | // FNV-1a of "a" is fixed, so the offsets cannot drift between releases and move every repository's restart. | |
| 139 | expect(offsetOf("a")).toBe(2_802_220); | |
| 140 | expect(repoPace(app, repos[0]!, midnight, true).slots).toBe(0); | |
| 141 | // Past the spread nothing waits. | |
| 142 | expect(staggeredUntil(app.db, repos[0]!, new Date(Date.UTC(2026, 10, 3, 7)))).toBeNull(); | |
| 143 | // A repository that did not hit its ceiling is never held. | |
| 144 | app.db.query("DELETE FROM run WHERE repo_id = ?1").run(repos[3]!.id); | |
| 145 | forgetStagger(); | |
| 146 | expect(staggeredUntil(app.db, repos[3]!, midnight)).toBeNull(); | |
| 147 | }); | |
| 148 | ||
| 149 | test("a ceiling of 0 allows no spend and is held by the ceiling, so it is never also held until an offset", () => { | |
| 150 | const midnight = new Date(Date.UTC(2026, 10, 3, 0, 5, 0)); | |
| 151 | forgetStagger(); | |
| 152 | expect(staggeredUntil(app.db, { id: repos[0]!.id, autonomyCeilingUsd: 0 }, midnight)).toBeNull(); | |
| 153 | }); | |
| 154 | ||
| 155 | test("the stagger reads yesterday's spend once for one instant", () => { | |
| 156 | const midnight = new Date(Date.UTC(2026, 10, 3, 0, 5, 0)); | |
| 157 | const run = createRun(app.db, { repoId: repos[3]!.id, kind: "agent" }); | |
| 158 | app.db.query("UPDATE run SET cost_usd = 1000, created_at = ?2, state = 'ok' WHERE id = ?1").run(run.id, new Date(Date.UTC(2026, 10, 2, 20, 0, 0)).toISOString()); | |
| 159 | forgetStagger(); | |
| 160 | const first = staggeredUntil(app.db, repos[3]!, midnight); | |
| 161 | expect(first).not.toBeNull(); | |
| 162 | app.db.query("DELETE FROM run WHERE id = ?1").run(run.id); | |
| 163 | expect(staggeredUntil(app.db, repos[3]!, midnight)).toEqual(first); | |
| 164 | forgetStagger(); | |
| 165 | expect(staggeredUntil(app.db, repos[3]!, midnight)).toBeNull(); | |
| 166 | }); | |
| 167 | ||
| 168 | test("picks made earlier in this tick in other repositories count against the crew's budget", () => { | |
| 169 | app.db.query("DELETE FROM run").run(); | |
| 170 | measured(); | |
| 171 | const picked = new Map<string, Set<string>>([[repos[0]!.id, new Set(["issue-x", "issue-y"])]]); | |
| 172 | expect(repoPace(app, repos[1]!, NOW).slots).toBe(2); | |
| 173 | expect(repoPace(app, repos[1]!, NOW, true, picked).slots).toBe(0); | |
| 174 | // Finishing work is not held by them. | |
| 175 | expect(repoPace(app, repos[1]!, NOW, false, picked).slots).toBeGreaterThan(0); | |
| 176 | }); | |
| 177 | ||
| 178 | test("repositories with a backed bet are driven first, the rest in their order", () => { | |
| 179 | const bet = insertBet(app.db, { repoId: repos[3]!.id, name: "Search", journeys: "", appetite: 5, leadId: null, issueIds: [] }, NOW); | |
| 180 | markBacked(app.db, bet.id, 5, "owner", NOW); | |
| 181 | const order = byBackedBets(app.db, repos).map((r) => r.name); | |
| 182 | expect(order).toEqual(["d", "a", "b", "c"]); | |
| 183 | app.db.query("DELETE FROM bet WHERE id = ?1").run(bet.id); | |
| 184 | expect(byBackedBets(app.db, repos).map((r) => r.name)).toEqual(["a", "b", "c", "d"]); | |
| 185 | }); | |
| 186 | ||
| 187 | test("the terminal's home prints the crew's number", async () => { | |
| 188 | measured(); | |
| 189 | const terminal = headlessTerminal(300, 32); | |
| 190 | const { app: tui } = createForgeApp(app, { sshUsername: "alice", remoteAddress: "127.0.0.1", fingerprint: "SHA256:t", user: alice }, { terminal, exitHooks: false }); | |
| 191 | tui.start(); | |
| 192 | await settleApp(tui); | |
| 193 | const text = tui.screen.toText(); | |
| 194 | tui.stop(); | |
| 195 | expect(text).toContain("whole crew"); | |
| 196 | }); |
new above
offline
A ceiling of 0 holds the repo every morning until its offset. Define that case. Also cache this query: repoPace calls it many times a tick.