Public: anyone can read this; the crew can change it.
The forge notices by itself when the week's allowance comes back early, and nothing about a release depends on a laptop #14334
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14254-the-forge-notices-by-itself-when-the-wee → main 6 commits fork
14 files
changed +401
−8
docs/features/meter-watch.md added +11
| @@ -0,0 +1,11 @@ | ||
| 1 | --- | |
| 2 | topic: What a forge is | |
| 3 | row: The meter watch: an early reset resumes the crew | |
| 4 | --- | |
| 5 | # The meter watch | |
| 6 | ||
| 7 | While a Claude account's window is held full, the forge reads the provider's usage meter (`api.anthropic.com/api/oauth/usage`, a status read, no model run) every 15 minutes. A window shown below 100% is lifted at once; the account wakes and its own paused runs go (other accounts' stay paused) only when none of its windows is still held full, and one log line says so. The response shape and the scale of `utilization` (a percent) are unconfirmed: the provider documents neither, so a value that cannot be a percent, or a body over 64 KiB, counts as an unreadable read. A refused read backs that account off to an hour. Nothing is read while no window is held full. | |
| 8 | ||
| 9 | `system` with `{ kind: "footing" }` answers `outside`: what still needs a hand outside the forge (today, the release keeper being off); empty when nothing does. The same list shows on /system under "What the forge stands on" and on the terminal's footing screen. /releases says "Nothing releases it unattended" while it is. | |
| 10 | ||
| 11 | Not done: releases cut and promoted by the forge itself; only the honest line is. |
scripts/unused-exports-allow.json −2
| @@ -163,7 +163,6 @@ | ||
| 163 | 163 | "src/spawn/clients.ts monthsSettled", |
| 164 | 164 | "src/spawn/clients.ts useChildFetch", |
| 165 | 165 | "src/spawn/door.ts usePlacement", |
| 166 | "src/spawn/door.ts useProbe", | |
| 167 | 166 | "src/spawn/host.ts provideCellsForTest", |
| 168 | 167 | "src/spawn/included_credit.ts worstCaseDailyUsd", |
| 169 | 168 | "src/spawn/parent_credential.ts keepParentCredential", |
| @@ -192,7 +191,6 @@ | ||
| 192 | 191 | "src/tui/ui/editor.ts expandLine", |
| 193 | 192 | "src/tui/ui/icons.ts iconNames", |
| 194 | 193 | "src/tui/ui/image.ts clearImage", |
| 195 | "src/tui/ui/markdown.ts lineText", | |
| 196 | 194 | "src/web/chrome.ts stickyAction", |
| 197 | 195 | "src/web/code_rows_pool.ts workersForTest", |
| 198 | 196 | "src/web/diff-cache.ts clearDiffCache", |
src/crew/meter_watch.ts added +172
| @@ -0,0 +1,172 @@ | ||
| 1 | /** | |
| 2 | * The provider's meter, read while the crew waits (#14254). A window noted | |
| 3 | * full stays full in our books until its recorded reset; if the provider | |
| 4 | * resets early nothing would notice, since no run is made to be refused. | |
| 5 | * Every 15 minutes, for each account whose window we hold full, one cheap | |
| 6 | * read of the provider's usage endpoint (no model run). A window it shows | |
| 7 | * below 100% is lifted at once: its note is closed, the account woken and | |
| 8 | * the paused runs released, and one line says so. A refused read (or one | |
| 9 | * that fails) backs that account off to an hour, so the probe never nags. | |
| 10 | */ | |
| 11 | ||
| 12 | import type { Database } from "bun:sqlite"; | |
| 13 | import type { LimitWindow } from "../core/standing.ts"; | |
| 14 | import { PLATFORM, subscriptionAad, wakeSubscription } from "../core/subscriptions.ts"; | |
| 15 | import { open } from "../kernel/secret.ts"; | |
| 16 | import { platformToken } from "./runner_env.ts"; | |
| 17 | ||
| 18 | export const METER_EVERY_MS = 15 * 60_000; | |
| 19 | export const METER_BACKOFF_MS = 60 * 60_000; | |
| 20 | export const METER_URL = "https://api.anthropic.com/api/oauth/usage"; | |
| 21 | const MAX_BACKOFFS = 64; | |
| 22 | const MAX_BODY_BYTES = 64 * 1024; | |
| 23 | ||
| 24 | export type Reading = Readonly<Partial<Record<LimitWindow, number>>>; | |
| 25 | ||
| 26 | /** | |
| 27 | * Percent used per window from the provider's body; anything unreadable is left out, and so is | |
| 28 | * a value that cannot be a percent (negative, over 100). The shape and scale are as the provider's | |
| 29 | * client reports them, not a documented contract: unconfirmed. | |
| 30 | */ | |
| 31 | export function parseUsage(body: unknown): Reading { | |
| 32 | const out: Partial<Record<LimitWindow, number>> = {}; | |
| 33 | if (typeof body !== "object" || body === null) return out; | |
| 34 | const b = body as Record<string, unknown>; | |
| 35 | for (const [key, window] of [["five_hour", "5h"], ["seven_day", "week"]] as const) { | |
| 36 | const w = b[key]; | |
| 37 | if (typeof w !== "object" || w === null) continue; | |
| 38 | const u = (w as Record<string, unknown>)["utilization"]; | |
| 39 | if (typeof u === "number" && Number.isFinite(u) && u >= 0 && u <= 100) out[window] = u; | |
| 40 | } | |
| 41 | return out; | |
| 42 | } | |
| 43 | ||
| 44 | export type Fetch = (url: string, init: RequestInit) => Promise<Response>; | |
| 45 | ||
| 46 | /** The body as text, or undefined when it runs past `max` bytes: never more than `max` is held. */ | |
| 47 | async function boundedText(res: Response, max: number): Promise<string | undefined> { | |
| 48 | if (res.body === null) return ""; | |
| 49 | const reader = res.body.getReader(); | |
| 50 | const chunks: Uint8Array[] = []; | |
| 51 | let size = 0; | |
| 52 | for (;;) { | |
| 53 | const { done, value } = await reader.read(); | |
| 54 | if (done) break; | |
| 55 | size += value.byteLength; | |
| 56 | if (size > max) { | |
| 57 | await reader.cancel(); | |
| 58 | return undefined; | |
| 59 | } | |
| 60 | chunks.push(value); | |
| 61 | } | |
| 62 | return Buffer.concat(chunks).toString("utf8"); | |
| 63 | } | |
| 64 | ||
| 65 | /** One read of the meter with `token`; undefined when the provider refuses or does not answer. */ | |
| 66 | export async function readMeter(token: string, fetcher: Fetch = fetch): Promise<Reading | undefined> { | |
| 67 | try { | |
| 68 | const res = await fetcher(METER_URL, { | |
| 69 | headers: { authorization: `Bearer ${token}`, "anthropic-beta": "oauth-2025-04-20", accept: "application/json" }, | |
| 70 | signal: AbortSignal.timeout(10_000), | |
| 71 | }); | |
| 72 | if (!res.ok) return undefined; | |
| 73 | const text = await boundedText(res, MAX_BODY_BYTES); | |
| 74 | if (text === undefined) return undefined; | |
| 75 | const read = parseUsage(JSON.parse(text)); | |
| 76 | return Object.keys(read).length === 0 ? undefined : read; | |
| 77 | } catch { | |
| 78 | return undefined; | |
| 79 | } | |
| 80 | } | |
| 81 | ||
| 82 | interface Held { | |
| 83 | readonly id: string; | |
| 84 | readonly name: string; | |
| 85 | readonly window: LimitWindow; | |
| 86 | } | |
| 87 | ||
| 88 | /** Windows we hold full: a read reset still ahead, on an account that is not API-key billed. */ | |
| 89 | export function heldFull(db: Database, now: Date): Held[] { | |
| 90 | return db | |
| 91 | .query<{ id: string; name: string | null; span: string }, [string]>( | |
| 92 | `SELECT l.subscription_id AS id, s.name AS name, l.span AS span | |
| 93 | FROM subscription_limit l LEFT JOIN subscription s ON s.id = l.subscription_id | |
| 94 | WHERE l.resets_at > ?1 AND (s.id IS NULL OR s.kind = 'oauth') ORDER BY l.subscription_id, l.span`, | |
| 95 | ) | |
| 96 | .all(now.toISOString()) | |
| 97 | .map((r) => ({ id: r.id, name: r.name ?? "the forge's own account", window: r.span as LimitWindow })); | |
| 98 | } | |
| 99 | ||
| 100 | /** Closes one noted window at `now`; waking the account is the caller's, once none is held. */ | |
| 101 | export function lift(db: Database, held: Held, now: Date): void { | |
| 102 | db.query("UPDATE subscription_limit SET resets_at = ?3, observed_at = ?3 WHERE subscription_id = ?1 AND span = ?2").run( | |
| 103 | held.id, | |
| 104 | held.window, | |
| 105 | now.toISOString(), | |
| 106 | ); | |
| 107 | } | |
| 108 | ||
| 109 | /** Queued agent runs paused on `account` are let go now; other accounts' runs stay paused. */ | |
| 110 | export function releasePaused(db: Database, account: string, now: Date): number { | |
| 111 | return db | |
| 112 | .query( | |
| 113 | `UPDATE run SET not_before = NULL | |
| 114 | WHERE kind = 'agent' AND state = 'queued' AND not_before > ?1 AND summary LIKE 'paused%' | |
| 115 | AND (subscription_id = ?2 OR (?2 = '${PLATFORM}' AND subscription_id IS NULL))`, | |
| 116 | ) | |
| 117 | .run(now.toISOString(), account).changes; | |
| 118 | } | |
| 119 | ||
| 120 | function tokenOf(db: Database, dataDir: string, id: string, env: Record<string, string | undefined>): string | undefined { | |
| 121 | if (id === PLATFORM) return platformToken(env); | |
| 122 | const row = db.query<{ token_enc: string }, [string]>("SELECT token_enc FROM subscription WHERE id = ?1").get(id); | |
| 123 | if (row === null) return undefined; | |
| 124 | const opened = open(dataDir, row.token_enc, subscriptionAad(id)); | |
| 125 | return opened.ok ? opened.value : undefined; | |
| 126 | } | |
| 127 | ||
| 128 | export type MeterWatch = ( | |
| 129 | db: Database, | |
| 130 | dataDir: string, | |
| 131 | opts?: { now?: Date; fetcher?: Fetch; env?: Record<string, string | undefined> }, | |
| 132 | ) => Promise<string[]>; | |
| 133 | ||
| 134 | /** | |
| 135 | * One look. Returns the lines said (one per account that came back). Reads | |
| 136 | * nothing when no window is held full. An account wakes, and its paused runs | |
| 137 | * go, only when none of its windows is still held full. | |
| 138 | * A watcher that remembers, in its own closure, which accounts are backed off. | |
| 139 | */ | |
| 140 | export function newMeterWatch(): MeterWatch { | |
| 141 | const backoff = new Map<string, number>(); | |
| 142 | return async (db, dataDir, opts = {}) => { | |
| 143 | const now = opts.now ?? new Date(); | |
| 144 | const held = heldFull(db, now); | |
| 145 | const lines: string[] = []; | |
| 146 | const byAccount = new Map<string, Held[]>(); | |
| 147 | for (const h of held) byAccount.set(h.id, [...(byAccount.get(h.id) ?? []), h]); | |
| 148 | for (const [id, windows] of byAccount) { | |
| 149 | if ((backoff.get(id) ?? 0) > now.getTime()) continue; | |
| 150 | const token = tokenOf(db, dataDir, id, opts.env ?? process.env); | |
| 151 | const read = token === undefined ? undefined : await readMeter(token, opts.fetcher); | |
| 152 | if (read === undefined) { | |
| 153 | if (backoff.size >= MAX_BACKOFFS) backoff.clear(); | |
| 154 | backoff.set(id, now.getTime() + METER_BACKOFF_MS); | |
| 155 | continue; | |
| 156 | } | |
| 157 | backoff.delete(id); | |
| 158 | const back = windows.filter((w) => (read[w.window] ?? 100) < 100); | |
| 159 | for (const w of back) lift(db, w, now); | |
| 160 | if (back.length === 0) continue; | |
| 161 | if (!heldFull(db, now).some((h) => h.id === id)) { | |
| 162 | if (id !== PLATFORM) wakeSubscription(db, id); | |
| 163 | releasePaused(db, id, now); | |
| 164 | } | |
| 165 | const w = back[0]!; | |
| 166 | lines.push( | |
| 167 | `${w.name}'s ${w.window === "week" ? "weekly" : "5-hour"} window has room again (${Math.round(read[w.window]!)}% used): the crew resumes`, | |
| 168 | ); | |
| 169 | } | |
| 170 | return lines; | |
| 171 | }; | |
| 172 | } |
src/crew/runner_env.ts +3
| @@ -88,6 +88,9 @@ export function runnerCredentialInEnv(env: Env): boolean { | ||
| 88 | 88 | return (runnerSettings(env).configDir ?? "").trim() !== ""; |
| 89 | 89 | } |
| 90 | 90 | |
| 91 | /** The platform account's own subscription token, when the process holds one. */ | |
| 92 | export const platformToken = (env: Env): string | undefined => (env["CLAUDE_CODE_OAUTH_TOKEN"] ?? "").trim() || undefined; | |
| 93 | ||
| 91 | 94 | /** Points a run at a parent's door, paying with the credit token. */ |
| 92 | 95 | export function lentEnv(door: string, token: string): Record<string, string> { |
| 93 | 96 | return { ANTHROPIC_BASE_URL: door, ANTHROPIC_API_KEY: token }; |
src/flow/alarm_main_ahead.ts +9 −3
| @@ -10,7 +10,7 @@ | ||
| 10 | 10 | |
| 11 | 11 | import type { Database } from "bun:sqlite"; |
| 12 | 12 | import { getRepo } from "../core/repos.ts"; |
| 13 | import { neverTickedLine } from "../keeper/at_boot.ts"; | |
| 13 | import { neverTickedLine, outsideDependencies } from "../keeper/at_boot.ts"; | |
| 14 | 14 | import { agoWords, lastBeat } from "../keeper/beat.ts"; |
| 15 | 15 | import { ownRepoPath } from "../kernel/own_repo.ts"; |
| 16 | 16 | import { running } from "../kernel/version.ts"; |
| @@ -95,8 +95,14 @@ export function mainAheadAlarm(db: Database, now: Date): MainAheadAlarm { | ||
| 95 | 95 | }; |
| 96 | 96 | } |
| 97 | 97 | |
| 98 | /** Main's state against production as one sentence for /releases and the terminal; undefined when main is not ahead and green. */ | |
| 98 | /** | |
| 99 | * Main's state against production as one sentence for /releases and the terminal; undefined when | |
| 100 | * main is not ahead and green. With the release keeper off it adds "Nothing releases it | |
| 101 | * unattended." | |
| 102 | */ | |
| 99 | 103 | export function aheadLine(db: Database, now: Date): string | undefined { |
| 100 | 104 | const a = mainAheadAlarm(db, now); |
| 101 | return a.fired || a.since !== undefined ? a.says : undefined; | |
| 105 | if (!a.fired && a.since === undefined) return undefined; | |
| 106 | // The keeper off: say plainly that no one releases main unattended (#14254). | |
| 107 | return outsideDependencies().length === 0 ? a.says : `${a.says} Nothing releases it unattended.`; | |
| 102 | 108 | } |
src/http/mcp/tools/system.ts +3 −2
Diff not shown — a line 2,879 characters wide (limit 2,000). +3 −2
View filesrc/keeper/at_boot.ts +9
| @@ -24,3 +24,12 @@ export function neverTickedLine(): string | undefined { | ||
| 24 | 24 | if (atBoot === undefined) return undefined; |
| 25 | 25 | return atBoot.on ? "the release keeper is on but has not ticked" : `the in-process release keeper is off: ${atBoot.why}`; |
| 26 | 26 | } |
| 27 | ||
| 28 | /** | |
| 29 | * What still needs a hand outside the forge, one sentence each; empty when nothing does (#14254). | |
| 30 | * Today only the release keeper being off is knowable from inside; the list says so rather than | |
| 31 | * claim more. | |
| 32 | */ | |
| 33 | export function outsideDependencies(): string[] { | |
| 34 | return atBoot !== undefined && !atBoot.on ? [`releases: nothing cuts or promotes one unattended; ${atBoot.why}`] : []; | |
| 35 | } |
src/main.ts +11
| @@ -137,6 +137,7 @@ import { census, censusWords, processRows } from "./kernel/census.ts"; | ||
| 137 | 137 | import { reapLeftoverBridges } from "./crew/leftovers.ts"; |
| 138 | 138 | import { crewRoot } from "./crew/workdir.ts"; |
| 139 | 139 | import { createLog } from "./kernel/log.ts"; |
| 140 | import { METER_EVERY_MS, newMeterWatch } from "./crew/meter_watch.ts"; | |
| 140 | 141 | import { watchStalls } from "./kernel/loop_lag.ts"; |
| 141 | 142 | import { createStallRelay } from "./kernel/stall_relay.ts"; |
| 142 | 143 | import { startPerfWatcher, TICK_MS } from "./perf/duty.ts"; |
| @@ -894,6 +895,15 @@ const deskTimer = setInterval(() => void during("desk", () => sweepDesk(app), un | ||
| 894 | 895 | // Every paid entitlement is re-read from Stripe once a day, so a missed webhook heals (#11247). |
| 895 | 896 | const entitlementTimer = setInterval(() => void during("entitlements", () => sweepEntitlements(app), undefined, app), 86_400_000); |
| 896 | 897 | void during("entitlements", () => sweepEntitlements(app), undefined, app); |
| 898 | // While the crew waits on a full window, the provider's meter is read every 15 minutes, so an early | |
| 899 | // reset resumes it with nobody touching it (#14254). | |
| 900 | const watchMeter = newMeterWatch(); | |
| 901 | const meterTimer = setInterval( | |
| 902 | () => void during("meter-watch", async () => { | |
| 903 | for (const line of await watchMeter(app.db, app.config.dataDir)) app.log.info(line); | |
| 904 | }, undefined, app), | |
| 905 | METER_EVERY_MS, | |
| 906 | ); | |
| 897 | 907 | // The cost ledger's chain is walked at boot and once a day (#12273), on this thread: |
| 898 | 908 | // health reads the verdict from this thread's memory. |
| 899 | 909 | const walkLedger = (): void => |
| @@ -1135,6 +1145,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 1135 | 1145 | clearInterval(staleTimer); |
| 1136 | 1146 | clearInterval(deskTimer); |
| 1137 | 1147 | clearInterval(entitlementTimer); |
| 1148 | clearInterval(meterTimer); | |
| 1138 | 1149 | if (ledgerTimer !== undefined) clearInterval(ledgerTimer); |
| 1139 | 1150 | clearInterval(followTimer); |
| 1140 | 1151 | landingsWatch(); |
src/tui/screens/footing.ts +3
| @@ -11,6 +11,7 @@ | ||
| 11 | 11 | */ |
| 12 | 12 | |
| 13 | 13 | import { BREAK_STEPS, declaredFooting, footing, offMachine, whereWords, type Footing } from "../../system/footing.ts"; |
| 14 | import { outsideDependencies } from "../../keeper/at_boot.ts"; | |
| 14 | 15 | import type { Ctx, Screen } from "../app/app.ts"; |
| 15 | 16 | import { binding } from "../app/keys.ts"; |
| 16 | 17 | import type { Session } from "../session.ts"; |
| @@ -31,6 +32,8 @@ export function footingLines(session: Session, f: Footing = current()): string[] | ||
| 31 | 32 | const where = |
| 32 | 33 | f.forgeNode === null ? `The forge's node was not read (${f.unread ?? "unknown"}); machines are as declared.` : `The forge runs on ${f.forgeNode}.`; |
| 33 | 34 | const lines = [where, ""]; |
| 35 | const outside = outsideDependencies(); | |
| 36 | if (outside.length > 0) lines.push("Needs a hand outside the forge:", ...outside.map((o) => ` ${o}`), ""); | |
| 34 | 37 | if (off.length === 0) lines.push("Every piece runs on the forge's own machine."); |
| 35 | 38 | else { |
| 36 | 39 | lines.push(`${String(off.length)} of ${String(f.rows.length)} pieces run on another machine. With pg-grand off, they fail in this order:`); |
src/web/pages/system.ts +11
| @@ -24,6 +24,7 @@ | ||
| 24 | 24 | import { askingOf, type Asker } from "../../perf/asking.ts"; |
| 25 | 25 | import { turnStatsOf } from "../../http/app_turn.ts"; |
| 26 | 26 | import { mayReadRefusals } from "../../core/first_ask_refusals.ts"; |
| 27 | import { outsideDependencies } from "../../keeper/at_boot.ts"; | |
| 27 | 28 | import { canSeeMemory, memoryHours } from "../../boards/memory.ts"; |
| 28 | 29 | import { telemetryReader } from "../../actions/telemetry.ts"; |
| 29 | 30 | import type { Ctx } from "../../http/context.ts"; |
| @@ -335,6 +336,15 @@ export function featuresSection(rows: readonly FeatureHealth[]): Html { | ||
| 335 | 336 | <p class="small"><a href="/settings/features">Settings › Features</a></p></section>`; |
| 336 | 337 | } |
| 337 | 338 | |
| 339 | /** | |
| 340 | * What still waits on a hand outside the forge (#14254), one line each; nothing when nothing does. | |
| 341 | */ | |
| 342 | function outsideNote(): Html | null { | |
| 343 | const outside = outsideDependencies(); | |
| 344 | if (outside.length === 0) return null; | |
| 345 | return html`<ul class="list small" id="sys-outside" aria-label="Needs a hand outside the forge">${outside.map((o) => html`<li>${o}</li>`)}</ul>`; | |
| 346 | } | |
| 347 | ||
| 338 | 348 | /** |
| 339 | 349 | * What the forge stands on (#11685): each piece it needs to answer, sign in, |
| 340 | 350 | * build, release and back up, and where it was found. The summary gives the |
| @@ -348,6 +358,7 @@ export function footingSection(f: Footing): Html { | ||
| 348 | 358 | // First sight (#11111): the heading and one count stay; the rows open behind one disclosure. |
| 349 | 359 | const count = off.length === 0 ? "Every piece runs on the forge's own machine" : `${off.length} of ${f.rows.length} pieces run on another machine`; |
| 350 | 360 | return html`<section class="settings-block" id="sys-footing" data-off="${String(off.length)}" data-forge-node="${f.forgeNode ?? ""}"><h2>What the forge stands on</h2> |
| 361 | ${outsideNote()} | |
| 351 | 362 | ${disclosure(count, html`<p class="small">${f.forgeNode === null ? html`The forge's node was not read (${f.unread ?? "unknown"}), so each piece shows the machine written in <code>docs/OPERATING.md</code>.` : html`The forge runs on <span class="mono">${f.forgeNode}</span>.`} |
| 352 | 363 | ${off.length === 0 ? null : "If that machine stops, each piece below stops with it; they are listed in the order the forge would lose them."}</p> |
| 353 | 364 | ${off.length === 0 ? null : html`<ul class="list" aria-label="Pieces on another machine">${off.map((r) => |
test/flow-alarm-main-ahead.test.ts +17 −1
| @@ -11,7 +11,7 @@ import { getIssueById } from "../src/core/issues.ts"; | ||
| 11 | 11 | import { getRepo } from "../src/core/repos.ts"; |
| 12 | 12 | import { hunt } from "../src/crew/sentinel.ts"; |
| 13 | 13 | import { firedAlarms } from "../src/flow/alarms.ts"; |
| 14 | import { aheadMinutes, mainAheadAlarm } from "../src/flow/alarm_main_ahead.ts"; | |
| 14 | import { aheadLine, aheadMinutes, mainAheadAlarm } from "../src/flow/alarm_main_ahead.ts"; | |
| 15 | 15 | import { lastBeat, recordBeat, stepWithBeat } from "../src/keeper/beat.ts"; |
| 16 | 16 | import { keeperPanel } from "../src/web/pages/keeper-panel.ts"; |
| 17 | 17 | import { keeperAnswer } from "../src/keeper/landed.ts"; |
| @@ -109,6 +109,22 @@ describe("main ahead alarm", () => { | ||
| 109 | 109 | expect(fired[0]!.since).toBe(ago(91)); |
| 110 | 110 | }); |
| 111 | 111 | |
| 112 | test("the line adds that nothing releases it unattended only while the keeper is off (#14254)", () => { | |
| 113 | reset(); | |
| 114 | check("ok", HEAD); | |
| 115 | merge(91, "2".repeat(40)); | |
| 116 | try { | |
| 117 | noteKeeperAtBoot({ on: true }); | |
| 118 | expect(aheadLine(app.db, NOW)).not.toContain("Nothing releases it unattended."); | |
| 119 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not set" }); | |
| 120 | expect(aheadLine(app.db, NOW)).toEndWith(" Nothing releases it unattended."); | |
| 121 | reset(); | |
| 122 | expect(aheadLine(app.db, NOW)).toBeUndefined(); | |
| 123 | } finally { | |
| 124 | noteKeeperAtBoot(undefined); | |
| 125 | } | |
| 126 | }); | |
| 127 | ||
| 112 | 128 | test("a red head, or production on the head, does not fire", () => { |
| 113 | 129 | reset(); |
| 114 | 130 | merge(200, "2".repeat(40)); |
test/footing.test.ts +29
| @@ -11,6 +11,7 @@ import { tmpdir } from "node:os"; | ||
| 11 | 11 | import { join } from "node:path"; |
| 12 | 12 | import { createApp, type App } from "../src/app.ts"; |
| 13 | 13 | import { SILENT_MS, TICK_EVERY_MS } from "../src/keeper/beat.ts"; |
| 14 | import { noteKeeperAtBoot } from "../src/keeper/at_boot.ts"; | |
| 14 | 15 | import { createToken } from "../src/core/tokens.ts"; |
| 15 | 16 | import { createUser } from "../src/core/users.ts"; |
| 16 | 17 | import { serve, type Servers } from "../src/http/server.ts"; |
| @@ -358,6 +359,25 @@ describe("where it shows", () => { | ||
| 358 | 359 | expect(lines.at(-1)).toStartWith("On the forge's own machine: forge, tunnel, tunnel-home, traefik, hosts"); |
| 359 | 360 | }); |
| 360 | 361 | |
| 362 | test("the page and the terminal say what waits on a hand outside, only while it does (#14254)", () => { | |
| 363 | const f = declaredFooting("x"); | |
| 364 | const alice = new Session(app, identity(getUserByName(app.db, "alice")!)); | |
| 365 | expect(String(footingSection(f))).not.toContain("sys-outside"); | |
| 366 | expect(footingLines(alice, f).join("\n")).not.toContain("outside the forge"); | |
| 367 | try { | |
| 368 | noteKeeperAtBoot({ on: false, why: "keeper <off>" }); | |
| 369 | const page = String(footingSection(f)); | |
| 370 | expect(page).toContain('id="sys-outside"'); | |
| 371 | expect(page).toContain("nothing cuts or promotes one unattended"); | |
| 372 | expect(page).not.toContain("<off>"); | |
| 373 | const lines = footingLines(alice, f); | |
| 374 | expect(lines).toContain("Needs a hand outside the forge:"); | |
| 375 | expect(lines.some((l) => l.includes("unattended"))).toBe(true); | |
| 376 | } finally { | |
| 377 | noteKeeperAtBoot(undefined); | |
| 378 | } | |
| 379 | }); | |
| 380 | ||
| 361 | 381 | test("a guest, a signed-in non-admin and nobody get no lines and no door", () => { |
| 362 | 382 | const f = declaredFooting("x"); |
| 363 | 383 | const doors = (user?: User): string[] => { |
| @@ -399,6 +419,15 @@ describe("where it shows", () => { | ||
| 399 | 419 | expect(out.rows.map((r: { id: string }) => r.id)).toEqual(FOOTING.map((p) => p.id)); |
| 400 | 420 | expect(Array.isArray(out.risks)).toBe(true); |
| 401 | 421 | expect(out.risks.length).toBe(out.rows.filter((r: { risk: boolean }) => r.risk).length); |
| 422 | expect(out.outside).toEqual([]); | |
| 423 | try { | |
| 424 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not set" }); | |
| 425 | const off = JSON.parse((await mcp("alice")).result.content[0].text); | |
| 426 | expect(off.outside).toHaveLength(1); | |
| 427 | expect(off.outside[0]).toContain("unattended"); | |
| 428 | } finally { | |
| 429 | noteKeeperAtBoot(undefined); | |
| 430 | } | |
| 402 | 431 | const refused = await mcp("bob"); |
| 403 | 432 | expect(JSON.stringify(refused)).toContain("only an admin of this forge reads what it stands on"); |
| 404 | 433 | }); |
test/job-slices.test.ts +1
| @@ -264,6 +264,7 @@ const UNTIMED: Record<string, string> = { | ||
| 264 | 264 | "runner-wait": "one-shot timers that only resolve a promise: a runner's claim held open, a check's look at its job each second (checks/runner_jobs.ts, runner_step.ts) and the runner program's own pauses; off unless FORGE_CHECKS_PLACEMENT=runner, and checks-runner-placement.test.ts drives all three", |
| 265 | 265 | "git-pace": "a one-shot timer in git/pace.ts that only resolves the promise a paced pack write waits on: no work of its own, armed per pack chunk while a page response is in flight; git-pace.test.ts drives the pacer with a fake clock", |
| 266 | 266 | "drill-watch": "a /proc read of the drill child's memory and a state write every WATCH_MS, only while a restore drill runs in its own process; drill-job tests drive runDrillInChild directly", |
| 267 | "meter-watch": "one indexed read of held-full windows every 15 minutes, and a status fetch only while one is held full; meter-watch.test.ts drives it with a fake fetcher; the fixture holds no window", | |
| 267 | 268 | desk: "expires notes past their time: one indexed delete on the desk table every 30 s (actions/desk.ts); the fixture has no notes", |
| 268 | 269 | ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger", |
| 269 | 270 | "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly", |
test/meter-watch.test.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** The meter watch (#14254): an early reset resumes the crew with nobody touching it. */ | |
| 2 | import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createUser } from "../src/core/users.ts"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { addSubscription, getSubscription, restSubscription } from "../src/core/subscriptions.ts"; | |
| 9 | import { noteLimit } from "../src/core/standing.ts"; | |
| 10 | import { noteKeeperAtBoot, outsideDependencies } from "../src/keeper/at_boot.ts"; | |
| 11 | import { newMeterWatch, parseUsage, type Fetch } from "../src/crew/meter_watch.ts"; | |
| 12 | ||
| 13 | const NOW = new Date("2026-11-17T10:00:00Z"); | |
| 14 | const LATER = "2026-11-22T02:00:00.000Z"; | |
| 15 | const body = (five: number, week: number): Fetch => async () => | |
| 16 | new Response(JSON.stringify({ five_hour: { utilization: five }, seven_day: { utilization: week } })); | |
| 17 | ||
| 18 | let watchMeter = newMeterWatch(); | |
| 19 | describe("the meter watch", () => { | |
| 20 | let dir: string; | |
| 21 | let app: App; | |
| 22 | let id: string; | |
| 23 | beforeAll(async () => { | |
| 24 | dir = await mkdtemp(join(tmpdir(), "forge-meter-")); | |
| 25 | app = createApp({ dataDir: dir, port: 0 }); | |
| 26 | const a = addSubscription(app.db, dir, { name: "main", token: "sk-ant-oat01-meter-test" }); | |
| 27 | if (!a.ok) throw new Error(a.error); | |
| 28 | id = a.value.id; | |
| 29 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", adminIfFirst: true }); | |
| 30 | if (!u.ok) throw new Error(u.error); | |
| 31 | app.db.query("INSERT INTO repo (id, owner_id, name, visibility, description, default_branch) VALUES ('x', ?1, 'one', 'private', '', 'main')").run(u.value.id); | |
| 32 | }); | |
| 33 | afterAll(async () => { | |
| 34 | app.close(); | |
| 35 | await rm(dir, { recursive: true, force: true }); | |
| 36 | }); | |
| 37 | beforeEach(() => { | |
| 38 | watchMeter = newMeterWatch(); | |
| 39 | noteLimit(app.db, id, { window: "week", resetsAt: LATER, observedAt: NOW.toISOString(), said: "full" }); | |
| 40 | restSubscription(app.db, id, LATER, "full"); | |
| 41 | app.db.query("DELETE FROM run").run(); | |
| 42 | }); | |
| 43 | ||
| 44 | test("parses what the provider says and ignores the rest", () => { | |
| 45 | expect(parseUsage({ seven_day: { utilization: 3 }, five_hour: { utilization: "x" } })).toEqual({ week: 3 }); | |
| 46 | expect(parseUsage(null)).toEqual({}); | |
| 47 | expect(parseUsage({ seven_day: { utilization: -1 }, five_hour: { utilization: 101 } })).toEqual({}); | |
| 48 | }); | |
| 49 | ||
| 50 | test("the week window back while the 5h window is still full: it is lifted, the account stays asleep, runs stay paused", async () => { | |
| 51 | noteLimit(app.db, id, { window: "5h", resetsAt: LATER, observedAt: NOW.toISOString(), said: "full" }); | |
| 52 | app.db.query("INSERT INTO run (id, repo_id, kind, state, not_before, summary, subscription_id) VALUES ('r1','x','agent','queued',?1,'paused: window',?2)").run(LATER, id); | |
| 53 | const lines = await watchMeter(app.db, dir, { now: NOW, fetcher: body(100, 1) }); | |
| 54 | expect(lines).toHaveLength(1); | |
| 55 | expect(app.db.query("SELECT resets_at FROM subscription_limit WHERE subscription_id = ?1 AND span = 'week'").get(id)).toEqual({ resets_at: NOW.toISOString() }); | |
| 56 | expect(app.db.query("SELECT resets_at FROM subscription_limit WHERE subscription_id = ?1 AND span = '5h'").get(id)).toEqual({ resets_at: LATER }); | |
| 57 | expect(getSubscription(app.db, id)!.restingUntil).toBe(LATER); | |
| 58 | expect(app.db.query("SELECT not_before FROM run WHERE id = 'r1'").get()).toEqual({ not_before: LATER }); | |
| 59 | }); | |
| 60 | ||
| 61 | test("only the lifted account's paused runs go", async () => { | |
| 62 | app.db.query("INSERT INTO run (id, repo_id, kind, state, not_before, summary, subscription_id) VALUES ('mine','x','agent','queued',?1,'paused: window',?2)").run(LATER, id); | |
| 63 | app.db.query("INSERT INTO run (id, repo_id, kind, state, not_before, summary, subscription_id) VALUES ('other','x','agent','queued',?1,'paused: window','someone-else')").run(LATER); | |
| 64 | await watchMeter(app.db, dir, { now: NOW, fetcher: body(0, 1) }); | |
| 65 | expect(app.db.query("SELECT not_before FROM run WHERE id = 'mine'").get()).toEqual({ not_before: null }); | |
| 66 | expect(app.db.query("SELECT not_before FROM run WHERE id = 'other'").get()).toEqual({ not_before: LATER }); | |
| 67 | }); | |
| 68 | ||
| 69 | test("a body over the cap is an unreadable read, and backs the account off", async () => { | |
| 70 | let calls = 0; | |
| 71 | const huge: Fetch = async () => (calls++, new Response(JSON.stringify({ pad: "x".repeat(70_000), seven_day: { utilization: 1 } }))); | |
| 72 | expect(await watchMeter(app.db, dir, { now: NOW, fetcher: huge })).toEqual([]); | |
| 73 | expect(getSubscription(app.db, id)!.restingUntil).toBe(LATER); | |
| 74 | await watchMeter(app.db, dir, { now: new Date(NOW.getTime() + 60_000), fetcher: huge }); | |
| 75 | expect(calls).toBe(1); | |
| 76 | }); | |
| 77 | ||
| 78 | test("an early reset wakes the account, closes the note and lets paused runs go", async () => { | |
| 79 | app.db.query("INSERT INTO run (id, repo_id, kind, state, not_before, summary, subscription_id) VALUES ('r1','x','agent','queued',?1,'paused: weekly window',?2)").run(LATER, id); | |
| 80 | const lines = await watchMeter(app.db, dir, { now: NOW, fetcher: body(0, 1) }); | |
| 81 | expect(lines).toHaveLength(1); | |
| 82 | expect(lines[0]).toContain("weekly window has room again"); | |
| 83 | expect(getSubscription(app.db, id)!.restingUntil).toBeNull(); | |
| 84 | expect(app.db.query("SELECT resets_at FROM subscription_limit WHERE subscription_id = ?1").get(id)).toEqual({ resets_at: NOW.toISOString() }); | |
| 85 | expect(app.db.query("SELECT not_before FROM run WHERE id = 'r1'").get()).toEqual({ not_before: null }); | |
| 86 | }); | |
| 87 | ||
| 88 | test("a window still full is left alone", async () => { | |
| 89 | expect(await watchMeter(app.db, dir, { now: NOW, fetcher: body(10, 100) })).toEqual([]); | |
| 90 | expect(getSubscription(app.db, id)!.restingUntil).toBe(LATER); | |
| 91 | }); | |
| 92 | ||
| 93 | test("a refused read backs the account off for an hour", async () => { | |
| 94 | let calls = 0; | |
| 95 | const refuse: Fetch = async () => (calls++, new Response("no", { status: 401 })); | |
| 96 | await watchMeter(app.db, dir, { now: NOW, fetcher: refuse }); | |
| 97 | await watchMeter(app.db, dir, { now: new Date(NOW.getTime() + 15 * 60_000), fetcher: refuse }); | |
| 98 | expect(calls).toBe(1); | |
| 99 | await watchMeter(app.db, dir, { now: new Date(NOW.getTime() + 61 * 60_000), fetcher: refuse }); | |
| 100 | expect(calls).toBe(2); | |
| 101 | }); | |
| 102 | ||
| 103 | test("nothing is read when no window is held full", async () => { | |
| 104 | app.db.query("DELETE FROM subscription_limit").run(); | |
| 105 | let calls = 0; | |
| 106 | await watchMeter(app.db, dir, { now: NOW, fetcher: async () => (calls++, new Response("{}")) }); | |
| 107 | expect(calls).toBe(0); | |
| 108 | }); | |
| 109 | }); | |
| 110 | ||
| 111 | describe("what still depends on something outside", () => { | |
| 112 | test("lists the keeper being off, and is empty when it is on", () => { | |
| 113 | try { | |
| 114 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not set" }); | |
| 115 | expect(outsideDependencies()[0]).toContain("unattended"); | |
| 116 | noteKeeperAtBoot({ on: true }); | |
| 117 | expect(outsideDependencies()).toEqual([]); | |
| 118 | } finally { | |
| 119 | noteKeeperAtBoot(undefined); | |
| 120 | } | |
| 121 | }); | |
| 122 | }); |
new above
offline