Public: anyone can read this; the crew can change it.
Release keeper: measure the write pause per release and let it set the cadence #13201
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/7836-release-keeper-measure-the-write-pause-p → main 11 commits fork 5 threads · 5 unresolved
docs/features/release-keeper.md +1 −1
| @@ -19,7 +19,7 @@ keyboard. It does what the steward's `release.sh`, `verify-after.sh` and | ||
| 19 | 19 | writes a new tree beside the live one; at 80% the tick says |
| 20 | 20 | "the disk is N% full; releases pause at 80% until room is made" and |
| 21 | 21 | cuts on the first tick with room; #11686); |
| 22 | - 30 minutes have passed since the last cut; | |
| 22 | - 30 minutes have passed since the last cut, or ten times the last release's write pause when that is longer (the new forge's hold on its queue work at boot, kept on the release and shown on /releases, in `release_keeper` and on /maintenance; a 720 s pause waits 120 min; unmeasured leaves 30 min); | |
| 23 | 23 | - no hold stands. |
| 24 | 24 | |
| 25 | 25 | Before anything moves, it records the known-good version and the exact |
scripts/unused-exports-allow.json +1
| @@ -125,6 +125,7 @@ | ||
| 125 | 125 | "src/http/strangers.ts setStrangerLimits", |
| 126 | 126 | "src/http/stripe_hook.ts refusedCount", |
| 127 | 127 | "src/http/stripe_hook.ts setStripeHookCaps", |
| 128 | "src/keeper/pause.ts forgetWritePause", | |
| 128 | 129 | "src/kernel/destroy_reach.ts unitsOf", |
| 129 | 130 | "src/kernel/db.ts provideEmptyImagesForTest", |
| 130 | 131 | "src/kernel/disk_reading.ts provideDiskForTest", |
src/core/migration-pins/20261115110000-keeper-write-pause.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 7f952df7e994536aea30692dd1b2a5d90dae30903a6cf2451deff0869c1d8c41 |
src/core/migrations/20261115110000-keeper-write-pause.ts added +6
| @@ -0,0 +1,6 @@ | ||
| 1 | // A roll should not pause queue work (#7836): the new forge times how long it held its queue work | |
| 2 | // at boot while the old one still wrote, and the keeper keeps it on the release. Null for a release | |
| 3 | // that did not measure it. Additive: one new nullable column. | |
| 4 | export default ` | |
| 5 | ALTER TABLE keeper_release ADD COLUMN write_pause_ms INTEGER; | |
| 6 | `; |
src/http/mcp/tools/release_keeper.ts +1 −1
| @@ -4,7 +4,7 @@ import { bad, good, type Tool } from "../shared.ts"; | ||
| 4 | 4 | |
| 5 | 5 | export const tool: Tool = { |
| 6 | 6 | name: "release_keeper", |
| 7 | doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042), interruptedRuns — the crew runs the drained process still had running at the switch, null when not counted (#11917)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases", | |
| 7 | doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042), interruptedRuns — the crew runs the drained process still had running at the switch, null when not counted (#11917), writePauseMs — how long the new forge held its queue work at boot, null when not measured; the next cut waits 10× it, #7836) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases", | |
| 8 | 8 | description: "The release keeper's releases, holds and what blocks the next cut.", |
| 9 | 9 | inputSchema: { type: "object", properties: {} }, |
| 10 | 10 | writes: false, |
src/keeper/cadence.ts +27 −6
| @@ -8,7 +8,8 @@ | ||
| 8 | 8 | * ∧ no merge-queue gate is running (the queue is not mid-landing) |
| 9 | 9 | * ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS` |
| 10 | 10 | * ∧ no crew run is running, unless the keeper has waited on them `CREW_WAIT_MS` |
| 11 | * ∧ now − the last cut ≥ the minimum interval | |
| 11 | * ∧ now − the last cut ≥ the minimum interval, or `PAUSE_CADENCE_FACTOR` × the last | |
| 12 | * release's write pause when that is longer (#7836) | |
| 12 | 13 | * ∧ no hold stands (a failed verify's, or an owner's) |
| 13 | 14 | * ∧ the forge's disk is less than `RELEASE_DISK_USED_MAX` full |
| 14 | 15 | * |
| @@ -19,6 +20,14 @@ | ||
| 19 | 20 | /** Two releases are at least this far apart: 30 minutes. */ |
| 20 | 21 | export const MIN_INTERVAL_MS = 30 * 60_000; |
| 21 | 22 | |
| 23 | import { pauseWords } from "./pause.ts"; | |
| 24 | ||
| 25 | /** | |
| 26 | * The next cut waits this many times the last release's write pause (#7836): 12 minutes held is | |
| 27 | * 120 minutes before the next roll, so queue work is held at most a tenth of the time. | |
| 28 | */ | |
| 29 | export const PAUSE_CADENCE_FACTOR = 10; | |
| 30 | ||
| 22 | 31 | /** |
| 23 | 32 | * A release drains the process a snapshot runs in, so a cut waits for the |
| 24 | 33 | * snapshot (#9653: swaps every few minutes cut the nightly short all |
| @@ -67,11 +76,17 @@ export interface CadenceInput { | ||
| 67 | 76 | readonly hold: string | undefined; |
| 68 | 77 | /** The share of the forge's disk in use, 0..1; undefined when it could not be read. */ |
| 69 | 78 | readonly diskUsed?: number | undefined; |
| 79 | /** | |
| 80 | * How long the last release's new forge held its queue work, ms (#7836); undefined when it was | |
| 81 | * not measured. A roll should not pause queue work: the cut waits ten times the pause. | |
| 82 | */ | |
| 83 | readonly writePauseMs?: number | undefined; | |
| 70 | 84 | readonly now: number; |
| 71 | 85 | readonly minIntervalMs?: number; |
| 72 | 86 | /** |
| 73 | 87 | * A person asked for a release now (#11839): the merge queue, a snapshot and the |
| 74 | * spacing no longer hold the cut, nor do running crew runs. A release in flight, a hold and a full disk still do. | |
| 88 | * spacing no longer hold the cut, nor do running crew runs. A release in flight, a hold and a | |
| 89 | * full disk still do. | |
| 75 | 90 | */ |
| 76 | 91 | readonly hotfix?: boolean; |
| 77 | 92 | } |
| @@ -80,7 +95,9 @@ export type Cadence = { readonly cut: true; readonly version: string } | { reado | ||
| 80 | 95 | |
| 81 | 96 | const SEMVER = /^(\d+)\.(\d+)\.(\d+)$/; |
| 82 | 97 | |
| 83 | /** The next patch version: `0.7.283` → `0.7.284`; undefined when `version` is not major.minor.patch. */ | |
| 98 | /** | |
| 99 | * The next patch version: `0.7.283` → `0.7.284`; undefined when `version` is not major.minor.patch. | |
| 100 | */ | |
| 84 | 101 | export function nextVersion(version: string): string | undefined { |
| 85 | 102 | const m = SEMVER.exec(version); |
| 86 | 103 | if (m === null) return undefined; |
| @@ -106,9 +123,13 @@ export function decideCut(input: CadenceInput): Cadence { | ||
| 106 | 123 | } |
| 107 | 124 | if (input.diskUsed !== undefined && input.diskUsed >= RELEASE_DISK_USED_MAX) |
| 108 | 125 | return no(`the disk is ${Math.round(input.diskUsed * 100)}% full; releases pause at ${RELEASE_DISK_USED_MAX * 100}% until room is made`); |
| 109 | const wait = input.minIntervalMs ?? MIN_INTERVAL_MS; | |
| 110 | if (!hotfix && input.lastCutAt !== undefined && input.now - input.lastCutAt < wait) | |
| 111 | return no(`the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`); | |
| 126 | const floor = input.minIntervalMs ?? MIN_INTERVAL_MS; | |
| 127 | const pause = input.writePauseMs !== undefined && input.writePauseMs > 0 ? input.writePauseMs * PAUSE_CADENCE_FACTOR : 0; | |
| 128 | const wait = Math.max(floor, pause); | |
| 129 | if (!hotfix && input.lastCutAt !== undefined && input.now - input.lastCutAt < wait) { | |
| 130 | const ago = `the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`; | |
| 131 | return no(pause > floor ? `${ago}, ${PAUSE_CADENCE_FACTOR} × the last release's write pause of ${pauseWords(input.writePauseMs!)}` : ago); | |
| 132 | } | |
| 112 | 133 | const version = nextVersion(input.newest.version); |
| 113 | 134 | if (version === undefined) return no(`${input.newest.version} is not major.minor.patch`); |
| 114 | 135 | return { cut: true, version }; |
src/keeper/keeper.ts +32 −10
| @@ -26,8 +26,8 @@ | ||
| 26 | 26 | * `release.verified_live`, `release.rolled_back`, `release.held`. |
| 27 | 27 | * |
| 28 | 28 | * Guardrail: the keeper's only writes are a tag, the Deployment's image, |
| 29 | * its own rows, issues, and the removal of its own old releases. It holds no account that may land a pull or | |
| 30 | * push main, and it has no code path to either. | |
| 29 | * its own rows, issues, and the removal of its own old releases. It holds no account that may land | |
| 30 | * a pull or push main, and it has no code path to either. | |
| 31 | 31 | */ |
| 32 | 32 | |
| 33 | 33 | import type { Database } from "bun:sqlite"; |
| @@ -44,7 +44,10 @@ import { rollBackToNamed } from "./rollback.ts"; | ||
| 44 | 44 | import type { Probe, Tally, Verdict } from "./verify.ts"; |
| 45 | 45 | import type { Step } from "./beat.ts"; |
| 46 | 46 | |
| 47 | /** A release not live this long after its cut never rolled: 3 h (the tag's check may take 2 h, the build some more). */ | |
| 47 | /** | |
| 48 | * A release not live this long after its cut never rolled: 3 h (the tag's check may take 2 h, the | |
| 49 | * build some more). | |
| 50 | */ | |
| 48 | 51 | export const ROLL_DEADLINE_MS = 3 * 3_600_000; |
| 49 | 52 | |
| 50 | 53 | export interface KeeperPorts { |
| @@ -66,15 +69,25 @@ export interface KeeperPorts { | ||
| 66 | 69 | /** The share of the forge's disk in use, 0..1; undefined when unread (#11686). */ |
| 67 | 70 | readonly diskUsed?: () => number | undefined; |
| 68 | 71 | readonly deployer: Deployer; |
| 69 | /** Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why not. */ | |
| 72 | /** | |
| 73 | * Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why | |
| 74 | * not. | |
| 75 | */ | |
| 70 | 76 | readonly tag: (version: string, sha: string) => Promise<string | undefined>; |
| 71 | 77 | /** One `/healthz` probe through the public URL. */ |
| 72 | 78 | readonly probe: () => Promise<Probe>; |
| 73 | 79 | /** Ten minutes under page load. */ |
| 74 | 80 | readonly verify: (onProbe: (t: Tally) => void) => Promise<Verdict>; |
| 75 | 81 | readonly minIntervalMs?: number; |
| 76 | /** The worst event-loop stall in this process's first five minutes, ms; none when unset (#8848). */ | |
| 82 | /** | |
| 83 | * The worst event-loop stall in this process's first five minutes, ms; none when unset (#8848). | |
| 84 | */ | |
| 77 | 85 | readonly bootStallMs?: () => number; |
| 86 | /** | |
| 87 | * How long this process held its queue work at boot for the old forge, ms; none when unset | |
| 88 | * (#7836). | |
| 89 | */ | |
| 90 | readonly writePauseMs?: () => number | undefined; | |
| 78 | 91 | } |
| 79 | 92 | |
| 80 | 93 | /** |
| @@ -139,7 +152,10 @@ function fileDark(p: KeeperPorts, r: KeeperRelease): number | undefined { | ||
| 139 | 152 | return made.value.number; |
| 140 | 153 | } |
| 141 | 154 | |
| 142 | /** The release each keeper is verifying; keyed by its ports, so one keeper (or one test) never sees another's. */ | |
| 155 | /** | |
| 156 | * The release each keeper is verifying; keyed by its ports, so one keeper (or one test) never sees | |
| 157 | * another's. | |
| 158 | */ | |
| 143 | 159 | const verifying = new WeakMap<KeeperPorts, number>(); |
| 144 | 160 | |
| 145 | 161 | /** The verify of an in-flight release this process is the new version of, to its end. */ |
| @@ -183,7 +199,8 @@ async function prune(p: KeeperPorts, r: KeeperRelease): Promise<{ pruned: string | ||
| 183 | 199 | const keep: string[] = []; |
| 184 | 200 | for (const ref of refs) { |
| 185 | 201 | const id = store.idOf(ref); |
| 186 | // Fail closed: a live, running or known-good release the store cannot name could be the one pruned. | |
| 202 | // Fail closed: a live, running or known-good release the store cannot name could be the one | |
| 203 | // pruned. | |
| 187 | 204 | if (id === undefined) return { pruned: [], pruneError: `${ref} names no stored release` }; |
| 188 | 205 | keep.push(id); |
| 189 | 206 | } |
| @@ -215,7 +232,10 @@ function holdStall(p: KeeperPorts, r: KeeperRelease): string | undefined { | ||
| 215 | 232 | return `${reason}; hold ${hold.id} set`; |
| 216 | 233 | } |
| 217 | 234 | |
| 218 | /** Hold with the numbers, then set the named known-good image. The hold first: this process may be the one replaced. */ | |
| 235 | /** | |
| 236 | * Hold with the numbers, then set the named known-good image. The hold first: this process may be | |
| 237 | * the one replaced. | |
| 238 | */ | |
| 219 | 239 | async function rollBack(p: KeeperPorts, r: KeeperRelease, reason: string): Promise<string> { |
| 220 | 240 | const hold = setHold(p.db, { kind: "failed", version: r.version, reason, probes: r.probes, badInRow: r.badInRow, worstMs: r.worstMs, at: p.now() }); |
| 221 | 241 | p.bus.append({ kind: "release.held", repoId: p.repo.id, payload: { version: r.version, hold: hold.id, reason, probes: r.probes, badInRow: r.badInRow, worstMs: r.worstMs } }); |
| @@ -244,7 +264,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 244 | 264 | const flying = inFlight(p.db); |
| 245 | 265 | if (flying !== undefined) { |
| 246 | 266 | if (p.live() === flying.version) { |
| 247 | markLive(p.db, flying.id, p.now()); | |
| 267 | markLive(p.db, flying.id, p.now(), p.writePauseMs?.()); | |
| 248 | 268 | return verifyAndDecide(p, getKeeperRelease(p.db, flying.id)!); |
| 249 | 269 | } |
| 250 | 270 | // A cut recorded but never tagged (the process died between the two): tag it now, or drop it. |
| @@ -276,6 +296,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 276 | 296 | crewSince: crewSince(p.db, crewRuns, p.now()), |
| 277 | 297 | diskUsed: p.diskUsed?.(), |
| 278 | 298 | lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt), |
| 299 | writePauseMs: last?.writePauseMs ?? undefined, | |
| 279 | 300 | hold: blocking(standingHolds(p.db)), |
| 280 | 301 | now: p.now(), |
| 281 | 302 | ...(asked === undefined ? {} : { hotfix: true }), |
| @@ -283,7 +304,8 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 283 | 304 | }); |
| 284 | 305 | if (!decision.cut) return { line: decision.why, outcome: "skipped" }; |
| 285 | 306 | if (main === undefined) return { line: "main is unreadable", outcome: "failed" }; |
| 286 | // A tree the deployer cannot release (dependencies changed, preflight red) is not cut: no tag, no row. | |
| 307 | // A tree the deployer cannot release (dependencies changed, preflight red) is not cut: no tag, no | |
| 308 | // row. | |
| 287 | 309 | const refused = await p.deployer.admits?.(main); |
| 288 | 310 | if (refused !== undefined) return { line: `not cut: ${refused}`, outcome: "failed" }; |
| 289 | 311 | const image = await p.deployer.image(); |
src/keeper/pause.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | /** | |
| 2 | * The write pause of this process's boot (#7836): how long the new forge held | |
| 3 | * its queue work while the forge it replaced still wrote to the same SQLite | |
| 4 | * file. `main.ts` times `waitForPeer` and notes it here; the keeper, in this | |
| 5 | * same process, keeps it on the release when the verify begins. | |
| 6 | */ | |
| 7 | ||
| 8 | let paused: number | undefined; | |
| 9 | ||
| 10 | /** Note the pause, in ms; a value that is not a finite, non-negative number is ignored. */ | |
| 11 | export function noteWritePause(ms: number): void { | |
| 12 | if (Number.isFinite(ms) && ms >= 0) paused = Math.round(ms); | |
| 13 | } | |
| 14 | ||
| 15 | /** The pause this process measured at boot, ms; undefined when it did not. */ | |
| 16 | export function writePause(): number | undefined { | |
| 17 | return paused; | |
| 18 | } | |
| 19 | ||
| 20 | /** A pause in plain words: whole seconds, `720 s`. */ | |
| 21 | export function pauseWords(ms: number): string { | |
| 22 | return `${Math.round(ms / 1000)} s`; | |
| 23 | } | |
| 24 | ||
| 25 | /** Forget it (tests). */ | |
| 26 | export function forgetWritePause(): void { | |
| 27 | paused = undefined; | |
| 28 | } |
src/keeper/record.ts +30 −7
| @@ -45,6 +45,11 @@ export interface KeeperRelease { | ||
| 45 | 45 | readonly rollbackMs: number | null; |
| 46 | 46 | /** Crew runs running in the drained process at the switch (#11917); null when not counted. */ |
| 47 | 47 | readonly interruptedRuns: number | null; |
| 48 | /** | |
| 49 | * How long the new forge held its queue work at boot while the old one still wrote, ms (#7836); | |
| 50 | * null when not measured. | |
| 51 | */ | |
| 52 | readonly writePauseMs: number | null; | |
| 48 | 53 | /** Why a person cut this release now, outside the spacing; empty for a cut on cadence. */ |
| 49 | 54 | readonly hotfix: string; |
| 50 | 55 | } |
| @@ -73,6 +78,7 @@ interface Row { | ||
| 73 | 78 | switch_ms: number | null; |
| 74 | 79 | rollback_ms: number | null; |
| 75 | 80 | interrupted_runs: number | null; |
| 81 | write_pause_ms: number | null; | |
| 76 | 82 | hotfix: string; |
| 77 | 83 | } |
| 78 | 84 | |
| @@ -99,12 +105,16 @@ const toRelease = (r: Row): KeeperRelease => ({ | ||
| 99 | 105 | switchMs: r.switch_ms, |
| 100 | 106 | rollbackMs: r.rollback_ms ?? null, |
| 101 | 107 | interruptedRuns: r.interrupted_runs, |
| 108 | writePauseMs: r.write_pause_ms ?? null, | |
| 102 | 109 | hotfix: r.hotfix, |
| 103 | 110 | }); |
| 104 | 111 | |
| 105 | 112 | const iso = (ms: number) => new Date(ms).toISOString(); |
| 106 | 113 | |
| 107 | /** The cut, written before the tag is pushed: the known-good image is on record before anything can need it. */ | |
| 114 | /** | |
| 115 | * The cut, written before the tag is pushed: the known-good image is on record before anything can | |
| 116 | * need it. | |
| 117 | */ | |
| 108 | 118 | export function recordCut( |
| 109 | 119 | db: Database, |
| 110 | 120 | input: { version: string; sha: string; knownGood: { version: string; image: string }; at: number; hotfix?: string }, |
| @@ -119,12 +129,18 @@ export function recordCut( | ||
| 119 | 129 | ); |
| 120 | 130 | } |
| 121 | 131 | |
| 122 | /** A cut whose tag was refused never happened: its row goes, so the cadence is not left waiting on it. */ | |
| 132 | /** | |
| 133 | * A cut whose tag was refused never happened: its row goes, so the cadence is not left waiting on | |
| 134 | * it. | |
| 135 | */ | |
| 123 | 136 | export function dropCut(db: Database, id: number): void { |
| 124 | 137 | db.query(`DELETE FROM keeper_release WHERE id = ?1 AND stage = 'rolling' AND live_at IS NULL`).run(id); |
| 125 | 138 | } |
| 126 | 139 | |
| 127 | /** What the deployer timed while promoting: kept on the release still rolling, so a rollback's swap never overwrites them. */ | |
| 140 | /** | |
| 141 | * What the deployer timed while promoting: kept on the release still rolling, so a rollback's swap | |
| 142 | * never overwrites them. | |
| 143 | */ | |
| 128 | 144 | export function keepTimings(db: Database, id: number, t: { buildMs: number; proofMs: number; switchMs: number }): void { |
| 129 | 145 | db.query(`UPDATE keeper_release SET build_ms = ?2, proof_ms = ?3, switch_ms = ?4 WHERE id = ?1 AND stage = 'rolling'`).run( |
| 130 | 146 | id, |
| @@ -143,7 +159,10 @@ export function keepRollback(db: Database, id: number, ms: number): void { | ||
| 143 | 159 | db.query(`UPDATE keeper_release SET rollback_ms = ?2 WHERE id = ?1 AND stage IN ('rolling', 'verifying')`).run(id, n); |
| 144 | 160 | } |
| 145 | 161 | |
| 146 | /** How many crew runs the old process still had running when the switch drained it: kept on the release still rolling. */ | |
| 162 | /** | |
| 163 | * How many crew runs the old process still had running when the switch drained it: kept on the | |
| 164 | * release still rolling. | |
| 165 | */ | |
| 147 | 166 | export function keepInterrupted(db: Database, id: number, runs: number): void { |
| 148 | 167 | const n = Number.isFinite(runs) ? Math.max(0, Math.round(runs)) : 0; |
| 149 | 168 | db.query(`UPDATE keeper_release SET interrupted_runs = ?2 WHERE id = ?1 AND stage = 'rolling'`).run(id, n); |
| @@ -157,9 +176,13 @@ export function countRollProbe(db: Database, id: number, ok: boolean): void { | ||
| 157 | 176 | ).run(id, ok ? 0 : 1); |
| 158 | 177 | } |
| 159 | 178 | |
| 160 | /** The new version answers: the verify begins. Only one forge moves it, so a second caller gets false. */ | |
| 161 | export function markLive(db: Database, id: number, at: number): boolean { | |
| 162 | return db.query(`UPDATE keeper_release SET stage = 'verifying', live_at = ?2 WHERE id = ?1 AND stage = 'rolling'`).run(id, iso(at)) | |
| 179 | /** | |
| 180 | * The new version answers: the verify begins, with the write pause its process measured at boot | |
| 181 | * (#7836). Only one forge moves it, so a second caller gets false. | |
| 182 | */ | |
| 183 | export function markLive(db: Database, id: number, at: number, writePauseMs?: number): boolean { | |
| 184 | const pause = writePauseMs !== undefined && Number.isFinite(writePauseMs) ? Math.max(0, Math.round(writePauseMs)) : null; | |
| 185 | return db.query(`UPDATE keeper_release SET stage = 'verifying', live_at = ?2, write_pause_ms = ?3 WHERE id = ?1 AND stage = 'rolling'`).run(id, iso(at), pause) | |
| 163 | 186 | .changes === 1; |
| 164 | 187 | } |
| 165 | 188 |
src/keeper/wire.ts +8 −4
| @@ -27,6 +27,7 @@ import { runGit } from "../git/env.ts"; | ||
| 27 | 27 | import { repoDir } from "../git/repos.ts"; |
| 28 | 28 | import { ownProbeHeaders } from "../kernel/probe.ts"; |
| 29 | 29 | import { bootStall } from "../perf/live.ts"; |
| 30 | import { writePause } from "./pause.ts"; | |
| 30 | 31 | import { running } from "../kernel/version.ts"; |
| 31 | 32 | import { compareVersions, readTags, recordFor } from "../release/shipped.ts"; |
| 32 | 33 | import { checkoutRelease, preflightRelease } from "../release/checkout.ts"; |
| @@ -83,8 +84,9 @@ export function loadPages(repo: Pick<Repo, "ownerName" | "name">): string[] { | ||
| 83 | 84 | } |
| 84 | 85 | |
| 85 | 86 | /** |
| 86 | * What the swap reports just before this process drains (#11839, #11917): the stage timings, and the crew | |
| 87 | * runs still running here, the ones the switch interrupts, kept on the release still rolling. | |
| 87 | * What the swap reports just before this process drains (#11839, #11917): the stage timings, and | |
| 88 | * the crew runs still running here, the ones the switch interrupts, kept on the release still | |
| 89 | * rolling. | |
| 88 | 90 | */ |
| 89 | 91 | export function switched(db: Database, runs: () => number): (t: Timing) => void { |
| 90 | 92 | return (t) => { |
| @@ -137,6 +139,7 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str | ||
| 137 | 139 | now: () => Date.now(), |
| 138 | 140 | live: () => running().version, |
| 139 | 141 | bootStallMs: () => bootStall.worst(), |
| 142 | writePauseMs: () => writePause(), | |
| 140 | 143 | async main() { |
| 141 | 144 | const head = await runGit(["rev-parse", "--verify", "--quiet", `refs/heads/${repo.defaultBranch}^{commit}`], { cwd: dir }); |
| 142 | 145 | return head.ok && OID.test(head.value) ? head.value : undefined; |
| @@ -159,8 +162,9 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str | ||
| 159 | 162 | deployer, |
| 160 | 163 | async tag(version, sha) { |
| 161 | 164 | if (!VERSION.test(version) || !OID.test(sha)) return "not a version and a commit"; |
| 162 | // The record `scripts/cut-release.ts` writes. The migrations are this process's: the keeper cuts only | |
| 163 | // when main has moved past a live tag, and each forge cuts the code it was built from at the earliest. | |
| 165 | // The record `scripts/cut-release.ts` writes. The migrations are this process's: the keeper | |
| 166 | // cuts only when main has moved past a live tag, and each forge cuts the code it was built | |
| 167 | // from at the earliest. | |
| 164 | 168 | const record = JSON.stringify(recordFor(version, migrationFiles, NUMBERED)); |
| 165 | 169 | const made = await runGit(["tag", "--annotate", "--file=-", `v${version}`, sha], { |
| 166 | 170 | cwd: dir, |
src/main.ts +41 −24
| @@ -142,6 +142,7 @@ import { startKeeping } from "./flow/keep.ts"; | ||
| 142 | 142 | import { keepForgePages } from "./flow/pages.ts"; |
| 143 | 143 | import { timeStatements } from "./kernel/slow_sql.ts"; |
| 144 | 144 | import { waitForPeer } from "./runs/peer.ts"; |
| 145 | import { noteWritePause } from "./keeper/pause.ts"; | |
| 145 | 146 | import { closeStaleSessions } from "./machines/shell.ts"; |
| 146 | 147 | |
| 147 | 148 | // `forge hook <name>` re-enters here from a git hook stub. It relays to the |
| @@ -383,11 +384,12 @@ loadTrustedHosts(app.db); | ||
| 383 | 384 | // and with nothing asked: one thread, as ever. |
| 384 | 385 | const engineOn = engineShape.mode === "worker"; |
| 385 | 386 | const tail = engineOn ? tailEvents(app.db, app.bus) : undefined; |
| 386 | // A write of the engine's is a few milliseconds. This thread waits for one with every request behind it, so | |
| 387 | // it waits seconds, not the fifteen of a forge whose only other writer is itself: a stuck engine is ended (`engine/host.ts`). | |
| 388 | // One side checkpoints the log, and it is the engine's (`engine/duties.ts`): a commit of this thread that crossed | |
| 389 | // the threshold paid for the checkpoint, fsync and all, with every request behind it. | |
| 390 | // Both are put back if the engine's Worker cannot be had (`engine/run.ts`). | |
| 387 | // A write of the engine's is a few milliseconds. This thread waits for one with every request | |
| 388 | // behind it, so it waits seconds, not the fifteen of a forge whose only other writer is itself: a | |
| 389 | // stuck engine is ended (`engine/host.ts`). One side checkpoints the log, and it is the engine's | |
| 390 | // (`engine/duties.ts`): a commit of this thread that crossed the threshold paid for the checkpoint, | |
| 391 | // fsync and all, with every request behind it. Both are put back if the engine's Worker cannot be | |
| 392 | // had (`engine/run.ts`). | |
| 391 | 393 | const loneWriter = engineOn ? tuneForSplit(app.db) : () => {}; |
| 392 | 394 | await refreshCatalogQuietly(app); |
| 393 | 395 | if (readOnly) { |
| @@ -438,9 +440,10 @@ const appDomain = appsOn | ||
| 438 | 440 | }) |
| 439 | 441 | : undefined; |
| 440 | 442 | if (appDomain?.ok === true) holdAppHosts(app, appDomain.domain); |
| 441 | // What the first visitors would pay for, read before the listener opens: while this boots the previous | |
| 442 | // release still answers, so the half second each of these costs on a cold thread is nobody's wait. Each | |
| 443 | // reads in slices on turns of its own; none of them is needed for the forge to be up. | |
| 443 | // What the first visitors would pay for, read before the listener opens: while this boots the | |
| 444 | // previous release still answers, so the half second each of these costs on a cold thread is | |
| 445 | // nobody's wait. Each reads in slices on turns of its own; none of them is needed for the forge to | |
| 446 | // be up. | |
| 444 | 447 | await warmFlow(app); |
| 445 | 448 | await warmPlans(app); |
| 446 | 449 | await warmDisplays(app).catch(() => undefined); |
| @@ -449,7 +452,8 @@ markReady(); | ||
| 449 | 452 | // From here the flow's forge-wide readings are read once and shared for half a minute |
| 450 | 453 | // (`flow/kept.ts`). |
| 451 | 454 | keepFlowReadings(KEPT_MS); |
| 452 | // The home's picture is read by a worker from here, so the first page finds it (web/pages/home_flow.ts). | |
| 455 | // The home's picture is read by a worker from here, so the first page finds it | |
| 456 | // (web/pages/home_flow.ts). | |
| 453 | 457 | warmHome(app); |
| 454 | 458 | warmBoards(app); |
| 455 | 459 | // The pictures the home, `/boards`, `/flow` and the front pages of the forge's own repositories |
| @@ -554,7 +558,11 @@ const leaveEarly = (): void => { | ||
| 554 | 558 | }; |
| 555 | 559 | const earlySignals = ["SIGINT", "SIGTERM", "SIGHUP"] as const; |
| 556 | 560 | for (const signal of earlySignals) process.once(signal, leaveEarly); |
| 561 | const peerWaitStarted = performance.now(); | |
| 557 | 562 | await waitForPeer(app.db, app.log); |
| 563 | // How long this forge held its queue work for the old one (#7836): the keeper keeps it on the | |
| 564 | // release. | |
| 565 | noteWritePause(performance.now() - peerWaitStarted); | |
| 558 | 566 | for (const signal of earlySignals) process.off(signal, leaveEarly); |
| 559 | 567 | // No shell outlives its forge: a row a stop or crash left open is closed. Only |
| 560 | 568 | // once the other forge has let go: its shells are live until then, and a write |
| @@ -623,9 +631,10 @@ const snapshots = startSnapshots( | ||
| 623 | 631 | // What the machine uses (boards/sampler.ts): the pod, each cell and the |
| 624 | 632 | // node, every 15 s into a ring the `machine` board source reads. |
| 625 | 633 | const sampler = startSampler(app, {}); |
| 626 | // What has left its window goes a bounded slice at a time on a pace (`system/retention.ts`). The drains no | |
| 627 | // page reads (machine samples, wake-up events, the old deploy ledger) are this thread's always, kept out of | |
| 628 | // the write clock; the age rule and the deploy backfill are the engine's when it has a thread of its own. | |
| 634 | // What has left its window goes a bounded slice at a time on a pace (`system/retention.ts`). The | |
| 635 | // drains no page reads (machine samples, wake-up events, the old deploy ledger) are this thread's | |
| 636 | // always, kept out of the write clock; the age rule and the deploy backfill are the engine's when | |
| 637 | // it has a thread of its own. | |
| 629 | 638 | const stopRetention = startRetention(app, { local: true, engine: !engineOn }); |
| 630 | 639 | const memorySampler = startMemorySampler(app); |
| 631 | 640 | // The starting boards, once each (actions/boards.ts). |
| @@ -680,9 +689,9 @@ if (crew !== undefined) { | ||
| 680 | 689 | // (crew/preview.ts). |
| 681 | 690 | const previewForge = (process.env.FORGE_APP_ENV ?? "").trim().toLowerCase() === "preview"; |
| 682 | 691 | void (previewForge ? Promise.resolve() : keepSandboxing(app)).finally(() => bootHostedChildren(app)); |
| 683 | // The pipeline answer and every repository's display read (#8978, #11007) were filled here, behind the | |
| 684 | // listener; they are read before it opens now (above). | |
| 685 | // The release probe's search, grepped once behind the listener so the first visit reads the cache (#11059). | |
| 692 | // The pipeline answer and every repository's display read (#8978, #11007) were filled here, behind | |
| 693 | // the listener; they are read before it opens now (above). The release probe's search, grepped once | |
| 694 | // behind the listener so the first visit reads the cache (#11059). | |
| 686 | 695 | void warmSearch(app); |
| 687 | 696 | // A spawned child's first boot: its direction, opened once for its planner |
| 688 | 697 | // (spawn/first-boot.ts). A forge that was not spawned is untouched. |
| @@ -701,7 +710,8 @@ await seedPreview(app) | ||
| 701 | 710 | const heartbeatTimer = startHeartbeat(app); |
| 702 | 711 | // ...and looks upstream once an hour (spawn/upstream.ts). |
| 703 | 712 | const upstreamTimer = startUpstreamWatch(app); |
| 704 | // ...and this forge asks every child's public addresses what a visitor would get, every few minutes (spawn/reach.ts). | |
| 713 | // ...and this forge asks every child's public addresses what a visitor would get, every few minutes | |
| 714 | // (spawn/reach.ts). | |
| 705 | 715 | const reachTimer = startReachWatch(app, undefined, (name) => !childAwake(app, name)); |
| 706 | 716 | // Prompts from Instagram (docs/APPS.md): once a day, for each repository |
| 707 | 717 | // that connected an account, the top comment on its latest Reel is told to |
| @@ -817,7 +827,8 @@ const followTimer = setInterval( | ||
| 817 | 827 | ); |
| 818 | 828 | // An app landing on this forge is told to the children awake here that follow it at once. |
| 819 | 829 | const landingsWatch = watchLandings(app); |
| 820 | // A child born before its apps were linked is offered the link, and a child offered one takes it: at boot, then every tick. | |
| 830 | // A child born before its apps were linked is offered the link, and a child offered one takes it: | |
| 831 | // at boot, then every tick. | |
| 821 | 832 | void followTick(app, true).catch((error) => app.log.error("could not link the apps", { error: String(error) })); |
| 822 | 833 | const staleTimer = setInterval(() => void during("asks", tickAsks, undefined, app), 120_000); |
| 823 | 834 | // A desk note whose time is up goes off the home line at once, over the stream. |
| @@ -864,9 +875,11 @@ const processTick = (): void => { | ||
| 864 | 875 | } |
| 865 | 876 | }; |
| 866 | 877 | // The clog watch (#6524, #6757) reads the flow every minute; its state is the database's. |
| 867 | // With an engine thread both are its duties (`engine/duties.ts`), reading what this thread observed from rows. | |
| 878 | // With an engine thread both are its duties (`engine/duties.ts`), reading what this thread observed | |
| 879 | // from rows. | |
| 868 | 880 | const stopClogTick = engineOn ? () => {} : startClogTick(app.db, app.bus, (error) => app.log.warn("clog tick failed", { error: String(error) }), 60_000); |
| 869 | // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to lift. | |
| 881 | // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to | |
| 882 | // lift. | |
| 870 | 883 | const stopKeeper = engineOn ? () => {} : startKeeper(app, () => sweepAssigned(app)); |
| 871 | 884 | // A forge that went down prepared for maintenance comes back by itself. The |
| 872 | 885 | // listener is up (`serve` above) and the checks service started; the tunnel |
| @@ -874,7 +887,8 @@ const stopKeeper = engineOn ? () => {} : startKeeper(app, () => sweepAssigned(ap | ||
| 874 | 887 | void resumeAtBoot(app, undefined, { forgeUrl, runner: checks?.lane }).catch((error) => |
| 875 | 888 | app.log.warn("resume at boot failed", { error: String(error) }), |
| 876 | 889 | ); |
| 877 | // Code scanning's nightly pass (#9593): each hour, the repositories whose last pass is a day old get secrets, advisories and static analysis over their default branch. | |
| 890 | // Code scanning's nightly pass (#9593): each hour, the repositories whose last pass is a day old | |
| 891 | // get secrets, advisories and static analysis over their default branch. | |
| 878 | 892 | const nightlyScanTimer = setInterval(() => { |
| 879 | 893 | void during("nightly-scan", async () => { |
| 880 | 894 | for (const id of nightlyDue(app.db, new Date(Date.now() - 24 * 3_600_000).toISOString(), 25)) { |
| @@ -893,7 +907,8 @@ const ledgerGitTimer = readOnly | ||
| 893 | 907 | ); |
| 894 | 908 | const processTimer = setInterval(() => during("processes", processTick, undefined, app), 60_000); |
| 895 | 909 | during("processes", processTick, undefined, app); |
| 896 | // A new version proves itself against the public URL; a failed one again, 5 min after and backing off to hourly (release/prove.ts). | |
| 910 | // A new version proves itself against the public URL; a failed one again, 5 min after and backing | |
| 911 | // off to hourly (release/prove.ts). | |
| 897 | 912 | proveRunning({ |
| 898 | 913 | dataDir: app.config.dataDir, |
| 899 | 914 | version: running().version, |
| @@ -918,7 +933,8 @@ proveRunning({ | ||
| 918 | 933 | }, app.log); |
| 919 | 934 | // The release tags of the forge's own repository are its shipped releases (release/shipped.ts); |
| 920 | 935 | // production behind the newest for over an hour is a card (release/behind.ts). |
| 921 | // A FORGE_REPO that is not owner/name is dropped by releaseEnv; say so, or the list stops at the frozen one unexplained. | |
| 936 | // A FORGE_REPO that is not owner/name is dropped by releaseEnv; say so, or the list stops at the | |
| 937 | // frozen one unexplained. | |
| 922 | 938 | if ((process.env.FORGE_REPO ?? "") !== "" && !app.config.forgeRepo) |
| 923 | 939 | app.log.warn("FORGE_REPO is not owner/name; release tags are not read", { value: process.env.FORGE_REPO }); |
| 924 | 940 | if (app.config.forgeRepo) { |
| @@ -936,8 +952,9 @@ if (app.config.forgeRepo) { | ||
| 936 | 952 | void during("landings", () => readLandings(app), undefined, app); |
| 937 | 953 | setInterval(() => void during("landings", () => readLandings(app), undefined, app), 60_000); |
| 938 | 954 | } |
| 939 | // The release keeper (#7557): cut on cadence, promote, verify under page load, roll back to the named | |
| 940 | // known-good image and hold. Off unless FORGE_RELEASE_KEEPER=on in a cluster (keeper/wire.ts). | |
| 955 | // The release keeper (#7557): cut on cadence, promote, verify under page load, roll back to the | |
| 956 | // named known-good image and hold. Off unless FORGE_RELEASE_KEEPER=on in a cluster | |
| 957 | // (keeper/wire.ts). | |
| 941 | 958 | const keeperOn = keeperEnv(process.env); |
| 942 | 959 | if (keeperOn.on) { |
| 943 | 960 | const ports = keeperPorts( |
src/tui/screens/keeper.ts +7 −2
| @@ -15,6 +15,7 @@ import { neverTickedLine } from "../../keeper/at_boot.ts"; | ||
| 15 | 15 | import { beatLine, lastBeat } from "../../keeper/beat.ts"; |
| 16 | 16 | import { stallWords } from "../../perf/boot_stall.ts"; |
| 17 | 17 | import { blocking, holdAs, liftAs, standingHolds } from "../../keeper/hold.ts"; |
| 18 | import { pauseWords } from "../../keeper/pause.ts"; | |
| 18 | 19 | import { secondsWords, withLandedToLive, type LandedRelease } from "../../keeper/landed.ts"; |
| 19 | 20 | import { releaseNowAs, pendingNow } from "../../keeper/now.ts"; |
| 20 | 21 | import { listKeeperReleases, type Stage } from "../../keeper/record.ts"; |
| @@ -38,7 +39,9 @@ const STAGE: Record<Stage, { label: string; tone: Tone }> = { | ||
| 38 | 39 | export interface KeeperRead { |
| 39 | 40 | /** What holds the next cut, or the line that says nothing does. */ |
| 40 | 41 | readonly blocked: string; |
| 41 | /** The release tick's last heartbeat as a sentence, said plainly when there is none or it is old. */ | |
| 42 | /** | |
| 43 | * The release tick's last heartbeat as a sentence, said plainly when there is none or it is old. | |
| 44 | */ | |
| 42 | 45 | readonly heartbeat: string; |
| 43 | 46 | /** Main green and ahead of production, and for how long; absent when it is not. */ |
| 44 | 47 | readonly ahead?: string; |
| @@ -79,6 +82,7 @@ export function keeperRows(releases: readonly LandedRelease[]): ListRow[] { | ||
| 79 | 82 | `worst ${r.worstMs} ms`, |
| 80 | 83 | r.worstStallMs === 0 ? "" : `boot stall ${stallWords(r.worstStallMs, releases[i + 1]?.worstStallMs)}`, |
| 81 | 84 | `${r.secondsDark} s dark`, |
| 85 | r.writePauseMs === null ? "" : `write pause ${pauseWords(r.writePauseMs)}`, | |
| 82 | 86 | ), |
| 83 | 87 | })); |
| 84 | 88 | } |
| @@ -117,7 +121,8 @@ export function keeperScreen(f: Session, ctx: Ctx): Screen { | ||
| 117 | 121 | emptyState(s, area, c, { title: "Not yours to see", body: "The release keeper is for the forge's administrators and its repository's readers." }); |
| 118 | 122 | return; |
| 119 | 123 | } |
| 120 | // What holds the cut ends with what to land ("release hold N"): wrapped, up to three lines, never cut short of it. | |
| 124 | // What holds the cut ends with what to land ("release hold N"): wrapped, up to three lines, | |
| 125 | // never cut short of it. | |
| 121 | 126 | const lines = [ |
| 122 | 127 | ...wrap(read.blocked, area.w).slice(0, 3), |
| 123 | 128 | ...wrap(read.heartbeat, area.w).slice(0, 2), |
src/web/pages/keeper-panel.ts +4 −2
| @@ -12,6 +12,7 @@ import { neverTickedLine } from "../../keeper/at_boot.ts"; | ||
| 12 | 12 | import { beatFact, lastBeat } from "../../keeper/beat.ts"; |
| 13 | 13 | import { blocking, MAX_REASON, standingHolds } from "../../keeper/hold.ts"; |
| 14 | 14 | import { pendingNow } from "../../keeper/now.ts"; |
| 15 | import { pauseWords } from "../../keeper/pause.ts"; | |
| 15 | 16 | import { secondsWords, stagesLine, withLandedToLive } from "../../keeper/landed.ts"; |
| 16 | 17 | import { listKeeperReleases, type Stage } from "../../keeper/record.ts"; |
| 17 | 18 | import { stallWords } from "../../perf/boot_stall.ts"; |
| @@ -54,7 +55,8 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date( | ||
| 54 | 55 | <td>${r.worstMs} ms</td> |
| 55 | 56 | <td>${stallWords(r.worstStallMs, releases[i + 1]?.worstStallMs)}</td> |
| 56 | 57 | <td>${r.secondsDark} s</td> |
outdated
Principle 3 and 8: a tenth column in a sticky-lead table; and '720000 ms' is hard to read. Show seconds ('720 s'), consistent with the 'Dark' column, or put it in the notes row rather than widening the table. |
||
| 57 | </tr>${when(notes(r) !== "", () => html`<tr><td colspan="9" class="small muted">${notes(r)}</td></tr>`)}`; | |
| 58 | <td>${r.writePauseMs === null ? "–" : pauseWords(r.writePauseMs)}</td> | |
| 59 | </tr>${when(notes(r) !== "", () => html`<tr><td colspan="10" class="small muted">${notes(r)}</td></tr>`)}`; | |
| 58 | 60 | }); |
| 59 | 61 | return html`<section class="settings-block" id="keeper"> |
| 60 | 62 | <h2>Release keeper</h2> |
| @@ -65,7 +67,7 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date( | ||
| 65 | 67 | ${when(ahead !== undefined, () => html`<div class="status-row"><dt>Main vs production</dt><dd id="keeper-ahead">${ahead}</dd></div>`)} |
| 66 | 68 | </dl> |
| 67 | 69 | ${when(rows.length > 0, () => html`<div class="table-scroll" role="region" tabindex="0" aria-label="Keeper releases"><table class="table" data-sticky-lead> |
| 68 | <thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th></tr></thead> | |
| 70 | <thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th><th>Write pause</th></tr></thead> | |
| 69 | 71 | <tbody>${rows}</tbody> |
| 70 | 72 | </table></div>`)} |
| 71 | 73 | ${when(mayHold, () => html`${holds.filter((h) => h.kind === "owner").map( |
src/web/pages/maintenance.ts +4 −1
| @@ -17,6 +17,7 @@ import { icon } from "../icons.ts"; | ||
| 17 | 17 | import { flashTo, page } from "../layout.ts"; |
| 18 | 18 | import { badge, btn, disclosure, field } from "../ui.ts"; |
| 19 | 19 | import { STALE_DAYS, STALE_WORDS, closeStale, nudge, stalePulls } from "../../maintain/stale_pulls.ts"; |
| 20 | import { writePauseFold } from "./maintenance_pause.ts"; | |
| 20 | 21 | import { budgetRows } from "./maintenance_budgets.ts"; |
| 21 | 22 | |
| 22 | 23 | function admin(ctx: Ctx): Response | undefined { |
| @@ -77,6 +78,7 @@ export async function maintenancePage(ctx: Ctx): Promise<Response> { | ||
| 77 | 78 | )} |
| 78 | 79 | </dl>`)} |
| 79 | 80 | ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${budgetRows(ctx.app, states.map((d) => d.name))}</dl>`)} |
| 81 | ${writePauseFold(ctx.app.db)} | |
| 80 | 82 | ${stale > 0 |
| 81 | 83 | ? fold("stale-pulls", `Pulls untouched for ${String(STALE_DAYS)} days`, String(stale), true, staleRows(ctx.auth.user!.kind !== "agent", stalePending)) |
| 82 | 84 | : null} |
| @@ -117,7 +119,8 @@ export async function maintenancePrepare(ctx: Ctx): Promise<Response> { | ||
| 117 | 119 | if (denied) return denied; |
| 118 | 120 | const form = await readForm(ctx, "admin"); |
| 119 | 121 | if (form instanceof Response) return form; |
| 120 | // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been agreed. | |
| 122 | // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been | |
| 123 | // agreed. | |
| 121 | 124 | if (status(ctx.app).phase === "idle" && formField(form, "confirm") !== "yes") |
| 122 | 125 | return flashTo(ctx, "/maintenance", { tone: "danger", message: "Tick the box to confirm: the crew and releases stop." }); |
| 123 | 126 | const r = await prepare(ctx.app, ctx.auth.user, formField(form, "reason")); |
src/web/pages/maintenance_pause.ts added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | /** | |
| 2 | * "Write pause per release" on /maintenance (#7836): how long each release's new forge held its | |
| 3 | * queue work while the old one still wrote, newest first, in seconds, with the wait the next cut | |
| 4 | * then keeps. Nothing is shown until a release has measured one. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { MIN_INTERVAL_MS, PAUSE_CADENCE_FACTOR } from "../../keeper/cadence.ts"; | |
| 9 | import { pauseWords } from "../../keeper/pause.ts"; | |
| 10 | import { listKeeperReleases } from "../../keeper/record.ts"; | |
| 11 | import { disclosure } from "../ui.ts"; | |
| 12 | import { html, type Html } from "../html.ts"; | |
| 13 | ||
| 14 | const SHOWN = 12; | |
| 15 | ||
| 16 | /** The panel, or an empty fragment when no release has measured a pause. */ | |
| 17 | export function writePauseFold(db: Database): Html { | |
| 18 | const rows = listKeeperReleases(db, SHOWN).filter((r) => r.writePauseMs !== null); | |
| 19 | if (rows.length === 0) return html``; | |
| 20 | const wait = Math.max(MIN_INTERVAL_MS, PAUSE_CADENCE_FACTOR * rows[0]!.writePauseMs!); | |
| 21 | return disclosure( | |
| 22 | html`Write pause per release <span class="muted">${pauseWords(rows[0]!.writePauseMs!)}</span>`, | |
| 23 | html`<p class="small muted" id="write-pause-says">The next cut waits ${String(wait / 60_000)} min.</p> | |
| 24 | <dl class="status-rows" id="write-pause"> | |
outdated
Title says '(ms)' and the value repeats in the muted span; show seconds, as the cadence reason does. |
||
| 25 | ${rows.map((r) => html`<div class="status-row"><dt>${r.version}</dt><dd>${pauseWords(r.writePauseMs!)}</dd></div>`)} | |
outdated
Principle 8 and 1: this sentence explains the formula the label already carries. Say it once, in seconds/minutes: 'The next cut waits 120 min.' |
||
| 26 | </dl>`, | |
| 27 | { id: "maintenance-write-pause", extra: "mb", open: false }, | |
| 28 | ); | |
outdated
Principles 2, 3 and 7: a fact is a |
||
| 29 | } |
test/keeper-language.test.ts +1 −1
| @@ -10,7 +10,7 @@ import { keeperPanel } from "../src/web/pages/keeper-panel.ts"; | ||
| 10 | 10 | import { keeperForge } from "./helpers/keeper.ts"; |
| 11 | 11 | |
| 12 | 12 | const HEAD = |
| 13 | "<thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th></tr></thead>"; | |
| 13 | "<thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th><th>Write pause</th></tr></thead>"; | |
| 14 | 14 | |
| 15 | 15 | test("the keeper is a section with an h2, no card, with its blocked line and its table", async () => { |
| 16 | 16 | const f = await keeperForge(); |
test/release-keeper-write-pause.test.ts added +71
| @@ -0,0 +1,71 @@ | ||
| 1 | // A roll should not pause queue work (#7836): the new forge's hold is measured, kept on the | |
| 2 | // release, and the next cut waits ten times it. | |
| 3 | import { afterEach, expect, test } from "bun:test"; | |
| 4 | import { decideCut } from "../src/keeper/cadence.ts"; | |
| 5 | import { keeperAnswer } from "../src/keeper/landed.ts"; | |
| 6 | import { forgetWritePause, noteWritePause } from "../src/keeper/pause.ts"; | |
| 7 | import { listKeeperReleases, markLive } from "../src/keeper/record.ts"; | |
| 8 | import { tick } from "../src/keeper/keeper.ts"; | |
| 9 | import { keeperPanel } from "../src/web/pages/keeper-panel.ts"; | |
| 10 | import { writePauseFold } from "../src/web/pages/maintenance_pause.ts"; | |
| 11 | import { keeperRows } from "../src/tui/screens/keeper.ts"; | |
| 12 | import { A, B, keeperForge } from "./helpers/keeper.ts"; | |
| 13 | ||
| 14 | afterEach(forgetWritePause); | |
| 15 | ||
| 16 | const base = { main: B, newest: { version: "0.7.283", sha: A }, live: "0.7.283", inFlight: false, gating: 0, hold: undefined } as const; | |
| 17 | const MIN = 60_000; | |
| 18 | const at = 1_000_000_000_000; | |
| 19 | ||
| 20 | test("a recorded 720 s pause makes the next cut wait 120 min, and says so", () => { | |
| 21 | const input = { ...base, lastCutAt: at, writePauseMs: 720_000 }; | |
| 22 | const d = decideCut({ ...input, now: at + 119 * MIN }); | |
| 23 | expect(d.cut).toBe(false); | |
| 24 | if (!d.cut) { | |
| 25 | expect(d.why).toContain("the interval is 120 min"); | |
| 26 | expect(d.why).toContain("write pause of 720 s"); | |
| 27 | } | |
| 28 | expect(decideCut({ ...input, now: at + 120 * MIN }).cut).toBe(true); | |
| 29 | }); | |
| 30 | ||
| 31 | test("an unmeasured pause leaves the 30 min floor, and a short one does not lengthen it", () => { | |
| 32 | const d = decideCut({ ...base, lastCutAt: at, now: at + 29 * MIN }); | |
| 33 | expect(d.cut).toBe(false); | |
| 34 | if (!d.cut) expect(d.why).toBe("the last cut was 29 min ago; the interval is 30 min"); | |
| 35 | expect(decideCut({ ...base, lastCutAt: at, now: at + 30 * MIN }).cut).toBe(true); | |
| 36 | const short = decideCut({ ...base, lastCutAt: at, writePauseMs: 60_000, now: at + 30 * MIN }); | |
| 37 | expect(short.cut).toBe(true); | |
| 38 | }); | |
| 39 | ||
| 40 | test("the pause this process measured is kept on the release when its verify begins, and shown", async () => { | |
| 41 | const f = await keeperForge(); | |
| 42 | await tick(f.ports); | |
| 43 | const r = listKeeperReleases(f.app.db)[0]!; | |
| 44 | expect(r.writePauseMs).toBeNull(); | |
| 45 | expect(markLive(f.app.db, r.id, f.world.now, 720_000)).toBe(true); | |
| 46 | const kept = listKeeperReleases(f.app.db)[0]!; | |
| 47 | expect(kept.writePauseMs).toBe(720_000); | |
| 48 | expect(keeperAnswer(f.app.db).releases[0]!.writePauseMs).toBe(720_000); | |
| 49 | expect(String(keeperPanel(f.app.db))).toContain("720 s"); | |
| 50 | expect(String(writePauseFold(f.app.db))).toContain("The next cut waits 120 min"); | |
| 51 | expect(String(writePauseFold(f.app.db))).toContain("Write pause per release"); | |
| 52 | expect(String(writePauseFold(f.app.db))).not.toContain("<svg"); | |
| 53 | const rows = keeperRows(keeperAnswer(f.app.db).releases); | |
| 54 | expect(JSON.stringify(rows)).toContain("write pause 720 s"); | |
| 55 | }); | |
| 56 | ||
| 57 | test("the keeper hands its process's pause to the release it marks live", async () => { | |
| 58 | const f = await keeperForge(); | |
| 59 | await tick(f.ports); | |
| 60 | noteWritePause(5_000); | |
| 61 | f.world.live = listKeeperReleases(f.app.db)[0]!.version; | |
| 62 | await tick({ ...f.ports, writePauseMs: () => 5_000 }); | |
| 63 | expect(listKeeperReleases(f.app.db)[0]!.writePauseMs).toBe(5_000); | |
| 64 | }); | |
| 65 | ||
| 66 | test("an unmeasured release shows no pause", async () => { | |
| 67 | const f = await keeperForge(); | |
| 68 | await tick(f.ports); | |
| 69 | expect(String(writePauseFold(f.app.db))).toBe(""); | |
| 70 | expect(String(keeperPanel(f.app.db))).toContain("<th>Write pause</th>"); | |
| 71 | }); |
Same unit: print 'write pause 720 s' not '720000 ms', matching the cadence reason; update the test's expectation.