Public: anyone can read this; the crew can change it.
A cut waits up to five minutes for running crew runs to finish, and the tick says how many it waited on #11920
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/11918-a-cut-waits-up-to-five-minutes-for-runni → main 1 commit fork
4 files
changed +66
−2
src/keeper/cadence.ts +19 −1
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | * tag back (nothing still promoting) |
| 8 | 8 | * ∧ no merge-queue gate is running (the queue is not mid-landing) |
| 9 | 9 | * ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS` |
| 10 | * ∧ no crew run is running, unless the keeper has waited on them `CREW_WAIT_MS` | |
| 10 | 11 | * ∧ now − the last cut ≥ the minimum interval |
| 11 | 12 | * ∧ no hold stands (a failed verify's, or an owner's) |
| 12 | 13 | * ∧ the forge's disk is less than `RELEASE_DISK_USED_MAX` full |
| @@ -26,6 +27,13 @@ export const MIN_INTERVAL_MS = 30 * 60_000; | ||
| 26 | 27 | */ |
| 27 | 28 | export const SNAPSHOT_WAIT_MS = 2 * 3_600_000; |
| 28 | 29 | |
| 30 | /** | |
| 31 | * A release drains the process crew runs run in, and a run cut short is lost | |
| 32 | * (#11918). A cut waits for them, but only this long from the first tick that | |
| 33 | * found them: a stuck run does not stop releases. | |
| 34 | */ | |
| 35 | export const CREW_WAIT_MS = 5 * 60_000; | |
| 36 | ||
| 29 | 37 | /** |
| 30 | 38 | * A release writes a new tree and its install beside the live one, so at |
| 31 | 39 | * this share of the forge's disk used no release is cut (#11686, item 6: |
| @@ -49,6 +57,10 @@ export interface CadenceInput { | ||
| 49 | 57 | readonly gating: number; |
| 50 | 58 | /** How long the snapshot being taken has run, ms; undefined when none is. */ |
| 51 | 59 | readonly snapshotMs?: number | undefined; |
| 60 | /** Crew runs running now; none when undefined. */ | |
| 61 | readonly crewRuns?: number | undefined; | |
| 62 | /** When the keeper first found crew runs running, epoch ms, in the stretch it is still in. */ | |
| 63 | readonly crewSince?: number | undefined; | |
| 52 | 64 | /** When the keeper last cut, epoch ms. */ |
| 53 | 65 | readonly lastCutAt: number | undefined; |
| 54 | 66 | /** The reason of a standing hold, when one stands. */ |
| @@ -59,7 +71,7 @@ export interface CadenceInput { | ||
| 59 | 71 | readonly minIntervalMs?: number; |
| 60 | 72 | /** |
| 61 | 73 | * A person asked for a release now (#11839): the merge queue, a snapshot and the |
| 62 | * spacing no longer hold the cut. A release in flight, a hold and a full disk still do. | |
| 74 | * spacing no longer hold the cut, nor do running crew runs. A release in flight, a hold and a full disk still do. | |
| 63 | 75 | */ |
| 64 | 76 | readonly hotfix?: boolean; |
| 65 | 77 | } |
| @@ -86,6 +98,12 @@ export function decideCut(input: CadenceInput): Cadence { | ||
| 86 | 98 | if (!hotfix && input.gating > 0) return no(`the merge queue is landing (${input.gating} gate${input.gating === 1 ? "" : "s"} running)`); |
| 87 | 99 | if (!hotfix && input.snapshotMs !== undefined && input.snapshotMs < SNAPSHOT_WAIT_MS) |
| 88 | 100 | return no(`a snapshot is being taken (${Math.floor(input.snapshotMs / 60_000)} min in); a release would cut it short`); |
| 101 | const runs = input.crewRuns ?? 0; | |
| 102 | if (!hotfix && runs > 0) { | |
| 103 | const waited = Math.max(0, input.now - (input.crewSince ?? input.now)); | |
| 104 | if (waited < CREW_WAIT_MS) | |
| 105 | return no(`waiting on ${runs} crew run${runs === 1 ? "" : "s"} (${Math.floor(waited / 60_000)}m of ${CREW_WAIT_MS / 60_000}m)`); | |
| 106 | } | |
| 89 | 107 | if (input.diskUsed !== undefined && input.diskUsed >= RELEASE_DISK_USED_MAX) |
| 90 | 108 | return no(`the disk is ${Math.round(input.diskUsed * 100)}% full; releases pause at ${RELEASE_DISK_USED_MAX * 100}% until room is made`); |
| 91 | 109 | const wait = input.minIntervalMs ?? MIN_INTERVAL_MS; |
src/keeper/keeper.ts +23
| @@ -57,6 +57,8 @@ export interface KeeperPorts { | ||
| 57 | 57 | readonly gating: () => number; |
| 58 | 58 | /** How long the snapshot being taken has run, ms; undefined when none is (#9653). */ |
| 59 | 59 | readonly snapshotting?: () => number | undefined; |
| 60 | /** Crew runs running now, which a release would cut short (#11918); none when unset. */ | |
| 61 | readonly crewRuns?: () => number; | |
| 60 | 62 | /** The share of the forge's disk in use, 0..1; undefined when unread (#11686). */ |
| 61 | 63 | readonly diskUsed?: () => number | undefined; |
| 62 | 64 | readonly deployer: Deployer; |
| @@ -71,6 +73,23 @@ export interface KeeperPorts { | ||
| 71 | 73 | readonly bootStallMs?: () => number; |
| 72 | 74 | } |
| 73 | 75 | |
| 76 | /** | |
| 77 | * When the keeper first found crew runs running, per forge database (#11918): | |
| 78 | * the wait counts from there, and starts over once none run. | |
| 79 | */ | |
| 80 | const crewSeen = new WeakMap<Database, number>(); | |
| 81 | ||
| 82 | /** The first tick of this stretch that found `runs` running; undefined when none run. */ | |
| 83 | function crewSince(db: Database, runs: number, now: number): number | undefined { | |
| 84 | if (runs <= 0) { | |
| 85 | crewSeen.delete(db); | |
| 86 | return undefined; | |
| 87 | } | |
| 88 | const since = crewSeen.get(db) ?? now; | |
| 89 | crewSeen.set(db, since); | |
| 90 | return since; | |
| 91 | } | |
| 92 | ||
| 74 | 93 | /** Pulls of the forge's repository merged since `since`: number, title and body. */ |
| 75 | 94 | function landedSince(db: Database, repoId: string, since: string): { number: number; text: string }[] { |
| 76 | 95 | return db |
| @@ -209,6 +228,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 209 | 228 | const newest = await p.newest(); |
| 210 | 229 | const last = listKeeperReleases(p.db, 1)[0]; |
| 211 | 230 | const asked = pendingNow(p.db, p.now()); |
| 231 | const crewRuns = p.crewRuns?.() ?? 0; | |
| 212 | 232 | const decision = decideCut({ |
| 213 | 233 | main: main ?? "", |
| 214 | 234 | newest, |
| @@ -217,6 +237,8 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 217 | 237 | inFlight: false, |
| 218 | 238 | gating: p.gating(), |
| 219 | 239 | snapshotMs: p.snapshotting?.(), |
| 240 | crewRuns, | |
| 241 | crewSince: crewSince(p.db, crewRuns, p.now()), | |
| 220 | 242 | diskUsed: p.diskUsed?.(), |
| 221 | 243 | lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt), |
| 222 | 244 | hold: blocking(standingHolds(p.db)), |
| @@ -235,6 +257,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> { | ||
| 235 | 257 | // whose tag the next tick sends (`tagged`), never a tag the keeper has no row for. |
| 236 | 258 | const r = recordCut(p.db, { version: decision.version, sha: main, knownGood, at: p.now(), ...(asked === undefined ? {} : { hotfix: `${asked.reason} (${asked.by})` }) }); |
| 237 | 259 | if (asked !== undefined) clearNow(p.db); |
| 260 | crewSeen.delete(p.db); | |
| 238 | 261 | const tagRefused = await tagged(p, r); |
| 239 | 262 | if (tagRefused !== undefined) return { line: tagRefused, outcome: "failed" }; |
| 240 | 263 | const sent = await promote(p, r.version, refFor(p, image, r.version, main)); |
src/keeper/wire.ts +2
| @@ -30,6 +30,7 @@ import { running } from "../kernel/version.ts"; | ||
| 30 | 30 | import { compareVersions, readTags, recordFor } from "../release/shipped.ts"; |
| 31 | 31 | import { checkoutRelease, preflightRelease } from "../release/checkout.ts"; |
| 32 | 32 | import { handover } from "../release/swap.ts"; |
| 33 | import { runningChildren } from "../boards/sampler.ts"; | |
| 33 | 34 | import { snapshotsOf } from "../snapshot/service.ts"; |
| 34 | 35 | import { kubeDeployer } from "./deploy.ts"; |
| 35 | 36 | import { inFlight, keepTimings } from "./record.ts"; |
| @@ -129,6 +130,7 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str | ||
| 129 | 130 | }, |
| 130 | 131 | gating: gatesInFlight, |
| 131 | 132 | snapshotting: () => snapshotsOf(app)?.takingMs(), |
| 133 | crewRuns: () => runningChildren(app.db).agents, | |
| 132 | 134 | diskUsed: () => { |
| 133 | 135 | const disk = diskUnder(app.config.dataDir); |
| 134 | 136 | return disk === undefined || !(disk.totalBytes > 0) || roomy(disk) ? undefined : 1 - disk.freeBytes / disk.totalBytes; |
test/release-keeper-cadence.test.ts +22 −1
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | // When the release keeper cuts (#7557): each rule refuses on its own, and a cut is a tag, never a commit to main. |
| 2 | 2 | import { describe, expect, test } from "bun:test"; |
| 3 | import { decideCut, MIN_INTERVAL_MS, nextVersion, RELEASE_DISK_USED_MAX, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts"; | |
| 3 | import { CREW_WAIT_MS, decideCut, MIN_INTERVAL_MS, nextVersion, RELEASE_DISK_USED_MAX, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts"; | |
| 4 | 4 | import { tick } from "../src/keeper/keeper.ts"; |
| 5 | import { askNow } from "../src/keeper/now.ts"; | |
| 5 | 6 | import { listKeeperReleases, recordCut } from "../src/keeper/record.ts"; |
| 6 | 7 | import { keeperPorts } from "../src/keeper/wire.ts"; |
| 7 | 8 | import { provideDiskForTest } from "../src/kernel/disk_reading.ts"; |
| @@ -58,6 +59,26 @@ describe("cadence", () => { | ||
| 58 | 59 | expect(await tick({ ...f.ports, snapshotting: () => undefined })).toContain("cut 0.7.284"); |
| 59 | 60 | }); |
| 60 | 61 | |
| 62 | test("crew runs in flight hold the cut five minutes from the first tick that found them; a hotfix is not held (#11918)", async () => { | |
| 63 | expect(CREW_WAIT_MS).toBe(5 * 60_000); | |
| 64 | expect(decideCut({ ...ready, crewRuns: 2, crewSince: NOW - 4 * 60_000 })).toEqual({ cut: false, why: "waiting on 2 crew runs (4m of 5m)" }); | |
| 65 | expect(decideCut({ ...ready, crewRuns: 2, crewSince: NOW - 5 * 60_000 })).toEqual({ cut: true, version: "0.7.284" }); | |
| 66 | expect(decideCut({ ...ready, crewRuns: 1, crewSince: NOW, hotfix: true })).toEqual({ cut: true, version: "0.7.284" }); | |
| 67 | ||
| 68 | const f = await keeperForge(); | |
| 69 | const ports = { ...f.ports, crewRuns: () => 2 }; | |
| 70 | expect(await tick(ports)).toBe("waiting on 2 crew runs (0m of 5m)"); | |
| 71 | f.world.now += 4 * 60_000; | |
| 72 | expect(await tick(ports)).toBe("waiting on 2 crew runs (4m of 5m)"); | |
| 73 | expect(f.world.tags).toEqual([]); | |
| 74 | f.world.now += 60_000; | |
| 75 | expect(await tick(ports)).toContain("cut 0.7.284"); | |
| 76 | ||
| 77 | const h = await keeperForge(); | |
| 78 | askNow(h.ports.db, "a hotfix", "alice", h.world.now); | |
| 79 | expect(await tick({ ...h.ports, crewRuns: () => 3 })).toContain("cut 0.7.284"); | |
| 80 | }); | |
| 81 | ||
| 61 | 82 | test("at 80% of the disk used no release is cut; under it, or unread, the cut goes (#11686)", async () => { |
| 62 | 83 | expect(RELEASE_DISK_USED_MAX).toBe(0.8); |
| 63 | 84 | expect(decideCut({ ...ready, diskUsed: 0.8 })).toEqual({ cut: false, why: "the disk is 80% full; releases pause at 80% until room is made" }); |
new above
offline