trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns10g rEventsFlows

Each release counts the crew runs it interrupted, and the release card shows the number #11922

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/11917-each-release-counts-the-crew-runs-it-int → main 3 commits fork

  • Merged. scout/11917-each-release-counts-the-crew-runs-it-int was merged into main as 8232739.

Review

Sign in to review this change.
7 files changed +80 −8
src/core/migration-pins/20261113120000-keeper-interrupted-runs.pin added +1
@@ -0,0 +1,1 @@
1 d7b0ba48eba8258669adf7d9faf5067efc2ac406f2763b8b1b4b4841d99b4956
src/core/migrations/20261113120000-keeper-interrupted-runs.ts added +6
@@ -0,0 +1,6 @@
1 // The switch disturbs nothing (#11839, #11917): each release counts the crew runs that were running in the
2 // process it drained, so the target of zero is measured first. Null for a release that did not count them.
3 // Additive: one new nullable column.
4 export default `
5 ALTER TABLE keeper_release ADD COLUMN interrupted_runs INTEGER;
6 `;
src/http/mcp/tools/release_keeper.ts +1 −1
@@ -4,7 +4,7 @@ import { bad, good, type Tool } from "../shared.ts";
44
55 export const tool: Tool = {
66 name: "release_keeper",
7 doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases",
7 doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042), interruptedRuns — the crew runs the drained process still had running at the switch, null when not counted (#11917)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases",
88 description: "The release keeper's releases, holds and what blocks the next cut.",
99 inputSchema: { type: "object", properties: {} },
1010 writes: false,
src/keeper/landed.ts +5 −2
@@ -65,6 +65,8 @@ export interface Stages {
6565 readonly buildMs: number | null;
6666 readonly proofMs: number | null;
6767 readonly switchMs: number | null;
68 /** Crew runs running in the drained process at the switch (#11917); null when not counted. */
69 readonly interruptedRuns: number | null;
6870 /** The new version answering to its verdict. */
6971 readonly verifyMs: number | null;
7072 }
@@ -75,12 +77,13 @@ const between = (from: string | null, to: string | null): number | null => {
7577 return Number.isFinite(ms) ? Math.max(0, ms) : null;
7678 };
7779
78 export function stagesOf(r: Pick<KeeperRelease, "cutAt" | "liveAt" | "finishedAt" | "buildMs" | "proofMs" | "switchMs">): Stages {
80 export function stagesOf(r: Pick<KeeperRelease, "cutAt" | "liveAt" | "finishedAt" | "buildMs" | "proofMs" | "switchMs"> & Partial<Pick<KeeperRelease, "interruptedRuns">>): Stages {
7981 return {
8082 cutToLiveMs: between(r.cutAt, r.liveAt),
8183 buildMs: r.buildMs,
8284 proofMs: r.proofMs,
8385 switchMs: r.switchMs,
86 interruptedRuns: r.interruptedRuns ?? null,
8487 verifyMs: between(r.liveAt, r.finishedAt),
8588 };
8689 }
@@ -88,7 +91,7 @@ export function stagesOf(r: Pick<KeeperRelease, "cutAt" | "liveAt" | "finishedAt
8891 /** "cut→live 1 min 2 s · build 4 s · proof 3 s · switch 1 s · verify 10 min 0 s"; a stage not measured is left out. */
8992 export function stagesLine(s: Stages): string {
9093 const w = (name: string, ms: number | null) => (ms === null ? [] : [`${name} ${secondsWords(Math.round(ms / 1000))}`]);
91 return [...w("cut→live", s.cutToLiveMs), ...w("build", s.buildMs), ...w("proof", s.proofMs), ...w("switch", s.switchMs), ...w("verify", s.verifyMs)].join(" · ");
94 return [...w("cut→live", s.cutToLiveMs), ...w("build", s.buildMs), ...w("proof", s.proofMs), ...w("switch", s.switchMs), ...(s.interruptedRuns === null ? [] : [`interrupted ${s.interruptedRuns} ${s.interruptedRuns === 1 ? "run" : "runs"}`]), ...w("verify", s.verifyMs)].join(" · ");
9295 }
9396
9497 export function withLandedToLive(db: Database, releases: readonly KeeperRelease[]): LandedRelease[] {
src/keeper/record.ts +10
@@ -38,6 +38,8 @@ export interface KeeperRelease {
3838 readonly buildMs: number | null;
3939 readonly proofMs: number | null;
4040 readonly switchMs: number | null;
41 /** Crew runs running in the drained process at the switch (#11917); null when not counted. */
42 readonly interruptedRuns: number | null;
4143 /** Why a person cut this release now, outside the spacing; empty for a cut on cadence. */
4244 readonly hotfix: string;
4345 }
@@ -64,6 +66,7 @@ interface Row {
6466 build_ms: number | null;
6567 proof_ms: number | null;
6668 switch_ms: number | null;
69 interrupted_runs: number | null;
6770 hotfix: string;
6871 }
6972
@@ -88,6 +91,7 @@ const toRelease = (r: Row): KeeperRelease => ({
8891 buildMs: r.build_ms,
8992 proofMs: r.proof_ms,
9093 switchMs: r.switch_ms,
94 interruptedRuns: r.interrupted_runs,
9195 hotfix: r.hotfix,
9296 });
9397
@@ -123,6 +127,12 @@ export function keepTimings(db: Database, id: number, t: { buildMs: number; proo
123127 );
124128 }
125129
130 /** How many crew runs the old process still had running when the switch drained it: kept on the release still rolling. */
131 export function keepInterrupted(db: Database, id: number, runs: number): void {
132 const n = Number.isFinite(runs) ? Math.max(0, Math.round(runs)) : 0;
133 db.query(`UPDATE keeper_release SET interrupted_runs = ?2 WHERE id = ?1 AND stage = 'rolling'`).run(id, n);
134 }
135
126136 /** One more `/healthz` probe while the roll runs. */
127137 export function countRollProbe(db: Database, id: number, ok: boolean): void {
128138 db.query(
src/keeper/wire.ts +17 −5
@@ -13,6 +13,7 @@
1313 * `FORGE_SELF_SWAP=1`: the keeper swaps main's tree in place (`swapper.ts`).
1414 */
1515
16 import type { Database } from "bun:sqlite";
1617 import { join } from "node:path";
1718 import type { App } from "../app.ts";
1819 import { originPush } from "../actions/branches.ts";
@@ -33,7 +34,8 @@ import { handover } from "../release/swap.ts";
3334 import { runningChildren } from "../boards/sampler.ts";
3435 import { snapshotsOf } from "../snapshot/service.ts";
3536 import { kubeDeployer } from "./deploy.ts";
36 import { inFlight, keepTimings } from "./record.ts";
37 import { inFlight, keepInterrupted, keepTimings } from "./record.ts";
38 import { runningAgentRuns } from "../crew/running.ts";
3739 import { swapper } from "./swapper.ts";
3840 import type { KeeperPorts } from "./keeper.ts";
3941 import { timedGet, verifyUnderLoad } from "./verify.ts";
@@ -80,6 +82,19 @@ export function loadPages(repo: Pick<Repo, "ownerName" | "name">): string[] {
8082 return ["/boards", `/${encodeURIComponent(repo.ownerName)}/${encodeURIComponent(repo.name)}`, "/explore", "/"];
8183 }
8284
85 /**
86 * What the swap reports just before this process drains (#11839, #11917): the stage timings, and the crew
87 * runs still running here, the ones the switch interrupts, kept on the release still rolling.
88 */
89 export function switched(db: Database, runs: () => number): (t: { buildMs: number; proofMs: number; switchMs: number }) => void {
90 return (t) => {
91 const flying = inFlight(db);
92 if (flying === undefined) return;
93 keepTimings(db, flying.id, t);
94 keepInterrupted(db, flying.id, runs());
95 };
96 }
97
8398 /** The pages looked at on a phone: home, the forge's repository, the apps page. */
8499 export function phonePages(repo: Pick<Repo, "ownerName" | "name">): string[] {
85100 return ["/", `/${encodeURIComponent(repo.ownerName)}/${encodeURIComponent(repo.name)}`, "/apps"];
@@ -104,10 +119,7 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str
104119 checkout: (sha) => checkoutRelease(app.config.dataDir, dir, sha),
105120 preflight: (tree) => preflightRelease(tree, app.config.dataDir),
106121 drain: () => handover.drain?.(),
107 timed: (t) => {
108 const flying = inFlight(app.db);
109 if (flying !== undefined) keepTimings(app.db, flying.id, t);
110 },
122 timed: switched(app.db, runningAgentRuns),
111123 probe: ownProbeHeaders(app.config.dataDir),
112124 ...(app.config.swapBudgetMs === undefined ? {} : { budgetMs: app.config.swapBudgetMs }),
113125 })
test/keeper-interrupted-runs.test.ts added +40
@@ -0,0 +1,40 @@
1 // The switch disturbs nothing (#11917): each release counts the crew runs that were running in the
2 // process it drained, and the release card and `release_keeper` say so beside the stage timings.
3 import { expect, test } from "bun:test";
4 import { tick } from "../src/keeper/keeper.ts";
5 import { keeperAnswer } from "../src/keeper/landed.ts";
6 import { listKeeperReleases } from "../src/keeper/record.ts";
7 import { switched } from "../src/keeper/wire.ts";
8 import { runningAgentRuns, trackAgentRun, untrackAgentRun } from "../src/crew/running.ts";
9 import { keeperPanel } from "../src/web/pages/keeper-panel.ts";
10 import { keeperForge } from "./helpers/keeper.ts";
11
12 test("a switch with two crew runs in flight records 2, and the release card says so", async () => {
13 const f = await keeperForge();
14 await tick(f.ports);
15 expect(listKeeperReleases(f.app.db)[0]!.stage).toBe("rolling");
16 expect(listKeeperReleases(f.app.db)[0]!.interruptedRuns).toBeNull();
17
18 trackAgentRun("run-one", new AbortController());
19 trackAgentRun("run-two", new AbortController());
20 try {
21 switched(f.app.db, runningAgentRuns)({ buildMs: 4_000, proofMs: 3_000, switchMs: 1_000 });
22 } finally {
23 untrackAgentRun("run-one");
24 untrackAgentRun("run-two");
25 }
26
27 const r = listKeeperReleases(f.app.db)[0]!;
28 expect(r.interruptedRuns).toBe(2);
29 expect(r.switchMs).toBe(1_000);
30 expect(keeperAnswer(f.app.db).releases[0]!.interruptedRuns).toBe(2);
31 expect(String(keeperPanel(f.app.db))).toContain("switch 1 s · interrupted 2 runs");
32 });
33
34 test("a quiet switch says interrupted 0 runs; a release that never counted says nothing", async () => {
35 const f = await keeperForge();
36 await tick(f.ports);
37 expect(String(keeperPanel(f.app.db))).not.toContain("interrupted");
38 switched(f.app.db, () => 0)({ buildMs: 0, proofMs: 0, switchMs: 0 });
39 expect(String(keeperPanel(f.app.db))).toContain("interrupted 0 runs");
40 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs