Public: anyone can read this; the crew can change it.

The keeper measures change-fail rate and revert rate per week #13120

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13110-the-keeper-measures-change-fail-rate-and → main 9 commits fork 6 threads · 6 unresolved

  • Merged. scout/13110-the-keeper-measures-change-fail-rate-and was merged into main as be551b3.

Review

Sign in to review this change.
6 files changed +466 −27
docs/features/keeper.md +3 −1
@@ -13,6 +13,8 @@ A `red_checks` issue lists the red pulls, oldest first, so whoever picks it up k
1313
1414 An `empty_pillar` issue lists the ready issues and the hold on each (`landed`, `open_pull`, `stalled`...), so a person sees why nothing can start.
1515
16 Not done: DORA change-fail rate, revert rate, queue-wait p95 and rework per agent from the issue discussion are not measured yet.
16 Two DORA numbers (`keeper_measures.ts`, #13110), over the last 7 days, each with the count behind it, on the flow panel (once the week has a reading), in the terminal's Flow dialog (which also says "No reading") and in `keeper.measures` of the MCP `flow` report and its API answer. **Change-fail rate**: landed pulls whose next finished main check was red, or, in the forge's own repository, whose next release was rolled back (a red production proof). **Revert rate**: landed pulls a revert pull landed for within seven days of their landing. Fewer than five landings in the week is "No reading", never a rate.
17
18 Still unmeasured: queue-wait p95 and rework per agent from the issue discussion. The two rates are approximations: "next check" is the first main check created within 24 hours after the landing (a release likewise: cut within 24 hours), so a red that one of several batched landings caused is counted against each; a pull landed in the last seven days may yet be reverted, so the revert rate can only rise; a release proof is matched by time, not by the commits it carried, and only for the forge's own repository; the week is rolling, not a calendar week, and no history of past weeks is kept.
1719
1820 Who reads what: a site admin reads the sentence with repository names and the question with its button. Everyone else, guests and agents included, reads the cause with no repository in it (`readKeeper(db, full)`). The Flow only switch is retired (#7771), so no act asks a question today; one already on the books is answered with its button, and the answer is a `keeper.answered` event carrying the owner's id.
src/flow/keeper.ts +27 −7
@@ -16,6 +16,7 @@ import { getRepo } from "../core/repos.ts";
1616 import { createLog } from "../kernel/log.ts";
1717 import { ownRepoPath } from "../kernel/own_repo.ts";
1818 import { KEEPER_CAUSES, pickCause, readKeeperFacts, type KeeperCause, type KeeperMove } from "./keeper_cause.ts";
19 import { measuresCached, type KeeperMeasures } from "./keeper_measures.ts";
1920 import { isLow, landedBetween, readRate, type Rate } from "./keeper_rate.ts";
2021 import { HOLDS, heldNow, heldWords } from "../crew/held.ts";
2122 import { oldestWaitingOn, redPulls } from "./pile.ts";
@@ -94,7 +95,10 @@ function skipped(db: Database, now: number): Set<KeeperCause> {
9495 return new Set(rows.map((r) => r.cause).filter((c): c is KeeperCause => (KEEPER_CAUSES as readonly string[]).includes(c)));
9596 }
9697
97 /** Checks every act whose time is up: landings rose and it stands, or it is undone and the owner is asked. */
98 /**
99 * Checks every act whose time is up: landings rose and it stands, or it is undone and the owner is
100 * asked.
101 */
98102 function settle(app: App, now: number): void {
99103 const { db } = app;
100104 const due = db
@@ -113,7 +117,10 @@ function settle(app: App, now: number): void {
113117 }
114118 }
115119
116 /** Files one issue for a cause the keeper cannot act on, in the forge's own repository; private repositories are named in it only when that repository is private too. */
120 /**
121 * Files one issue for a cause the keeper cannot act on, in the forge's own repository; private
122 * repositories are named in it only when that repository is private too.
123 */
117124 export function fileOnce(app: App, found: { cause: KeeperCause; says: string }, rate: Rate, now: number): void {
118125 const [owner, name] = ownRepoPath().split("/");
119126 const repo = owner && name ? getRepo(app.db, owner, name) : undefined;
@@ -164,7 +171,8 @@ export function tickKeeper(app: App, now = new Date(), onLift: () => void = () =
164171 saveState(db, "", "", t);
165172 return { cause: null, acted: null };
166173 }
167 // One act at a time: an act waiting on its check is left to show its effect, and its cause stays named.
174 // One act at a time: an act waiting on its check is left to show its effect, and its cause stays
175 // named.
168176 const waiting = db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM keeper_attempt WHERE outcome = 'pending'").get();
169177 if ((waiting?.n ?? 0) > 0) return { cause: found.cause, acted: null };
170178 saveState(db, found.cause, found.says, t);
@@ -209,6 +217,11 @@ export interface KeeperRead {
209217 /** The last thing it did, one sentence; null before it did anything. */
210218 readonly last: string | null;
211219 readonly ask: KeeperAsk | null;
220 /**
221 * Change-fail and revert rate over the last 7 days, each with the count behind it (#13110);
222 * public repositories only unless `full`.
223 */
224 readonly measures: KeeperMeasures;
212225 }
213226
214227 const ACTION_WORDS: Record<string, string> = {
@@ -218,7 +231,9 @@ const ACTION_WORDS: Record<string, string> = {
218231
219232 const clock = (ms: number): string => new Date(ms).toISOString().slice(11, 16);
220233
221 /** The cause with no repository named in it: what a reader who may not see every repository is told. */
234 /**
235 * The cause with no repository named in it: what a reader who may not see every repository is told.
236 */
222237 const GENERIC: Record<KeeperCause, string> = {
223238 ceiling: "A daily ceiling is reached: builders start again at 00:00 UTC.",
224239 usage: "Usage is the limit: no subscription has headroom.",
@@ -235,7 +250,7 @@ const GENERIC: Record<KeeperCause, string> = {
235250 * `full` (a site admin) reads the sentence that names repositories and the
236251 * question; anyone else reads the cause with no repository in it.
237252 */
238 export function readKeeper(db: Database, full: boolean): KeeperRead {
253 export function readKeeper(db: Database, full: boolean, now: Date = new Date()): KeeperRead {
239254 const state = db.query<{ cause: string; says: string }, []>("SELECT cause, says FROM keeper_state WHERE id = 1").get();
240255 const acts = db
241256 .query<AttemptRow, []>(`SELECT ${COLS} FROM keeper_attempt WHERE outcome IN ('pending','worked','undone','filed') ORDER BY id DESC LIMIT 1`)
@@ -249,12 +264,17 @@ export function readKeeper(db: Database, full: boolean): KeeperRead {
249264 cause: state && state.cause !== "" ? (full ? state.says : (GENERIC[state.cause as KeeperCause] ?? "")) || null : null,
250265 name: state && state.cause !== "" ? (state.cause as KeeperCause) : null,
251266 last,
252 // No act asks a question since FLOW ONLY left (#7771); a question already on the books is still answerable.
267 // No act asks a question since FLOW ONLY left (#7771); a question already on the books is still
268 // answerable.
253269 ask: full && asked ? { id: asked.id, question: asked.says, button: "Got it" } : null,
270 measures: measuresCached(db, now, !full),
254271 };
255272 }
256273
257 /** The line for a page: the cause and the last act, in one sentence run; empty when there is neither. */
274 /**
275 * The line for a page: the cause and the last act, in one sentence run; empty when there is
276 * neither.
277 */
258278 export function keeperLine(read: KeeperRead): string {
259279 return [read.cause === null ? "" : `Binding landings: ${read.cause}`, read.last ?? ""].filter((s) => s !== "").join(" ");
260280 }
src/flow/keeper_measures.ts added +163
@@ -0,0 +1,163 @@
1 /**
2 * The keeper's two DORA numbers for the last seven days (#13110), read from
3 * what the forge already records, no table of its own:
4 *
5 * - change-fail rate: landed pulls (`pull.merged_at`, default branch, not
6 * carried) whose next finished main check (`run`, kind `check`, no issue,
7 * env the default branch) failed, or, in the forge's own repository, whose
8 * next release (`keeper_release.cut_at`) was `rolled_back` (its proof red).
9 * - revert rate: landed pulls a revert pull (`pull.reverts_issue_id`) landed
10 * within seven days of.
11 *
12 * Fewer than {@link MIN_LANDINGS} landings in the week is no reading: a rate
13 * of three pulls is a story, not a number. A reader who is not a site admin
14 * counts landings in public repositories only.
15 */
16
17 import type { Database } from "bun:sqlite";
18 import { writeStamp } from "../core/write_clock.ts";
19 import { ownRepoPath } from "../kernel/own_repo.ts";
20
21 const DAY_MS = 86_400_000;
22 export const MIN_LANDINGS = 5;
23
24 export interface Measure {
25 /** Pulls behind the rate's numerator. */
26 readonly count: number;
27 /** Null when the week has no reading. */
28 readonly rate: number | null;
29 }
30
31 export interface KeeperMeasures {
32 readonly landed: number;
33 readonly changeFail: Measure;
34 readonly revert: Measure;
35 }
36
37 /** A measure as a row: its name, and its rate with the count behind it. */
38 export interface MeasureRow {
39 readonly label: string;
40 readonly value: string;
41 }
42
43 interface Landed {
44 issue_id: string;
45 repo_id: string;
46 branch: string;
47 merged_at: string;
48 path: string;
49 }
50
51 interface Check {
52 ms: number;
53 state: string;
54 }
55
56 const measure = (count: number, landed: number): Measure => ({ count, rate: landed < MIN_LANDINGS ? null : count / landed });
57
58 /** The first check at or after `from` and before `to`, in a time-ordered list (binary search). */
59 function firstIn(list: readonly Check[], from: number, to: number): Check | undefined {
60 let lo = 0;
61 let hi = list.length;
62 while (lo < hi) {
63 const mid = (lo + hi) >> 1;
64 if (list[mid]!.ms < from) lo = mid + 1;
65 else hi = mid;
66 }
67 const c = list[lo];
68 return c !== undefined && c.ms < to ? c : undefined;
69 }
70
71 /**
72 * Read the week. `publicOnly` (any reader but a site admin) counts landings in public repositories
outdated

This takes any later release with no upper bound, so a rollback days later is counted against this landing. Bound it, for example cut_at < merged_at + 24h, and test it.

73 * only: a private repository's landings are not a guest's to count.
outdated

Medium (DoS/perf): checks.find scans the whole week of main checks for every landing, and within(at) builds a Date on every comparison. Group checks by repo+env once (a Map of sorted arrays) and compute the bound once per landing. With 100k runs this already timed /boards out once.

74 */
75 export function readMeasures(db: Database, now: Date = new Date(), publicOnly = false): KeeperMeasures {
76 const since = new Date(now.getTime() - 7 * DAY_MS).toISOString();
77 const rows = db
78 .query<Landed, [string, string, number]>(
79 `SELECT p.issue_id, i.repo_id, r.default_branch AS branch, p.merged_at, u.username || '/' || r.name AS path
80 FROM pull p JOIN issue i ON i.id = p.issue_id JOIN repo r ON r.id = i.repo_id JOIN user u ON u.id = r.owner_id
81 WHERE (?3 = 0 OR r.visibility = 'public') AND p.base_branch = r.default_branch AND p.carried = 0
82 AND p.merged_at >= ?1 AND p.merged_at <= ?2`,
83 )
84 .all(since, now.toISOString(), publicOnly ? 1 : 0);
85 // The week's finished main checks, read once and grouped by repository and branch in time
86 // order: each landing finds its next one by binary search, not a scan.
87 const byLane = new Map<string, Check[]>();
88 for (const c of db
89 .query<{ repo_id: string; env: string; state: string; created_at: string }, [string]>(
90 `SELECT repo_id, env, state, created_at FROM run WHERE kind = 'check' AND issue_id IS NULL
91 AND state IN ('ok','failed') AND created_at >= ?1 ORDER BY created_at, id`,
92 )
93 .all(since)) {
94 const key = `${c.repo_id}\0${c.env}`;
95 const list = byLane.get(key) ?? [];
96 list.push({ ms: Date.parse(c.created_at), state: c.state });
97 byLane.set(key, list);
98 }
99 const nextRelease = db.query<{ stage: string }, [string, string]>(
outdated

Design principle 7 (a fact is a dl row, not a sentence run) and 1/8: this joins two measures and their sentences into one paragraph that repeats "7 days" and "landed pulls" twice. Return the two Measure rows and render them as a dl (Change-fail rate: 57%, 4 of 7; Revert rate: 14%, 1 of 7), one sentence stating the window once.

100 "SELECT stage FROM keeper_release WHERE cut_at >= ?1 AND cut_at < ?2 ORDER BY cut_at LIMIT 1",
101 );
102 const reverted = db.query<{ n: number }, [string, string, string]>(
103 `SELECT COUNT(*) AS n FROM pull WHERE reverts_issue_id = ?1 AND merged_at IS NOT NULL AND merged_at >= ?2 AND merged_at <= ?3`,
104 );
105 const own = ownRepoPath().toLowerCase();
106 let failed = 0;
107 let back = 0;
108 for (const l of rows) {
109 const at = Date.parse(l.merged_at);
110 // A landing is blamed only for what follows within a day: a red a week later is another
111 // change's.
112 const until = at + DAY_MS;
113 const check = firstIn(byLane.get(`${l.repo_id}\0${l.branch}`) ?? [], at, until);
114 const release = l.path.toLowerCase() === own ? nextRelease.get(l.merged_at, new Date(until).toISOString()) : null;
115 if (check?.state === "failed" || release?.stage === "rolled_back") failed += 1;
116 if ((reverted.get(l.issue_id, l.merged_at, new Date(at + 7 * DAY_MS).toISOString())?.n ?? 0) > 0) back += 1;
117 }
118 return { landed: rows.length, changeFail: measure(failed, rows.length), revert: measure(back, rows.length) };
119 }
120
121 export const MEASURES_TTL_MS = 60_000;
122
123 interface Entry {
124 readonly at: number;
125 readonly stamp: string;
126 readonly measures: KeeperMeasures;
127 }
128
129 const kept = new WeakMap<Database, Map<boolean, Entry>>();
130
131 /**
132 * {@link readMeasures} kept for a minute while nothing is written, apart by `publicOnly`: what
133 * /boards and the Flow dialog read on every paint.
134 */
135 export function measuresCached(db: Database, now: Date = new Date(), publicOnly = false): KeeperMeasures {
136 const mine = kept.get(db) ?? new Map<boolean, Entry>();
137 kept.set(db, mine);
138 const stamp = writeStamp(db);
139 const t = now.getTime();
140 const hit = mine.get(publicOnly);
141 if (stamp !== undefined && hit && hit.stamp === stamp && t >= hit.at && t - hit.at < MEASURES_TTL_MS) return hit.measures;
142 const measures = readMeasures(db, now, publicOnly);
143 if (stamp === undefined) mine.delete(publicOnly);
144 else mine.set(publicOnly, { at: t, stamp, measures });
145 return measures;
146 }
147
148 const pct = (n: number, of: number): string => `${Math.round((n / of) * 100)}%`;
149
150 /**
151 * The window, said once: the sentence over the rows, or the whole answer while there is no reading.
152 */
153 export const measuresWindow = (m: KeeperMeasures): string =>
154 m.changeFail.rate === null
155 ? `No reading: ${m.landed} landed in the last 7 days, fewer than ${MIN_LANDINGS}.`
156 : `Of ${m.landed} pulls landed in the last 7 days:`;
157
158 /** The two measures as rows; none while there is no reading. */
159 export function measureRows(m: KeeperMeasures): MeasureRow[] {
160 if (m.changeFail.rate === null) return [];
161 const row = (label: string, x: Measure): MeasureRow => ({ label, value: `${pct(x.count, m.landed)}, ${x.count} of ${m.landed}` });
162 return [row("Change-fail rate", m.changeFail), row("Revert rate", m.revert)];
163 }
src/tui/screens/flow.ts +62 −17
@@ -15,10 +15,10 @@
1515 * filter that holds the rest (`flow/held.ts`, #2244), as the flow panel draws it;
1616 * then how long each main has gone unverified after landing (`flow/verified.ts`, #7786);
1717 * then one line for the full-suite report's open issues (`checks/report.ts`, #7780);
18 * the red batches the queue named (`flow/culprits.ts`, #7639, as MCP `flow` carries them as `redBatches`);
19 * the clog sentence, the test failing on many pulls and the pulls on the same files (`flow/clog_view.ts`, #11454, an admin's);
20 * the hour's CPU stops and the pulls closed unmerged after one (`./flow_cpu.ts`, #11904, an admin's);
21 * last, the duplicate lines (`duplicateLines`).
18 * the red batches the queue named (`flow/culprits.ts`, #7639, as MCP `flow` carries them as
19 * `redBatches`); the clog sentence, the test failing on many pulls and the pulls on the same files
20 * (`flow/clog_view.ts`, #11454, an admin's); the hour's CPU stops and the pulls closed unmerged
21 * after one (`./flow_cpu.ts`, #11904, an admin's); last, the duplicate lines (`duplicateLines`).
2222 * Read-only; Enter, Esc or q closes it.
2323 */
2424
@@ -31,6 +31,7 @@ import { pullCounts } from "../../crew/hot_files.ts";
3131 import { warmingLine } from "../../crew/hot_history.ts";
3232 import { ago } from "../../feed/rank.ts";
3333 import type { HeldRow } from "../../flow/held.ts";
34 import { measureRows, measuresCached, measuresWindow, type KeeperMeasures } from "../../flow/keeper_measures.ts";
3435 import { keeperLine, type KeeperAsk, type KeeperRead } from "../../flow/keeper.ts";
3536 import { verifiedSays, type VerifiedRow } from "../../flow/verified.ts";
3637 import { conflictedSays, rebaseLaneSays } from "../../flow/conflicted.ts";
@@ -66,7 +67,10 @@ import { confirmScreen } from "./common.ts";
6667 import { duplicateLines } from "./system.ts";
6768 import type { HandoverLists, Named } from "../../flow/handover_read.ts";
6869
69 /** The hand-over lists as dialog lines: only lists with rows; the rebase rows are numbered, and the number is the key that updates one. */
70 /**
71 * The hand-over lists as dialog lines: only lists with rows; the rebase rows are numbered, and the
72 * number is the key that updates one.
73 */
7074 export function handoverLines(l: HandoverLists, inner: number): string[] {
7175 const list = (title: string, rows: readonly Named[], num = false): string[] =>
7276 rows.length === 0 ? [] : ["", title, ...rows.map((n, i) => truncate(` ${num ? `${String(i + 1)} ` : ""}${n.repo} #${String(n.number)}`, inner))];
@@ -85,17 +89,31 @@ export function flowLines(
8589 db: Database,
8690 now = new Date(),
8791 dups = live().read().duplicates,
88 /** What an agent reader may see: its queue lines, conflicted pulls and changes-requested waits come already cut to public. */
92 /**
93 * What an agent reader may see: its queue lines, conflicted pulls and changes-requested waits
94 * come already cut to public.
95 */
8996 agent?: { queues: ReturnType<typeof readQueues> } | undefined,
90 /** Why ready issues wait, per autonomous repository (`readHeld`, already cut to what the reader may see). */
97 /**
98 * Why ready issues wait, per autonomous repository (`readHeld`, already cut to what the reader
99 * may see).
100 */
91101 held?: readonly HeldRow[],
92 /** The forge's daily ceiling, which the idle-crew alarm needs to know it is not paused on purpose. */
102 /**
103 * The forge's daily ceiling, which the idle-crew alarm needs to know it is not paused on purpose.
104 */
93105 ceilingUsd: number | null = null,
94 /** How long each main has gone unverified after landing (`readVerified`, already cut to what the reader may see). */
106 /**
107 * How long each main has gone unverified after landing (`readVerified`, already cut to what the
108 * reader may see).
109 */
95110 verified?: readonly VerifiedRow[],
96111 /** The keeper's cause and last act (`readKeeper`, already cut to what the reader may see). */
97112 keeper?: KeeperRead,
98 /** Where the backup outcome files are kept; without it the off-box backup alarm cannot be read (the terminal passes the app's). */
113 /**
114 * Where the backup outcome files are kept; without it the off-box backup alarm cannot be read
115 * (the terminal passes the app's).
116 */
99117 dataDir: string | null = null,
100118 /** The release tick's heartbeat line, an owner's only (#11507). */
101119 heartbeat?: string,
@@ -106,7 +124,8 @@ export function flowLines(
106124 const queues = agent?.queues ?? readQueues(db, now, { batches: true });
107125 // The row is full at 58 columns: the red-batch rate (#4235) is a line under it.
108126 const red = queues.filter((q) => (q.batches?.batches ?? 0) > 0);
109 // The figures the /boards panel keeps (`flow/panel_cache.ts`): an agent's public-only set is apart from an admin's.
127 // The figures the /boards panel keeps (`flow/panel_cache.ts`): an agent's public-only set is
128 // apart from an admin's.
110129 const fig = panelFigures(db, agent === undefined ? "owner" : "agent", now);
111130 const conflicted = conflictedSays(fig.conflicted, now.getTime());
112131 const lane = rebaseLaneSays(fig.lane);
@@ -121,6 +140,7 @@ export function flowLines(
121140 return [
122141 ...wrap(verdict(stages, now.getTime()), inner),
123142 ...(keeper === undefined || keeperLine(keeper) === "" ? [] : ["", ...wrap(keeperLine(keeper), inner)]),
143 ...measureLines(measuresCached(db, now, agent !== undefined), inner),
124144 ...(heartbeat === undefined ? [] : ["", ...wrap(heartbeat, inner)]),
125145 ...(keeper?.ask == null ? [] : wrap(`${keeper.ask.question} Press l to ${keeper.ask.button.toLowerCase()}.`, inner)),
126146 "",
@@ -187,7 +207,10 @@ export function flowLines(
187207 ];
188208 }
189209
190 /** What red batches were (#7639): the counts and newest sentence, then each recent verdict with its tests and what was done; nothing when none. */
210 /**
211 * What red batches were (#7639): the counts and newest sentence, then each recent verdict with its
212 * tests and what was done; nothing when none.
213 */
191214 export function verdictLines(v: ReturnType<typeof readVerdicts>, inner: number): string[] {
192215 if (v.recent.length === 0) return [];
193216 return [
@@ -199,7 +222,10 @@ export function verdictLines(v: ReturnType<typeof readVerdicts>, inner: number):
199222 ];
200223 }
201224
202 /** What reviewers repeated (#8716): the line, only once a finding was proposed or a review comment seen. */
225 /**
226 * What reviewers repeated (#8716): the line, only once a finding was proposed or a review comment
227 * seen.
228 */
203229 export function repeatLines(r: ReturnType<typeof readRepeats>, inner: number): string[] {
204230 if (r.proposed === 0 && r.thisWeek.comments === 0 && r.lastWeek.comments === 0) return [];
205231 return ["", ...wrap(`Review repeats: ${r.says}.`, inner)];
@@ -209,6 +235,13 @@ export function repeatLines(r: ReturnType<typeof readRepeats>, inner: number): s
209235 const GATES_SHOWN = 5;
210236
211237 /** Each `owner/name` the queue lists, to its repository id: one query. */

Non-blocking: this doc comment belongs to repoIds below, but measureLines was inserted between them. Put measureLines above this comment.

238 /** The keeper's rates (#13110): the window once, then a row a measure, label and value aligned. */
239 function measureLines(m: KeeperMeasures, inner: number): string[] {
240 const rows = measureRows(m);
241 const width = Math.max(0, ...rows.map((r) => r.label.length));
242 return ["", ...wrap(measuresWindow(m), inner), ...rows.map((r) => truncate(` ${pad(r.label, width)} ${r.value}`, inner))];
243 }
244
212245 function repoIds(db: Database, names: readonly string[]): Map<string, string> {
213246 if (names.length === 0) return new Map();
214247 const rows = db
@@ -245,7 +278,10 @@ export function gateLines(gates: readonly KeptResult[], now: number, inner: numb
245278 /** The doctor's acts shown under a repository's gates. */
246279 const DOCTOR_SHOWN = 3;
247280
248 /** One indented line per batch-doctor act (#7555), newest first: `doctor #14 held · functional · how long ago · the error line`; none when it has not acted. */
281 /**
282 * One indented line per batch-doctor act (#7555), newest first:
283 * `doctor #14 held · functional · how long ago · the error line`; none when it has not acted.
284 */
249285 export function doctorLines(runs: readonly DoctorRun[], now: number, inner: number): string[] {
250286 return runs.map((r) => {
251287 const pull = r.sealed || r.culprit === null ? "a sealed pull" : `#${r.culprit}`;
@@ -256,7 +292,10 @@ export function doctorLines(runs: readonly DoctorRun[], now: number, inner: numb
256292 /** Idle pulls shown before `+N more`. */
257293 const IDLE_SHOWN = 5;
258294
259 /** A blank line, `idle: N`, then a line per idle crew pull (`flow/idle_pulls.ts`, #6479), oldest first; nothing when none. */
295 /**
296 * A blank line, `idle: N`, then a line per idle crew pull (`flow/idle_pulls.ts`, #6479), oldest
297 * first; nothing when none.
298 */
260299 export function idleLines(idle: readonly IdlePull[], inner: number): string[] {
261300 if (idle.length === 0) return [];
262301 return [
@@ -310,9 +349,15 @@ export function heldLines(rows: readonly HeldRow[], inner: number): string[] {
310349 export function flowScreen(
311350 ctx: Ctx,
312351 lines: readonly string[],
313 /** The keeper's open question (a site admin's); `answer` presses its one button and says what happened. */
352 /**
353 * The keeper's open question (a site admin's); `answer` presses its one button and says what
354 * happened.
355 */
314356 keeper?: { ask: KeeperAsk; answer: () => { done: boolean; message: string } },
315 /** The hand-over lists (a person-admin's): read when the screen opens; a digit presses that rebase row's update, as `updateSubmit` does. */
357 /**
358 * The hand-over lists (a person-admin's): read when the screen opens; a digit presses that rebase
359 * row's update, as `updateSubmit` does.
360 */
316361 handover?: { read: () => Promise<HandoverLists>; update: (n: Named) => { ok: boolean; says: string } },
317362 ): Screen {
318363 let lists: HandoverLists | undefined;
src/web/studio/keeper_row.ts +11 −2
@@ -6,6 +6,7 @@
66 */
77
88 import type { Database } from "bun:sqlite";
9 import { measureRows, measuresWindow } from "../../flow/keeper_measures.ts";
910 import { answerKeeper, keeperLine, readKeeper } from "../../flow/keeper.ts";
1011 import type { Ctx } from "../../http/context.ts";
1112 import { signInFirst, NOT_ADMIN_HERE, forbiddenPage } from "../chrome.ts";
@@ -17,8 +18,16 @@ import { btn } from "../ui.ts";
1718 export function keeperRow(db: Database, full: boolean): Html {
1819 const read = readKeeper(db, full);
1920 const line = keeperLine(read);
20 if (line === "" && read.ask === null) return html``;
21 return html`${line === "" ? null : html`<p class="small muted break" id="keeper">${line}</p>`}
21 // Silent until the week has a reading: a page's first view has a budget, and "No reading" says
22 // nothing. A reading is the window said once, then a dl row a measure (principle 7).
23 const rows = measureRows(read.measures);
outdated

Principle 1 and the issue's box 2: hiding the line leaves the page unable to say 'no reading'. The terminal also lacks this line. Show the same sentence in src/tui/ Flow.

24 return html`${rows.length === 0
outdated

Design principle 7: a muted p holding a run-on line where two facts belong in a dl row. Also leak: read.measures counts landings in private repositories and is shown to guests here; scope the landed set to public repositories when full is false.

25 ? null
26 : html`<section class="small" id="keeper-measures">
27 <p class="muted">${measuresWindow(read.measures)}</p>
28 <dl>${rows.map((r) => html`<dt>${r.label}</dt><dd>${r.value}</dd>`)}</dl>
29 </section>`}
30 ${line === "" ? null : html`<p class="small muted break" id="keeper">${line}</p>`}
2231 ${read.ask === null
2332 ? null
2433 : html`<form method="post" action="/flow/keeper/answer" id="keeper-ask" class="bar small">
test/keeper-measures.test.ts added +200
@@ -0,0 +1,200 @@
1 /** The keeper's change-fail and revert rate over a week (#13110), from fixtures, and no reading. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { openIssue } from "../src/actions/issues.ts";
8 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
9 import { getRepoById, type Repo } from "../src/core/repos.ts";
10 import { readKeeper } from "../src/flow/keeper.ts";
11 import { MIN_LANDINGS, measureRows, measuresCached, measuresWindow, readMeasures } from "../src/flow/keeper_measures.ts";
12 import { flowLines } from "../src/tui/screens/flow.ts";
13 import { keeperRow } from "../src/web/studio/keeper_row.ts";
14 import { seedAdmin } from "./helpers/admin.ts";
15 import { quietLogs } from "./helpers/globals.ts";
16
17 quietLogs();
18
19 let dir: string;
20 let app: App;
21 let repo: Repo;
22 let was: string | undefined;
23 const now = new Date("2026-11-10T12:00:00.000Z");
24 const at = (minutesBefore: number): string => new Date(now.getTime() - minutesBefore * 60_000).toISOString();
25 const issues: string[] = [];
26
27 function pull(issueId: string, mergedAt: string | null, reverts: string | null = null): void {
28 app.db
29 .query(
30 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, reverts_issue_id, merged_at)
31 VALUES (?1, ?2, ?3, '', '', ?4, ?5, ?6)`,
32 )
33 .run(issueId, repo.defaultBranch, `b/${issueId}`, repo.id, reverts, mergedAt);
34 }
35 function check(createdAt: string, state: string): void {
36 app.db
37 .query("INSERT INTO run (repo_id, kind, env, state, created_at) VALUES (?1, 'check', ?2, ?3, ?4)")
38 .run(repo.id, repo.defaultBranch, state, createdAt);
39 }
40 function release(version: string, stage: string, cutAt: string): void {
41 app.db
42 .query("INSERT INTO keeper_release (version, sha, known_good_version, known_good_image, stage, cut_at) VALUES (?1, 'abc', 'v0', 'img', ?2, ?3)")
43 .run(version, stage, cutAt);
44 }
45 function newIssue(title: string): string {
46 const made = openIssue(app, admin, repo, { title });
47 if (!made.ok) throw new Error(String(made.error));
48 issues.push(made.value.id);
49 return made.value.id;
50 }
51 let admin: Awaited<ReturnType<typeof seedAdmin>> extends infer R ? (R extends { ok: true; value: infer V } ? V : never) : never;
52
53 beforeAll(async () => {
54 was = process.env.FORGE_SENTINEL_REPO;
55 process.env.FORGE_SENTINEL_REPO = "alice/engine";
56 dir = await mkdtemp(join(tmpdir(), "forge-measures-"));
57 app = createApp({ dataDir: dir, port: 0 });
58 const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-passphrase" });
59 if (!a.ok) throw new Error("signup failed");
60 admin = a.value;
61 const made = await createRepo(app, admin, { name: "engine", visibility: "public" });
62 if (!made.ok) throw new Error(String(made.error));
63 repo = made.value;
64 const seeded = await commitToBranch(app, admin, repo, { message: "Start", files: [{ path: "README.md", content: "# engine\n" }] });
65 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
66 repo = getRepoById(app.db, repo.id)!;
67
68 // Six pulls landed an hour apart, p0 oldest (6 h before `now`) to p5 (1 h before).
69 const p = Array.from({ length: 6 }, (_, i) => newIssue(`p${i}`));
70 p.forEach((id, i) => pull(id, at((6 - i) * 60)));
71 // Each landing's check runs ten minutes after it: only p4's is red.
72 for (let i = 0; i < 6; i++) check(at((6 - i) * 60 - 10), i === 4 ? "failed" : "ok");
73 // Releases: one verified after p0; one rolled back after p3 (it stands for p1, p2, p3).
74 release("v1", "verified", at(5 * 60 + 30));
75 release("v2", "rolled_back", at(3 * 60 - 30));
76 // A revert of p2 landed half an hour ago; a revert of p5 is still open and does not count.
77 const rev = newIssue("Revert p2");
78 pull(rev, at(30), p[2]!);
79 check(at(20), "ok");
80 const open = newIssue("Revert p5");
81 pull(open, null, p[5]!);
82 }, 60_000);
83
84 afterAll(async () => {
85 if (was === undefined) delete process.env.FORGE_SENTINEL_REPO;
86 else process.env.FORGE_SENTINEL_REPO = was;
87 app.close();
88 await rm(dir, { recursive: true, force: true });
89 });
90
91 test("change-fail rate counts a red next main check or a rolled-back next release, once per pull", () => {
92 const m = readMeasures(app.db, now);
93 expect(m.landed).toBe(7);
94 // p1, p2, p3 by the rolled-back release; p4 by its red check.
95 expect(m.changeFail.count).toBe(4);
96 expect(m.changeFail.rate).toBeCloseTo(4 / 7);
97 expect(measureRows(m)[0]).toEqual({ label: "Change-fail rate", value: "57%, 4 of 7" });
98 });
99
100 test("revert rate counts landed pulls a landed revert took back within seven days", () => {
101 const m = readMeasures(app.db, now);
102 expect(m.revert.count).toBe(1);
103 expect(m.revert.rate).toBeCloseTo(1 / 7);
104 expect(measureRows(m)[1]).toEqual({ label: "Revert rate", value: "14%, 1 of 7" });
105 expect(measuresWindow(m)).toBe("Of 7 pulls landed in the last 7 days:");
106 });
107
108 test("fewer than five landings in the week says there is no reading", () => {
109 const early = new Date(now.getTime() - 150 * 60_000);
110 const m = readMeasures(app.db, early);
111 expect(m.landed).toBeLessThan(MIN_LANDINGS);
112 expect(m.changeFail.rate).toBeNull();
113 expect(m.revert.rate).toBeNull();
114 expect(measureRows(m)).toEqual([]);
115 expect(measuresWindow(m)).toContain("No reading");
116 expect(measuresWindow(m)).toContain("fewer than 5");
117 });
118
119 test("the keeper's answer and its page carry both, with the counts", () => {
120 const read = readKeeper(app.db, false, now);
121 expect(read.measures.changeFail.count).toBe(4);
122 expect(read.measures.revert.count).toBe(1);
123 const html = String(keeperRow(app.db, false));
124 expect(html).toContain("<dt>Change-fail rate</dt><dd>57%, 4 of 7</dd>");
125 expect(html).toContain("<dt>Revert rate</dt><dd>14%, 1 of 7</dd>");
126 // The window is said once.
127 expect(html.match(/7 days/g)?.length).toBe(1);
128 });
129
130 test("a page says nothing of the rates while the week has no reading", async () => {
131 const dir = await mkdtemp(join(tmpdir(), "keeper-quiet-"));
132 const fresh = createApp({ dataDir: dir, port: 0 });
133 try {
134 expect(String(keeperRow(fresh.db, false))).not.toContain("keeper-measures");
135 } finally {
136 fresh.close();
137 await rm(dir, { recursive: true, force: true });
138 }
139 });
140
141 test("a red check or rollback more than a day after a landing is not counted against it", () => {
142 const old = newIssue("Landed long ago");
143 pull(old, at(6 * 24 * 60));
144 check(at(4 * 24 * 60), "failed");
145 release("v-late", "rolled_back", at(4 * 24 * 60));
146 try {
147 const m = readMeasures(app.db, now);
148 expect(m.landed).toBe(8);
149 expect(m.changeFail.count).toBe(4);
150 } finally {
151 app.db.query("DELETE FROM keeper_release WHERE version = 'v-late'").run();
152 app.db.query("DELETE FROM pull WHERE issue_id = ?1").run(old);
153 app.db.query("DELETE FROM run WHERE created_at = ?1").run(at(4 * 24 * 60));
154 }
155 });
156
157 const flat = (lines: string[]): string => lines.join(" ").replace(/\s+/g, " ");
158
159 test("the terminal's Flow dialog says both rates, and says there is no reading below five landings", () => {
160 const said = flat(flowLines(app.db, now));
161 expect(said).toContain("Of 7 pulls landed in the last 7 days:");
162 expect(said).toMatch(/Change-fail rate 57%, 4 of 7/);
163 expect(said).toMatch(/Revert rate 14%, 1 of 7/);
164 const early = new Date(now.getTime() - 150 * 60_000);
165 expect(flat(flowLines(app.db, early))).toMatch(/No reading: \d landed in the last 7 days, fewer than 5\./);
166 });
167
168 test("a reader who is not a site admin counts landings in public repositories only", async () => {
169 const hidden = await createRepo(app, admin, { name: "hidden", visibility: "private" });
170 if (!hidden.ok) throw new Error(String(hidden.error));
171 const made = openIssue(app, admin, hidden.value, { title: "private landing" });
172 if (!made.ok) throw new Error(String(made.error));
173 app.db
174 .query(
175 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at)
176 VALUES (?1, ?2, 'b/x', '', '', ?3, ?4)`,
177 )
178 .run(made.value.id, hidden.value.defaultBranch, hidden.value.id, at(90));
179 try {
180 expect(readMeasures(app.db, now, false).landed).toBe(8);
181 expect(readMeasures(app.db, now, true).landed).toBe(7);
182 expect(readKeeper(app.db, false, now).measures.landed).toBe(7);
183 expect(readKeeper(app.db, true, now).measures.landed).toBe(8);
184 } finally {
185 app.db.query("DELETE FROM pull WHERE issue_id = ?1").run(made.value.id);
186 }
187 });
188
189 test("the kept measures hold for a minute while nothing is written, and read again after a write", () => {
190 const first = measuresCached(app.db, now, true);
191 expect(measuresCached(app.db, new Date(now.getTime() + 30_000), true)).toBe(first);
192 expect(measuresCached(app.db, new Date(now.getTime() + 61_000), true)).not.toBe(first);
193 const later = measuresCached(app.db, now, true);
194 check(at(5), "ok");
195 try {
196 expect(measuresCached(app.db, now, true)).not.toBe(later);
197 } finally {
198 app.db.query("DELETE FROM run WHERE created_at = ?1").run(at(5));
199 }
200 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs