Public: anyone can read this; the crew can change it.
Measure each step of the path: time and red rate, posted as the map in #7703 #8250
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/7784-measure-each-step-of-the-path-time-and-r → main 3 commits fork 4 threads · 4 unresolved
7 files
changed +349
−3
docs/FLOW.md +12
| @@ -27,6 +27,18 @@ bottleneck; the knobs stay on Settings → Capacity. The same sentence | ||
| 27 | 27 | (`verdict`) is the terminal's admin-only Flow command and the MCP `flow` tool, |
| 28 | 28 | which also returns the stages; anyone but a forge admin is refused. |
| 29 | 29 | |
| 30 | `readStepStats` (same module, #7784) measures the road's steps over the last | |
| 31 | 30 days: median time (finished − started) and red rate (failed ÷ ok + failed) | |
| 32 | for the pull check, review, proof, merge-queue gate (time only: `check_gate` | |
| 33 | keeps a week and no verdict), main's check, release build and verify. The boot | |
| 34 | check runs inside the release build and has no run of its own. Review's rate | |
| 35 | counts review runs that errored, not changes asked, and says "errored". `/pipeline` | |
| 36 | shows them under each step's details, from public repositories alone, held in | |
| 37 | memory for the minute so the open page costs one read a minute. The same | |
| 38 | numbers, private repositories included for the owner, are the terminal's Flow | |
| 39 | dialog ("Steps, 30 days") and the `steps` array of the MCP `flow` tool and | |
| 40 | `/api/v1/telemetry/flow` (public repositories alone for an agent). | |
| 41 | ||
| 30 | 42 | ## Rebases |
| 31 | 43 | |
| 32 | 44 | Approved work lands before new work is built, in the check lane too (#6140). |
docs/design/areas/pipeline.md added +5
| @@ -0,0 +1,5 @@ | ||
| 1 | # Pipeline | |
| 2 | ||
| 3 | ## Measured steps (#7784) | |
| 4 | ||
| 5 | Under each step's folded details: one muted `small` line saying what is measured, then a `.stats` grid of `stat()` (docs/design/sections/5-components.md), one per measured part — label "Pull check", value the median ("8m", or "—"), note the red rate ("25% red (1 of 4)") or why there is no number. No new class, no colour; server-rendered, no JavaScript. |
src/flow/pipeline.ts +124
| @@ -242,3 +242,127 @@ export function verdict(stages: readonly PipelineStage[], now = Date.now()): str | ||
| 242 | 242 | const n = worst.waiting; |
| 243 | 243 | return `${STAGE_LABEL[worst.key]} is the bottleneck: its oldest has waited ${waited(worst, now) ?? ""}, ${n} ${n === 1 ? "waits" : "wait"}${worst.capacity === null ? "" : `, ${worst.capacity}`}.`; |
| 244 | 244 | } |
| 245 | ||
| 246 | /** The window the per-step numbers are read over (#7784). */ | |
| 247 | export const STEP_WINDOW_DAYS = 30; | |
| 248 | ||
| 249 | /** One measured step of the path: what it is, and how long and how often red it has been. */ | |
| 250 | export interface StepStat { | |
| 251 | readonly key: "pull-check" | "review" | "proof" | "gate" | "main-check" | "release" | "boot" | "verify"; | |
| 252 | readonly label: string; | |
| 253 | /** Finished runs counted (green + red); cancelled ones are no verdict and not counted. */ | |
| 254 | readonly runs: number; | |
| 255 | readonly failed: number; | |
| 256 | /** Median of finished − started, in ms; null when nothing finished or the step keeps no trace. */ | |
| 257 | readonly medianMs: number | null; | |
| 258 | /** failed / runs, or null with no runs. */ | |
| 259 | readonly redRate: number | null; | |
| 260 | /** Why there is no number, when there is none to read. */ | |
| 261 | readonly note: string | null; | |
| 262 | /** What a red outcome means for this step: a check that failed is "red"; a review run that failed "errored" (a changes-asked verdict is not counted here). */ | |
| 263 | readonly redWord: "red" | "errored"; | |
| 264 | } | |
| 265 | ||
| 266 | interface Timed { | |
| 267 | state: string; | |
| 268 | ms: number | null; | |
| 269 | } | |
| 270 | ||
| 271 | /** Median of the sorted middle: the mean of the two middle values for an even count. */ | |
| 272 | export function median(xs: readonly number[]): number | null { | |
| 273 | if (xs.length === 0) return null; | |
| 274 | const s = [...xs].sort((a, b) => a - b); | |
| 275 | const m = s.length >> 1; | |
| 276 | return s.length % 2 === 1 ? (s[m] as number) : ((s[m - 1] as number) + (s[m] as number)) / 2; | |
| 277 | } | |
| 278 | ||
| 279 | function stat(key: StepStat["key"], label: string, rows: readonly Timed[], note: string | null = null): StepStat { | |
| 280 | const done = rows.filter((r) => r.state === "ok" || r.state === "failed"); | |
| 281 | const failed = done.filter((r) => r.state === "failed").length; | |
| 282 | const times = done.flatMap((r) => (r.ms !== null && r.ms >= 0 ? [r.ms] : [])); | |
| 283 | return { | |
| 284 | key, | |
| 285 | label, | |
| 286 | runs: done.length, | |
| 287 | failed, | |
| 288 | medianMs: median(times), | |
| 289 | redRate: done.length === 0 ? null : failed / done.length, | |
| 290 | note, | |
| 291 | redWord: "red", | |
| 292 | }; | |
| 293 | } | |
| 294 | ||
| 295 | const MS = `(julianday(x.finished_at) - julianday(x.started_at)) * 86400000.0`; | |
| 296 | ||
| 297 | /** | |
| 298 | * Each step's median time and red rate over the last 30 days, from the runs | |
| 299 | * table (`run`, `proof_run`) and the merge-queue gate's trace (`check_gate`, | |
| 300 | * kept a week and without a verdict, so time only). `publicOnly` reads public | |
| 301 | * repositories alone, for a reader who must not learn what private ones hold. | |
| 302 | * | |
| 303 | * time = finished_at − started_at; red rate = failed ÷ (ok + failed) | |
| 304 | */ | |
| 305 | export function readStepStats(db: Database, now = new Date(), publicOnly = false): StepStat[] { | |
| 306 | const since = new Date(now.getTime() - STEP_WINDOW_DAYS * DAY_MS).toISOString(); | |
| 307 | const vis = publicOnly ? `AND r.visibility = 'public'` : ""; | |
| 308 | const runs = (from: string, where: string): Timed[] => | |
| 309 | db | |
| 310 | .query<Timed, [string]>( | |
| 311 | `SELECT x.state AS state, CASE WHEN x.started_at IS NULL THEN NULL ELSE ${MS} END AS ms | |
| 312 | FROM ${from} x JOIN repo r ON r.id = x.repo_id | |
| 313 | WHERE x.finished_at >= ?1 AND x.state IN ('ok','failed') ${vis} AND ${where}`, | |
| 314 | ) | |
| 315 | .all(since); | |
| 316 | const verifyLane = `x.env IN ('verify','verify-bisect')`; | |
| 317 | const gate = db | |
| 318 | .query<{ ms: number }, [string]>( | |
| 319 | `SELECT ${MS} AS ms FROM check_gate x JOIN repo r ON r.id = x.repo_id | |
| 320 | WHERE x.finished_at IS NOT NULL AND x.finished_at >= ?1 ${vis}`, | |
| 321 | ) | |
| 322 | .all(since); | |
| 323 | const gateStat = stat("gate", "Merge-queue gate", []); | |
| 324 | return [ | |
| 325 | stat("pull-check", "Pull check", runs("run", `x.kind = 'check' AND x.issue_id IS NOT NULL AND NOT ${verifyLane}`)), | |
| 326 | { ...stat("review", "Review run", runs("run", `x.kind = 'agent' AND x.reason = 'review'`)), redWord: "errored" }, | |
| 327 | stat("proof", "Proof", runs("proof_run", "1")), | |
| 328 | { | |
| 329 | ...gateStat, | |
| 330 | runs: gate.length, | |
| 331 | medianMs: median(gate.map((g) => g.ms)), | |
| 332 | note: "time only: the gate keeps no verdict, and only the last week", | |
| 333 | }, | |
| 334 | stat("main-check", "Main's check", runs("run", `x.kind = 'check' AND x.issue_id IS NULL AND x.env = r.default_branch`)), | |
| 335 | stat("release", "Release build", runs("run", `x.kind = 'deploy' AND x.env = 'production'`)), | |
| 336 | { | |
| 337 | ...stat("boot", "Boot check", []), | |
| 338 | note: "runs inside the release build: a failed boot is a failed release build", | |
| 339 | }, | |
| 340 | stat("verify", "Verify", runs("run", `x.kind = 'check' AND ${verifyLane}`)), | |
| 341 | ]; | |
outdated
Notes are joined after 'Label: ' as sentences; make them read as clauses (lowercase) or place them after a dash. |
||
| 342 | } | |
| 343 | ||
| 344 | /** A step as a figure and a note under it: "8m" over "25% red (1 of 4)", or "—" over why there is no number. */ | |
| 345 | export function statParts(s: StepStat): { value: string; note: string } { | |
| 346 | const value = s.medianMs === null ? "—" : span(s.medianMs); | |
| 347 | if (s.medianMs === null && s.redRate === null) return { value, note: s.note ?? `no runs in the last ${STEP_WINDOW_DAYS} days` }; | |
| 348 | const red = s.redRate === null ? (s.note ?? "no verdict kept") : `${Math.round(s.redRate * 100)}% ${s.redWord} (${s.failed} of ${s.runs})`; | |
| 349 | return { value, note: red }; | |
| 350 | } | |
| 351 | ||
| 352 | /** "median 12m · 8% red (1 of 12)", or why there is no number: the terminal and MCP say this. */ | |
| 353 | export function statWords(s: StepStat): string { | |
| 354 | const p = statParts(s); | |
| 355 | return s.medianMs === null && s.redRate === null ? p.note : `median ${p.value} · ${p.note}`; | |
| 356 | } | |
| 357 | ||
| 358 | const stepCache = new WeakMap<Database, { minute: number; publicOnly: boolean; stats: StepStat[] }>(); | |
| 359 | ||
| 360 | /** `readStepStats` held for the minute it was read in, per database: the open page runs it at most once a minute, however often it is asked. */ | |
| 361 | export function cachedStepStats(db: Database, now = new Date(), publicOnly = false): StepStat[] { | |
| 362 | const minute = Math.floor(now.getTime() / 60_000); | |
| 363 | const kept = stepCache.get(db); | |
| 364 | if (kept !== undefined && kept.minute === minute && kept.publicOnly === publicOnly) return kept.stats; | |
| 365 | const stats = readStepStats(db, now, publicOnly); | |
| 366 | stepCache.set(db, { minute, publicOnly, stats }); | |
| 367 | return stats; | |
| 368 | } |
src/system/telemetry.ts +3 −1
| @@ -12,7 +12,7 @@ import { firedAlarms } from "../flow/alarms.ts"; | ||
| 12 | 12 | import { readVitals, vitalsFor, vitalsLine, vitalsMisses } from "../flow/vitals.ts"; |
| 13 | 13 | import { parkedSays, readParked } from "../flow/parked.ts"; |
| 14 | 14 | import { readKeeper } from "../flow/keeper.ts"; |
| 15 | import { readPipeline, verdict } from "../flow/pipeline.ts"; | |
| 15 | import { readPipeline, readStepStats, statWords, verdict } from "../flow/pipeline.ts"; | |
| 16 | 16 | import { type QueueReading, readQueues } from "../flow/queue.ts"; |
| 17 | 17 | import { reviewersBusy, reviewingSays } from "../flow/reviewing.ts"; |
| 18 | 18 | import { requestedSays, requestedWaits } from "../flow/requested.ts"; |
| @@ -40,6 +40,8 @@ export function flowFor(app: App, reader: Reader, now = new Date()) { | ||
| 40 | 40 | const parked = readParked(app.db, now, reader.kind === "agent"); |
| 41 | 41 | return { |
| 42 | 42 | says: verdict(stages, now.getTime()), |
| 43 | // Each step of the path: median time and red rate over 30 days (#7784); private repositories for the owner only. | |
| 44 | steps: readStepStats(app.db, now, reader.kind === "agent").map((m) => ({ ...m, says: statWords(m) })), | |
| 43 | 45 | // The cause binding landings now and the keeper's last act (#7645). |
| 44 | 46 | keeper: readKeeper(app.db, reader.kind === "owner"), |
| 45 | 47 | // How long each main has gone unverified after landing (#7786); private repositories for the owner only. |
src/tui/screens/flow.ts +4 −1
| @@ -30,7 +30,7 @@ import { conflictedSays, rebaseLaneSays } from "../../flow/conflicted.ts"; | ||
| 30 | 30 | import { IDLE_LIMIT, type IdlePull } from "../../flow/idle_pulls.ts"; |
| 31 | 31 | import { pileSays } from "../../flow/pile.ts"; |
| 32 | 32 | import { panelFigures } from "../../flow/panel_cache.ts"; |
| 33 | import { readPipeline, STAGE_LABEL, verdict } from "../../flow/pipeline.ts"; | |
| 33 | import { readPipeline, readStepStats, STAGE_LABEL, statWords, verdict } from "../../flow/pipeline.ts"; | |
| 34 | 34 | import { parkedSays } from "../../flow/parked.ts"; |
| 35 | 35 | import { firedAlarms } from "../../flow/alarms.ts"; |
| 36 | 36 | import { readVitals, vitalsFor, vitalsLine } from "../../flow/vitals.ts"; |
| @@ -112,6 +112,9 @@ export function flowLines( | ||
| 112 | 112 | `${pad(STAGE_LABEL[s.key], 12)} ${pad(String(s.lastHour), 9, "right")} ${pad(String(s.day), 5, "right")} ${s.label}`, |
| 113 | 113 | ), |
| 114 | 114 | "", |
| 115 | "Steps, 30 days:", | |
| 116 | ...readStepStats(db, now, agent !== undefined).flatMap((m) => wrap(`${m.label}: ${statWords(m)}`, inner)), | |
| 117 | "", | |
| 115 | 118 | `${pad("Merge queue", 15)} ${pad("queued", 6, "right")} ${pad("gating", 6, "right")} ${pad("gates", 5, "right")} ${pad("landed 1h", 9, "right")} ${pad("to land, min", 12, "right")}`, |
| 116 | 119 | ...(queues.length === 0 |
| 117 | 120 | ? ["Nothing queued, gating or landed in the last hour."] |
src/web/pages/pipeline.ts +19 −1
| @@ -8,10 +8,11 @@ | ||
| 8 | 8 | */ |
| 9 | 9 | import type { Ctx } from "../../http/context.ts"; |
| 10 | 10 | import { STEP_MEANS, STEPS, type Step } from "../../flow/path.ts"; |
| 11 | import { cachedStepStats, statParts, STEP_WINDOW_DAYS, type StepStat } from "../../flow/pipeline.ts"; | |
| 11 | 12 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; |
| 12 | 13 | import { html } from "../html.ts"; |
| 13 | 14 | import { page } from "../layout.ts"; |
| 14 | import { card, row } from "../ui.ts"; | |
| 15 | import { card, row, stat } from "../ui.ts"; | |
| 15 | 16 | |
| 16 | 17 | /** What a pull waits on at each step, what only that step catches, and the parts inside it. */ |
| 17 | 18 | export const STEP_DETAIL: Record<Step, { waits: string; catches: string; inside: readonly string[] }> = { |
| @@ -52,12 +53,22 @@ export const STEP_DETAIL: Record<Step, { waits: string; catches: string; inside: | ||
| 52 | 53 | }, |
| 53 | 54 | }; |
| 54 | 55 | |
| 56 | /** Which measured parts sit inside each step of the path. */ | |
| 57 | export const STEP_MEASURES: Record<Step, readonly StepStat["key"][]> = { | |
| 58 | check: ["pull-check"], | |
| 59 | review: ["review"], | |
| 60 | land: ["proof", "gate"], | |
| 61 | live: ["main-check", "release", "boot", "verify"], | |
| 62 | }; | |
| 63 | ||
| 55 | 64 | const title = (s: Step) => `${s.charAt(0).toUpperCase()}${s.slice(1)}`; |
| 56 | 65 | |
| 57 | 66 | // GET /pipeline |
| 58 | 67 | export function pipelinePage(ctx: Ctx): Response { |
| 59 | 68 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 60 | 69 | const rendering = performance.now(); |
| 70 | // Public repositories alone: this page is open to anyone. | |
| 71 | const stats = new Map(cachedStepStats(ctx.app.db, new Date(), true).map((m) => [m.key, m])); | |
| 61 | 72 | const body = html`<div class="page-head"> |
| 62 | 73 | <h1>How a change lands</h1> |
| 63 | 74 | <p class="muted"> |
| @@ -80,6 +91,13 @@ export function pipelinePage(ctx: Ctx): Response { | ||
| 80 | 91 | body: html`<div class="card-body"> |
| 81 | 92 | <p class="small">It waits for ${d.waits} Only this step catches ${d.catches}</p> |
| 82 | 93 | <ul class="small">${d.inside.map((line) => html`<li>${line}</li>`)}</ul> |
| 94 | <p class="small muted">Measured, last ${STEP_WINDOW_DAYS} days, public repositories: median time, then how often red.</p> | |
outdated
h3.small and the bare ul[data-measures] are not in docs/DESIGN.md. Use a named component or declare it in the area doc. |
||
| 95 | <div class="stats" data-measures="${s}">${STEP_MEASURES[s].map((k) => { | |
| 96 | const m = stats.get(k); | |
| 97 | if (m === undefined) return ""; | |
| 98 | const p = statParts(m); | |
| 99 | return stat({ label: m.label, value: p.value, title: m.medianMs === null ? "no median" : `median ${p.value}`, note: p.note, noteTitle: p.note }); | |
| 100 | })}</div> | |
| 83 | 101 | </div>`, |
| 84 | 102 | }, |
| 85 | 103 | }); |
test/pipeline-steps.test.ts added +182
| @@ -0,0 +1,182 @@ | ||
| 1 | /** Each step of the path, measured (#7784): median time and red rate over 30 days, from seeded runs, on the page. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { openIssue } from "../src/actions/issues.ts"; | |
| 8 | import { createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { createRun } from "../src/core/runs.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { flowFor } from "../src/system/telemetry.ts"; | |
| 12 | import { flowLines } from "../src/tui/screens/flow.ts"; | |
| 13 | import { cachedStepStats, median, readStepStats, statWords } from "../src/flow/pipeline.ts"; | |
| 14 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 15 | ||
| 16 | const MIN = 60_000; | |
| 17 | const DAY = 86_400_000; | |
| 18 | ||
| 19 | let dataDir: string; | |
| 20 | let app: App; | |
| 21 | let servers: Servers; | |
| 22 | let repoId: string; | |
| 23 | let privateId: string; | |
| 24 | ||
| 25 | /** A finished run that took `mins` minutes, ended `daysAgo` days ago. */ | |
| 26 | function seed(kind: string, state: string, mins: number, daysAgo: number, set: Record<string, string | null> = {}, repo = repoId) { | |
| 27 | const run = createRun(app.db, { repoId: repo, kind: kind as "check", reason: set["reason"] ?? "push" }); | |
| 28 | const end = Date.now() - daysAgo * DAY; | |
| 29 | app.db | |
| 30 | .query("UPDATE run SET state = ?1, started_at = ?2, finished_at = ?3, issue_id = ?4, env = ?5 WHERE id = ?6") | |
outdated
Dead: |
||
| 31 | .run(state, new Date(end - mins * MIN).toISOString(), new Date(end).toISOString(), set["issue"] ?? null, set["env"] ?? "", run.id); | |
| 32 | } | |
| 33 | ||
| 34 | function proof(state: string, mins: number, daysAgo: number, repo = repoId) { | |
| 35 | const end = Date.now() - daysAgo * DAY; | |
| 36 | app.db | |
| 37 | .query("INSERT INTO proof_run (id, repo_id, issue_id, head, base, state, summary, started_at, finished_at) VALUES (?1, ?2, ?3, 'h', 'b', ?4, '', ?5, ?6)") | |
| 38 | .run(`p${Math.random()}`, repo, repo === repoId ? pullIssueId : privIssueId, state, new Date(end - mins * MIN).toISOString(), new Date(end).toISOString()); | |
| 39 | } | |
| 40 | ||
| 41 | function gate(mins: number | null, daysAgo: number, repo = repoId) { | |
| 42 | const end = Date.now() - daysAgo * DAY; | |
| 43 | app.db | |
| 44 | .query("INSERT INTO check_gate (id, repo_id, started_at, finished_at) VALUES (?1, ?2, ?3, ?4)") | |
| 45 | .run(`g${Math.random()}`, repo, new Date(end - (mins ?? 0) * MIN).toISOString(), mins === null ? null : new Date(end).toISOString()); | |
| 46 | } | |
| 47 | ||
| 48 | let pullIssueId = ""; | |
| 49 | let privIssueId = ""; | |
| 50 | ||
| 51 | beforeAll(async () => { | |
| 52 | dataDir = await mkdtemp(join(tmpdir(), "forge-steps-")); | |
| 53 | app = createApp({ dataDir, port: 0 }); | |
| 54 | servers = await serve(app); | |
| 55 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 56 | if (!u.ok) throw new Error(u.error); | |
| 57 | const a = await createRepo(app, u.value, { name: "open" }); | |
| 58 | const b = await createRepo(app, u.value, { name: "closed" }); | |
| 59 | if (!a.ok || !b.ok) throw new Error("repo"); | |
| 60 | repoId = a.value.id; | |
| 61 | privateId = b.value.id; | |
| 62 | app.db.query("UPDATE repo SET visibility = 'public' WHERE id = ?1").run(repoId); | |
| 63 | const opened = openIssue(app, u.value, a.value, { title: "A change" }); | |
| 64 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 65 | const pullIssue = opened.value.id; | |
| 66 | const po = openIssue(app, u.value, b.value, { title: "Hidden" }); | |
| 67 | if (!po.ok) throw new Error(String(po.error)); | |
| 68 | const privIssue = po.value.id; | |
| 69 | pullIssueId = pullIssue; | |
| 70 | privIssueId = privIssue; | |
| 71 | // Pull checks: 4, 6, 10 minutes green and one 20-minute red → median 8m, 25% red. | |
| 72 | seed("check", "ok", 4, 1, { issue: pullIssue }); | |
| 73 | seed("check", "ok", 6, 2, { issue: pullIssue }); | |
| 74 | seed("check", "ok", 10, 3, { issue: pullIssue }); | |
| 75 | seed("check", "failed", 20, 4, { issue: pullIssue }); | |
| 76 | seed("check", "ok", 99, 40, { issue: pullIssue }); // outside 30 days | |
| 77 | seed("check", "cancelled", 99, 1, { issue: pullIssue }); // no verdict | |
| 78 | seed("check", "failed", 5, 1, { issue: privIssue }, privateId); // private: hidden from the page | |
| 79 | // Main's check: 2 and 4 minutes, one red. | |
| 80 | seed("check", "ok", 2, 1, { env: "main" }); | |
| 81 | seed("check", "failed", 4, 1, { env: "main" }); | |
| 82 | // Verify lane: a verify and a verify-bisect check; even one tied to a pull is not a pull check. | |
| 83 | seed("check", "ok", 10, 1, { env: "verify" }); | |
| 84 | seed("check", "failed", 30, 1, { env: "verify-bisect", issue: pullIssue }); | |
| 85 | // Proofs: 5 and 15 minutes green, 40 red → median 15m, 1 of 3 red; a private one is hidden. | |
| 86 | proof("ok", 5, 1); | |
| 87 | proof("ok", 15, 2); | |
| 88 | proof("failed", 40, 3); | |
| 89 | proof("failed", 1, 1, privateId); | |
| 90 | // Merge-queue gates: 6 and 12 minutes; an open one and a private one do not count. | |
| 91 | gate(6, 1); | |
| 92 | gate(12, 2); | |
| 93 | gate(null, 0); | |
| 94 | gate(60, 1, privateId); | |
| 95 | // Release build. | |
| 96 | seed("deploy", "ok", 3, 1, { env: "production" }); | |
| 97 | // A review that failed. | |
| 98 | seed("agent", "failed", 7, 1, { reason: "review" }); | |
| 99 | }); | |
| 100 | ||
| 101 | afterAll(async () => { | |
| 102 | await servers.stop(); | |
| 103 | app.close(); | |
| 104 | await rm(dataDir, { recursive: true, force: true, maxRetries: 20, retryDelay: 100 }); | |
| 105 | }); | |
| 106 | ||
| 107 | test("median is the middle, or the mean of the two middle values", () => { | |
| 108 | expect(median([])).toBeNull(); | |
| 109 | expect(median([5, 1, 3])).toBe(3); | |
| 110 | expect(median([1, 2, 3, 10])).toBe(2.5); | |
| 111 | }); | |
| 112 | ||
| 113 | test("a step's median time and red rate follow from its runs in the last 30 days", () => { | |
| 114 | const by = Object.fromEntries(readStepStats(app.db, new Date(), true).map((s) => [s.key, s])); | |
| 115 | // Hand calculation: durations 4, 6, 10, 20 → median (6+10)/2 = 8 min; 1 of 4 red. | |
| 116 | expect(by["pull-check"]?.runs).toBe(4); | |
| 117 | expect(by["pull-check"]?.medianMs).toBeCloseTo(8 * MIN, -2); | |
| 118 | expect(by["pull-check"]?.redRate).toBe(0.25); | |
| 119 | expect(by["main-check"]?.medianMs).toBeCloseTo(3 * MIN, -2); | |
| 120 | expect(by["main-check"]?.redRate).toBe(0.5); | |
| 121 | expect(by["release"]?.redRate).toBe(0); | |
| 122 | expect(by["review"]?.redRate).toBe(1); | |
| 123 | // Only the verify lane's rows: the plain verify (10m, green) and the bisect (30m, red) → median 20m, 50% red. | |
| 124 | expect(by["verify"]?.runs).toBe(2); | |
| 125 | expect(by["verify"]?.medianMs).toBeCloseTo(20 * MIN, -2); | |
| 126 | expect(by["verify"]?.redRate).toBe(0.5); | |
| 127 | // 5, 15, 40 → median 15m; 1 of 3 red; the private proof is cut. | |
| 128 | expect(by["proof"]?.runs).toBe(3); | |
| 129 | expect(by["proof"]?.medianMs).toBeCloseTo(15 * MIN, -2); | |
| 130 | expect(by["proof"]?.redRate).toBeCloseTo(1 / 3, 10); | |
| 131 | // Gates: time only — 6 and 12 → median 9m; no verdict; the open and private rows are cut. | |
| 132 | expect(by["gate"]?.runs).toBe(2); | |
| 133 | expect(by["gate"]?.medianMs).toBeCloseTo(9 * MIN, -2); | |
| 134 | expect(by["gate"]?.redRate).toBeNull(); | |
| 135 | expect(by["gate"]?.note).toBe("time only: the gate keeps no verdict, and only the last week"); | |
| 136 | expect(statWords(by["gate"] as never)).toBe("median 9m · time only: the gate keeps no verdict, and only the last week"); | |
| 137 | // A review run that failed is an error, not a changes-asked verdict, and says so. | |
| 138 | expect(statWords(by["review"] as never)).toBe("median 7m · 100% errored (1 of 1)"); | |
| 139 | expect(statWords(by["boot"] as never)).toBe("runs inside the release build: a failed boot is a failed release build"); | |
| 140 | expect(statWords(by["pull-check"] as never)).toBe("median 8m · 25% red (1 of 4)"); | |
| 141 | const all = Object.fromEntries(readStepStats(app.db, new Date(), false).map((s) => [s.key, s])); | |
| 142 | expect(all["pull-check"]?.runs).toBe(5); | |
| 143 | expect(all["proof"]?.runs).toBe(4); | |
| 144 | expect(all["gate"]?.runs).toBe(3); | |
| 145 | }); | |
| 146 | ||
| 147 | test("/pipeline shows each step's median and red rate under its details", async () => { | |
| 148 | const body = await (await fetch(`${servers.url}/pipeline`)).text(); | |
| 149 | const statOf = (label: string) => | |
| 150 | body.match(new RegExp(`<span class="stat-label">${label}</span><span class="stat-value num"[^>]*>([^<]*)</span><span class="stat-note"[^>]*>([^<]*)</span>`))?.slice(1); | |
| 151 | expect(statOf("Pull check")).toEqual(["8m", "25% red (1 of 4)"]); | |
| 152 | expect(statOf("Main's check")).toEqual(["3m", "50% red (1 of 2)"]); | |
| 153 | expect(statOf("Review run")).toEqual(["7m", "100% errored (1 of 1)"]); | |
| 154 | expect(statOf("Boot check")).toEqual(["—", "runs inside the release build: a failed boot is a failed release build"]); | |
| 155 | expect(body.match(/data-measures="/g)?.length).toBe(4); | |
| 156 | }); | |
| 157 | ||
| 158 | test("the open page reads the runs once a minute", () => { | |
| 159 | const at = new Date(); | |
| 160 | const first = cachedStepStats(app.db, at, true); | |
| 161 | seed("check", "failed", 1, 0, { issue: pullIssueId }); | |
| 162 | expect(cachedStepStats(app.db, new Date(at.getTime() + 1), true)).toBe(first); | |
| 163 | expect(cachedStepStats(app.db, new Date(at.getTime() + 61_000), true)).not.toBe(first); | |
| 164 | }); | |
| 165 | ||
| 166 | test("the terminal's Flow lines carry the steps, private repositories for an admin only", () => { | |
| 167 | const admin = flowLines(app.db).join(" ").replace(/\s+/g, " "); | |
| 168 | expect(admin).toContain("Steps, 30 days:"); | |
| 169 | expect(admin).toContain("Merge-queue gate: median 12m"); | |
| 170 | const agent = flowLines(app.db, new Date(), undefined, { queues: [] }).join(" ").replace(/\s+/g, " "); | |
| 171 | expect(agent).toContain("Merge-queue gate: median 9m"); | |
| 172 | expect(admin).toContain("Proof: median"); | |
| 173 | }); | |
| 174 | ||
| 175 | test("the MCP flow tool and the API return the steps, an agent's cut to public repositories", () => { | |
| 176 | const owner = flowFor(app, { kind: "owner" } as never).steps; | |
| 177 | const agent = flowFor(app, { kind: "agent" } as never).steps; | |
| 178 | const proofOf = (steps: typeof owner) => steps.find((s) => s.key === "proof"); | |
| 179 | expect(proofOf(owner)?.runs).toBe(4); | |
| 180 | expect(proofOf(agent)?.runs).toBe(3); | |
| 181 | expect(proofOf(owner)?.says).toContain("median"); | |
| 182 | }); |
new above
offline
Review is 'agent run failed', not 'review asked for changes'; name that in the label so the percentage is not misread.