Public: anyone can read this; the crew can change it.
Slow public pages alarm by name: past one second warm, the forge says which layer and which setting holds the limit #14034
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13636-slow-public-pages-alarm-by-name-past-one → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +186
−3
src/maintain/duties/wait_split.ts +19 −3
| @@ -4,13 +4,20 @@ | ||
| 4 | 4 | * loopback, time the handshake with the edge, and split each wait into its layer |
| 5 | 5 | * (`../wait_split.ts`). GET only; it changes nothing; each read has the proof's own 15 s and the |
| 6 | 6 | * whole run 25 s, so it is done before the runner's 30 s deadline gives up on it. |
| 7 | * A slow page is a finding; every page is one line on the record. | |
| 7 | * A slow page is a finding; every page is one line on the record. A slow page also files one | |
| 8 | * issue per layer (#13636), naming the setting that bounds that layer's time with its reason (or | |
| 9 | * that none is held, and what to check) (`../wait_limits.ts`), so a repeat is loud, not a line | |
| 10 | * nobody reads. | |
| 8 | 11 | */ |
| 9 | 12 | |
| 10 | 13 | import { connect as tcpConnect } from "node:net"; |
| 11 | 14 | import { connect as tlsConnect } from "node:tls"; |
| 15 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 16 | import { getRepo } from "../../core/repos.ts"; | |
| 17 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 12 | 18 | import type { Duty, Finding } from "../duty.ts"; |
| 13 | import { serverAppMs, splitLine, waitLayer, type Timings } from "../wait_split.ts"; | |
| 19 | import { alarmBody, alarmTitle } from "../wait_limits.ts"; | |
| 20 | import { serverAppMs, splitLine, waitLayer, type Layer, type Timings } from "../wait_split.ts"; | |
| 14 | 21 | |
| 15 | 22 | export const PATHS = ["/healthz", "/", "/pricing", "/terms", "/privacy"] as const; |
| 16 | 23 | |
| @@ -78,13 +85,14 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS): | ||
| 78 | 85 | name: "wait-split", |
| 79 | 86 | intervalMs: 6 * 3_600_000, |
| 80 | 87 | budget: { runsPerDay: 8, dollarsPerDay: 1, reason: "every six hours is four a day; the rest is room for a catch-up, and it spends no money" }, |
| 81 | async run({ now, app }) { | |
| 88 | async run({ db, bus, now, app }) { | |
| 82 | 89 | const url = app?.config.url; |
| 83 | 90 | if (url === undefined || url === "") return { findings: [], actions: ["skipped: this forge does not know its public origin (FORGE_URL)"] }; |
| 84 | 91 | const origin = new URL(url); |
| 85 | 92 | const loopback = `http://127.0.0.1:${app?.config.port ?? 3000}`; |
| 86 | 93 | const findings: Finding[] = []; |
| 87 | 94 | const actions: string[] = []; |
| 95 | const slow = new Map<Exclude<Layer, "none">, string[]>(); | |
| 88 | 96 | const run = AbortSignal.timeout(runMs); |
| 89 | 97 | for (const [i, path] of PATHS.entries()) { |
| 90 | 98 | if (run.aborted) { |
| @@ -104,6 +112,7 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS): | ||
| 104 | 112 | const said = splitLine(path, t, pub.totalMs); |
| 105 | 113 | actions.push(said); |
| 106 | 114 | const layer = waitLayer(t); |
| 115 | if (layer !== "none") slow.set(layer, [...(slow.get(layer) ?? []), said]); | |
| 107 | 116 | if (layer !== "none") findings.push({ summary: `${path} is slow from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: ms to first byte, waiting in ${layer}`, number: Math.round(t.ttfbMs), at: now.toISOString() } }); |
| 108 | 117 | } catch (e) { |
| 109 | 118 | if (run.aborted) { |
| @@ -115,6 +124,13 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS): | ||
| 115 | 124 | findings.push({ summary: `${path} is unreachable from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: 1 is no answer`, number: 1, at: now.toISOString() } }); |
| 116 | 125 | } |
| 117 | 126 | } |
| 127 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 128 | const repo = slow.size === 0 ? undefined : getRepo(db, owner, name); | |
| 129 | if (repo !== undefined) | |
| 130 | for (const [layer, lines] of slow) { | |
| 131 | const filed = fileIssueOnce(db, bus, repo, { title: alarmTitle(layer), body: alarmBody(layer, lines) }); | |
| 132 | if (filed.opened) actions.push(`filed slow ${layer} issue #${String(filed.number)}`); | |
| 133 | } | |
| 118 | 134 | return { findings, actions }; |
| 119 | 135 | }, |
| 120 | 136 | }; |
src/maintain/wait_limits.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * What bounds each layer's wait (#13636, part 2 of #13633): when a public page waits past | |
| 3 | * `SLOW_MS` in a layer (`wait_split.ts`), the alarm names the setting that bounds time there, its | |
| 4 | * value as the code holds it now, and the reason beside it. Where no forge-held setting bounds a | |
| 5 | * layer's time, the row says so and names what to check instead, so the alarm never calls a | |
| 6 | * routing target or a logging threshold a limit. Plain data in, plain data out. | |
| 7 | */ | |
| 8 | ||
| 9 | import { SLICE_MS } from "../kernel/breathe.ts"; | |
| 10 | import { SLOW_SQL_MS } from "../kernel/slow_sql.ts"; | |
| 11 | import type { Layer } from "./wait_split.ts"; | |
| 12 | ||
| 13 | /** A forge-held setting that bounds how long this layer may take. */ | |
| 14 | export interface Bound { | |
| 15 | readonly kind: "bound"; | |
| 16 | /** The setting's name, as a person searches for it. */ | |
| 17 | readonly setting: string; | |
| 18 | /** Where it is set. */ | |
| 19 | readonly where: string; | |
| 20 | /** Its value now. */ | |
| 21 | readonly value: string; | |
| 22 | /** Why it is that, and what it bounds. */ | |
| 23 | readonly reason: string; | |
| 24 | } | |
| 25 | ||
| 26 | /** No forge-held setting bounds this layer's time: why, and what to check instead. */ | |
| 27 | export interface Unbound { | |
| 28 | readonly kind: "none"; | |
| 29 | readonly why: string; | |
| 30 | readonly check: string; | |
| 31 | } | |
| 32 | ||
| 33 | export type Limit = Bound | Unbound; | |
| 34 | ||
| 35 | export const LIMITS: Readonly<Record<Exclude<Layer, "none">, Limit>> = { | |
| 36 | edge: { | |
| 37 | kind: "none", | |
| 38 | why: "the TLS handshake ends at Cloudflare's edge before any byte reaches the host, and the forge holds no setting there", | |
| 39 | check: "Cloudflare's status page and the zone's analytics for the forge's hostnames; a slow handshake from one place is the client's route", | |
| 40 | }, | |
| 41 | tunnel: { | |
| 42 | kind: "none", | |
| 43 | why: "the rendered cloudflared config (src/system/tunnel_config.ts, /etc/forge/cloudflared.yml) sets no originRequest timeouts, so cloudflared's own defaults apply; it only routes every hostname to the loopback origin", | |
| 44 | check: "`journalctl -u cloudflared` on the host for reconnects and origin errors, and whether the host's loopback answer (the accept layer) was fast at the same time", | |
| 45 | }, | |
| 46 | accept: { | |
| 47 | kind: "bound", | |
outdated
SLOW_SQL_MS is a logging threshold; it does not hold the page's limit. Name the actual handler or request time bound, or state that none exists and the card should point to the slow-query log instead of calling this the limit. |
||
| 48 | setting: "SLICE_MS", | |
| 49 | where: "src/kernel/breathe.ts", | |
| 50 | value: `${String(SLICE_MS)} ms`, | |
| 51 | reason: "the longest a background pass may hold the one serving thread before a waiting request is answered; a request queued for seconds means some work runs without breathing", | |
| 52 | }, | |
| 53 | app: { | |
| 54 | kind: "none", | |
| 55 | why: "nothing bounds one handler's time below the listener's idleTimeout of 120 s (src/http/server.ts), which only closes a silent connection, far past 1 s", | |
| 56 | check: `the slow-query log, where a query past SLOW_SQL_MS (${String(SLOW_SQL_MS)} ms, src/kernel/slow_sql.ts) is logged by name, and the page's \`Server-Timing: app\``, | |
| 57 | }, | |
| 58 | }; | |
| 59 | ||
| 60 | /** The alarm's title: one open issue stands for a layer, however many runs see it. */ | |
| 61 | export const alarmTitle = (layer: Exclude<Layer, "none">): string => `A public page waits past 1 s in the ${layer}`; | |
| 62 | ||
| 63 | /** The limit's lines: the bounding setting with its reason, or plainly that none is held here. */ | |
| 64 | function limitLines(l: Limit): string { | |
| 65 | if (l.kind === "bound") | |
| 66 | return `The setting that bounds this layer's time: \`${l.setting}\` = \`${l.value}\` (${l.where}).\nWhy: ${l.reason}.`; | |
| 67 | return `No forge-held setting bounds this layer's time: ${l.why}.\nCheck instead: ${l.check}.`; | |
| 68 | } | |
| 69 | ||
| 70 | /** | |
| 71 | * The alarm's body: the layer, what bounds it with its reason beside it, and each slow page's line. | |
| 72 | */ | |
| 73 | export function alarmBody(layer: Exclude<Layer, "none">, lines: readonly string[]): string { | |
| 74 | return ( | |
| 75 | `_Filed by the wait-split duty from the forge's own host, not written by a person._\n\n` + | |
| 76 | `A warm GET of a public page took over 1 s, and the wait sits in the **${layer}**.\n\n` + | |
| 77 | `${limitLines(LIMITS[layer])}\n\n` + | |
| 78 | `Measured:\n${lines.map((s) => `- ${s}`).join("\n")}` | |
| 79 | ); | |
| 80 | } |
test/maintain-wait-alarm.test.ts added +87
| @@ -0,0 +1,87 @@ | ||
| 1 | /** | |
| 2 | * A slow public page alarms by name (#13636): past 1 s warm, the wait-split duty files one issue | |
| 3 | * per layer naming the setting that bounds that layer's time with its reason, or saying plainly | |
| 4 | * that none is held there and what to check instead; once. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 8 | import { waitSplit, type Meter, type Read } from "../src/maintain/duties/wait_split.ts"; | |
| 9 | import { LIMITS, alarmBody, alarmTitle } from "../src/maintain/wait_limits.ts"; | |
| 10 | import { SLICE_MS } from "../src/kernel/breathe.ts"; | |
| 11 | ||
| 12 | // Set for this file's tests only: bun runs many files in one process. | |
| 13 | const before = process.env.FORGE_SENTINEL_REPO; | |
| 14 | beforeAll(() => { | |
| 15 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 16 | }); | |
| 17 | afterAll(() => { | |
| 18 | if (before === undefined) delete process.env.FORGE_SENTINEL_REPO; | |
| 19 | else process.env.FORGE_SENTINEL_REPO = before; | |
| 20 | }); | |
| 21 | const w = rulesWorld(); | |
| 22 | ||
| 23 | const read = (ttfbMs: number, app: number): Read => ({ status: 200, ttfbMs, totalMs: ttfbMs + 10, serverTiming: `app;dur=${String(app)}` }); | |
| 24 | /** /pricing waits 7 s in the tunnel; /terms waits 4 s on loopback, so in the accept queue. */ | |
| 25 | const meter = (slow: boolean): Meter => ({ | |
| 26 | handshake: () => Promise.resolve(150), | |
| 27 | get: (url) => { | |
| 28 | const local = url.startsWith("http://127.0.0.1"); | |
| 29 | if (slow && url.endsWith("/pricing") && !local) return Promise.resolve(read(7000, 3.5)); | |
| 30 | if (slow && url.endsWith("/terms")) return Promise.resolve(read(local ? 4000 : 4300, 2)); | |
| 31 | return Promise.resolve(read(local ? 3 : 300, 2)); | |
| 32 | }, | |
| 33 | }); | |
| 34 | const run = (slow: boolean) => { | |
| 35 | (w.app.config as { url?: string }).url = "https://forge.example"; | |
| 36 | return waitSplit(meter(slow)).run({ db: w.app.db, bus: w.app.bus, now: new Date("2026-11-16T12:00:00Z"), app: w.app }); | |
| 37 | }; | |
| 38 | const bodyOf = (title: string): string | undefined => | |
| 39 | w.app.db.query<{ body: string }, [string, string]>("SELECT body FROM issue WHERE repo_id = ?1 AND title = ?2").get(w.repoId, title)?.body; | |
| 40 | ||
| 41 | test("fast pages file nothing", async () => { | |
| 42 | const r = await run(false); | |
| 43 | expect(r.findings).toEqual([]); | |
| 44 | expect(w.titles()).toEqual([]); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a slow page files one issue per layer, naming the setting and its reason, once", async () => { | |
| 48 | const first = await run(true); | |
| 49 | expect(first.actions.filter((a) => a.startsWith("filed slow"))).toHaveLength(2); | |
| 50 | expect(w.titles().sort()).toEqual([alarmTitle("accept"), alarmTitle("tunnel")].sort()); | |
| 51 | const tunnel = bodyOf(alarmTitle("tunnel"))!; | |
| 52 | // The tunnel has no forge-held bound: the card says so, and does not dress up a route as a limit. | |
| 53 | expect(tunnel).toContain("No forge-held setting bounds this layer's time"); | |
| 54 | expect(tunnel).not.toContain("FORGE_ORIGIN"); | |
| 55 | expect(tunnel).toContain("journalctl -u cloudflared"); | |
| 56 | expect(tunnel).toContain("/pricing connect 150 ms, first byte 7000 ms"); | |
| 57 | expect(tunnel).not.toContain("/terms"); | |
| 58 | const accept = bodyOf(alarmTitle("accept"))!; | |
| 59 | expect(accept).toContain(`The setting that bounds this layer's time: \`SLICE_MS\` = \`${String(SLICE_MS)} ms\``); | |
| 60 | expect(accept).toContain("/terms "); | |
| 61 | // The next run sees the same: the open issues stand for it. | |
| 62 | const again = await run(true); | |
| 63 | expect(again.findings).toHaveLength(2); | |
| 64 | expect(again.actions.filter((a) => a.startsWith("filed slow"))).toEqual([]); | |
| 65 | expect(w.titles()).toHaveLength(2); | |
| 66 | }); | |
| 67 | ||
| 68 | test("every layer names its bound with a reason, or says none is held and what to check", () => { | |
| 69 | for (const layer of ["edge", "tunnel", "accept", "app"] as const) { | |
| 70 | const l = LIMITS[layer]; | |
| 71 | const body = alarmBody(layer, ["/ line"]); | |
| 72 | expect(body).toContain(`**${layer}**`); | |
| 73 | expect(body).toContain("- / line"); | |
| 74 | if (l.kind === "bound") { | |
| 75 | for (const s of [l.setting, l.where, l.value, l.reason]) expect(s.length).toBeGreaterThan(0); | |
| 76 | expect(body).toContain(`\`${l.setting}\` = \`${l.value}\``); | |
| 77 | expect(body).toContain(`Why: ${l.reason}.`); | |
| 78 | } else { | |
| 79 | expect(body).toContain(`No forge-held setting bounds this layer's time: ${l.why}.`); | |
| 80 | expect(body).toContain(`Check instead: ${l.check}.`); | |
| 81 | } | |
| 82 | } | |
| 83 | // A logging threshold is never named as the app's limit, only as where to look. | |
| 84 | expect(LIMITS.app.kind).toBe("none"); | |
| 85 | expect(alarmBody("app", [])).not.toContain("bounds this layer's time: `SLOW_SQL_MS`"); | |
| 86 | expect(alarmBody("app", [])).toContain("slow-query log"); | |
| 87 | }); |
new above
offline
FORGE_ORIGIN is a routing target, not a limit on time. Name the setting that bounds the tunnel hop (a cloudflared timeout/keepalive/connection setting in the rendered config), or say no forge-held limit exists.