Public: anyone can read this; the crew can change it.

Slow public pages alarm by name: past one second warm, the forge says which layer and which setting holds the limit #14034

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13636-slow-public-pages-alarm-by-name-past-one → main 2 commits fork 2 threads · 2 unresolved

  • Merged. scout/13636-slow-public-pages-alarm-by-name-past-one was merged into main as e409249.

Review

Sign in to review this change.
3 files changed +186 −3
src/maintain/duties/wait_split.ts +19 −3
@@ -4,13 +4,20 @@
44 * loopback, time the handshake with the edge, and split each wait into its layer
55 * (`../wait_split.ts`). GET only; it changes nothing; each read has the proof's own 15 s and the
66 * whole run 25 s, so it is done before the runner's 30 s deadline gives up on it.
7 * A slow page is a finding; every page is one line on the record.
7 * A slow page is a finding; every page is one line on the record. A slow page also files one
8 * issue per layer (#13636), naming the setting that bounds that layer's time with its reason (or
9 * that none is held, and what to check) (`../wait_limits.ts`), so a repeat is loud, not a line
10 * nobody reads.
811 */
912
1013 import { connect as tcpConnect } from "node:net";
1114 import { connect as tlsConnect } from "node:tls";
15 import { fileIssueOnce } from "../../checks/flake.ts";
16 import { getRepo } from "../../core/repos.ts";
17 import { ownRepoPath } from "../../kernel/own_repo.ts";
1218 import type { Duty, Finding } from "../duty.ts";
13 import { serverAppMs, splitLine, waitLayer, type Timings } from "../wait_split.ts";
19 import { alarmBody, alarmTitle } from "../wait_limits.ts";
20 import { serverAppMs, splitLine, waitLayer, type Layer, type Timings } from "../wait_split.ts";
1421
1522 export const PATHS = ["/healthz", "/", "/pricing", "/terms", "/privacy"] as const;
1623
@@ -78,13 +85,14 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS):
7885 name: "wait-split",
7986 intervalMs: 6 * 3_600_000,
8087 budget: { runsPerDay: 8, dollarsPerDay: 1, reason: "every six hours is four a day; the rest is room for a catch-up, and it spends no money" },
81 async run({ now, app }) {
88 async run({ db, bus, now, app }) {
8289 const url = app?.config.url;
8390 if (url === undefined || url === "") return { findings: [], actions: ["skipped: this forge does not know its public origin (FORGE_URL)"] };
8491 const origin = new URL(url);
8592 const loopback = `http://127.0.0.1:${app?.config.port ?? 3000}`;
8693 const findings: Finding[] = [];
8794 const actions: string[] = [];
95 const slow = new Map<Exclude<Layer, "none">, string[]>();
8896 const run = AbortSignal.timeout(runMs);
8997 for (const [i, path] of PATHS.entries()) {
9098 if (run.aborted) {
@@ -104,6 +112,7 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS):
104112 const said = splitLine(path, t, pub.totalMs);
105113 actions.push(said);
106114 const layer = waitLayer(t);
115 if (layer !== "none") slow.set(layer, [...(slow.get(layer) ?? []), said]);
107116 if (layer !== "none") findings.push({ summary: `${path} is slow from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: ms to first byte, waiting in ${layer}`, number: Math.round(t.ttfbMs), at: now.toISOString() } });
108117 } catch (e) {
109118 if (run.aborted) {
@@ -115,6 +124,13 @@ export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS):
115124 findings.push({ summary: `${path} is unreachable from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: 1 is no answer`, number: 1, at: now.toISOString() } });
116125 }
117126 }
127 const [owner = "", name = ""] = ownRepoPath().split("/");
128 const repo = slow.size === 0 ? undefined : getRepo(db, owner, name);
129 if (repo !== undefined)
130 for (const [layer, lines] of slow) {
131 const filed = fileIssueOnce(db, bus, repo, { title: alarmTitle(layer), body: alarmBody(layer, lines) });
132 if (filed.opened) actions.push(`filed slow ${layer} issue #${String(filed.number)}`);
133 }
118134 return { findings, actions };
119135 },
120136 };
src/maintain/wait_limits.ts added +80
@@ -0,0 +1,80 @@
1 /**
2 * What bounds each layer's wait (#13636, part 2 of #13633): when a public page waits past
3 * `SLOW_MS` in a layer (`wait_split.ts`), the alarm names the setting that bounds time there, its
4 * value as the code holds it now, and the reason beside it. Where no forge-held setting bounds a
5 * layer's time, the row says so and names what to check instead, so the alarm never calls a
6 * routing target or a logging threshold a limit. Plain data in, plain data out.
7 */
8
9 import { SLICE_MS } from "../kernel/breathe.ts";
10 import { SLOW_SQL_MS } from "../kernel/slow_sql.ts";
11 import type { Layer } from "./wait_split.ts";
12
13 /** A forge-held setting that bounds how long this layer may take. */
14 export interface Bound {
15 readonly kind: "bound";
16 /** The setting's name, as a person searches for it. */
17 readonly setting: string;
18 /** Where it is set. */
19 readonly where: string;
20 /** Its value now. */
21 readonly value: string;
22 /** Why it is that, and what it bounds. */
23 readonly reason: string;
24 }
25
26 /** No forge-held setting bounds this layer's time: why, and what to check instead. */
27 export interface Unbound {
28 readonly kind: "none";
29 readonly why: string;
30 readonly check: string;
31 }
32
33 export type Limit = Bound | Unbound;
outdated

FORGE_ORIGIN is a routing target, not a limit on time. Name the setting that bounds the tunnel hop (a cloudflared timeout/keepalive/connection setting in the rendered config), or say no forge-held limit exists.

34
35 export const LIMITS: Readonly<Record<Exclude<Layer, "none">, Limit>> = {
36 edge: {
37 kind: "none",
38 why: "the TLS handshake ends at Cloudflare's edge before any byte reaches the host, and the forge holds no setting there",
39 check: "Cloudflare's status page and the zone's analytics for the forge's hostnames; a slow handshake from one place is the client's route",
40 },
41 tunnel: {
42 kind: "none",
43 why: "the rendered cloudflared config (src/system/tunnel_config.ts, /etc/forge/cloudflared.yml) sets no originRequest timeouts, so cloudflared's own defaults apply; it only routes every hostname to the loopback origin",
44 check: "`journalctl -u cloudflared` on the host for reconnects and origin errors, and whether the host's loopback answer (the accept layer) was fast at the same time",
45 },
46 accept: {
47 kind: "bound",
outdated

SLOW_SQL_MS is a logging threshold; it does not hold the page's limit. Name the actual handler or request time bound, or state that none exists and the card should point to the slow-query log instead of calling this the limit.

48 setting: "SLICE_MS",
49 where: "src/kernel/breathe.ts",
50 value: `${String(SLICE_MS)} ms`,
51 reason: "the longest a background pass may hold the one serving thread before a waiting request is answered; a request queued for seconds means some work runs without breathing",
52 },
53 app: {
54 kind: "none",
55 why: "nothing bounds one handler's time below the listener's idleTimeout of 120 s (src/http/server.ts), which only closes a silent connection, far past 1 s",
56 check: `the slow-query log, where a query past SLOW_SQL_MS (${String(SLOW_SQL_MS)} ms, src/kernel/slow_sql.ts) is logged by name, and the page's \`Server-Timing: app\``,
57 },
58 };
59
60 /** The alarm's title: one open issue stands for a layer, however many runs see it. */
61 export const alarmTitle = (layer: Exclude<Layer, "none">): string => `A public page waits past 1 s in the ${layer}`;
62
63 /** The limit's lines: the bounding setting with its reason, or plainly that none is held here. */
64 function limitLines(l: Limit): string {
65 if (l.kind === "bound")
66 return `The setting that bounds this layer's time: \`${l.setting}\` = \`${l.value}\` (${l.where}).\nWhy: ${l.reason}.`;
67 return `No forge-held setting bounds this layer's time: ${l.why}.\nCheck instead: ${l.check}.`;
68 }
69
70 /**
71 * The alarm's body: the layer, what bounds it with its reason beside it, and each slow page's line.
72 */
73 export function alarmBody(layer: Exclude<Layer, "none">, lines: readonly string[]): string {
74 return (
75 `_Filed by the wait-split duty from the forge's own host, not written by a person._\n\n` +
76 `A warm GET of a public page took over 1 s, and the wait sits in the **${layer}**.\n\n` +
77 `${limitLines(LIMITS[layer])}\n\n` +
78 `Measured:\n${lines.map((s) => `- ${s}`).join("\n")}`
79 );
80 }
test/maintain-wait-alarm.test.ts added +87
@@ -0,0 +1,87 @@
1 /**
2 * A slow public page alarms by name (#13636): past 1 s warm, the wait-split duty files one issue
3 * per layer naming the setting that bounds that layer's time with its reason, or saying plainly
4 * that none is held there and what to check instead; once.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { rulesWorld } from "./helpers/rules-app.ts";
8 import { waitSplit, type Meter, type Read } from "../src/maintain/duties/wait_split.ts";
9 import { LIMITS, alarmBody, alarmTitle } from "../src/maintain/wait_limits.ts";
10 import { SLICE_MS } from "../src/kernel/breathe.ts";
11
12 // Set for this file's tests only: bun runs many files in one process.
13 const before = process.env.FORGE_SENTINEL_REPO;
14 beforeAll(() => {
15 process.env.FORGE_SENTINEL_REPO = "alice/rules";
16 });
17 afterAll(() => {
18 if (before === undefined) delete process.env.FORGE_SENTINEL_REPO;
19 else process.env.FORGE_SENTINEL_REPO = before;
20 });
21 const w = rulesWorld();
22
23 const read = (ttfbMs: number, app: number): Read => ({ status: 200, ttfbMs, totalMs: ttfbMs + 10, serverTiming: `app;dur=${String(app)}` });
24 /** /pricing waits 7 s in the tunnel; /terms waits 4 s on loopback, so in the accept queue. */
25 const meter = (slow: boolean): Meter => ({
26 handshake: () => Promise.resolve(150),
27 get: (url) => {
28 const local = url.startsWith("http://127.0.0.1");
29 if (slow && url.endsWith("/pricing") && !local) return Promise.resolve(read(7000, 3.5));
30 if (slow && url.endsWith("/terms")) return Promise.resolve(read(local ? 4000 : 4300, 2));
31 return Promise.resolve(read(local ? 3 : 300, 2));
32 },
33 });
34 const run = (slow: boolean) => {
35 (w.app.config as { url?: string }).url = "https://forge.example";
36 return waitSplit(meter(slow)).run({ db: w.app.db, bus: w.app.bus, now: new Date("2026-11-16T12:00:00Z"), app: w.app });
37 };
38 const bodyOf = (title: string): string | undefined =>
39 w.app.db.query<{ body: string }, [string, string]>("SELECT body FROM issue WHERE repo_id = ?1 AND title = ?2").get(w.repoId, title)?.body;
40
41 test("fast pages file nothing", async () => {
42 const r = await run(false);
43 expect(r.findings).toEqual([]);
44 expect(w.titles()).toEqual([]);
45 });
46
47 test("a slow page files one issue per layer, naming the setting and its reason, once", async () => {
48 const first = await run(true);
49 expect(first.actions.filter((a) => a.startsWith("filed slow"))).toHaveLength(2);
50 expect(w.titles().sort()).toEqual([alarmTitle("accept"), alarmTitle("tunnel")].sort());
51 const tunnel = bodyOf(alarmTitle("tunnel"))!;
52 // The tunnel has no forge-held bound: the card says so, and does not dress up a route as a limit.
53 expect(tunnel).toContain("No forge-held setting bounds this layer's time");
54 expect(tunnel).not.toContain("FORGE_ORIGIN");
55 expect(tunnel).toContain("journalctl -u cloudflared");
56 expect(tunnel).toContain("/pricing connect 150 ms, first byte 7000 ms");
57 expect(tunnel).not.toContain("/terms");
58 const accept = bodyOf(alarmTitle("accept"))!;
59 expect(accept).toContain(`The setting that bounds this layer's time: \`SLICE_MS\` = \`${String(SLICE_MS)} ms\``);
60 expect(accept).toContain("/terms ");
61 // The next run sees the same: the open issues stand for it.
62 const again = await run(true);
63 expect(again.findings).toHaveLength(2);
64 expect(again.actions.filter((a) => a.startsWith("filed slow"))).toEqual([]);
65 expect(w.titles()).toHaveLength(2);
66 });
67
68 test("every layer names its bound with a reason, or says none is held and what to check", () => {
69 for (const layer of ["edge", "tunnel", "accept", "app"] as const) {
70 const l = LIMITS[layer];
71 const body = alarmBody(layer, ["/ line"]);
72 expect(body).toContain(`**${layer}**`);
73 expect(body).toContain("- / line");
74 if (l.kind === "bound") {
75 for (const s of [l.setting, l.where, l.value, l.reason]) expect(s.length).toBeGreaterThan(0);
76 expect(body).toContain(`\`${l.setting}\` = \`${l.value}\``);
77 expect(body).toContain(`Why: ${l.reason}.`);
78 } else {
79 expect(body).toContain(`No forge-held setting bounds this layer's time: ${l.why}.`);
80 expect(body).toContain(`Check instead: ${l.check}.`);
81 }
82 }
83 // A logging threshold is never named as the app's limit, only as where to look.
84 expect(LIMITS.app.kind).toBe("none");
85 expect(alarmBody("app", [])).not.toContain("bounds this layer's time: `SLOW_SQL_MS`");
86 expect(alarmBody("app", [])).toContain("slow-query log");
87 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs