Public: anyone can read this; the crew can change it.

Production's slow public pages: name where the wait sits, from the forge's own host and from outside, and write it on #13633 #13638

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13635-production-s-slow-public-pages-name-wher → main 4 commits fork 1 thread · 1 unresolved

  • Merged. torque/13635-production-s-slow-public-pages-name-wher was merged into main as 6b4a119.

Review

Sign in to review this change.
6 files changed +328 −2
docs/features/maintenance-duties.md +6
@@ -22,6 +22,12 @@ The `merged-branches` duty finds branches of pulls that merged over seven days a
2222
2323 Tests: `test/maintain-merged-branches.test.ts`, `test/maintain-merged-web.test.ts`.
2424
25 ## Wait split (#13635)
26
27 The `wait-split` duty finds where a slow public page waits. Every six hours, from the forge's own host, it does three things for each of `/healthz`, `/`, `/pricing`, `/terms` and `/privacy`: it times a TCP+TLS handshake with the public origin (the edge), a GET of the public URL (to first byte and to the last byte), and a GET of the same path on `127.0.0.1:<port>`, which skips the edge and the tunnel. It also reads the `app;dur=` the forge wrote in `Server-Timing`. `src/maintain/wait_split.ts` splits the first-byte time into edge = connect, tunnel = first byte − connect − loopback, accept = loopback − app, and app. The largest share names the layer. A page under 1 s waits in `none`. It sends GET only, and its timeout is the proof's own 15 s. A slow or unanswered page is a finding, and every page is one line on `/maintenance`. Inside a cell, the bare handshake cannot pass the egress proxy: that line says so and counts the handshake as 0 ms.
28
29 Tests: `test/maintain-wait-split.test.ts`.
30
2531 ## Duty idle (#13406)
2632
2733 The `duty-idle` duty watches the other duties for silence. Hourly, it looks at when each duty last started (budget stops do not count as a start). A duty whose last start is more than twice its interval ago is idle; exactly 2x is not. A duty that has never run is skipped, because there is no last run to name. The watcher never watches itself.
forge.json +2 −1
@@ -1,7 +1,8 @@
11 {
22 "egress": [
33 "api.anthropic.com",
4 "api.github.com"
4 "api.github.com",
5 "lab--forge-trevato-apps.open-platform.sh"
56 ],
67 "app": {
78 "looks": ["/", "/docs", "/explore", "/board", "/notes", "/review", "/releases", "/apps", "/security", "/system", "/maintenance", "/talk", "/flow", "/resilience", "/boards?help", "/settings/crew", "/settings/crew?help", "/settings/security/controls", "/settings/grants", "/preview-admin/sample", "/preview-admin/sample/drop", "/preview-admin/sample/data-move", "/preview-admin/sample/pulls/3", "/preview-admin/sample/next?pick", "/preview-admin/sample/bets", "/preview-admin/sample/waves", "/preview-admin/sample/space", "/looks/missing", "/looks/failed"],
src/maintain/duties/wait_split.ts added +123
@@ -0,0 +1,123 @@
1 /**
2 * Where production's public pages wait (#13635, part 1 of #13633): every six hours, from the
3 * forge's own host, GET /healthz, /, /pricing, /terms and /privacy at the public origin and on
4 * loopback, time the handshake with the edge, and split each wait into its layer
5 * (`../wait_split.ts`). GET only; it changes nothing; each read has the proof's own 15 s and the
6 * whole run 25 s, so it is done before the runner's 30 s deadline gives up on it.
7 * A slow page is a finding; every page is one line on the record.
8 */
9
10 import { connect as tcpConnect } from "node:net";
11 import { connect as tlsConnect } from "node:tls";
12 import type { Duty, Finding } from "../duty.ts";
13 import { serverAppMs, splitLine, waitLayer, type Timings } from "../wait_split.ts";
14
15 export const PATHS = ["/healthz", "/", "/pricing", "/terms", "/privacy"] as const;
16
17 /** The proof's own per-request timeout (`core/v1_proof.ts`); not widened (#13633). */
18 const TIMEOUT_MS = 15_000;
19
20 /**
21 * The whole run's bound, under the runner's 30 s (`../runner.ts` DUTY_TIMEOUT_MS): ten reads of
22 * up to 15 s each would outlive it and keep reading after the runner had moved on. Pages left
23 * when it runs out are named as unread, not as unreachable.
24 */
25 export const RUN_MS = 25_000;
26
27 export interface Read {
28 readonly status: number;
29 readonly ttfbMs: number;
30 readonly totalMs: number;
31 readonly serverTiming: string | null;
32 }
33
34 /** How the duty reads the network; the test hands in its own. */
35 export interface Meter {
36 /** ms to open TCP (and TLS on https) to the origin, then close; given up when `run` aborts. */
37 readonly handshake: (origin: URL, run: AbortSignal) => Promise<number>;
38 /** One GET: ms to its headers, ms to its last byte; given up when `run` aborts. */
39 readonly get: (url: string, run: AbortSignal) => Promise<Read>;
40 }
41
42 /** Both deadlines on one call: the proof's per-request 15 s and the run's own. */
43 const bounded = (run: AbortSignal): AbortSignal => AbortSignal.any([run, AbortSignal.timeout(TIMEOUT_MS)]);
44
45 const handshake = (origin: URL, run: AbortSignal): Promise<number> =>
46 new Promise((resolve, reject) => {
47 const start = performance.now();
48 const secure = origin.protocol === "https:";
49 const port = Number(origin.port || (secure ? 443 : 80));
50 const signal = bounded(run);
51 if (signal.aborted) return reject(signal.reason);
52 const socket = secure ? tlsConnect({ host: origin.hostname, port, servername: origin.hostname }) : tcpConnect({ host: origin.hostname, port });
53 const settle = (fn: () => void): void => {
54 signal.removeEventListener("abort", onAbort);
55 socket.destroy();
56 fn();
57 };
58 const onAbort = (): void => settle(() => reject(signal.reason));
59 signal.addEventListener("abort", onAbort, { once: true });
60 socket.once(secure ? "secureConnect" : "connect", () => settle(() => resolve(performance.now() - start)));
61 socket.once("error", (e) => settle(() => reject(e)));
62 });
63
64 async function get(url: string, run: AbortSignal): Promise<Read> {
65 const start = performance.now();
66 const res = await fetch(url, { method: "GET", redirect: "manual", signal: bounded(run) });
67 const ttfbMs = performance.now() - start;
68 await res.arrayBuffer();
69 return { status: res.status, ttfbMs, totalMs: performance.now() - start, serverTiming: res.headers.get("server-timing") };
70 }
71
72 export const networkMeter: Meter = { handshake, get };
73
74 const why = (e: unknown): string => (e instanceof Error ? `${e.name}: ${e.message}` : String(e));
75
76 export function waitSplit(meter: Meter = networkMeter, runMs: number = RUN_MS): Duty {
77 return {
78 name: "wait-split",
79 intervalMs: 6 * 3_600_000,
80 budget: { runsPerDay: 8, dollarsPerDay: 1, reason: "every six hours is four a day; the rest is room for a catch-up, and it spends no money" },
81 async run({ now, app }) {
82 const url = app?.config.url;
83 if (url === undefined || url === "") return { findings: [], actions: ["skipped: this forge does not know its public origin (FORGE_URL)"] };
84 const origin = new URL(url);
85 const loopback = `http://127.0.0.1:${app?.config.port ?? 3000}`;
86 const findings: Finding[] = [];
87 const actions: string[] = [];
88 const run = AbortSignal.timeout(runMs);
89 for (const [i, path] of PATHS.entries()) {
90 if (run.aborted) {
91 actions.push(`not read within this run's ${runMs} ms: ${PATHS.slice(i).join(", ")}`);
92 break;
93 }
94 try {
95 // A cell reaches out only through the egress proxy, where a bare handshake cannot go:
96 // the GETs are still timed, and the line says the edge went unmeasured.
97 const connectMs = await meter.handshake(origin, run).catch((e: unknown) => {

appMs falls back to 0 when no Server-Timing is read; that makes accept absorb all loopback time. Use an unknown value rather than 0.

98 actions.push(`${path} handshake unmeasured, counted as 0 ms: ${why(e)}`);
99 return 0;
100 });
101 const pub = await meter.get(new URL(path, origin).href, run);
102 const local = await meter.get(`${loopback}${path}`, run);
103 const t: Timings = { connectMs, ttfbMs: pub.ttfbMs, loopbackMs: local.ttfbMs, appMs: serverAppMs(pub.serverTiming) ?? serverAppMs(local.serverTiming) ?? 0 };
104 const said = splitLine(path, t, pub.totalMs);
105 actions.push(said);
106 const layer = waitLayer(t);
107 if (layer !== "none") findings.push({ summary: `${path} is slow from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: ms to first byte, waiting in ${layer}`, number: Math.round(t.ttfbMs), at: now.toISOString() } });
108 } catch (e) {
109 if (run.aborted) {
110 actions.push(`not read within this run's ${runMs} ms: ${PATHS.slice(i).join(", ")}`);
111 break;
112 }
113 const said = `${path} got no answer within ${TIMEOUT_MS} ms: ${why(e)}`;
114 actions.push(said);
115 findings.push({ summary: `${path} is unreachable from the forge's own host: ${said}`, probe: { kind: "measurement", what: `GET ${origin.origin}${path}: 1 is no answer`, number: 1, at: now.toISOString() } });
116 }
117 }
118 return { findings, actions };
119 },
120 };
121 }
122
123 export const duty: Duty = waitSplit();
src/maintain/registry.ts +2 −1
@@ -12,9 +12,10 @@ import { duty as backupAge } from "./duties/backup_age.ts";
1212 import { duty as mergedBranches } from "./duties/merged_branches.ts";
1313 import { duty as diskFree } from "./duties/disk_free.ts";
1414 import { duty as staleIssues } from "./duties/stale_issues.ts";
15 import { duty as waitSplit } from "./duties/wait_split.ts";
1516 import { dutyIdle } from "./duties/duty_idle.ts";
1617
17 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, staleIssues];
18 const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, staleIssues, waitSplit];
1819
1920 /** The duties plus the one that watches them for silence. */
2021 export const duties: readonly Duty[] = [...others, dutyIdle(others)];
src/maintain/wait_split.ts added +75
@@ -0,0 +1,75 @@
1 /**
2 * Where a slow public page waits (#13635, part 1 of #13633): one request timed in four places,
3 * split into the layer that holds the wait. The path is edge (Cloudflare) → tunnel (cloudflared on
4 * the host) → the forge's listener (accept queue, event loop) → the app's handler. Measured from
5 * the forge's own host:
6 *
7 * - `connectMs` TCP + TLS handshake with the public origin: the edge.
8 * - `ttfbMs` GET of the public URL until its headers arrive (fetch resolves).
9 * - `loopbackMs` GET of the same path on 127.0.0.1:<port> until its headers: no edge, no tunnel.
10 * - `appMs` the `app;dur=` the forge itself wrote in `Server-Timing`.
11 *
12 * So, each share clamped at 0:
13 * edge = connectMs
14 * tunnel = ttfbMs − connectMs − loopbackMs
15 * accept = loopbackMs − appMs (waiting to be accepted and scheduled on the host)
16 * app = appMs
17 * The layer is the largest share; a page whose public answer is under `SLOW_MS` waits nowhere.
18 * Plain data in, plain data out.
19 */
20
21 /** Under this a warm public GET is not slow: #13633's bar, 1 s. */
22 export const SLOW_MS = 1000;
23
24 export type Layer = "none" | "edge" | "tunnel" | "accept" | "app";
25
26 export interface Timings {
27 readonly connectMs: number;
28 readonly ttfbMs: number;
29 readonly loopbackMs: number;
30 readonly appMs: number;
31 }
32
33 const pos = (n: number): number => (Number.isFinite(n) && n > 0 ? n : 0);
34
35 /** Each layer's share of the public wait, in ms. */
36 export function shares(t: Timings): Record<Exclude<Layer, "none">, number> {
37 return {
38 edge: pos(t.connectMs),
39 tunnel: pos(t.ttfbMs - pos(t.connectMs) - pos(t.loopbackMs)),
40 accept: pos(t.loopbackMs - pos(t.appMs)),
41 app: pos(t.appMs),
42 };
43 }
44
45 /** The layer the wait sits in: the largest share, or `none` when the page is not slow. */
46 export function waitLayer(t: Timings): Layer {
47 if (pos(t.ttfbMs) < SLOW_MS && pos(t.connectMs) < SLOW_MS) return "none";
48 let best: Layer = "none";
49 let most = -1;
50 for (const [layer, ms] of Object.entries(shares(t)) as [Exclude<Layer, "none">, number][]) {
51 if (ms > most) [best, most] = [layer, ms];
52 }
53 return best;
54 }
55
56 /** The `app;dur=` of a `Server-Timing` header, in ms; undefined when it carries none. */
57 export function serverAppMs(header: string | null | undefined): number | undefined {
58 if (typeof header !== "string") return undefined;
59 for (const metric of header.split(",")) {
60 const [name, ...params] = metric.trim().split(";");
61 if (name?.trim() !== "app") continue;
62 for (const p of params) {
63 const m = /^\s*dur=([0-9]+(?:\.[0-9]+)?)\s*$/.exec(p);
64 if (m?.[1] !== undefined) return Number(m[1]);
65 }
66 }
67 return undefined;
68 }
69
70 const ms = (n: number): string => `${Math.round(n)} ms`;
71
72 /** One line for the record: the path, the four timings, the total and the layer. */
73 export function splitLine(path: string, t: Timings, totalMs: number): string {
74 return `${path} connect ${ms(t.connectMs)}, first byte ${ms(t.ttfbMs)}, total ${ms(totalMs)}, loopback ${ms(t.loopbackMs)}, app ${t.appMs.toFixed(1)} ms: waits in ${waitLayer(t)}`;
75 }
test/maintain-wait-split.test.ts added +120
@@ -0,0 +1,120 @@
1 /**
2 * Where a slow public page waits (#13635): four timings in, the layer's name out, checked by hand;
3 * and the duty, on a meter of its own, GETs each page at the public origin and on loopback.
4 */
5 import { afterAll, expect, test } from "bun:test";
6 import { mkdtempSync, rmSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { duties } from "../src/maintain/registry.ts";
11 import { DUTY_TIMEOUT_MS } from "../src/maintain/runner.ts";
12 import { PATHS, RUN_MS, waitSplit, type Meter, type Read } from "../src/maintain/duties/wait_split.ts";
13 import { serverAppMs, shares, splitLine, waitLayer } from "../src/maintain/wait_split.ts";
14
15 test("each layer is named by the largest share of the wait, by hand", () => {
16 // 9580 ms public, 0.4 ms app (#13633's /healthz): 9580 − 200 − 9000 = 380 tunnel; 9000 − 0.4
17 // accept.
18 expect(shares({ connectMs: 200, ttfbMs: 9580, loopbackMs: 9000, appMs: 0.4 })).toEqual({ edge: 200, tunnel: 380, accept: 8999.6, app: 0.4 });
19 expect(waitLayer({ connectMs: 200, ttfbMs: 9580, loopbackMs: 9000, appMs: 0.4 })).toBe("accept");
20 // Loopback is fast, the public read is not: the tunnel between the edge and the host.
21 expect(waitLayer({ connectMs: 200, ttfbMs: 8320, loopbackMs: 20, appMs: 16.8 })).toBe("tunnel");
22 // The handshake itself takes the seconds: the edge.
23 expect(waitLayer({ connectMs: 6000, ttfbMs: 6950, loopbackMs: 5, appMs: 3.5 })).toBe("edge");
24 // The handler is the time: the app.
25 expect(waitLayer({ connectMs: 100, ttfbMs: 2300, loopbackMs: 2100, appMs: 2050 })).toBe("app");
26 });
27
28 test("a page under 1 s waits nowhere, and a share never goes negative", () => {
29 expect(waitLayer({ connectMs: 200, ttfbMs: 341, loopbackMs: 2, appMs: 1.1 })).toBe("none");
30 // A reused connection: fetch's first byte can come before a fresh handshake's.
31 expect(shares({ connectMs: 300, ttfbMs: 120, loopbackMs: 2, appMs: 3 })).toEqual({ edge: 300, tunnel: 0, accept: 0, app: 3 });
32 });
33
34 test("the app's own time is read from Server-Timing, and nothing else is taken for it", () => {
35 expect(serverAppMs("db;dur=0.1, app;dur=0.4")).toBe(0.4);
36 expect(serverAppMs("app;dur=16.8, render;dur=11.7")).toBe(16.8);
37 expect(serverAppMs("render;dur=11.7")).toBeUndefined();
38 expect(serverAppMs("app;desc=x")).toBeUndefined();
39 expect(serverAppMs(null)).toBeUndefined();
40 });
41
42 test("the line names the path, the four timings, the total and the layer", () => {
43 expect(splitLine("/", { connectMs: 200, ttfbMs: 8320, loopbackMs: 20, appMs: 16.8 }, 8400)).toBe(
44 "/ connect 200 ms, first byte 8320 ms, total 8400 ms, loopback 20 ms, app 16.8 ms: waits in tunnel",
45 );
46 });
47
48 const made: { app: App; dir: string }[] = [];
49 /** A real app at this public origin and port; the duty reads only its config. */
50 const appAt = (url: string, port = 3000): App => {
51 const dir = mkdtempSync(join(tmpdir(), "wait-split-"));
52 const app = createApp({ dataDir: dir, port, url });
53 made.push({ app, dir });
54 return app;
55 };
56 const runOn = (meter: Meter, app: App, now = new Date(), runMs?: number) => waitSplit(meter, runMs).run({ db: app.db, bus: app.bus, now, app });
57 afterAll(() => {
58 for (const { app, dir } of made) {
59 app.close();
60 rmSync(dir, { recursive: true, force: true });
61 }
62 });
63
64 const read = (ttfbMs: number, app: number): Read => ({ status: 200, ttfbMs, totalMs: ttfbMs + 10, serverTiming: `app;dur=${app}` });
65
66 test("the duty GETs every page at the public origin and on loopback, and a slow one is a finding", async () => {
67 const asked: string[] = [];
68 const meter: Meter = {
69 handshake: () => Promise.resolve(150),
70 get: (url) => {
71 asked.push(url);
72 return Promise.resolve(url === "https://forge.example/pricing" ? read(7000, 3.5) : read(url.startsWith("http://127.0.0.1") ? 3 : 300, 2));
73 },
74 };
75 const r = await runOn(meter, appAt("https://forge.example", 4123), new Date("2026-11-15T12:00:00Z"));
76 expect(asked).toEqual(PATHS.flatMap((p) => [`https://forge.example${p}`, `http://127.0.0.1:4123${p}`]));
77 expect(r.actions).toHaveLength(PATHS.length);
78 expect(r.actions).toContain("/pricing connect 150 ms, first byte 7000 ms, total 7010 ms, loopback 3 ms, app 3.5 ms: waits in tunnel");
79 expect(r.findings.map((f) => f.summary)).toEqual([expect.stringContaining("/pricing is slow from the forge's own host")]);
80 expect(r.findings[0]?.probe?.number).toBe(7000);
81 });
82
83 test("a page that never answers is a finding, a failed handshake is said, and no origin is a skip", async () => {
84 const meter: Meter = {
85 handshake: () => Promise.reject(new Error("refused")),
86 get: (url) => (url.endsWith("/terms") ? Promise.reject(new DOMException("timed out", "TimeoutError")) : Promise.resolve(read(100, 1))),
87 };
88 const run = (url: string) => runOn(meter, appAt(url));
89 const r = await run("https://forge.example");
90 expect(r.findings.map((f) => f.summary)).toEqual([expect.stringContaining("/terms is unreachable")]);
91 expect(r.actions).toContain("/healthz handshake unmeasured, counted as 0 ms: Error: refused");
92 expect((await run("")).actions).toEqual(["skipped: this forge does not know its public origin (FORGE_URL)"]);
93 });
94
95 test("the run stops reading at its own bound, gives up the read in flight, and names the pages it left", async () => {
96 const signals: AbortSignal[] = [];
97 // A public origin that never answers, as a cell's refused egress can: each read waits for its
98 // abort.
99 const hang = (run: AbortSignal) =>
100 new Promise<never>((_, reject) => {
101 signals.push(run);
102 run.addEventListener("abort", () => reject(run.reason), { once: true });
103 });
104 const meter: Meter = { handshake: () => Promise.resolve(10), get: (url, run) => (url.endsWith("/healthz") ? Promise.resolve(read(100, 1)) : hang(run)) };
105 const started = performance.now();
106 const r = await runOn(meter, appAt("https://forge.example"), new Date(), 50);
107 expect(performance.now() - started).toBeLessThan(2_000);
108 expect(signals.length).toBeGreaterThan(0);
109 expect(signals.every((s) => s.aborted)).toBe(true);
110 expect(r.findings).toEqual([]);
111 expect(r.actions.at(-1)).toBe("not read within this run's 50 ms: /, /pricing, /terms, /privacy");
112 });
113
114 test("the run's bound sits under the runner's deadline", () => {
115 expect(RUN_MS).toBeLessThan(DUTY_TIMEOUT_MS);
116 });
117
118 test("the forge runs it", () => {
119 expect(duties.map((d) => d.name)).toContain("wait-split");
120 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs