Public: anyone can read this; the crew can change it.

A maintenance duty notices when the edge is not keeping the pages the forge marks shareable #14682

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14663-a-maintenance-duty-notices-when-the-edge → main 4 commits fork

  • Merged. scout/14663-a-maintenance-duty-notices-when-the-edge was merged into main as 4c73853.

Review

Sign in to review this change.
5 files changed +254 −0
docs/features/maintenance-duties.md +6
@@ -32,6 +32,12 @@ The `wait-split` duty finds where a slow public page waits. Every six hours, fro
3232
3333 Tests: `test/maintain-wait-split.test.ts`.
3434
35 ## Edge kept (#14663)
36
37 The `edge-kept` duty notices when the edge is not keeping the pages the forge marks shareable. Hourly, it sends two signed-out GETs (no cookie) to each of `/` and `/explore`, and only at `config.url` (the forge's own public origin), never a host from a request. If the forge marked the answer shareable (`Cache-Control: public` with a positive `s-maxage`) and the `cf-cache-status` header says the edge went to the forge both times (`DYNAMIC`, `MISS` or `BYPASS`), it files one issue per page, once, and leaves it open. A `HIT`, `STALE` or `REVALIDATED` on either read counts as kept. No edge header, an unshared page, a non-200 answer, a host the forge's gate refuses, or a forge with no `FORGE_URL` say nothing but an action line; so does a forge without its own repository row, which has nowhere to file. The one fix is the owner's, outside this repository: an edge cache rule that caches signed-out HTML and keys on `Cookie, Authorization`. The duty's lines (one per page) show on the maintenance runs lists (`/api/v1/maintenance/runs`, the terminal) after its first run, an hour after boot.
38
39 Tests: `test/maintain-edge-kept.test.ts`.
40
3541 ## Duty idle (#13406)
3642
3743 The `duty-idle` duty watches the other duties for silence. Hourly, it looks at when each duty last started (budget stops do not count as a start). A duty whose last start is more than twice its interval ago is idle; exactly 2x is not. A duty that has never run is skipped, because there is no last run to name. The watcher never watches itself.
src/maintain/duties/edge_kept.ts added +91
@@ -0,0 +1,91 @@
1 /**
2 * Is the edge keeping the pages the forge marks shareable (#14663)? Hourly, GET the forge's own
3 * public `/` and `/explore` signed out, twice each (no cookie is sent). When the forge marked the
4 * answer shareable (`sharedCache`, `http/edge_cache.ts`) and both answers show the edge went to
5 * the forge, file one issue per page: the page, the header seen, and the one fix (an edge cache
6 * rule for signed-out HTML keyed on EDGE_VARY). A forge with no edge header says nothing. The
7 * URL is `config.url` only, never a host from a request; two small requests a page an hour. Where
8 * the forge's gate refuses the host (`X-Forge-Egress: refused`) the answer is unknown and silent.
9 */
10
11 import { fileIssueOnce } from "../../checks/flake.ts";
12 import { getRepo } from "../../core/repos.ts";
13 import { EDGE_VARY } from "../../http/edge_cache.ts";
14 import { ownRepoPath } from "../../kernel/own_repo.ts";
15 import type { Duty, Finding } from "../duty.ts";
16 import { edgeHeader, edgeVerdict, marked } from "../edge_kept.ts";
17
18 export const PATHS = ["/", "/explore"] as const;
19 const TIMEOUT_MS = 8_000;
20
21 /** One signed-out GET; the test hands in its own. */
22 export type Get = (url: string) => Promise<Headers | undefined>;
23
24 const get: Get = async (url) => {
25 const res = await fetch(url, { method: "GET", redirect: "manual", headers: { accept: "text/html" }, signal: AbortSignal.timeout(TIMEOUT_MS) });
26 await res.body?.cancel();
27 return res.headers.has("x-forge-egress") || res.status !== 200 ? undefined : res.headers;
28 };
29
30 export const title = (path: string): string => `The edge is not keeping ${path}, which the forge marks shareable`;
31
32 const body = (path: string, seen: string): string =>
33 `A signed-out GET of ${path} was marked shareable by the forge (\`Cache-Control: public, s-maxage\`), but twice in a row the edge did not serve it from its copy (${seen}).\n\n` +
34 `The one fix is the owner's, outside this repository: an edge cache rule that caches signed-out HTML and keys on \`${EDGE_VARY}\`.\n\n` +
35 "This issue is filed once per page and left open until it is fixed; the hourly duty `edge-kept` checks again.";
36
37 export function edgeKept(read: Get = get): Duty {
38 return {
39 name: "edge-kept",
40 intervalMs: 3_600_000,
41 budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 runs a day; the rest is room for a catch-up, and it spends no money" },
42 async run({ db, bus, now, app }) {
43 const url = app?.config.url;
44 if (url === undefined || url === "") return { findings: [], actions: ["skipped: this forge does not know its public origin (FORGE_URL)"] };
45 const origin = new URL(url);
46 const findings: Finding[] = [];
47 const actions: string[] = [];
48 const [owner = "", name = ""] = ownRepoPath().split("/");
49 const repo = getRepo(db, owner, name);
50 for (const path of PATHS) {
51 const at = new URL(path, origin).href;
52 let first: Headers | undefined;
53 let second: Headers | undefined;
54 try {
55 first = await read(at);
56 second = first === undefined ? undefined : await read(at);
57 } catch (e) {
58 actions.push(`${path}: not read: ${e instanceof Error ? e.name : "error"}`);
59 continue;
60 }
61 if (first === undefined || second === undefined) {
62 actions.push(`${path}: no plain 200 answer; nothing judged`);
63 continue;
64 }
65 if (!marked(second)) {
66 actions.push(`${path}: the forge did not mark it shareable; nothing judged`);
67 continue;
68 }
69 const verdict = edgeVerdict(first, second);
70 const seen = edgeHeader(second) ?? "no edge header";
71 actions.push(`${path}: ${verdict} (${seen})`);
72 if (verdict !== "not") continue;
73 const summary = `${path} is marked shareable but the edge did not keep it (${seen})`;
74 const probe = { kind: "measurement", what: `GET ${origin.origin}${path} twice, signed out: 1 is not served from the edge`, number: 1, at: now.toISOString() } as const;
75 if (repo === undefined) {
76 actions.push(`${path}: not kept, but this forge's own repository is not here to file it in`);
77 continue;
78 }
79 const filed = fileIssueOnce(db, bus, repo, { title: title(path), body: body(path, seen) });
80 if (filed.opened) {
81 findings.push({ summary: `${summary}; filed ${owner}/${name}#${String(filed.number)}`, probe });
82 actions.push(`filed issue #${String(filed.number)}`);
83 } else if (filed.number === 0) actions.push(`${path}: not kept, and the issue could not be filed`);
84 else actions.push(`already filed as issue #${String(filed.number)}`);
85 }
86 return { findings, actions };
87 },
88 };
89 }
90
91 export const duty: Duty = edgeKept();
src/maintain/edge_kept.ts added +42
@@ -0,0 +1,42 @@
1 /**
2 * Did the edge keep a page the forge marked shareable (#14663)? Plain data in, plain data out:
3 * one answer's headers say "kept", "not" or "unknown"; two answers in a row say whether to
4 * speak. A forge with no edge in front sends no edge header, which is unknown, never a fault.
5 */
6
7 export type Kept = "kept" | "not" | "unknown";
8
9 /** `cf-cache-status` values that mean the edge answered from its copy. */
10 const KEPT = new Set(["HIT", "STALE", "REVALIDATED"]);
11 /** Values that mean it went to the forge for this answer. */
12 const NOT = new Set(["DYNAMIC", "MISS", "BYPASS"]);
13
14 /**
15 * The edge header the verdict rests on, as seen (`cf-cache-status: DYNAMIC`), or undefined when
16 * there is none.
17 */
18 export function edgeHeader(headers: Headers): string | undefined {
19 const status = headers.get("cf-cache-status")?.trim();
20 return status ? `cf-cache-status: ${status.slice(0, 32)}` : undefined;
21 }
22
23 /** One answer: was it served from the edge's copy? */
24 export function edgeKept(headers: Headers): Kept {
25 const status = headers.get("cf-cache-status")?.trim().toUpperCase();
26 if (status) return KEPT.has(status) ? "kept" : NOT.has(status) ? "not" : "unknown";
27 return "unknown";
28 }
29
30 /** Whether the forge marked this answer shareable: `public` with a shared lifetime. */
31 export function marked(headers: Headers): boolean {
32 const cc = (headers.get("cache-control") ?? "").toLowerCase();
33 return /(^|,)\s*public\b/.test(cc) && /\bs-maxage=[1-9]/.test(cc);
34 }
35
36 /** Two answers in a row: kept if either was; not only when both were not; else unknown. */
37 export function edgeVerdict(first: Headers, second: Headers): Kept {
38 const a = edgeKept(first);
39 const b = edgeKept(second);
40 if (a === "kept" || b === "kept") return "kept";
41 return a === "not" && b === "not" ? "not" : "unknown";
42 }
test/maintain-edge-kept.test.ts added +114
@@ -0,0 +1,114 @@
1 import { afterAll, expect, test } from "bun:test";
2 import { mkdtempSync, rmSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { createApp, type App } from "../src/app.ts";
6 import { startMaintain } from "../src/maintain/runner.ts";
7 import { latest } from "../src/maintain/runs.ts";
8 import { duties } from "../src/maintain/registry.ts";
9 import { edgeKept as edgeDuty, PATHS, title, type Get } from "../src/maintain/duties/edge_kept.ts";
10 import { edgeKept, edgeVerdict, marked } from "../src/maintain/edge_kept.ts";
11 import { sharedCache } from "../src/http/edge_cache.ts";
12 import { createUser } from "../src/core/users.ts";
13 import { createRepo } from "../src/actions/repos.ts";
14 import { ownRepoPath } from "../src/kernel/own_repo.ts";
15
16 const h = (o: Record<string, string>) => new Headers(o);
17 const cf = (v: string) => h({ "cf-cache-status": v });
18
19 test("HIT, STALE and REVALIDATED are kept; DYNAMIC, MISS and BYPASS are not; no header is unknown", () => {
20 for (const v of ["HIT", "stale", "REVALIDATED"]) expect(edgeKept(cf(v))).toBe("kept");
21 for (const v of ["DYNAMIC", "MISS", "BYPASS"]) expect(edgeKept(cf(v))).toBe("not");
22 expect(edgeKept(h({}))).toBe("unknown");
23 expect(edgeKept(cf("WEIRD"))).toBe("unknown");
24 expect(edgeKept(h({ age: "0" }))).toBe("unknown");
25 expect(edgeKept(h({ age: "12" }))).toBe("unknown");
26 });
27
28 test("not kept only when both answers were not; a hit on either keeps; no edge is silent", () => {
29 expect(edgeVerdict(cf("MISS"), cf("MISS"))).toBe("not");
30 expect(edgeVerdict(cf("DYNAMIC"), cf("BYPASS"))).toBe("not");
31 expect(edgeVerdict(cf("MISS"), cf("HIT"))).toBe("kept");
32 expect(edgeVerdict(h({}), h({}))).toBe("unknown");
33 expect(edgeVerdict(cf("MISS"), h({}))).toBe("unknown");
34 });
35
36 test("marked reads the header sharedCache sends", () => {
37 const anon = new Request("http://x/");
38 expect(marked(h({ "cache-control": sharedCache(anon, "/", () => false) }))).toBe(true);
39 expect(marked(h({ "cache-control": "no-cache" }))).toBe(false);
40 expect(marked(h({ "cache-control": "private, s-maxage=60" }))).toBe(false);
41 });
42
43 const dirs: string[] = [];
44 afterAll(() => dirs.forEach((d) => rmSync(d, { recursive: true, force: true })));
45
46 async function run(read: Get, url: string | undefined = "https://forge.example", withRepo = true) {
47 const dir = mkdtempSync(join(tmpdir(), "edge-kept-"));
48 dirs.push(dir);
49 const app: App = createApp({ dataDir: dir, port: 0, ...(url === undefined ? {} : { url }) });
50 const u = await createUser(app.db, { origin: "seed", trusted: true, username: ownRepoPath().split("/")[0]!, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
51 if (!u.ok) throw new Error(u.error);
52 const [, name = "forge"] = ownRepoPath().split("/");
53 if (withRepo) await createRepo(app, u.value, { name, visibility: "private" });
54 const duty = edgeDuty(read);
55 const once = () => duty.run({ db: app.db, bus: app.bus, now: new Date(), app });
56 return { app, once };
57 }
58
59 const shareable = (edge: Record<string, string>): Get => async () => h({ "cache-control": "public, max-age=0, s-maxage=60", ...edge });
60
61 test("the duty is on the roster, hourly, and reads only the configured URL", async () => {
62 expect(duties.find((d) => d.name === "edge-kept")?.intervalMs).toBe(3_600_000);
63 const seen: string[] = [];
64 const { once } = await run(async (u) => (seen.push(u), h({ "cache-control": "public, s-maxage=60", "cf-cache-status": "HIT" })));
65 const r = await once();
66 expect(r.findings).toEqual([]);
67 expect(seen).toEqual(["https://forge.example/", "https://forge.example/", "https://forge.example/explore", "https://forge.example/explore"]);
68 expect(PATHS.length).toBe(2);
69 });
70
71 test("not kept files one issue per page, and a second run adds nothing", async () => {
72 const { app, once } = await run(shareable({ "cf-cache-status": "DYNAMIC" }));
73 const first = await once();
74 expect(first.findings.length).toBe(2);
75 expect(first.findings[0]!.summary).toContain("cf-cache-status: DYNAMIC");
76 const second = await once();
77 expect(second.findings).toEqual([]);
78 const n = app.db.query("SELECT COUNT(*) AS n FROM issue WHERE title = ?1").get(title("/")) as { n: number };
79 expect(n.n).toBe(1);
80 const body = app.db.query("SELECT body FROM issue WHERE title = ?1").get(title("/")) as { body: string };
81 expect(body.body).toContain("Cookie, Authorization");
82 });
83
84 test("no edge header, an unshared page, a refused host or no URL say nothing", async () => {
85 expect((await (await run(shareable({}))).once()).findings).toEqual([]);
86 expect((await (await run(async () => h({ "cache-control": "no-cache", "cf-cache-status": "DYNAMIC" }))).once()).findings).toEqual([]);
87 expect((await (await run(async () => undefined)).once()).findings).toEqual([]);
88 const none = await (await run(shareable({ "cf-cache-status": "MISS" }), "")).once();
89 expect(none.findings).toEqual([]);
90 expect(none.actions[0]).toContain("skipped");
91 });
92
93 test("without the forge's own repository it files nothing and says why, every run", async () => {
94 const { app, once } = await run(shareable({ "cf-cache-status": "DYNAMIC" }), "https://forge.example", false);
95 for (let i = 0; i < 2; i++) {
96 const r = await once();
97 expect(r.findings).toEqual([]);
98 expect(r.actions.some((a) => a.includes("not here to file it"))).toBe(true);
99 }
100 expect((app.db.query("SELECT COUNT(*) AS n FROM issue").get() as { n: number }).n).toBe(0);
101 });
102
103 test("an age header alone is not an edge: no verdict from it", async () => {
104 expect((await (await run(shareable({ age: "0" }))).once()).findings).toEqual([]);
105 });
106
107 test("a run through the maintainer lands its action lines where /maintenance runs, the API and the terminal read them (latest)", async () => {
108 const { app } = await run(shareable({ "cf-cache-status": "DYNAMIC" }));
109 await startMaintain(app, { duties: [edgeDuty(shareable({ "cf-cache-status": "DYNAMIC" }))] }).tick();
110 const row = latest(app.db, 10).find((r) => r.duty === "edge-kept");
111 expect(row?.outcome).toBe("ok");
112 expect(row?.actions.some((a) => a.startsWith("/: not (cf-cache-status: DYNAMIC)"))).toBe(true);
113 expect(row?.findings.length).toBe(2);
114 });
test/outbound-sweep.test.ts +1
@@ -54,6 +54,7 @@ const ALLOWED: Record<string, { count: number; why: string }> = {
5454 "src/maintenance/standby.ts": { count: 1, why: "the operator's standby page URL" },
5555 "src/maintain/keeper/down.ts": { count: 1, why: "services the operator listed in the keeper's environment" },
5656 "src/maintain/duties/wait_split.ts": { count: 1, why: "the forge's own public URL, timed" },
57 "src/maintain/duties/edge_kept.ts": { count: 1, why: "the forge's own public URL (config.url), never a host from a request" },
5758 };
5859
5960 /** Every non-literal fetch in src, as `file:line`. */

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs