Public: anyone can read this; the crew can change it.
A maintenance duty notices when the edge is not keeping the pages the forge marks shareable #14682
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14663-a-maintenance-duty-notices-when-the-edge → main 4 commits fork
5 files
changed +254
−0
docs/features/maintenance-duties.md +6
| @@ -32,6 +32,12 @@ The `wait-split` duty finds where a slow public page waits. Every six hours, fro | ||
| 32 | 32 | |
| 33 | 33 | Tests: `test/maintain-wait-split.test.ts`. |
| 34 | 34 | |
| 35 | ## Edge kept (#14663) | |
| 36 | ||
| 37 | The `edge-kept` duty notices when the edge is not keeping the pages the forge marks shareable. Hourly, it sends two signed-out GETs (no cookie) to each of `/` and `/explore`, and only at `config.url` (the forge's own public origin), never a host from a request. If the forge marked the answer shareable (`Cache-Control: public` with a positive `s-maxage`) and the `cf-cache-status` header says the edge went to the forge both times (`DYNAMIC`, `MISS` or `BYPASS`), it files one issue per page, once, and leaves it open. A `HIT`, `STALE` or `REVALIDATED` on either read counts as kept. No edge header, an unshared page, a non-200 answer, a host the forge's gate refuses, or a forge with no `FORGE_URL` say nothing but an action line; so does a forge without its own repository row, which has nowhere to file. The one fix is the owner's, outside this repository: an edge cache rule that caches signed-out HTML and keys on `Cookie, Authorization`. The duty's lines (one per page) show on the maintenance runs lists (`/api/v1/maintenance/runs`, the terminal) after its first run, an hour after boot. | |
| 38 | ||
| 39 | Tests: `test/maintain-edge-kept.test.ts`. | |
| 40 | ||
| 35 | 41 | ## Duty idle (#13406) |
| 36 | 42 | |
| 37 | 43 | The `duty-idle` duty watches the other duties for silence. Hourly, it looks at when each duty last started (budget stops do not count as a start). A duty whose last start is more than twice its interval ago is idle; exactly 2x is not. A duty that has never run is skipped, because there is no last run to name. The watcher never watches itself. |
src/maintain/duties/edge_kept.ts added +91
| @@ -0,0 +1,91 @@ | ||
| 1 | /** | |
| 2 | * Is the edge keeping the pages the forge marks shareable (#14663)? Hourly, GET the forge's own | |
| 3 | * public `/` and `/explore` signed out, twice each (no cookie is sent). When the forge marked the | |
| 4 | * answer shareable (`sharedCache`, `http/edge_cache.ts`) and both answers show the edge went to | |
| 5 | * the forge, file one issue per page: the page, the header seen, and the one fix (an edge cache | |
| 6 | * rule for signed-out HTML keyed on EDGE_VARY). A forge with no edge header says nothing. The | |
| 7 | * URL is `config.url` only, never a host from a request; two small requests a page an hour. Where | |
| 8 | * the forge's gate refuses the host (`X-Forge-Egress: refused`) the answer is unknown and silent. | |
| 9 | */ | |
| 10 | ||
| 11 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 12 | import { getRepo } from "../../core/repos.ts"; | |
| 13 | import { EDGE_VARY } from "../../http/edge_cache.ts"; | |
| 14 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 15 | import type { Duty, Finding } from "../duty.ts"; | |
| 16 | import { edgeHeader, edgeVerdict, marked } from "../edge_kept.ts"; | |
| 17 | ||
| 18 | export const PATHS = ["/", "/explore"] as const; | |
| 19 | const TIMEOUT_MS = 8_000; | |
| 20 | ||
| 21 | /** One signed-out GET; the test hands in its own. */ | |
| 22 | export type Get = (url: string) => Promise<Headers | undefined>; | |
| 23 | ||
| 24 | const get: Get = async (url) => { | |
| 25 | const res = await fetch(url, { method: "GET", redirect: "manual", headers: { accept: "text/html" }, signal: AbortSignal.timeout(TIMEOUT_MS) }); | |
| 26 | await res.body?.cancel(); | |
| 27 | return res.headers.has("x-forge-egress") || res.status !== 200 ? undefined : res.headers; | |
| 28 | }; | |
| 29 | ||
| 30 | export const title = (path: string): string => `The edge is not keeping ${path}, which the forge marks shareable`; | |
| 31 | ||
| 32 | const body = (path: string, seen: string): string => | |
| 33 | `A signed-out GET of ${path} was marked shareable by the forge (\`Cache-Control: public, s-maxage\`), but twice in a row the edge did not serve it from its copy (${seen}).\n\n` + | |
| 34 | `The one fix is the owner's, outside this repository: an edge cache rule that caches signed-out HTML and keys on \`${EDGE_VARY}\`.\n\n` + | |
| 35 | "This issue is filed once per page and left open until it is fixed; the hourly duty `edge-kept` checks again."; | |
| 36 | ||
| 37 | export function edgeKept(read: Get = get): Duty { | |
| 38 | return { | |
| 39 | name: "edge-kept", | |
| 40 | intervalMs: 3_600_000, | |
| 41 | budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 runs a day; the rest is room for a catch-up, and it spends no money" }, | |
| 42 | async run({ db, bus, now, app }) { | |
| 43 | const url = app?.config.url; | |
| 44 | if (url === undefined || url === "") return { findings: [], actions: ["skipped: this forge does not know its public origin (FORGE_URL)"] }; | |
| 45 | const origin = new URL(url); | |
| 46 | const findings: Finding[] = []; | |
| 47 | const actions: string[] = []; | |
| 48 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 49 | const repo = getRepo(db, owner, name); | |
| 50 | for (const path of PATHS) { | |
| 51 | const at = new URL(path, origin).href; | |
| 52 | let first: Headers | undefined; | |
| 53 | let second: Headers | undefined; | |
| 54 | try { | |
| 55 | first = await read(at); | |
| 56 | second = first === undefined ? undefined : await read(at); | |
| 57 | } catch (e) { | |
| 58 | actions.push(`${path}: not read: ${e instanceof Error ? e.name : "error"}`); | |
| 59 | continue; | |
| 60 | } | |
| 61 | if (first === undefined || second === undefined) { | |
| 62 | actions.push(`${path}: no plain 200 answer; nothing judged`); | |
| 63 | continue; | |
| 64 | } | |
| 65 | if (!marked(second)) { | |
| 66 | actions.push(`${path}: the forge did not mark it shareable; nothing judged`); | |
| 67 | continue; | |
| 68 | } | |
| 69 | const verdict = edgeVerdict(first, second); | |
| 70 | const seen = edgeHeader(second) ?? "no edge header"; | |
| 71 | actions.push(`${path}: ${verdict} (${seen})`); | |
| 72 | if (verdict !== "not") continue; | |
| 73 | const summary = `${path} is marked shareable but the edge did not keep it (${seen})`; | |
| 74 | const probe = { kind: "measurement", what: `GET ${origin.origin}${path} twice, signed out: 1 is not served from the edge`, number: 1, at: now.toISOString() } as const; | |
| 75 | if (repo === undefined) { | |
| 76 | actions.push(`${path}: not kept, but this forge's own repository is not here to file it in`); | |
| 77 | continue; | |
| 78 | } | |
| 79 | const filed = fileIssueOnce(db, bus, repo, { title: title(path), body: body(path, seen) }); | |
| 80 | if (filed.opened) { | |
| 81 | findings.push({ summary: `${summary}; filed ${owner}/${name}#${String(filed.number)}`, probe }); | |
| 82 | actions.push(`filed issue #${String(filed.number)}`); | |
| 83 | } else if (filed.number === 0) actions.push(`${path}: not kept, and the issue could not be filed`); | |
| 84 | else actions.push(`already filed as issue #${String(filed.number)}`); | |
| 85 | } | |
| 86 | return { findings, actions }; | |
| 87 | }, | |
| 88 | }; | |
| 89 | } | |
| 90 | ||
| 91 | export const duty: Duty = edgeKept(); |
src/maintain/edge_kept.ts added +42
| @@ -0,0 +1,42 @@ | ||
| 1 | /** | |
| 2 | * Did the edge keep a page the forge marked shareable (#14663)? Plain data in, plain data out: | |
| 3 | * one answer's headers say "kept", "not" or "unknown"; two answers in a row say whether to | |
| 4 | * speak. A forge with no edge in front sends no edge header, which is unknown, never a fault. | |
| 5 | */ | |
| 6 | ||
| 7 | export type Kept = "kept" | "not" | "unknown"; | |
| 8 | ||
| 9 | /** `cf-cache-status` values that mean the edge answered from its copy. */ | |
| 10 | const KEPT = new Set(["HIT", "STALE", "REVALIDATED"]); | |
| 11 | /** Values that mean it went to the forge for this answer. */ | |
| 12 | const NOT = new Set(["DYNAMIC", "MISS", "BYPASS"]); | |
| 13 | ||
| 14 | /** | |
| 15 | * The edge header the verdict rests on, as seen (`cf-cache-status: DYNAMIC`), or undefined when | |
| 16 | * there is none. | |
| 17 | */ | |
| 18 | export function edgeHeader(headers: Headers): string | undefined { | |
| 19 | const status = headers.get("cf-cache-status")?.trim(); | |
| 20 | return status ? `cf-cache-status: ${status.slice(0, 32)}` : undefined; | |
| 21 | } | |
| 22 | ||
| 23 | /** One answer: was it served from the edge's copy? */ | |
| 24 | export function edgeKept(headers: Headers): Kept { | |
| 25 | const status = headers.get("cf-cache-status")?.trim().toUpperCase(); | |
| 26 | if (status) return KEPT.has(status) ? "kept" : NOT.has(status) ? "not" : "unknown"; | |
| 27 | return "unknown"; | |
| 28 | } | |
| 29 | ||
| 30 | /** Whether the forge marked this answer shareable: `public` with a shared lifetime. */ | |
| 31 | export function marked(headers: Headers): boolean { | |
| 32 | const cc = (headers.get("cache-control") ?? "").toLowerCase(); | |
| 33 | return /(^|,)\s*public\b/.test(cc) && /\bs-maxage=[1-9]/.test(cc); | |
| 34 | } | |
| 35 | ||
| 36 | /** Two answers in a row: kept if either was; not only when both were not; else unknown. */ | |
| 37 | export function edgeVerdict(first: Headers, second: Headers): Kept { | |
| 38 | const a = edgeKept(first); | |
| 39 | const b = edgeKept(second); | |
| 40 | if (a === "kept" || b === "kept") return "kept"; | |
| 41 | return a === "not" && b === "not" ? "not" : "unknown"; | |
| 42 | } |
test/maintain-edge-kept.test.ts added +114
| @@ -0,0 +1,114 @@ | ||
| 1 | import { afterAll, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp, type App } from "../src/app.ts"; | |
| 6 | import { startMaintain } from "../src/maintain/runner.ts"; | |
| 7 | import { latest } from "../src/maintain/runs.ts"; | |
| 8 | import { duties } from "../src/maintain/registry.ts"; | |
| 9 | import { edgeKept as edgeDuty, PATHS, title, type Get } from "../src/maintain/duties/edge_kept.ts"; | |
| 10 | import { edgeKept, edgeVerdict, marked } from "../src/maintain/edge_kept.ts"; | |
| 11 | import { sharedCache } from "../src/http/edge_cache.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { ownRepoPath } from "../src/kernel/own_repo.ts"; | |
| 15 | ||
| 16 | const h = (o: Record<string, string>) => new Headers(o); | |
| 17 | const cf = (v: string) => h({ "cf-cache-status": v }); | |
| 18 | ||
| 19 | test("HIT, STALE and REVALIDATED are kept; DYNAMIC, MISS and BYPASS are not; no header is unknown", () => { | |
| 20 | for (const v of ["HIT", "stale", "REVALIDATED"]) expect(edgeKept(cf(v))).toBe("kept"); | |
| 21 | for (const v of ["DYNAMIC", "MISS", "BYPASS"]) expect(edgeKept(cf(v))).toBe("not"); | |
| 22 | expect(edgeKept(h({}))).toBe("unknown"); | |
| 23 | expect(edgeKept(cf("WEIRD"))).toBe("unknown"); | |
| 24 | expect(edgeKept(h({ age: "0" }))).toBe("unknown"); | |
| 25 | expect(edgeKept(h({ age: "12" }))).toBe("unknown"); | |
| 26 | }); | |
| 27 | ||
| 28 | test("not kept only when both answers were not; a hit on either keeps; no edge is silent", () => { | |
| 29 | expect(edgeVerdict(cf("MISS"), cf("MISS"))).toBe("not"); | |
| 30 | expect(edgeVerdict(cf("DYNAMIC"), cf("BYPASS"))).toBe("not"); | |
| 31 | expect(edgeVerdict(cf("MISS"), cf("HIT"))).toBe("kept"); | |
| 32 | expect(edgeVerdict(h({}), h({}))).toBe("unknown"); | |
| 33 | expect(edgeVerdict(cf("MISS"), h({}))).toBe("unknown"); | |
| 34 | }); | |
| 35 | ||
| 36 | test("marked reads the header sharedCache sends", () => { | |
| 37 | const anon = new Request("http://x/"); | |
| 38 | expect(marked(h({ "cache-control": sharedCache(anon, "/", () => false) }))).toBe(true); | |
| 39 | expect(marked(h({ "cache-control": "no-cache" }))).toBe(false); | |
| 40 | expect(marked(h({ "cache-control": "private, s-maxage=60" }))).toBe(false); | |
| 41 | }); | |
| 42 | ||
| 43 | const dirs: string[] = []; | |
| 44 | afterAll(() => dirs.forEach((d) => rmSync(d, { recursive: true, force: true }))); | |
| 45 | ||
| 46 | async function run(read: Get, url: string | undefined = "https://forge.example", withRepo = true) { | |
| 47 | const dir = mkdtempSync(join(tmpdir(), "edge-kept-")); | |
| 48 | dirs.push(dir); | |
| 49 | const app: App = createApp({ dataDir: dir, port: 0, ...(url === undefined ? {} : { url }) }); | |
| 50 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: ownRepoPath().split("/")[0]!, email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 51 | if (!u.ok) throw new Error(u.error); | |
| 52 | const [, name = "forge"] = ownRepoPath().split("/"); | |
| 53 | if (withRepo) await createRepo(app, u.value, { name, visibility: "private" }); | |
| 54 | const duty = edgeDuty(read); | |
| 55 | const once = () => duty.run({ db: app.db, bus: app.bus, now: new Date(), app }); | |
| 56 | return { app, once }; | |
| 57 | } | |
| 58 | ||
| 59 | const shareable = (edge: Record<string, string>): Get => async () => h({ "cache-control": "public, max-age=0, s-maxage=60", ...edge }); | |
| 60 | ||
| 61 | test("the duty is on the roster, hourly, and reads only the configured URL", async () => { | |
| 62 | expect(duties.find((d) => d.name === "edge-kept")?.intervalMs).toBe(3_600_000); | |
| 63 | const seen: string[] = []; | |
| 64 | const { once } = await run(async (u) => (seen.push(u), h({ "cache-control": "public, s-maxage=60", "cf-cache-status": "HIT" }))); | |
| 65 | const r = await once(); | |
| 66 | expect(r.findings).toEqual([]); | |
| 67 | expect(seen).toEqual(["https://forge.example/", "https://forge.example/", "https://forge.example/explore", "https://forge.example/explore"]); | |
| 68 | expect(PATHS.length).toBe(2); | |
| 69 | }); | |
| 70 | ||
| 71 | test("not kept files one issue per page, and a second run adds nothing", async () => { | |
| 72 | const { app, once } = await run(shareable({ "cf-cache-status": "DYNAMIC" })); | |
| 73 | const first = await once(); | |
| 74 | expect(first.findings.length).toBe(2); | |
| 75 | expect(first.findings[0]!.summary).toContain("cf-cache-status: DYNAMIC"); | |
| 76 | const second = await once(); | |
| 77 | expect(second.findings).toEqual([]); | |
| 78 | const n = app.db.query("SELECT COUNT(*) AS n FROM issue WHERE title = ?1").get(title("/")) as { n: number }; | |
| 79 | expect(n.n).toBe(1); | |
| 80 | const body = app.db.query("SELECT body FROM issue WHERE title = ?1").get(title("/")) as { body: string }; | |
| 81 | expect(body.body).toContain("Cookie, Authorization"); | |
| 82 | }); | |
| 83 | ||
| 84 | test("no edge header, an unshared page, a refused host or no URL say nothing", async () => { | |
| 85 | expect((await (await run(shareable({}))).once()).findings).toEqual([]); | |
| 86 | expect((await (await run(async () => h({ "cache-control": "no-cache", "cf-cache-status": "DYNAMIC" }))).once()).findings).toEqual([]); | |
| 87 | expect((await (await run(async () => undefined)).once()).findings).toEqual([]); | |
| 88 | const none = await (await run(shareable({ "cf-cache-status": "MISS" }), "")).once(); | |
| 89 | expect(none.findings).toEqual([]); | |
| 90 | expect(none.actions[0]).toContain("skipped"); | |
| 91 | }); | |
| 92 | ||
| 93 | test("without the forge's own repository it files nothing and says why, every run", async () => { | |
| 94 | const { app, once } = await run(shareable({ "cf-cache-status": "DYNAMIC" }), "https://forge.example", false); | |
| 95 | for (let i = 0; i < 2; i++) { | |
| 96 | const r = await once(); | |
| 97 | expect(r.findings).toEqual([]); | |
| 98 | expect(r.actions.some((a) => a.includes("not here to file it"))).toBe(true); | |
| 99 | } | |
| 100 | expect((app.db.query("SELECT COUNT(*) AS n FROM issue").get() as { n: number }).n).toBe(0); | |
| 101 | }); | |
| 102 | ||
| 103 | test("an age header alone is not an edge: no verdict from it", async () => { | |
| 104 | expect((await (await run(shareable({ age: "0" }))).once()).findings).toEqual([]); | |
| 105 | }); | |
| 106 | ||
| 107 | test("a run through the maintainer lands its action lines where /maintenance runs, the API and the terminal read them (latest)", async () => { | |
| 108 | const { app } = await run(shareable({ "cf-cache-status": "DYNAMIC" })); | |
| 109 | await startMaintain(app, { duties: [edgeDuty(shareable({ "cf-cache-status": "DYNAMIC" }))] }).tick(); | |
| 110 | const row = latest(app.db, 10).find((r) => r.duty === "edge-kept"); | |
| 111 | expect(row?.outcome).toBe("ok"); | |
| 112 | expect(row?.actions.some((a) => a.startsWith("/: not (cf-cache-status: DYNAMIC)"))).toBe(true); | |
| 113 | expect(row?.findings.length).toBe(2); | |
| 114 | }); |
test/outbound-sweep.test.ts +1
| @@ -54,6 +54,7 @@ const ALLOWED: Record<string, { count: number; why: string }> = { | ||
| 54 | 54 | "src/maintenance/standby.ts": { count: 1, why: "the operator's standby page URL" }, |
| 55 | 55 | "src/maintain/keeper/down.ts": { count: 1, why: "services the operator listed in the keeper's environment" }, |
| 56 | 56 | "src/maintain/duties/wait_split.ts": { count: 1, why: "the forge's own public URL, timed" }, |
| 57 | "src/maintain/duties/edge_kept.ts": { count: 1, why: "the forge's own public URL (config.url), never a host from a request" }, | |
| 57 | 58 | }; |
| 58 | 59 | |
| 59 | 60 | /** Every non-literal fetch in src, as `file:line`. */ |
new above
offline