Public: anyone can read this; the crew can change it.
Server-rendered pages answer a repeat visit with 304: / and /explore carry an ETag today only on some pages #14300
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14277-server-rendered-pages-answer-a-repeat-vi → main 8 commits fork 4 threads · 4 unresolved
6 files
changed +180
−2
src/http/etag.ts +19
| @@ -31,3 +31,22 @@ export function holds(request: Request, etag: string): boolean { | ||
| 31 | 31 | const bare = (t: string) => t.trim().replace(/^W\//, ""); |
| 32 | 32 | return held.split(",").some((t) => bare(t) === bare(etag)); |
| 33 | 33 | } |
| 34 | ||
| 35 | /** | |
| 36 | * Whether the copy the request holds is this one: by its tag when it sends | |
| 37 | * `If-None-Match` (which then wins, RFC 9110 §13.1.3), else by its date when | |
| 38 | * it sends `If-Modified-Since` and the page has a `Last-Modified`. | |
| 39 | * | |
| 40 | * The date matters because the edge in front of production strips the ETag | |
| 41 | * from every text/html answer (it keeps it on text/plain, JSON and Atom: | |
| 42 | * `test/etag.test.ts` names what was measured) and keeps Last-Modified; a | |
| 43 | * browser behind it can only revalidate a page by date. | |
| 44 | */ | |
| 45 | export function unchanged(request: Request, etag: string, lastModified?: string | null): boolean { | |
| 46 | if (request.headers.has("if-none-match")) return holds(request, etag); | |
| 47 | const since = request.headers.get("if-modified-since"); | |
| 48 | if (since === null || lastModified === undefined || lastModified === null) return false; | |
| 49 | const held = Date.parse(since); | |
| 50 | const stamped = Date.parse(lastModified); | |
| 51 | return Number.isFinite(held) && Number.isFinite(stamped) && held >= stamped; | |
| 52 | } |
src/http/page_stamp.ts added +49
| @@ -0,0 +1,49 @@ | ||
| 1 | /** | |
| 2 | * A page's Last-Modified, for the edge that keeps no ETag on HTML | |
| 3 | * (`http/etag.ts` `unchanged`). | |
| 4 | * | |
| 5 | * The forge does not know when a page's rows last changed, so the stamp is | |
| 6 | * when this process first sent these bytes for this URL: kept per URL with | |
| 7 | * the tag it was sent with, and moved to a later second whenever the tag | |
| 8 | * differs. A client's date can therefore only match while the bytes do: a | |
| 9 | * page that changes and changes back (A, B, A) is stamped three times, each | |
| 10 | * one later than the last. A restart or an evicted URL starts again at now, | |
| 11 | * which costs a 200, never a stale 304. | |
| 12 | * | |
| 13 | * Plain data kept per forge (an app object, #12569), bounded to | |
| 14 | * `MAX_PAGES` URLs, oldest dropped. | |
| 15 | */ | |
| 16 | ||
| 17 | export const MAX_PAGES = 2048; | |
| 18 | ||
| 19 | type Stamps = Map<string, { etag: string; second: number }>; | |
| 20 | ||
| 21 | const byApp = new WeakMap<object, Stamps>(); | |
| 22 | ||
| 23 | function stampsOf(app: object): Stamps { | |
| 24 | let stamps = byApp.get(app); | |
| 25 | if (stamps === undefined) byApp.set(app, (stamps = new Map())); | |
| 26 | return stamps; | |
| 27 | } | |
| 28 | ||
| 29 | /** The HTTP date this URL's current bytes (named by `etag`) were first sent by this forge. */ | |
| 30 | export function pageStamp(app: object, url: string, etag: string, now: number = Date.now()): string { | |
| 31 | const stamps = stampsOf(app); | |
| 32 | const second = Math.floor(now / 1000); | |
| 33 | const held = stamps.get(url); | |
| 34 | let stamp = held; | |
| 35 | if (held === undefined) stamp = { etag, second }; | |
| 36 | else if (held.etag !== etag) stamp = { etag, second: Math.max(second, held.second + 1) }; | |
| 37 | stamps.delete(url); | |
| 38 | stamps.set(url, stamp!); | |
| 39 | if (stamps.size > MAX_PAGES) stamps.delete(stamps.keys().next().value as string); | |
| 40 | return new Date(stamp!.second * 1000).toUTCString(); | |
| 41 | } | |
| 42 | ||
| 43 | /** | |
| 44 | * The date this URL was last stamped with, for a page that answers its own 304 before rendering. | |
| 45 | */ | |
| 46 | export function heldStamp(app: object, url: string): string | undefined { | |
| 47 | const held = byApp.get(app)?.get(url); | |
| 48 | return held === undefined ? undefined : new Date(held.second * 1000).toUTCString(); | |
| 49 | } |
src/web/layout.ts +5 −2
| @@ -37,7 +37,8 @@ import type { Ctx } from "../http/context.ts"; | ||
| 37 | 37 | import { activeEdgeToken, edgeLoginUrl, edgeLogoutUrl, edgeStrangerLogin } from "../http/edge.ts"; |
| 38 | 38 | import { json } from "../http/respond.ts"; |
| 39 | 39 | import { sha256 } from "../kernel/hash.ts"; |
| 40 | import { holds, taggable, weakEtag } from "../http/etag.ts"; | |
| 40 | import { taggable, unchanged, weakEtag } from "../http/etag.ts"; | |
| 41 | import { pageStamp } from "../http/page_stamp.ts"; | |
| 41 | 42 | import { EDGE_VARY, sharedCache } from "../http/edge_cache.ts"; |
| 42 | 43 | import { runningLine } from "../kernel/version.ts"; |
| 43 | 44 | import { |
| @@ -996,8 +997,10 @@ export function page(options: PageOptions, status = 200): Response { | ||
| 996 | 997 | if (cacheControl.startsWith("public")) headers.set("vary", EDGE_VARY); |
| 997 | 998 | if (tagged) { |
| 998 | 999 | const etag = weakEtag(text); |
| 1000 | const lastModified = pageStamp(ctx.app, `${ctx.url.pathname}${ctx.url.search}`, etag); | |
| 999 | 1001 | headers.set("etag", etag); |
| 1000 | if (holds(ctx.request, etag)) | |
| 1002 | headers.set("last-modified", lastModified); | |
| 1003 | if (unchanged(ctx.request, etag, lastModified)) | |
| 1001 | 1004 | return new Response(null, { status: 304, headers }); |
| 1002 | 1005 | } |
| 1003 | 1006 | return new Response(text, { status, headers }); |
src/web/pages/changelog.ts +4
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | * a new commit. Public and server-rendered. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { heldStamp } from "../../http/page_stamp.ts"; | |
| 9 | 10 | import { MONTH_NAMES, monthLabel } from "../../release/month.ts"; |
| 10 | 11 | import { notePhase, takeSessionTime, timePhaseAsync } from "../../core/route_timing.ts"; |
| 11 | 12 | import type { Ctx } from "../../http/context.ts"; |
| @@ -432,6 +433,9 @@ function cacheHeaders(ctx: Ctx, etag: string): { matched: boolean; response: Res | ||
| 432 | 433 | headers.set("vary", EDGE_VARY); |
| 433 | 434 | } else headers.set("cache-control", "private, no-cache"); |
| 434 | 435 | const matched = holds(h.get("if-none-match"), etag); |
| 436 | // The date the page was last sent with, so a 304 here says what the page's own would. | |
| 437 | const stamped = matched ? heldStamp(ctx.app, `${ctx.url.pathname}${ctx.url.search}`) : undefined; | |
| 438 | if (stamped !== undefined) headers.set("last-modified", stamped); | |
| 435 | 439 | // The 304 carries the floor headers; the stored copy keeps the CSP it came with. |
| 436 | 440 | const response = matched ? securityFloor(new Response(null, { status: 304, headers: new Headers(headers) })) : new Response(null); |
| 437 | 441 | return { matched, response, headers }; |
test/etag.test.ts +70
| @@ -9,6 +9,8 @@ import { tmpdir } from "node:os"; | ||
| 9 | 9 | import { join } from "node:path"; |
| 10 | 10 | import { createApp, type App } from "../src/app.ts"; |
| 11 | 11 | import { forgeHttp, type ForgeHttp } from "../src/http/server.ts"; |
| 12 | import { HEADS } from "../src/http/route_marks.ts"; | |
| 13 | import { setCrawlClock } from "../src/http/crawl.ts"; | |
| 12 | 14 | import { adminCookie } from "./helpers/admin.ts"; |
| 13 | 15 | import { quietLogs } from "./helpers/globals.ts"; |
| 14 | 16 | |
| @@ -26,6 +28,10 @@ beforeAll(async () => { | ||
| 26 | 28 | dir = await mkdtemp(join(tmpdir(), "forge-etag-")); |
| 27 | 29 | app = createApp({ dataDir: dir, port: 0 }); |
| 28 | 30 | http = forgeHttp(app); |
| 31 | // The walk below is one address asking for every page: each ask is a | |
| 32 | // fresh minute to the crawler budget, which is not what this file tests. | |
| 33 | let t = Date.now(); | |
| 34 | setCrawlClock(app, () => (t += 61_000)); | |
| 29 | 35 | cookie = await adminCookie(app, { username: "ada", email: "[email protected]" }); |
| 30 | 36 | }); |
| 31 | 37 | |
| @@ -84,3 +90,67 @@ test("a signed-out 404 carries no ETag", async () => { | ||
| 84 | 90 | expect(res.status).toBe(404); |
outdated
Hand-kept route list; derive it from the route table or add a guard test so a new public page cannot be missed. Name the exclusions (/offline, signed-in pages). |
||
| 85 | 91 | expect(res.headers.has("etag")).toBe(false); |
| 86 | 92 | }); |
| 93 | ||
| 94 | // Every open page in the route table that answers a stranger 200 HTML names | |
| 95 | // its bytes twice: a weak ETag and a Last-Modified. Both are needed in | |
| 96 | // production: measured on 2026-11-19 against production and the preview of | |
| 97 | // #14300 (curl -sD- with and without accept-encoding), the edge in front of | |
| 98 | // the forge strips ETag from every text/html answer, weak or strong (/, | |
| 99 | // /explore, /changelog), and keeps it on text/plain (/llms.txt), JSON | |
| 100 | // (/api/v1/repos) and Atom (/changelog.atom). A browser behind it can only | |
| 101 | // revalidate a page by date, so Last-Modified is what turns its repeat visit | |
| 102 | // into a 304 (`http/page_stamp.ts`). | |
| 103 | // | |
| 104 | // The list is derived from `HEADS` (`http/route_marks.ts`): a new open page | |
| 105 | // is walked without being named here. Admin and signed-in routes are not open | |
| 106 | // and are not walked; the exclusions below are the open heads that are not a | |
| 107 | // stranger's 200 HTML page or carry no tag on purpose. | |
| 108 | const NOT_TAGGED: Record<string, string> = { | |
outdated
no-cache|private is too loose. Assert the exact value, and that If-None-Match on a signed-in request does not yield 304. |
||
| 109 | "/offline": "the service worker's fallback, cached by the worker, not revalidated", | |
| 110 | "/status": "the standalone health page: fresh for 10 s by max-age, live state after that", | |
| 111 | }; | |
| 112 | const openPages = Object.entries(HEADS) | |
| 113 | .filter(([path, mark]) => mark === "open" && !path.includes(":") && !/\.[a-z]+$/.test(path) && !(path in NOT_TAGGED)) | |
| 114 | .map(([path]) => path); | |
| 115 | ||
| 116 | test("the walked list is derived from the route table and holds the pages people open first", () => { | |
| 117 | for (const path of ["/", "/explore", "/changelog", "/docs", "/pricing", "/security", "/privacy", "/terms"]) { | |
| 118 | expect(openPages).toContain(path); | |
| 119 | } | |
| 120 | for (const path of Object.keys(NOT_TAGGED)) expect(HEADS[path]).toBe("open"); | |
| 121 | }); | |
| 122 | ||
| 123 | for (const path of openPages) { | |
| 124 | test(`${path}: a stranger's 200 HTML carries a weak tag and a date, and either one held is an empty 304`, async () => { | |
| 125 | const first = await get(path); | |
| 126 | expect(first.status).toBeLessThan(429); | |
| 127 | if (first.status !== 200 || !(first.headers.get("content-type") ?? "").startsWith("text/html")) return; | |
| 128 | const etag = first.headers.get("etag")!; | |
| 129 | expect(etag).toMatch(/^W\/"[0-9a-z]+"$/); | |
| 130 | const date = first.headers.get("last-modified")!; | |
| 131 | expect(Number.isFinite(Date.parse(date))).toBe(true); | |
| 132 | const head = await http.handle(new Request(`http://forge.test${path}`, { method: "HEAD" })); | |
| 133 | expect(head.headers.get("etag")).toBe(etag); | |
| 134 | for (const held of [{ "if-none-match": etag }, { "if-modified-since": date }]) { | |
| 135 | const again = await get(path, held); | |
| 136 | expect(again.status).toBe(304); | |
| 137 | expect(await again.text()).toBe(""); | |
| 138 | for (const h of ["cache-control", "vary", "etag", "last-modified"]) expect(again.headers.get(h)).toBe(first.headers.get(h)); | |
| 139 | } | |
| 140 | // A tag that does not match wins over a date that would (RFC 9110 §13.1.3). | |
| 141 | expect((await get(path, { "if-none-match": 'W/"other"', "if-modified-since": date })).status).toBe(200); | |
| 142 | expect((await get(path, { "if-modified-since": new Date(Date.parse(date) - 1000).toUTCString() })).status).toBe(200); | |
| 143 | }); | |
| 144 | } | |
| 145 | ||
| 146 | test("a signed-in page is no-cache, carries no validator, and a matching tag or date still gets a 200", async () => { | |
| 147 | const out = await get("/"); | |
| 148 | const res = await get("/", { cookie }); | |
| 149 | expect(res.status).toBe(200); | |
| 150 | expect(res.headers.get("cache-control")).toBe("no-cache"); | |
| 151 | expect(res.headers.has("etag")).toBe(false); | |
| 152 | expect(res.headers.has("last-modified")).toBe(false); | |
| 153 | const held = await get("/", { cookie, "if-none-match": out.headers.get("etag")!, "if-modified-since": out.headers.get("last-modified")! }); | |
| 154 | expect(held.status).toBe(200); | |
| 155 | expect(held.headers.get("cache-control")).toBe("no-cache"); | |
| 156 | }); |
test/page-stamp.test.ts added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | /** A page's Last-Modified moves whenever its bytes do (`http/page_stamp.ts`). */ | |
| 2 | ||
| 3 | import { expect, test } from "bun:test"; | |
| 4 | import { pageStamp } from "../src/http/page_stamp.ts"; | |
| 5 | ||
| 6 | const app = {}; | |
| 7 | const at = Date.UTC(2026, 10, 19, 12, 0, 0); | |
| 8 | const secs = (d: string) => Date.parse(d) / 1000; | |
| 9 | ||
| 10 | test("the same bytes keep their first date; new bytes get a later one, even in the same second", () => { | |
| 11 | const a = pageStamp(app, "/stamp-1", 'W/"a"', at); | |
| 12 | expect(pageStamp(app, "/stamp-1", 'W/"a"', at + 5000)).toBe(a); | |
| 13 | const b = pageStamp(app, "/stamp-1", 'W/"b"', at + 200); | |
| 14 | expect(secs(b)).toBe(secs(a) + 1); | |
| 15 | // Back to the first bytes: a third date, later than both, so a copy held from B is not | |
| 16 | // "unchanged". | |
| 17 | const again = pageStamp(app, "/stamp-1", 'W/"a"', at + 300); | |
| 18 | expect(secs(again)).toBe(secs(b) + 1); | |
| 19 | }); | |
| 20 | ||
| 21 | test("each URL is stamped on its own", () => { | |
| 22 | const one = pageStamp(app, "/stamp-2", 'W/"x"', at); | |
| 23 | const two = pageStamp(app, "/stamp-3", 'W/"x"', at + 60_000); | |
| 24 | expect(secs(two) - secs(one)).toBe(60); | |
| 25 | }); | |
| 26 | ||
| 27 | test("each forge keeps its own stamps", () => { | |
| 28 | const other = {}; | |
| 29 | const mine = pageStamp(app, "/stamp-4", 'W/"y"', at); | |
| 30 | const theirs = pageStamp(other, "/stamp-4", 'W/"z"', at); | |
| 31 | expect(theirs).toBe(mine); | |
| 32 | expect(pageStamp(app, "/stamp-4", 'W/"y"', at + 10_000)).toBe(mine); | |
| 33 | }); |
new above
offline
The test server applies no brotli, so 'survives br' cannot fail. Prove it against the preview or drop the claim; assert the signed-in page is private, not just not public.