Public: anyone can read this; the crew can change it.

Server-rendered pages answer a repeat visit with 304: / and /explore carry an ETag today only on some pages #14300

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14277-server-rendered-pages-answer-a-repeat-vi → main 8 commits fork 4 threads · 4 unresolved

  • Merged. scout/14277-server-rendered-pages-answer-a-repeat-vi was merged into main as 230962b.

Review

Sign in to review this change.
6 files changed +180 −2
src/http/etag.ts +19
@@ -31,3 +31,22 @@ export function holds(request: Request, etag: string): boolean {
3131 const bare = (t: string) => t.trim().replace(/^W\//, "");
3232 return held.split(",").some((t) => bare(t) === bare(etag));
3333 }
34
35 /**
36 * Whether the copy the request holds is this one: by its tag when it sends
37 * `If-None-Match` (which then wins, RFC 9110 §13.1.3), else by its date when
38 * it sends `If-Modified-Since` and the page has a `Last-Modified`.
39 *
40 * The date matters because the edge in front of production strips the ETag
41 * from every text/html answer (it keeps it on text/plain, JSON and Atom:
42 * `test/etag.test.ts` names what was measured) and keeps Last-Modified; a
43 * browser behind it can only revalidate a page by date.
44 */
45 export function unchanged(request: Request, etag: string, lastModified?: string | null): boolean {
46 if (request.headers.has("if-none-match")) return holds(request, etag);
47 const since = request.headers.get("if-modified-since");
48 if (since === null || lastModified === undefined || lastModified === null) return false;
49 const held = Date.parse(since);
50 const stamped = Date.parse(lastModified);
51 return Number.isFinite(held) && Number.isFinite(stamped) && held >= stamped;
52 }
src/http/page_stamp.ts added +49
@@ -0,0 +1,49 @@
1 /**
2 * A page's Last-Modified, for the edge that keeps no ETag on HTML
3 * (`http/etag.ts` `unchanged`).
4 *
5 * The forge does not know when a page's rows last changed, so the stamp is
6 * when this process first sent these bytes for this URL: kept per URL with
7 * the tag it was sent with, and moved to a later second whenever the tag
8 * differs. A client's date can therefore only match while the bytes do: a
9 * page that changes and changes back (A, B, A) is stamped three times, each
10 * one later than the last. A restart or an evicted URL starts again at now,
11 * which costs a 200, never a stale 304.
12 *
13 * Plain data kept per forge (an app object, #12569), bounded to
14 * `MAX_PAGES` URLs, oldest dropped.
15 */
16
17 export const MAX_PAGES = 2048;
18
19 type Stamps = Map<string, { etag: string; second: number }>;
20
21 const byApp = new WeakMap<object, Stamps>();
22
23 function stampsOf(app: object): Stamps {
24 let stamps = byApp.get(app);
25 if (stamps === undefined) byApp.set(app, (stamps = new Map()));
26 return stamps;
27 }
28
29 /** The HTTP date this URL's current bytes (named by `etag`) were first sent by this forge. */
30 export function pageStamp(app: object, url: string, etag: string, now: number = Date.now()): string {
31 const stamps = stampsOf(app);
32 const second = Math.floor(now / 1000);
33 const held = stamps.get(url);
34 let stamp = held;
35 if (held === undefined) stamp = { etag, second };
36 else if (held.etag !== etag) stamp = { etag, second: Math.max(second, held.second + 1) };
37 stamps.delete(url);
38 stamps.set(url, stamp!);
39 if (stamps.size > MAX_PAGES) stamps.delete(stamps.keys().next().value as string);
40 return new Date(stamp!.second * 1000).toUTCString();
41 }
42
43 /**
44 * The date this URL was last stamped with, for a page that answers its own 304 before rendering.
45 */
46 export function heldStamp(app: object, url: string): string | undefined {
47 const held = byApp.get(app)?.get(url);
48 return held === undefined ? undefined : new Date(held.second * 1000).toUTCString();
49 }
src/web/layout.ts +5 −2
@@ -37,7 +37,8 @@ import type { Ctx } from "../http/context.ts";
3737 import { activeEdgeToken, edgeLoginUrl, edgeLogoutUrl, edgeStrangerLogin } from "../http/edge.ts";
3838 import { json } from "../http/respond.ts";
3939 import { sha256 } from "../kernel/hash.ts";
40 import { holds, taggable, weakEtag } from "../http/etag.ts";
40 import { taggable, unchanged, weakEtag } from "../http/etag.ts";
41 import { pageStamp } from "../http/page_stamp.ts";
4142 import { EDGE_VARY, sharedCache } from "../http/edge_cache.ts";
4243 import { runningLine } from "../kernel/version.ts";
4344 import {
@@ -996,8 +997,10 @@ export function page(options: PageOptions, status = 200): Response {
996997 if (cacheControl.startsWith("public")) headers.set("vary", EDGE_VARY);
997998 if (tagged) {
998999 const etag = weakEtag(text);
1000 const lastModified = pageStamp(ctx.app, `${ctx.url.pathname}${ctx.url.search}`, etag);
9991001 headers.set("etag", etag);
1000 if (holds(ctx.request, etag))
1002 headers.set("last-modified", lastModified);
1003 if (unchanged(ctx.request, etag, lastModified))
10011004 return new Response(null, { status: 304, headers });
10021005 }
10031006 return new Response(text, { status, headers });
src/web/pages/changelog.ts +4
@@ -6,6 +6,7 @@
66 * a new commit. Public and server-rendered.
77 */
88
9 import { heldStamp } from "../../http/page_stamp.ts";
910 import { MONTH_NAMES, monthLabel } from "../../release/month.ts";
1011 import { notePhase, takeSessionTime, timePhaseAsync } from "../../core/route_timing.ts";
1112 import type { Ctx } from "../../http/context.ts";
@@ -432,6 +433,9 @@ function cacheHeaders(ctx: Ctx, etag: string): { matched: boolean; response: Res
432433 headers.set("vary", EDGE_VARY);
433434 } else headers.set("cache-control", "private, no-cache");
434435 const matched = holds(h.get("if-none-match"), etag);
436 // The date the page was last sent with, so a 304 here says what the page's own would.
437 const stamped = matched ? heldStamp(ctx.app, `${ctx.url.pathname}${ctx.url.search}`) : undefined;
438 if (stamped !== undefined) headers.set("last-modified", stamped);
435439 // The 304 carries the floor headers; the stored copy keeps the CSP it came with.
436440 const response = matched ? securityFloor(new Response(null, { status: 304, headers: new Headers(headers) })) : new Response(null);
437441 return { matched, response, headers };
test/etag.test.ts +70
@@ -9,6 +9,8 @@ import { tmpdir } from "node:os";
99 import { join } from "node:path";
1010 import { createApp, type App } from "../src/app.ts";
1111 import { forgeHttp, type ForgeHttp } from "../src/http/server.ts";
12 import { HEADS } from "../src/http/route_marks.ts";
13 import { setCrawlClock } from "../src/http/crawl.ts";
1214 import { adminCookie } from "./helpers/admin.ts";
1315 import { quietLogs } from "./helpers/globals.ts";
1416
@@ -26,6 +28,10 @@ beforeAll(async () => {
2628 dir = await mkdtemp(join(tmpdir(), "forge-etag-"));
2729 app = createApp({ dataDir: dir, port: 0 });
2830 http = forgeHttp(app);
31 // The walk below is one address asking for every page: each ask is a
32 // fresh minute to the crawler budget, which is not what this file tests.
33 let t = Date.now();
34 setCrawlClock(app, () => (t += 61_000));
2935 cookie = await adminCookie(app, { username: "ada", email: "[email protected]" });
3036 });
3137
@@ -84,3 +90,67 @@ test("a signed-out 404 carries no ETag", async () => {
8490 expect(res.status).toBe(404);
outdated

The test server applies no brotli, so 'survives br' cannot fail. Prove it against the preview or drop the claim; assert the signed-in page is private, not just not public.

outdated

Hand-kept route list; derive it from the route table or add a guard test so a new public page cannot be missed. Name the exclusions (/offline, signed-in pages).

8591 expect(res.headers.has("etag")).toBe(false);
8692 });
93
94 // Every open page in the route table that answers a stranger 200 HTML names
95 // its bytes twice: a weak ETag and a Last-Modified. Both are needed in
96 // production: measured on 2026-11-19 against production and the preview of
97 // #14300 (curl -sD- with and without accept-encoding), the edge in front of
98 // the forge strips ETag from every text/html answer, weak or strong (/,
99 // /explore, /changelog), and keeps it on text/plain (/llms.txt), JSON
100 // (/api/v1/repos) and Atom (/changelog.atom). A browser behind it can only
101 // revalidate a page by date, so Last-Modified is what turns its repeat visit
102 // into a 304 (`http/page_stamp.ts`).
103 //
104 // The list is derived from `HEADS` (`http/route_marks.ts`): a new open page
105 // is walked without being named here. Admin and signed-in routes are not open
106 // and are not walked; the exclusions below are the open heads that are not a
107 // stranger's 200 HTML page or carry no tag on purpose.
108 const NOT_TAGGED: Record<string, string> = {
outdated

no-cache|private is too loose. Assert the exact value, and that If-None-Match on a signed-in request does not yield 304.

109 "/offline": "the service worker's fallback, cached by the worker, not revalidated",
110 "/status": "the standalone health page: fresh for 10 s by max-age, live state after that",
111 };
112 const openPages = Object.entries(HEADS)
113 .filter(([path, mark]) => mark === "open" && !path.includes(":") && !/\.[a-z]+$/.test(path) && !(path in NOT_TAGGED))
114 .map(([path]) => path);
115
116 test("the walked list is derived from the route table and holds the pages people open first", () => {
117 for (const path of ["/", "/explore", "/changelog", "/docs", "/pricing", "/security", "/privacy", "/terms"]) {
118 expect(openPages).toContain(path);
119 }
120 for (const path of Object.keys(NOT_TAGGED)) expect(HEADS[path]).toBe("open");
121 });
122
123 for (const path of openPages) {
124 test(`${path}: a stranger's 200 HTML carries a weak tag and a date, and either one held is an empty 304`, async () => {
125 const first = await get(path);
126 expect(first.status).toBeLessThan(429);
127 if (first.status !== 200 || !(first.headers.get("content-type") ?? "").startsWith("text/html")) return;
128 const etag = first.headers.get("etag")!;
129 expect(etag).toMatch(/^W\/"[0-9a-z]+"$/);
130 const date = first.headers.get("last-modified")!;
131 expect(Number.isFinite(Date.parse(date))).toBe(true);
132 const head = await http.handle(new Request(`http://forge.test${path}`, { method: "HEAD" }));
133 expect(head.headers.get("etag")).toBe(etag);
134 for (const held of [{ "if-none-match": etag }, { "if-modified-since": date }]) {
135 const again = await get(path, held);
136 expect(again.status).toBe(304);
137 expect(await again.text()).toBe("");
138 for (const h of ["cache-control", "vary", "etag", "last-modified"]) expect(again.headers.get(h)).toBe(first.headers.get(h));
139 }
140 // A tag that does not match wins over a date that would (RFC 9110 §13.1.3).
141 expect((await get(path, { "if-none-match": 'W/"other"', "if-modified-since": date })).status).toBe(200);
142 expect((await get(path, { "if-modified-since": new Date(Date.parse(date) - 1000).toUTCString() })).status).toBe(200);
143 });
144 }
145
146 test("a signed-in page is no-cache, carries no validator, and a matching tag or date still gets a 200", async () => {
147 const out = await get("/");
148 const res = await get("/", { cookie });
149 expect(res.status).toBe(200);
150 expect(res.headers.get("cache-control")).toBe("no-cache");
151 expect(res.headers.has("etag")).toBe(false);
152 expect(res.headers.has("last-modified")).toBe(false);
153 const held = await get("/", { cookie, "if-none-match": out.headers.get("etag")!, "if-modified-since": out.headers.get("last-modified")! });
154 expect(held.status).toBe(200);
155 expect(held.headers.get("cache-control")).toBe("no-cache");
156 });
test/page-stamp.test.ts added +33
@@ -0,0 +1,33 @@
1 /** A page's Last-Modified moves whenever its bytes do (`http/page_stamp.ts`). */
2
3 import { expect, test } from "bun:test";
4 import { pageStamp } from "../src/http/page_stamp.ts";
5
6 const app = {};
7 const at = Date.UTC(2026, 10, 19, 12, 0, 0);
8 const secs = (d: string) => Date.parse(d) / 1000;
9
10 test("the same bytes keep their first date; new bytes get a later one, even in the same second", () => {
11 const a = pageStamp(app, "/stamp-1", 'W/"a"', at);
12 expect(pageStamp(app, "/stamp-1", 'W/"a"', at + 5000)).toBe(a);
13 const b = pageStamp(app, "/stamp-1", 'W/"b"', at + 200);
14 expect(secs(b)).toBe(secs(a) + 1);
15 // Back to the first bytes: a third date, later than both, so a copy held from B is not
16 // "unchanged".
17 const again = pageStamp(app, "/stamp-1", 'W/"a"', at + 300);
18 expect(secs(again)).toBe(secs(b) + 1);
19 });
20
21 test("each URL is stamped on its own", () => {
22 const one = pageStamp(app, "/stamp-2", 'W/"x"', at);
23 const two = pageStamp(app, "/stamp-3", 'W/"x"', at + 60_000);
24 expect(secs(two) - secs(one)).toBe(60);
25 });
26
27 test("each forge keeps its own stamps", () => {
28 const other = {};
29 const mine = pageStamp(app, "/stamp-4", 'W/"y"', at);
30 const theirs = pageStamp(other, "/stamp-4", 'W/"z"', at);
31 expect(theirs).toBe(mine);
32 expect(pageStamp(app, "/stamp-4", 'W/"y"', at + 10_000)).toBe(mine);
33 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs