cabcdec · a547777 · 2929eab · 0b605bf
@@ -0,0 +1,107 @@
/**
 * Content-Encoding for the bodies the forge writes itself (#2915 check 7): pages,
      
        sentinelcrew
        26d ago
        outdated
      
      Low: this cites #2940, but this work is #2915 check 7 (pull #2975).
    
 * stylesheets, scripts and JSON, as br or gzip when `Accept-Encoding` allows.
 * Both come from the runtime (`Bun.gzipSync`, `node:zlib` brotli); no
 * dependency. A body under MIN_BYTES is left alone, and so is anything that
 * is not on the type list: SSE, git streams, log follows and tarballs keep
 * their bytes and their flush.
 *
 * Bodies that never change (hashed assets) are encoded once and kept (brotli 9: q11 blocks the loop ~0.3 s on the stylesheet); the rest
 * are encoded per response at a speed-first level.
 */
import { brotliCompressSync, constants } from "node:zlib";
export type Encoding = "br" | "gzip";
/** Below this the headers cost more than the bytes saved. */
export const MIN_BYTES = 1024;
const TYPES = new Set([
  "text/html",
  "text/css",
  "text/javascript",
  "application/javascript",
  "application/json",
  "application/manifest+json",
  "image/svg+xml",
]);
/** The encoding to use: br over gzip, a `q=0` or absent coding refused. */
export function negotiate(header: string | null): Encoding | null {
  if (header === null) return null;
  const q: Record<string, number> = {};
  for (const part of header.split(",")) {
    const [name, ...params] = part.split(";");
    let value = 1;
    for (const p of params) {
      const [k, v] = p.split("=").map((x) => x.trim().toLowerCase());
      if (k === "q") value = Number(v);
    }
    const key = (name ?? "").trim().toLowerCase();
    if (key && Number.isFinite(value)) q[key] = value;
  }
  if ((q["br"] ?? 0) > 0) return "br";
  if ((q["gzip"] ?? q["x-gzip"] ?? 0) > 0) return "gzip";
  return null;
}
const mediaType = (type: string | null) => (type ?? "").split(";")[0]!.trim().toLowerCase();
function encode(bytes: Uint8Array<ArrayBuffer>, encoding: Encoding, best: boolean): Uint8Array {
  if (encoding === "gzip") return Bun.gzipSync(bytes, { level: best ? 9 : 4 });
  return brotliCompressSync(bytes, {
    params: {
      [constants.BROTLI_PARAM_QUALITY]: best ? 9 : 4,
      
        sentinelcrew
        26d ago
        outdated
      
      Quality 11 on the 150 KB stylesheet takes ~312–358 ms of synchronous CPU, and encodeOnce calls this inside a request, so the event loop stalls for every client on the first hit after each deploy. Quality 9 produces 30.2 KB vs 27.3 KB in 23 ms. Use 9 for best, or warm the assets at boot with the async brotliCompress.
    
      [constants.BROTLI_PARAM_SIZE_HINT]: bytes.length,
    },
  });
}
const kept = new Map<string, Uint8Array>();
const KEPT_MAX = 64;
/** Encoded once per `key` (a content hash) and `encoding`, for bodies that never change. */
export function encodeOnce(key: string, bytes: Uint8Array<ArrayBuffer>, encoding: Encoding): Uint8Array {
  const id = `${encoding}:${key}`;
  let out = kept.get(id);
  if (out === undefined) {
    out = encode(bytes, encoding, true);
    if (kept.size >= KEPT_MAX) kept.clear();
    kept.set(id, out);
  }
  return out;
}
/**
 * The response with its body encoded if the request allows it, else the
 * response as it came. `Vary: Accept-Encoding` goes on every compressible type,
 * encoded or not, so a cache never hands a small plain copy's key to the wrong
 * client. Headers are rewritten in place, as `securityFloor` does.
      
        sentinelcrew
        26d ago
        
      
      Low (BREACH, for the record): authenticated HTML is now compressed. CSRF here is enforced by same-origin checks, not tokens in the page, and no session secret is in a body, so there is nothing a size side-channel can recover today. If a page ever puts a stable secret next to text an attacker controls (a CSRF token, an API key shown on every view), have that response send Cache-Control: no-transform so compress() leaves it alone. That opt-out already exists. A one-line note about it in this doc comment would keep the next builder from forgetting it.
    
 */
export async function compress(request: Request, response: Response): Promise<Response> {
  try {
    if (!TYPES.has(mediaType(response.headers.get("content-type")))) return response;
    const vary = response.headers.get("vary");
    if (!/(^|,)\s*(accept-encoding|\*)\s*(,|$)/i.test(vary ?? ""))
      response.headers.set("vary", vary ? `${vary}, Accept-Encoding` : "Accept-Encoding");
    const encoding = negotiate(request.headers.get("accept-encoding"));
    if (
      encoding === null ||
      request.method === "HEAD" ||
      response.body === null ||
      response.status === 206 ||
      response.headers.has("content-encoding") ||
      /\bno-transform\b/i.test(response.headers.get("cache-control") ?? "")
    )
      return response;
    const bytes = new Uint8Array(await response.clone().arrayBuffer());
    if (bytes.length < MIN_BYTES) return new Response(bytes, response);
    const out = new Response(encode(bytes, encoding, false), response);
    out.headers.set("content-encoding", encoding);
    out.headers.delete("content-length");
    return out;
  } catch {
    return response;
  }
}
@@ -485,6 +485,7 @@
import {
  prefersHtml,
  securityFloor,
} from "./respond.ts";
import { compress } from "./compress.ts";
import { RateLimiter, type Verdict } from "../core/ratelimit.ts";
import {
  MAX_KEY_LENGTH,
@@ -2115,7 +2116,7 @@
export async function serve(app: App): Promise<Servers> {
    if (request.signal.aborted) onAbort();
    else request.signal.addEventListener("abort", onAbort, { once: true });
    const work = run().then(
      (response) => securityFloor(response),
      (response) => compress(request, response).then(securityFloor),
      
        sentinelcrew
        26d ago
        
      
      This is the change's main behaviour, and no test covers it: reverting this line to securityFloor(response) leaves bun test green. Add a serve()-level test: GET / with accept-encoding: br gives content-encoding: br, Vary and CSP, and SSE or git responses are not encoded.
    
      (error) => failure(owner, error),
    );
    try {
@@ -1,5 +1,5 @@
/**
 * The website specification as a report (#2915, checks 1 and 2). `webspec()`
 * The website specification as a report (#2915, checks 1, 2 and 7). `webspec()`
 * fetches each path of a site once and returns what the page's head and
 * response headers get wrong, each finding citing its specification.website
 * section. It reports; it never throws on a bad page and never fails a run.
@@ -106,6 +106,32 @@
export function headerProblems(h: Headers, https: boolean): Problem[] {
  return out;
}
const COMPRESS_MIN = 1024;
const ACCEPT_ENCODING = "br, gzip";
/** The stylesheets the head links, as absolute URLs. */
function stylesheetTargets(html: string, pageUrl: string): string[] {
  const out: string[] = [];
  for (const t of links(html, "stylesheet")) {
    try {
      const u = new URL(attr(t, "href") ?? "", pageUrl).href;
      if (!out.includes(u)) out.push(u);
    } catch {}
  }
  return out;
}
/** A text body big enough to be worth it must leave encoded, and say it varies. */
export function encodingProblems(h: Headers, bodyBytes: number): Problem[] {
  if (bodyBytes < COMPRESS_MIN) return [];
  const out: Problem[] = [];
  const encoding = (h.get("content-encoding") ?? "").toLowerCase();
  if (encoding !== "br" && encoding !== "gzip")
    out.push({ check: "compression", message: `a ${bodyBytes}-byte body was sent without br or gzip though Accept-Encoding: ${ACCEPT_ENCODING}`, section: SPEC, edge: true });
  else if (!/accept-encoding/i.test(h.get("vary") ?? "")) out.push({ check: "compression", message: "a compressed response has no Vary: Accept-Encoding", section: SPEC, edge: true });
  return out;
}
/** Fetches `paths` of `base` and reports head and header findings. */
export async function webspec(base: string, paths: string[], fetcher: Fetcher = fetch): Promise<Finding[]> {
  const findings: Finding[] = [];
@@ -122,9 +148,11 @@
export async function webspec(base: string, paths: string[], fetcher: Fetcher =
    }
    let res: Response;
    let html: string;
    let htmlBytes = 0;
    try {
      res = await fetcher(url, { headers: { accept: "text/html" }, redirect: "manual", signal: AbortSignal.timeout(TIMEOUT_MS) });
      res = await fetcher(url, { headers: { accept: "text/html", "accept-encoding": ACCEPT_ENCODING }, redirect: "manual", signal: AbortSignal.timeout(TIMEOUT_MS) });
      html = await res.text();
      htmlBytes = new TextEncoder().encode(html).length;
    } catch (e) {
      report([{ check: "fetch", message: `could not fetch ${url}: ${e instanceof Error ? e.message : String(e)}`, section: SPEC }]);
      continue;
@@ -134,6 +162,14 @@
export async function webspec(base: string, paths: string[], fetcher: Fetcher =
      continue;
    }
    report(headProblems(html));
    report(encodingProblems(res.headers, htmlBytes));
    for (const sheet of stylesheetTargets(html, url)) {
      try {
        const r = await fetcher(sheet, { headers: { "accept-encoding": ACCEPT_ENCODING }, signal: AbortSignal.timeout(TIMEOUT_MS) });
        const size = new TextEncoder().encode(await r.text()).length;
        if (r.ok) report(encodingProblems(r.headers, size).map((p) => ({ ...p, message: `${sheet}: ${p.message}` })));
      } catch {}
    }
    const { urls, bad } = headTargets(html, url);
    for (const h of bad) report([{ check: "head-links", message: `"${h}" is not a URL`, section: s("icons") }]);
    for (const target of urls) {
@@ -17,6 +17,7 @@
 */
import { sha256 } from "../kernel/hash.ts";
import { encodeOnce, MIN_BYTES, negotiate } from "../http/compress.ts";
import { createLog } from "../kernel/log.ts";
import { beaconJs } from "./beacon.ts";
import { shellJs } from "./client.ts";
@@ -73,15 +74,20 @@
async function build(): Promise<BuiltAsset | null> {
 * immutable.
 */
export function staticResponse(request: Request, body: string, type: string, etag: string): Response {
  const headers = {
  const headers: Record<string, string> = {
    "content-type": type,
    "cache-control": "public, max-age=31536000, immutable",
    "x-content-type-options": "nosniff",
    vary: "Accept-Encoding",
    etag,
  };
  if (matches(request.headers.get("if-none-match"), etag))
    return new Response(null, { status: 304, headers });
  return new Response(body, { headers });
  const encoding = negotiate(request.headers.get("accept-encoding"));
  const bytes = new TextEncoder().encode(body);
  if (encoding === null || bytes.length < MIN_BYTES || request.method === "HEAD") return new Response(body, { headers });
  // Hashed, so the encoded bytes are as immutable as the plain ones: made once.
  return new Response(encodeOnce(etag, bytes, encoding), { headers: { ...headers, "content-encoding": encoding } });
      
        sentinelcrew
        26d ago
        
      
      This is where the q11 stall lands: on the first request for the stylesheet, shell JS and highlight JS, roughly 0.4 s together. If you keep q11, encode these when the assets are built rather than here.
    
}
function matches(header: string | null, etag: string): boolean {
@@ -0,0 +1,55 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { adminCookie } from "./helpers/admin.ts";
import { quietLogs } from "./helpers/globals.ts";
// The seam in `answered()`: compression runs on what the server really sends,
// ahead of the security floor, and leaves streams alone.
let dataDir: string;
let app: App;
let servers: Servers;
let cookie: string;
quietLogs();
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-compress-"));
  app = createApp({ dataDir, port: 0 });
  servers = await serve(app);
  cookie = await adminCookie(app, { username: "ada", email: "[email protected]", password: "analytical-engine" });
});
afterAll(async () => {
  servers.stop();
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
const get = (path: string, headers: Record<string, string>) => fetch(`${servers.url}${path}`, { headers: { cookie, ...headers } });
test("a page is br-encoded with Vary and the security floor, and decodes to HTML", async () => {
  const res = await get("/", { "accept-encoding": "br" });
  expect(res.status).toBe(200);
  expect(res.headers.get("content-encoding")).toBe("br");
  expect(res.headers.get("vary")).toMatch(/accept-encoding/i);
  expect(res.headers.get("content-security-policy")).toBeTruthy();
  expect(res.headers.get("x-content-type-options")).toBe("nosniff");
  expect(await res.text()).toMatch(/^\s*<!doctype html>/i);
});
test("without Accept-Encoding the page is sent plain", async () => {
  const res = await get("/", { "accept-encoding": "identity" });
  expect(res.headers.get("content-encoding")).toBe(null);
  expect(res.headers.get("vary")).toMatch(/accept-encoding/i);
});
test("a live event stream is never encoded, even when br is asked for", async () => {
  const res = await get("/events", { "accept-encoding": "br" });
  expect(res.headers.get("content-type")).toContain("text/event-stream");
  expect(res.headers.get("content-encoding")).toBe(null);
  await res.body?.cancel();
});
@@ -0,0 +1,64 @@
import { expect, test } from "bun:test";
import { brotliDecompressSync, gunzipSync } from "node:zlib";
import { compress, MIN_BYTES, negotiate } from "../src/http/compress.ts";
import { staticResponse } from "../src/web/asset.ts";
const big = "<p>hello forge</p>".repeat(200);
const req = (enc?: string, method = "GET") =>
  new Request("http://x.test/", { method, headers: enc ? { "accept-encoding": enc } : {} });
const html = (body: string, headers: Record<string, string> = {}) =>
  new Response(body, { headers: { "content-type": "text/html; charset=utf-8", ...headers } });
test("negotiate prefers br, honours q=0, and ignores the unknown", () => {
  expect(negotiate("gzip, br")).toBe("br");
  expect(negotiate("gzip")).toBe("gzip");
  expect(negotiate("br;q=0, gzip")).toBe("gzip");
  expect(negotiate("deflate, identity")).toBe(null);
  expect(negotiate(null)).toBe(null);
});
test("a large page is encoded with Vary and decodes to the original", async () => {
  const br = await compress(req("br, gzip"), html(big));
  expect(br.headers.get("content-encoding")).toBe("br");
  expect(br.headers.get("vary")).toBe("Accept-Encoding");
  expect(brotliDecompressSync(new Uint8Array(await br.arrayBuffer())).toString()).toBe(big);
  const gz = await compress(req("gzip"), html(big));
  expect(gz.headers.get("content-encoding")).toBe("gzip");
  expect(gunzipSync(new Uint8Array(await gz.arrayBuffer())).toString()).toBe(big);
});
test("JSON is encoded too", async () => {
  const r = await compress(req("gzip"), new Response(JSON.stringify({ a: "x".repeat(MIN_BYTES) }), { headers: { "content-type": "application/json" } }));
  expect(r.headers.get("content-encoding")).toBe("gzip");
});
test("small bodies, other types, HEAD, and clients without support are left alone", async () => {
  expect((await compress(req("br"), html("tiny"))).headers.get("content-encoding")).toBe(null);
  expect((await compress(req("br"), html("tiny"))).headers.get("vary")).toBe("Accept-Encoding");
  const sse = new Response(big, { headers: { "content-type": "text/event-stream" } });
  const s = await compress(req("br"), sse);
  expect(s.headers.get("content-encoding")).toBe(null);
  expect(s.headers.get("vary")).toBe(null);
  const git = new Response(big, { headers: { "content-type": "application/x-git-upload-pack-result" } });
  expect((await compress(req("br"), git)).headers.get("content-encoding")).toBe(null);
  expect((await compress(req("br", "HEAD"), html(big))).headers.get("content-encoding")).toBe(null);
  expect((await compress(req(), html(big))).headers.get("content-encoding")).toBe(null);
  expect((await compress(req("br"), html(big, { "content-encoding": "gzip" }))).headers.get("content-encoding")).toBe("gzip");
});
test("a Vary the page already set is extended, not replaced", async () => {
  expect((await compress(req("br"), html("t", { vary: "Cookie" }))).headers.get("vary")).toBe("Cookie, Accept-Encoding");
});
test("a hashed asset is encoded once and the bytes are reused", async () => {
  const css = "a{color:red}\n".repeat(200);
  const a = staticResponse(req("br"), css, "text/css", '"h1"');
  const b = staticResponse(req("br"), css, "text/css", '"h1"');
  expect(a.headers.get("content-encoding")).toBe("br");
  expect(a.headers.get("vary")).toBe("Accept-Encoding");
  const bytes = new Uint8Array(await a.arrayBuffer());
  expect(brotliDecompressSync(bytes).toString()).toBe(css);
  expect(new Uint8Array(await b.arrayBuffer())).toEqual(bytes);
  expect(staticResponse(req(), css, "text/css", '"h1"').headers.get("content-encoding")).toBe(null);
  expect(staticResponse(req("br"), "a{}", "text/css", '"h2"').headers.get("content-encoding")).toBe(null);
});
@@ -143,3 +143,22 @@
test("every fetch carries a timeout signal", async () => {
  expect(seen.length).toBeGreaterThan(3);
  expect(seen.every(Boolean)).toBe(true);
});
test("compression: a big page and stylesheet must leave encoded with Vary", async () => {
  const big = GOOD_HTML.replace("</head>", `<link rel="stylesheet" href="/a.css"></head>`).replace(">hi<", `>${"x".repeat(2000)}<`);
  const fetcher = (encoded: boolean) => async (input: string, init?: RequestInit) => {
    const u = new URL(input);
    const h = new Headers(init?.headers);
    if (u.pathname === "/icon.svg" || u.pathname === "/m.json") return new Response("x");
    if (h.get("if-none-match")) return new Response(null, { status: 304 });
    expect(h.get("accept-encoding")).toBe("br, gzip");
    const enc: Record<string, string> = encoded ? { "content-encoding": "br", vary: "Accept-Encoding" } : {};
    if (u.pathname === "/a.css") return new Response("a{}".repeat(500), { headers: enc });
    return new Response(big, { headers: { ...GOOD_HEADERS, ...enc } });
  };
  expect(await webspec("https://x.test", ["/"], fetcher(true))).toEqual([]);
  const found = await webspec("https://x.test", ["/"], fetcher(false));
  expect(found.map((f) => f.check)).toEqual(["compression", "compression"]);
  expect(found[1]!.message).toContain("/a.css");
  expect(found[0]!.section).toBe("https://specification.website/");
});
		@@ -0,0 +1,107 @@
	1	/**
	2	* Content-Encoding for the bodies the forge writes itself (#2915 check 7): pages,
		sentinelcrew 26d ago outdated Low: this cites #2940, but this work is #2915 check 7 (pull #2975).
	3	* stylesheets, scripts and JSON, as br or gzip when `Accept-Encoding` allows.
	4	* Both come from the runtime (`Bun.gzipSync`, `node:zlib` brotli); no
	5	* dependency. A body under MIN_BYTES is left alone, and so is anything that
	6	* is not on the type list: SSE, git streams, log follows and tarballs keep
	7	* their bytes and their flush.
	8	*
	9	* Bodies that never change (hashed assets) are encoded once and kept (brotli 9: q11 blocks the loop ~0.3 s on the stylesheet); the rest
	10	* are encoded per response at a speed-first level.
	11	*/
	12
	13	import { brotliCompressSync, constants } from "node:zlib";
	14
	15	export type Encoding = "br" \| "gzip";
	16
	17	/** Below this the headers cost more than the bytes saved. */
	18	export const MIN_BYTES = 1024;
	19
	20	const TYPES = new Set([
	21	"text/html",
	22	"text/css",
	23	"text/javascript",
	24	"application/javascript",
	25	"application/json",
	26	"application/manifest+json",
	27	"image/svg+xml",
	28	]);
	29
	30	/** The encoding to use: br over gzip, a `q=0` or absent coding refused. */
	31	export function negotiate(header: string \| null): Encoding \| null {
	32	if (header === null) return null;
	33	const q: Record<string, number> = {};
	34	for (const part of header.split(",")) {
	35	const [name, ...params] = part.split(";");
	36	let value = 1;
	37	for (const p of params) {
	38	const [k, v] = p.split("=").map((x) => x.trim().toLowerCase());
	39	if (k === "q") value = Number(v);
	40	}
	41	const key = (name ?? "").trim().toLowerCase();
	42	if (key && Number.isFinite(value)) q[key] = value;
	43	}
	44	if ((q["br"] ?? 0) > 0) return "br";
	45	if ((q["gzip"] ?? q["x-gzip"] ?? 0) > 0) return "gzip";
	46	return null;
	47	}
	48
	49	const mediaType = (type: string \| null) => (type ?? "").split(";")[0]!.trim().toLowerCase();
	50
	51	function encode(bytes: Uint8Array<ArrayBuffer>, encoding: Encoding, best: boolean): Uint8Array {
	52	if (encoding === "gzip") return Bun.gzipSync(bytes, { level: best ? 9 : 4 });
	53	return brotliCompressSync(bytes, {
	54	params: {
	55	[constants.BROTLI_PARAM_QUALITY]: best ? 9 : 4,
		sentinelcrew 26d ago outdated Quality 11 on the 150 KB stylesheet takes ~312–358 ms of synchronous CPU, and `encodeOnce` calls this inside a request, so the event loop stalls for every client on the first hit after each deploy. Quality 9 produces 30.2 KB vs 27.3 KB in 23 ms. Use 9 for `best`, or warm the assets at boot with the async `brotliCompress`.
	56	[constants.BROTLI_PARAM_SIZE_HINT]: bytes.length,
	57	},
	58	});
	59	}
	60
	61	const kept = new Map<string, Uint8Array>();
	62	const KEPT_MAX = 64;
	63
	64	/** Encoded once per `key` (a content hash) and `encoding`, for bodies that never change. */
	65	export function encodeOnce(key: string, bytes: Uint8Array<ArrayBuffer>, encoding: Encoding): Uint8Array {
	66	const id = `${encoding}:${key}`;
	67	let out = kept.get(id);
	68	if (out === undefined) {
	69	out = encode(bytes, encoding, true);
	70	if (kept.size >= KEPT_MAX) kept.clear();
	71	kept.set(id, out);
	72	}
	73	return out;
	74	}
	75
	76	/**
	77	* The response with its body encoded if the request allows it, else the
	78	* response as it came. `Vary: Accept-Encoding` goes on every compressible type,
	79	* encoded or not, so a cache never hands a small plain copy's key to the wrong
	80	* client. Headers are rewritten in place, as `securityFloor` does.
		sentinelcrew 26d ago Low (BREACH, for the record): authenticated HTML is now compressed. CSRF here is enforced by same-origin checks, not tokens in the page, and no session secret is in a body, so there is nothing a size side-channel can recover today. If a page ever puts a stable secret next to text an attacker controls (a CSRF token, an API key shown on every view), have that response send `Cache-Control: no-transform` so `compress()` leaves it alone. That opt-out already exists. A one-line note about it in this doc comment would keep the next builder from forgetting it.
	81	*/
	82	export async function compress(request: Request, response: Response): Promise<Response> {
	83	try {
	84	if (!TYPES.has(mediaType(response.headers.get("content-type")))) return response;
	85	const vary = response.headers.get("vary");
	86	if (!/(^\|,)\s(accept-encoding\|\)\s*(,\|$)/i.test(vary ?? ""))
	87	response.headers.set("vary", vary ? `${vary}, Accept-Encoding` : "Accept-Encoding");
	88	const encoding = negotiate(request.headers.get("accept-encoding"));
	89	if (
	90	encoding === null \|\|
	91	request.method === "HEAD" \|\|
	92	response.body === null \|\|
	93	response.status === 206 \|\|
	94	response.headers.has("content-encoding") \|\|
	95	/\bno-transform\b/i.test(response.headers.get("cache-control") ?? "")
	96	)
	97	return response;
	98	const bytes = new Uint8Array(await response.clone().arrayBuffer());
	99	if (bytes.length < MIN_BYTES) return new Response(bytes, response);
	100	const out = new Response(encode(bytes, encoding, false), response);
	101	out.headers.set("content-encoding", encoding);
	102	out.headers.delete("content-length");
	103	return out;
	104	} catch {
	105	return response;
	106	}
	107	}
		@@ -485,6 +485,7 @@ import {
485	485	prefersHtml,
486	486	securityFloor,
487	487	} from "./respond.ts";
	488	import { compress } from "./compress.ts";
488	489	import { RateLimiter, type Verdict } from "../core/ratelimit.ts";
489	490	import {
490	491	MAX_KEY_LENGTH,
		@@ -2115,7 +2116,7 @@ export async function serve(app: App): Promise<Servers> {
2115	2116	if (request.signal.aborted) onAbort();
2116	2117	else request.signal.addEventListener("abort", onAbort, { once: true });
2117	2118	const work = run().then(
2118		(response) => securityFloor(response),
	2119	(response) => compress(request, response).then(securityFloor),
		sentinelcrew 26d ago This is the change's main behaviour, and no test covers it: reverting this line to `securityFloor(response)` leaves `bun test` green. Add a `serve()`-level test: `GET /` with `accept-encoding: br` gives `content-encoding: br`, `Vary` and CSP, and SSE or git responses are not encoded.
2119	2120	(error) => failure(owner, error),
2120	2121	);
2121	2122	try {
		@@ -1,5 +1,5 @@
1	1	/**
2		* The website specification as a report (#2915, checks 1 and 2). `webspec()`
	2	* The website specification as a report (#2915, checks 1, 2 and 7). `webspec()`
3	3	* fetches each path of a site once and returns what the page's head and
4	4	* response headers get wrong, each finding citing its specification.website
5	5	* section. It reports; it never throws on a bad page and never fails a run.
		@@ -106,6 +106,32 @@ export function headerProblems(h: Headers, https: boolean): Problem[] {
106	106	return out;
107	107	}
108	108
	109	const COMPRESS_MIN = 1024;
	110	const ACCEPT_ENCODING = "br, gzip";
	111
	112	/** The stylesheets the head links, as absolute URLs. */
	113	function stylesheetTargets(html: string, pageUrl: string): string[] {
	114	const out: string[] = [];
	115	for (const t of links(html, "stylesheet")) {
	116	try {
	117	const u = new URL(attr(t, "href") ?? "", pageUrl).href;
	118	if (!out.includes(u)) out.push(u);
	119	} catch {}
	120	}
	121	return out;
	122	}
	123
	124	/** A text body big enough to be worth it must leave encoded, and say it varies. */
	125	export function encodingProblems(h: Headers, bodyBytes: number): Problem[] {
	126	if (bodyBytes < COMPRESS_MIN) return [];
	127	const out: Problem[] = [];
	128	const encoding = (h.get("content-encoding") ?? "").toLowerCase();
	129	if (encoding !== "br" && encoding !== "gzip")
	130	out.push({ check: "compression", message: `a ${bodyBytes}-byte body was sent without br or gzip though Accept-Encoding: ${ACCEPT_ENCODING}`, section: SPEC, edge: true });
	131	else if (!/accept-encoding/i.test(h.get("vary") ?? "")) out.push({ check: "compression", message: "a compressed response has no Vary: Accept-Encoding", section: SPEC, edge: true });
	132	return out;
	133	}
	134
109	135	/** Fetches `paths` of `base` and reports head and header findings. */
110	136	export async function webspec(base: string, paths: string[], fetcher: Fetcher = fetch): Promise<Finding[]> {
111	137	const findings: Finding[] = [];
		@@ -122,9 +148,11 @@ export async function webspec(base: string, paths: string[], fetcher: Fetcher =
122	148	}
123	149	let res: Response;
124	150	let html: string;
	151	let htmlBytes = 0;
125	152	try {
126		res = await fetcher(url, { headers: { accept: "text/html" }, redirect: "manual", signal: AbortSignal.timeout(TIMEOUT_MS) });
	153	res = await fetcher(url, { headers: { accept: "text/html", "accept-encoding": ACCEPT_ENCODING }, redirect: "manual", signal: AbortSignal.timeout(TIMEOUT_MS) });
127	154	html = await res.text();
	155	htmlBytes = new TextEncoder().encode(html).length;
128	156	} catch (e) {
129	157	report([{ check: "fetch", message: `could not fetch ${url}: ${e instanceof Error ? e.message : String(e)}`, section: SPEC }]);
130	158	continue;
		@@ -134,6 +162,14 @@ export async function webspec(base: string, paths: string[], fetcher: Fetcher =
134	162	continue;
135	163	}
136	164	report(headProblems(html));
	165	report(encodingProblems(res.headers, htmlBytes));
	166	for (const sheet of stylesheetTargets(html, url)) {
	167	try {
	168	const r = await fetcher(sheet, { headers: { "accept-encoding": ACCEPT_ENCODING }, signal: AbortSignal.timeout(TIMEOUT_MS) });
	169	const size = new TextEncoder().encode(await r.text()).length;
	170	if (r.ok) report(encodingProblems(r.headers, size).map((p) => ({ ...p, message: `${sheet}: ${p.message}` })));
	171	} catch {}
	172	}
137	173	const { urls, bad } = headTargets(html, url);
138	174	for (const h of bad) report([{ check: "head-links", message: `"${h}" is not a URL`, section: s("icons") }]);
139	175	for (const target of urls) {
		@@ -17,6 +17,7 @@
17	17	*/
18	18
19	19	import { sha256 } from "../kernel/hash.ts";
	20	import { encodeOnce, MIN_BYTES, negotiate } from "../http/compress.ts";
20	21	import { createLog } from "../kernel/log.ts";
21	22	import { beaconJs } from "./beacon.ts";
22	23	import { shellJs } from "./client.ts";
		@@ -73,15 +74,20 @@ async function build(): Promise<BuiltAsset \| null> {
73	74	* immutable.
74	75	*/
75	76	export function staticResponse(request: Request, body: string, type: string, etag: string): Response {
76		const headers = {
	77	const headers: Record<string, string> = {
77	78	"content-type": type,
78	79	"cache-control": "public, max-age=31536000, immutable",
79	80	"x-content-type-options": "nosniff",
	81	vary: "Accept-Encoding",
80	82	etag,
81	83	};
82	84	if (matches(request.headers.get("if-none-match"), etag))
83	85	return new Response(null, { status: 304, headers });
84		return new Response(body, { headers });
	86	const encoding = negotiate(request.headers.get("accept-encoding"));
	87	const bytes = new TextEncoder().encode(body);
	88	if (encoding === null \|\| bytes.length < MIN_BYTES \|\| request.method === "HEAD") return new Response(body, { headers });
	89	// Hashed, so the encoded bytes are as immutable as the plain ones: made once.
	90	return new Response(encodeOnce(etag, bytes, encoding), { headers: { ...headers, "content-encoding": encoding } });
		sentinelcrew 26d ago This is where the q11 stall lands: on the first request for the stylesheet, shell JS and highlight JS, roughly 0.4 s together. If you keep q11, encode these when the assets are built rather than here.
85	91	}
86	92
87	93	function matches(header: string \| null, etag: string): boolean {
		@@ -0,0 +1,55 @@
	1	import { afterAll, beforeAll, expect, test } from "bun:test";
	2	import { mkdtemp, rm } from "node:fs/promises";
	3	import { tmpdir } from "node:os";
	4	import { join } from "node:path";
	5	import { createApp, type App } from "../src/app.ts";
	6	import { serve, type Servers } from "../src/http/server.ts";
	7	import { adminCookie } from "./helpers/admin.ts";
	8	import { quietLogs } from "./helpers/globals.ts";
	9
	10	// The seam in `answered()`: compression runs on what the server really sends,
	11	// ahead of the security floor, and leaves streams alone.
	12	let dataDir: string;
	13	let app: App;
	14	let servers: Servers;
	15	let cookie: string;
	16
	17	quietLogs();
	18
	19	beforeAll(async () => {
	20	dataDir = await mkdtemp(join(tmpdir(), "forge-compress-"));
	21	app = createApp({ dataDir, port: 0 });
	22	servers = await serve(app);
	23	cookie = await adminCookie(app, { username: "ada", email: "[email protected]", password: "analytical-engine" });
	24	});
	25
	26	afterAll(async () => {
	27	servers.stop();
	28	app.close();
	29	await rm(dataDir, { recursive: true, force: true });
	30	});
	31
	32	const get = (path: string, headers: Record<string, string>) => fetch(`${servers.url}${path}`, { headers: { cookie, ...headers } });
	33
	34	test("a page is br-encoded with Vary and the security floor, and decodes to HTML", async () => {
	35	const res = await get("/", { "accept-encoding": "br" });
	36	expect(res.status).toBe(200);
	37	expect(res.headers.get("content-encoding")).toBe("br");
	38	expect(res.headers.get("vary")).toMatch(/accept-encoding/i);
	39	expect(res.headers.get("content-security-policy")).toBeTruthy();
	40	expect(res.headers.get("x-content-type-options")).toBe("nosniff");
	41	expect(await res.text()).toMatch(/^\s*<!doctype html>/i);
	42	});
	43
	44	test("without Accept-Encoding the page is sent plain", async () => {
	45	const res = await get("/", { "accept-encoding": "identity" });
	46	expect(res.headers.get("content-encoding")).toBe(null);
	47	expect(res.headers.get("vary")).toMatch(/accept-encoding/i);
	48	});
	49
	50	test("a live event stream is never encoded, even when br is asked for", async () => {
	51	const res = await get("/events", { "accept-encoding": "br" });
	52	expect(res.headers.get("content-type")).toContain("text/event-stream");
	53	expect(res.headers.get("content-encoding")).toBe(null);
	54	await res.body?.cancel();
	55	});
		@@ -0,0 +1,64 @@
	1	import { expect, test } from "bun:test";
	2	import { brotliDecompressSync, gunzipSync } from "node:zlib";
	3	import { compress, MIN_BYTES, negotiate } from "../src/http/compress.ts";
	4	import { staticResponse } from "../src/web/asset.ts";
	5
	6	const big = "<p>hello forge</p>".repeat(200);
	7	const req = (enc?: string, method = "GET") =>
	8	new Request("http://x.test/", { method, headers: enc ? { "accept-encoding": enc } : {} });
	9	const html = (body: string, headers: Record<string, string> = {}) =>
	10	new Response(body, { headers: { "content-type": "text/html; charset=utf-8", ...headers } });
	11
	12	test("negotiate prefers br, honours q=0, and ignores the unknown", () => {
	13	expect(negotiate("gzip, br")).toBe("br");
	14	expect(negotiate("gzip")).toBe("gzip");
	15	expect(negotiate("br;q=0, gzip")).toBe("gzip");
	16	expect(negotiate("deflate, identity")).toBe(null);
	17	expect(negotiate(null)).toBe(null);
	18	});
	19
	20	test("a large page is encoded with Vary and decodes to the original", async () => {
	21	const br = await compress(req("br, gzip"), html(big));
	22	expect(br.headers.get("content-encoding")).toBe("br");
	23	expect(br.headers.get("vary")).toBe("Accept-Encoding");
	24	expect(brotliDecompressSync(new Uint8Array(await br.arrayBuffer())).toString()).toBe(big);
	25	const gz = await compress(req("gzip"), html(big));
	26	expect(gz.headers.get("content-encoding")).toBe("gzip");
	27	expect(gunzipSync(new Uint8Array(await gz.arrayBuffer())).toString()).toBe(big);
	28	});
	29
	30	test("JSON is encoded too", async () => {
	31	const r = await compress(req("gzip"), new Response(JSON.stringify({ a: "x".repeat(MIN_BYTES) }), { headers: { "content-type": "application/json" } }));
	32	expect(r.headers.get("content-encoding")).toBe("gzip");
	33	});
	34
	35	test("small bodies, other types, HEAD, and clients without support are left alone", async () => {
	36	expect((await compress(req("br"), html("tiny"))).headers.get("content-encoding")).toBe(null);
	37	expect((await compress(req("br"), html("tiny"))).headers.get("vary")).toBe("Accept-Encoding");
	38	const sse = new Response(big, { headers: { "content-type": "text/event-stream" } });
	39	const s = await compress(req("br"), sse);
	40	expect(s.headers.get("content-encoding")).toBe(null);
	41	expect(s.headers.get("vary")).toBe(null);
	42	const git = new Response(big, { headers: { "content-type": "application/x-git-upload-pack-result" } });
	43	expect((await compress(req("br"), git)).headers.get("content-encoding")).toBe(null);
	44	expect((await compress(req("br", "HEAD"), html(big))).headers.get("content-encoding")).toBe(null);
	45	expect((await compress(req(), html(big))).headers.get("content-encoding")).toBe(null);
	46	expect((await compress(req("br"), html(big, { "content-encoding": "gzip" }))).headers.get("content-encoding")).toBe("gzip");
	47	});
	48
	49	test("a Vary the page already set is extended, not replaced", async () => {
	50	expect((await compress(req("br"), html("t", { vary: "Cookie" }))).headers.get("vary")).toBe("Cookie, Accept-Encoding");
	51	});
	52
	53	test("a hashed asset is encoded once and the bytes are reused", async () => {
	54	const css = "a{color:red}\n".repeat(200);
	55	const a = staticResponse(req("br"), css, "text/css", '"h1"');
	56	const b = staticResponse(req("br"), css, "text/css", '"h1"');
	57	expect(a.headers.get("content-encoding")).toBe("br");
	58	expect(a.headers.get("vary")).toBe("Accept-Encoding");
	59	const bytes = new Uint8Array(await a.arrayBuffer());
	60	expect(brotliDecompressSync(bytes).toString()).toBe(css);
	61	expect(new Uint8Array(await b.arrayBuffer())).toEqual(bytes);
	62	expect(staticResponse(req(), css, "text/css", '"h1"').headers.get("content-encoding")).toBe(null);
	63	expect(staticResponse(req("br"), "a{}", "text/css", '"h2"').headers.get("content-encoding")).toBe(null);
	64	});
		@@ -143,3 +143,22 @@ test("every fetch carries a timeout signal", async () => {
143	143	expect(seen.length).toBeGreaterThan(3);
144	144	expect(seen.every(Boolean)).toBe(true);
145	145	});
	146
	147	test("compression: a big page and stylesheet must leave encoded with Vary", async () => {
	148	const big = GOOD_HTML.replace("</head>", `<link rel="stylesheet" href="/a.css"></head>`).replace(">hi<", `>${"x".repeat(2000)}<`);
	149	const fetcher = (encoded: boolean) => async (input: string, init?: RequestInit) => {
	150	const u = new URL(input);
	151	const h = new Headers(init?.headers);
	152	if (u.pathname === "/icon.svg" \|\| u.pathname === "/m.json") return new Response("x");
	153	if (h.get("if-none-match")) return new Response(null, { status: 304 });
	154	expect(h.get("accept-encoding")).toBe("br, gzip");
	155	const enc: Record<string, string> = encoded ? { "content-encoding": "br", vary: "Accept-Encoding" } : {};
	156	if (u.pathname === "/a.css") return new Response("a{}".repeat(500), { headers: enc });
	157	return new Response(big, { headers: { ...GOOD_HEADERS, ...enc } });
	158	};
	159	expect(await webspec("https://x.test", ["/"], fetcher(true))).toEqual([]);
	160	const found = await webspec("https://x.test", ["/"], fetcher(false));
	161	expect(found.map((f) => f.check)).toEqual(["compression", "compression"]);
	162	expect(found[1]!.message).toContain("/a.css");
	163	expect(found[0]!.section).toBe("https://specification.website/");
	164	});