Public: anyone can read this; the crew can change it.
Every compressed response says Vary: Accept-Encoding: /llms.txt and /sitemap.xml come back br-encoded without it #8340
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/8328-every-compressed-response-says-vary-acce → main 1 commit fork
3 files
changed +34
−2
src/http/compress.ts +9 −1
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * Content-Encoding for the bodies the forge writes itself (#2915 check 7): pages, |
| 3 | * stylesheets, scripts and JSON, as br or gzip when `Accept-Encoding` allows. | |
| 3 | * stylesheets, scripts, JSON, text and XML, as br or gzip when `Accept-Encoding` allows. | |
| 4 | 4 | * Both come from the runtime (`Bun.gzipSync`, `node:zlib` brotli); no |
| 5 | 5 | * dependency. A body under MIN_BYTES is left alone, and so is anything that |
| 6 | 6 | * is not on the type list: SSE, git streams, log follows and tarballs keep |
| @@ -25,6 +25,14 @@ const TYPES = new Set([ | ||
| 25 | 25 | "application/json", |
| 26 | 26 | "application/manifest+json", |
| 27 | 27 | "image/svg+xml", |
| 28 | // The site's words for crawlers and agents: /llms.txt, /robots.txt, the | |
| 29 | // sitemap and the feeds. Left off, the edge encoded them with no Vary (#8328). | |
| 30 | "text/markdown", | |
| 31 | "text/plain", | |
| 32 | "text/xml", | |
| 33 | "application/xml", | |
| 34 | "application/atom+xml", | |
| 35 | "application/rss+xml", | |
| 28 | 36 | ]); |
| 29 | 37 | |
| 30 | 38 | /** The encoding to use: br over gzip, a `q=0` or absent coding refused. */ |
test/compress-serve.test.ts +24
| @@ -2,7 +2,9 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; | ||
| 2 | 2 | import { mkdtemp, rm } from "node:fs/promises"; |
| 3 | 3 | import { tmpdir } from "node:os"; |
| 4 | 4 | import { join } from "node:path"; |
| 5 | import { brotliDecompressSync } from "node:zlib"; | |
| 5 | 6 | import { createApp, type App } from "../src/app.ts"; |
| 7 | import { MIN_BYTES } from "../src/http/compress.ts"; | |
| 6 | 8 | import { serve, type Servers } from "../src/http/server.ts"; |
| 7 | 9 | import { adminCookie } from "./helpers/admin.ts"; |
| 8 | 10 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -53,3 +55,25 @@ test("a live event stream is never encoded, even when br is asked for", async () | ||
| 53 | 55 | expect(res.headers.get("content-encoding")).toBe(null); |
| 54 | 56 | await res.body?.cancel(); |
| 55 | 57 | }); |
| 58 | ||
| 59 | // #8328: the edge br-encoded these with no Vary, because the forge left them alone. | |
| 60 | for (const path of ["/llms.txt", "/sitemap.xml"]) { | |
| 61 | test(`${path} says Vary: Accept-Encoding once, plain or br, and is encoded only when asked`, async () => { | |
| 62 | const plain = await fetch(`${servers.url}${path}`, { headers: { "accept-encoding": "identity" } }); | |
| 63 | expect(plain.status).toBe(200); | |
| 64 | expect(plain.headers.get("content-encoding")).toBe(null); | |
| 65 | expect(plain.headers.get("vary")?.match(/accept-encoding/gi)).toHaveLength(1); | |
| 66 | const text = await plain.text(); | |
| 67 | ||
| 68 | const br = await fetch(`${servers.url}${path}`, { headers: { "accept-encoding": "br" }, decompress: false }); | |
| 69 | expect(br.status).toBe(200); | |
| 70 | expect(br.headers.get("vary")?.match(/accept-encoding/gi)).toHaveLength(1); | |
| 71 | expect(br.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 72 | if (new TextEncoder().encode(text).length >= MIN_BYTES) { | |
| 73 | expect(br.headers.get("content-encoding")).toBe("br"); | |
| 74 | expect(brotliDecompressSync(new Uint8Array(await br.arrayBuffer())).toString("utf8")).toBe(text); | |
| 75 | } else { | |
| 76 | expect(br.headers.get("content-encoding")).toBe(null); | |
| 77 | } | |
| 78 | }); | |
| 79 | } |
test/feeds.test.ts +1 −1
| @@ -184,7 +184,7 @@ describe("a repository's feed", () => { | ||
| 184 | 184 | const first = await fetch(url); |
| 185 | 185 | const etag = first.headers.get("etag")!; |
| 186 | 186 | expect(etag).toMatch(/^W\//); |
| 187 | expect(first.headers.get("vary")).toBe("authorization, cookie"); | |
| 187 | expect(first.headers.get("vary")).toBe("authorization, cookie, Accept-Encoding"); | |
| 188 | 188 | await first.text(); |
| 189 | 189 | const again = await fetch(url, { headers: { "if-none-match": etag } }); |
| 190 | 190 | expect(again.status).toBe(304); |
new above
offline