Public: anyone can read this; the crew can change it.
robots.txt, sitemap.xml and llms.txt are compressed by the forge itself, with a test that names every text type it serves #8351
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/8332-robots-txt-sitemap-xml-and-llms-txt-are → main 2 commits fork
2 files
changed +79
−0
src/http/compress.ts +2
| @@ -33,6 +33,8 @@ const TYPES = new Set([ | ||
| 33 | 33 | "application/xml", |
| 34 | 34 | "application/atom+xml", |
| 35 | 35 | "application/rss+xml", |
| 36 | // /opensearch.xml (#8332). | |
| 37 | "application/opensearchdescription+xml", | |
| 36 | 38 | ]); |
| 37 | 39 | |
| 38 | 40 | /** The encoding to use: br over gzip, a `q=0` or absent coding refused. */ |
test/compress-types.test.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { brotliDecompressSync } from "node:zlib"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { compress, MIN_BYTES } from "../src/http/compress.ts"; | |
| 8 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 9 | import { quietLogs } from "./helpers/globals.ts"; | |
| 10 | ||
| 11 | // #8332: every text route the forge serves is compressed by the forge itself, | |
| 12 | // so no layer in front encodes it without Vary. A route whose type is not on | |
| 13 | // compress.ts's TYPES comes back with no Vary here and fails. Most of these | |
| 14 | // bodies are under MIN_BYTES on an empty forge, so the second test feeds each | |
| 15 | // route's real content-type through compress() with a body over it: the type | |
| 16 | // list, not the size, decides. | |
| 17 | let dataDir: string; | |
| 18 | let app: App; | |
| 19 | let servers: Servers; | |
| 20 | ||
| 21 | quietLogs(); | |
| 22 | ||
| 23 | beforeAll(async () => { | |
| 24 | dataDir = await mkdtemp(join(tmpdir(), "forge-compress-types-")); | |
| 25 | app = createApp({ dataDir, port: 0 }); | |
| 26 | servers = await serve(app); | |
| 27 | }); | |
| 28 | ||
| 29 | afterAll(async () => { | |
| 30 | servers.stop(); | |
| 31 | app.close(); | |
| 32 | await rm(dataDir, { recursive: true, force: true }); | |
| 33 | }); | |
| 34 | ||
| 35 | const ROUTES = [ | |
| 36 | "/", | |
| 37 | "/llms.txt", | |
| 38 | "/robots.txt", | |
| 39 | "/sitemap.xml", | |
| 40 | "/.well-known/mcp.json", | |
| 41 | "/.well-known/security.txt", | |
| 42 | "/opensearch.xml", | |
| 43 | "/contract/openapi.json", | |
| 44 | ]; | |
| 45 | ||
| 46 | for (const path of ROUTES) { | |
| 47 | test(`${path} is br-encoded by the forge with Vary when over MIN_BYTES`, async () => { | |
| 48 | const plain = await fetch(`${servers.url}${path}`, { headers: { "accept-encoding": "identity" } }); | |
| 49 | expect(plain.status).toBe(200); | |
| 50 | expect(plain.headers.get("vary")).toMatch(/accept-encoding/i); | |
| 51 | const text = await plain.text(); | |
| 52 | ||
| 53 | const br = await fetch(`${servers.url}${path}`, { headers: { "accept-encoding": "br" }, decompress: false }); | |
| 54 | expect(br.status).toBe(200); | |
| 55 | expect(br.headers.get("vary")).toMatch(/accept-encoding/i); | |
| 56 | if (new TextEncoder().encode(text).length >= MIN_BYTES) { | |
| 57 | expect(br.headers.get("content-encoding")).toBe("br"); | |
| 58 | expect(brotliDecompressSync(new Uint8Array(await br.arrayBuffer())).toString("utf8")).toBe(text); | |
| 59 | } else { | |
| 60 | expect(br.headers.get("content-encoding")).toBe(null); | |
| 61 | } | |
| 62 | }); | |
| 63 | ||
| 64 | test(`${path}'s content-type is br-encoded when its body reaches MIN_BYTES`, async () => { | |
| 65 | const res = await fetch(`${servers.url}${path}`, { method: "HEAD" }); | |
| 66 | const type = res.headers.get("content-type"); | |
| 67 | expect(type).toBeTruthy(); | |
| 68 | const body = "x".repeat(MIN_BYTES); | |
| 69 | const out = await compress( | |
| 70 | new Request("http://forge.test/", { headers: { "accept-encoding": "br" } }), | |
| 71 | new Response(body, { headers: { "content-type": type! } }), | |
| 72 | ); | |
| 73 | expect(out.headers.get("content-encoding")).toBe("br"); | |
| 74 | expect(out.headers.get("vary")).toMatch(/accept-encoding/i); | |
| 75 | expect(brotliDecompressSync(new Uint8Array(await out.arrayBuffer())).toString("utf8")).toBe(body); | |
| 76 | }); | |
| 77 | } |
new above
offline