Public: anyone can read this; the crew can change it.
The app proxy compresses a page after the forge pill is injected #2966
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2961-the-app-proxy-compresses-a-page-after-th → main 7 commits fork 5 threads · 5 unresolved
4 files
changed +104
−7
src/apps/proxy.ts +18 −4
| @@ -14,6 +14,7 @@ | ||
| 14 | 14 | * directory — so guessing a hostname yields the forge's own "not found". |
| 15 | 15 | */ |
| 16 | 16 | |
| 17 | import { compressBytes, MIN_BYTES, negotiate } from "../http/compress.ts"; | |
| 17 | 18 | import type { Database } from "bun:sqlite"; |
| 18 | 19 | import { getIssue, type Issue } from "../core/issues.ts"; |
| 19 | 20 | import { mayView } from "../core/sealed.ts"; |
| @@ -649,7 +650,8 @@ export async function proxyApp( | ||
| 649 | 650 | headers.set(name, value); |
| 650 | 651 | |
| 651 | 652 | // A page the pill may be written into is asked for plain: the rewrite |
| 652 | // cannot read a compressed body, and the app compresses what it is allowed. | |
| 653 | // cannot read a compressed body, and the proxy compresses what it wrote. | |
| 654 | const encoding = negotiate(request.headers.get("accept-encoding")); | |
| 653 | 655 | if ( |
| 654 | 656 | request.method === "GET" && |
| 655 | 657 | !target.nested && |
| @@ -718,7 +720,7 @@ export async function proxyApp( | ||
| 718 | 720 | response.body !== null && |
| 719 | 721 | rewritable(out, response.status) |
| 720 | 722 | ) { |
| 721 | let page: Uint8Array | undefined; | |
| 723 | let page: Uint8Array<ArrayBuffer> | undefined; | |
| 722 | 724 | try { |
| 723 | 725 | page = withToolbar( |
| 724 | 726 | new Uint8Array(await response.arrayBuffer()), |
| @@ -728,10 +730,22 @@ export async function proxyApp( | ||
| 728 | 730 | } catch { |
| 729 | 731 | return unavailable(runtime, target, status, "the app stopped answering"); |
| 730 | 732 | } |
| 731 | out.set("content-length", String(page.length)); | |
| 733 | let sent = page; | |
| 734 | // Whether it is gzipped depends on the client, so every copy says so. | |
| 735 | out.append("vary", "accept-encoding"); | |
Nit: |
||
| 736 | if ( | |
| 737 | encoding !== null && | |
| 738 | page.length >= MIN_BYTES && | |
| 739 | !out.has("content-encoding") && | |
Small: |
||
| 740 | !/\bno-transform\b/i.test(out.get("cache-control") ?? "") | |
| 741 | ) { | |
| 742 | sent = compressBytes(page, encoding) as Uint8Array<ArrayBuffer>; | |
| 743 | out.set("content-encoding", encoding); | |
| 744 | } | |
| 745 | out.set("content-length", String(sent.length)); | |
| 732 | 746 | out.delete("etag"); |
| 733 | 747 | out.delete("content-md5"); |
| 734 | return new Response(page, { | |
| 748 | return new Response(sent, { | |
| 735 | 749 | status: response.status, |
| 736 | 750 | statusText: response.statusText, |
| 737 | 751 | headers: out, |
src/apps/toolbar.ts +1 −1
| @@ -227,7 +227,7 @@ export function withToolbar( | ||
| 227 | 227 | bytes: Uint8Array, |
| 228 | 228 | forgeUrl: string, |
| 229 | 229 | where?: PillWhere, |
| 230 | ): Uint8Array { | |
| 230 | ): Uint8Array<ArrayBuffer> { | |
| 231 | 231 | const tag = Buffer.from(toolbarTag(forgeUrl, where)); |
| 232 | 232 | const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); |
| 233 | 233 | // latin1 is one character per byte, so an index in it is a byte offset. |
src/http/compress.ts +3
| @@ -58,6 +58,9 @@ function encode(bytes: Uint8Array<ArrayBuffer>, encoding: Encoding, best: boolea | ||
| 58 | 58 | }); |
| 59 | 59 | } |
| 60 | 60 | |
| 61 | /** Speed-first encoding of a body built per response. */ | |
| 62 | export const compressBytes = (bytes: Uint8Array<ArrayBuffer>, encoding: Encoding) => encode(bytes, encoding, false); | |
| 63 | ||
| 61 | 64 | const kept = new Map<string, Uint8Array>(); |
| 62 | 65 | const KEPT_MAX = 64; |
| 63 | 66 |
test/apps-toolbar.test.ts +82 −2
| @@ -9,7 +9,7 @@ import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; | ||
| 9 | 9 | import { mkdtemp, rm } from "node:fs/promises"; |
| 10 | 10 | import { tmpdir } from "node:os"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import { gzipSync } from "node:zlib"; | |
| 12 | import { brotliDecompressSync, gzipSync } from "node:zlib"; | |
| 13 | 13 | import { createApp, type App } from "../src/app.ts"; |
| 14 | 14 | import { newApp } from "../src/actions/apps.ts"; |
| 15 | 15 | import { commitToBranch } from "../src/actions/repos.ts"; |
| @@ -27,6 +27,7 @@ quietLogs(); | ||
| 27 | 27 | const DOMAIN = "apps.test"; |
| 28 | 28 | const PNG = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 0, 1, 2, 3]); |
| 29 | 29 | const HTML = "<html><body><h1>hi</h1></body></html>"; |
| 30 | const BIG = `<html><body>${"<p>hello hello hello</p>".repeat(200)}</body></html>`; | |
| 30 | 31 | |
| 31 | 32 | describe("the forge pill", () => { |
| 32 | 33 | let dataDir: string; |
| @@ -34,6 +35,7 @@ describe("the forge pill", () => { | ||
| 34 | 35 | let servers: Servers; |
| 35 | 36 | let runtime: AppsRuntime; |
| 36 | 37 | let backend: ReturnType<typeof Bun.serve>; |
| 38 | let seen: string | null = null; | |
| 37 | 39 | let toolbar: boolean | undefined = undefined; |
| 38 | 40 | let status: ReturnType<typeof spyOn>; |
| 39 | 41 | |
| @@ -56,10 +58,12 @@ describe("the forge pill", () => { | ||
| 56 | 58 | if (path === "/json") return Response.json({ a: 1 }); |
| 57 | 59 | if (path === "/img") return new Response(PNG, { headers: { "content-type": "image/png" } }); |
| 58 | 60 | if (path === "/bare") return new Response("<p>no body tag</p>", { headers: html }); |
| 59 | if (path === "/gz") | |
| 61 | if (path === "/gz") { | |
| 62 | seen = request.headers.get("accept-encoding"); | |
| 60 | 63 | return new Response(gzipSync(HTML), { |
| 61 | 64 | headers: { ...html, "content-encoding": "gzip" }, |
| 62 | 65 | }); |
| 66 | } | |
| 63 | 67 | if (path === "/stream") |
| 64 | 68 | return new Response( |
| 65 | 69 | new ReadableStream({ |
| @@ -83,6 +87,10 @@ describe("the forge pill", () => { | ||
| 83 | 87 | return new Response(HTML, { |
| 84 | 88 | headers: { "content-type": "text/html; charset=shift_jis" }, |
| 85 | 89 | }); |
| 90 | if (path === "/long") { | |
| 91 | seen = request.headers.get("accept-encoding"); | |
| 92 | return new Response(BIG, { headers: html }); | |
| 93 | } | |
| 86 | 94 | return new Response(HTML, { headers: html }); |
| 87 | 95 | }, |
| 88 | 96 | }); |
| @@ -183,6 +191,78 @@ describe("the forge pill", () => { | ||
| 183 | 191 | expect(await (await at("/sjis")).text()).toBe(HTML); |
| 184 | 192 | }); |
| 185 | 193 | |
| 194 | test("a rewritten page is gzipped by the proxy for a client that accepts gzip", async () => { | |
| 195 | const res = await fetch(`${servers.url}/long`, { | |
| 196 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "gzip" }, | |
| 197 | redirect: "manual", | |
| 198 | decompress: false, | |
| 199 | } as RequestInit); | |
| 200 | expect(seen).toBe("identity"); | |
| 201 | expect(res.headers.get("content-encoding")).toBe("gzip"); | |
| 202 | expect(res.headers.get("vary")).toContain("accept-encoding"); | |
| 203 | const raw = new Uint8Array(await res.arrayBuffer()); | |
| 204 | expect(res.headers.get("content-length")).toBe(String(raw.length)); | |
| 205 | expect(new TextDecoder().decode(Bun.gunzipSync(raw))).toBe( | |
| 206 | BIG.replace("</body>", `${tag()}</body>`), | |
| 207 | ); | |
| 208 | const plain = await fetch(`${servers.url}/long`, { | |
| 209 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "identity" }, | |
| 210 | redirect: "manual", | |
| 211 | }); | |
| 212 | expect(plain.headers.get("content-encoding")).toBeNull(); | |
| 213 | expect(plain.headers.get("vary")).toContain("accept-encoding"); | |
| 214 | expect(await plain.text()).toContain(TOOLBAR_PATH); | |
| 215 | }); | |
| 216 | ||
| 217 | test("a client that refuses gzip with q=0 gets the page plain", async () => { | |
| 218 | const res = await fetch(`${servers.url}/long`, { | |
| 219 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "gzip;q=0" }, | |
| 220 | redirect: "manual", | |
| 221 | decompress: false, | |
| 222 | } as RequestInit); | |
| 223 | expect(res.headers.get("content-encoding")).toBeNull(); | |
| 224 | expect(res.headers.get("vary")).toContain("accept-encoding"); | |
| 225 | expect(await res.text()).toContain(TOOLBAR_PATH); | |
| 226 | }); | |
| 227 | ||
| 228 | test("a client that accepts br is sent brotli", async () => { | |
| 229 | const res = await fetch(`${servers.url}/long`, { | |
| 230 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "gzip, br" }, | |
| 231 | redirect: "manual", | |
| 232 | decompress: false, | |
| 233 | } as RequestInit); | |
| 234 | expect(res.headers.get("content-encoding")).toBe("br"); | |
| 235 | const raw = new Uint8Array(await res.arrayBuffer()); | |
| 236 | expect(res.headers.get("content-length")).toBe(String(raw.length)); | |
| 237 | expect(new TextDecoder().decode(brotliDecompressSync(raw))).toBe( | |
| 238 | BIG.replace("</body>", `${tag()}</body>`), | |
| 239 | ); | |
| 240 | }); | |
| 241 | ||
| 242 | test("a page under the threshold stays plain for a gzip client, and still varies", async () => { | |
| 243 | const res = await fetch(`${servers.url}/`, { | |
| 244 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "gzip" }, | |
| 245 | redirect: "manual", | |
| 246 | decompress: false, | |
| 247 | } as RequestInit); | |
| 248 | expect(res.headers.get("content-encoding")).toBeNull(); | |
| 249 | expect(res.headers.get("vary")).toContain("accept-encoding"); | |
| 250 | expect(await res.text()).toContain(TOOLBAR_PATH); | |
| 251 | }); | |
| 252 | ||
| 253 | test("an app with the pill off is sent the client's encoding and its gzip passes through", async () => { | |
| 254 | toolbar = false; | |
| 255 | const res = await fetch(`${servers.url}/gz`, { | |
| 256 | headers: { host: `notes.${DOMAIN}`, accept: "text/html", "accept-encoding": "gzip" }, | |
| 257 | redirect: "manual", | |
| 258 | decompress: false, | |
| 259 | } as RequestInit); | |
| 260 | toolbar = undefined; | |
| 261 | expect(seen).toBe("gzip"); | |
| 262 | expect(res.headers.get("content-encoding")).toBe("gzip"); | |
| 263 | expect(new TextDecoder().decode(Bun.gunzipSync(new Uint8Array(await res.arrayBuffer())))).toBe(HTML); | |
| 264 | }); | |
| 265 | ||
| 186 | 266 | test("toolbar: false turns it off", async () => { |
| 187 | 267 | toolbar = false; |
| 188 | 268 | expect(await (await at("/")).text()).toBe(HTML); |
new above
offline
This regex accepts
gzip;q=0and ignores br. Usenegotiate()from src/http/compress.ts.