eb8bf1f · 365b49b
@@ -841,6 +841,15 @@
file), and if you named an agent, the sentence became issue #1, assigned.
  (`, M the edge owns`). The terminal's pull screen ends its checks row with the
  same words. Nothing here fails a deploy or the check; see
  [`docs/WEBSPEC-PREVIEWS.md`](docs/WEBSPEC-PREVIEWS.md).
- **A streamed page gets the pill too.** An HTML answer with no
  `Content-Length` is read up to 4 MiB for at most 100 ms, and only while its
  first chunk starts like markup; if it ends in that window it gets the pill
  like a sized page. Otherwise what was read is sent first and the rest of the
  stream follows unchanged. The cost: a streamed page that is still going
  waits up to those 100 ms before its first byte reaches the browser, so a slow
  server-rendered page paints up to 100 ms later (a page sent with a
  `Content-Length`, or one that starts with something other than markup, does
  not wait). Event streams and WebSockets are untouched.
- **The forge pill acts.** The pill the proxy adds to an app's pages
  (`"toolbar": false` in `forge.json` turns it off; to write it in, the proxy asks the app for the page uncompressed, so a page that carries the pill goes out without gzip) links to a forge page,
  `/:owner/:name/toolbar?env=production|pull-N`, opened in a tab of its own: the
@@ -37,7 +37,7 @@
import { previewEnv, type EnvName } from "./data.ts";
import type { AppsRuntime, EnvStatus } from "./runtime.ts";
import { crewNow, tellLink } from "./crew.ts";
import { appMeasure } from "./measure.ts";
import { rewritable, withToolbar, type PillWhere } from "./toolbar.ts";
import { htmlAnswer, readPage, rewritable, withToolbar, type PillWhere } from "./toolbar.ts";
export interface AppTarget {
  readonly app: AppRecord;
@@ -718,18 +718,32 @@
export async function proxyApp(
    !target.nested &&
    status.toolbar !== false &&
    response.body !== null &&
    rewritable(out, response.status)
    (rewritable(out, response.status) ||
      (!out.has("content-length") && htmlAnswer(out, response.status)))
  ) {
    let page: Uint8Array<ArrayBuffer> | undefined;
    let bytes: Uint8Array<ArrayBuffer>;
    try {
      page = withToolbar(
        new Uint8Array(await response.arrayBuffer()),
        runtime.options.forgeUrl,
        pillWhere(runtime, target, vouched?.who),
      );
      if (out.has("content-length")) bytes = new Uint8Array(await response.arrayBuffer());
      else {
        // A streamed page is held only while it is small, quick and HTML;
        // past that it goes on as it came, what was read first.
        const read = await readPage(response.body);
        if (!read.whole)
          return new Response(read.rest, {
            status: response.status,
            statusText: response.statusText,
            headers: out,
          });
        bytes = read.bytes;
      }
    } catch {
      return unavailable(runtime, target, status, "the app stopped answering");
    }
    const page = withToolbar(
      bytes,
      runtime.options.forgeUrl,
      pillWhere(runtime, target, vouched?.who),
    );
    let sent = page;
    // Whether it is gzipped depends on the client, so every copy says so.
    out.append("vary", "accept-encoding");
@@ -161,12 +161,15 @@
export const REWRITE_MAX = 4 * 1024 * 1024;
const SAFE_CHARSETS = new Set(["utf-8", "utf8", "us-ascii", "iso-8859-1", "latin1", "windows-1252"]);
/** How long a streamed page may take to show its first bytes and its end. */
export const STREAM_WAIT_MS = 100;
/**
 * Whether a response's headers say its body can be rewritten byte-wise:
 * HTML, not compressed, not an attachment, in a charset where `</body>`
 * is plain ASCII, and of a length that is declared and bounded.
 * Whether a response's headers say its body can be rewritten byte-wise,
 * length aside: HTML, not compressed, not an attachment, in a charset where
 * `</body>` is plain ASCII.
 */
export function rewritable(headers: Headers, status: number): boolean {
export function htmlAnswer(headers: Headers, status: number): boolean {
  if (status < 200 || status === 204 || status === 205 || status === 206 || status === 304)
    return false;
  // A byte range cannot be rewritten: Content-Range would stop matching.
@@ -179,6 +182,12 @@
export function rewritable(headers: Headers, status: number): boolean {
  if (encoding !== "" && encoding !== "identity") return false;
  if ((headers.get("content-disposition") ?? "").toLowerCase().startsWith("attachment"))
    return false;
  return true;
}
/** An `htmlAnswer` whose length is declared and bounded. */
export function rewritable(headers: Headers, status: number): boolean {
  if (!htmlAnswer(headers, status)) return false;
  const length = headers.get("content-length");
  if (length === null || !/^\d{1,9}$/.test(length)) return false;
  return Number(length) <= REWRITE_MAX;
@@ -235,3 +244,75 @@
export function withToolbar(
  const cut = at < 0 ? buffer.length : at;
  return Buffer.concat([buffer.subarray(0, cut), tag, buffer.subarray(cut)]);
}
type Read = { done: boolean; value?: Uint8Array | undefined };
/** What `readPage` found: the whole page, or every byte as a stream. */
export type PageRead =
  | { whole: true; bytes: Uint8Array<ArrayBuffer> }
  | { whole: false; rest: ReadableStream<Uint8Array> };
/** Whether a first chunk looks like markup: optional BOM and space, then `<`. */
function htmlShaped(chunk: Uint8Array): boolean {
  let i = 0;
  if (chunk[0] === 0xef && chunk[1] === 0xbb && chunk[2] === 0xbf) i = 3;
  while (i < chunk.length && [0x20, 0x09, 0x0a, 0x0d].includes(chunk[i]!)) i++;
  return chunk[i] === 0x3c;
}
/**
 * Read a streamed page of unknown length. If it ends within `max` bytes and
 * `wait` milliseconds, and begins like HTML, it is returned whole. Otherwise
 * `rest` carries every byte, those already read first, in order.
 */
export async function readPage(
  body: ReadableStream<Uint8Array>,
  max = REWRITE_MAX,
  wait = STREAM_WAIT_MS,
): Promise<PageRead> {
  const reader = body.getReader();
  const chunks: Uint8Array[] = [];
  let size = 0;
  let pending: Promise<Read> | undefined;
  let timer: ReturnType<typeof setTimeout> | undefined;
  const late = new Promise<"late">((resolve) => {
    timer = setTimeout(resolve, wait, "late");
  });
  let whole = false;
  try {
    for (;;) {
      pending ??= reader.read();
      const got = await Promise.race([pending, late]);
      if (got === "late") break;
      pending = undefined;
      if (got.done) {
        whole = true;
        break;
      }
      const first = chunks.length === 0;
      chunks.push(got.value!);
      size += got.value!.length;
      if ((first && !htmlShaped(got.value!)) || size > max) break;
    }
  } catch (error) {
    reader.cancel(error).catch(() => {});
    throw error;
  } finally {
    clearTimeout(timer);
  }
  if (whole) return { whole: true, bytes: new Uint8Array(Buffer.concat(chunks)) };
  return {
    whole: false,
    rest: new ReadableStream<Uint8Array>({
      async pull(controller) {
        const next = chunks.shift();
        if (next !== undefined) return controller.enqueue(next);
        const got = await (pending ?? reader.read());
        pending = undefined;
        if (got.done) controller.close();
        else controller.enqueue(got.value!);
      },
      cancel: (reason) => reader.cancel(reason),
    }),
  };
}
@@ -17,7 +17,7 @@
import { getAppByRepo } from "../src/core/apps.ts";
import { listRuns } from "../src/core/runs.ts";
import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
import { parseManifest } from "../src/apps/manifest.ts";
import { REWRITE_MAX, TOOLBAR_PATH, toolbarJs, withToolbar } from "../src/apps/toolbar.ts";
import { REWRITE_MAX, TOOLBAR_PATH, readPage, toolbarJs, withToolbar } from "../src/apps/toolbar.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -76,6 +76,46 @@
describe("the forge pill", () => {
            }),
            { headers: html },
          );
        if (path === "/chunked-big") {
          const piece = new TextEncoder().encode("<p>" + "b".repeat(1024 * 1024) + "</p>");
          return new Response(
            new ReadableStream({
              start(c) {
                c.enqueue(new TextEncoder().encode("<html><body>"));
                for (let i = 0; i < 5; i++) c.enqueue(piece);
                c.enqueue(new TextEncoder().encode("</body></html>"));
                c.close();
              },
            }),
            { headers: html },
          );
        }
        if (path === "/slow") {
          return new Response(
            new ReadableStream({
              async start(c) {
                c.enqueue(new TextEncoder().encode("<html><body>one"));
                await Bun.sleep(250);
                c.enqueue(new TextEncoder().encode("two</body></html>"));
                c.close();
              },
            }),
            { headers: html },
          );
        }
        if (path === "/notmarkup") {
          return new Response(
            new ReadableStream({
              async start(c) {
                c.enqueue(new TextEncoder().encode('{"a":1}'));
                await Bun.sleep(10);
                c.enqueue(new TextEncoder().encode("</body>"));
                c.close();
              },
            }),
            { headers: html },
          );
        }
        if (path === "/range")
          return new Response(HTML.slice(0, 20), {
            status: 206,
@@ -174,7 +214,29 @@
describe("the forge pill", () => {
    expect(gz.headers.get("content-encoding")).toBe("gzip");
    // The client unzips what the app zipped; no tag went in before that.
    expect(await gz.text()).toBe(HTML);
    expect(await (await at("/stream")).text()).toBe(HTML);
  });
  test("a chunked page under the limit gets the pill", async () => {
    const text = await (await at("/stream")).text();
    expect(text).toContain(tag());
    expect(text.replace(tag(), "")).toBe(HTML);
  });
  test("a chunked page over the limit arrives byte-identical with no pill", async () => {
    const res = await at("/chunked-big");
    const got = Buffer.from(await res.arrayBuffer()).toString();
    const piece = "<p>" + "b".repeat(1024 * 1024) + "</p>";
    expect(got).toBe("<html><body>" + piece.repeat(5) + "</body></html>");
    expect(got).not.toContain(TOOLBAR_PATH);
  });
  test("a streamed page still open after the wait arrives whole with no pill", async () => {
    const got = await (await at("/slow")).text();
    expect(got).toBe("<html><body>onetwo</body></html>");
  });
  test("a streamed text/html answer that does not start like markup is untouched", async () => {
    expect(await (await at("/notmarkup")).text()).toBe('{"a":1}</body>');
  });
  test("a ranged answer, a HEAD, a huge page and a foreign charset pass through", async () => {
@@ -501,3 +563,56 @@
describe("the opt-out, from forge.json to the proxy", () => {
    expect(html).not.toContain(TOOLBAR_PATH);
  }, 180_000);
});
describe("readPage", () => {
  const enc = new TextEncoder();
  const stream = (parts: (string | Uint8Array | number)[]) =>
    new ReadableStream<Uint8Array>({
      async start(c) {
        for (const part of parts) {
          if (typeof part === "number") await Bun.sleep(part);
          else c.enqueue(typeof part === "string" ? enc.encode(part) : part);
        }
        c.close();
      },
    });
  const text = async (body: ReadableStream<Uint8Array>) => await new Response(body).text();
  test("a page that ends in time and in size is returned whole", async () => {
    const got = await readPage(stream(["<p>a", "b</p>"]), 100, 1000);
    expect(got.whole).toBe(true);
    if (got.whole) expect(new TextDecoder().decode(got.bytes)).toBe("<p>ab</p>");
  });
  test("a page still open after the wait is passed on in order", async () => {
    const got = await readPage(stream(["<p>a", 200, "b</p>"]), 100, 30);
    expect(got.whole).toBe(false);
    if (!got.whole) expect(await text(got.rest)).toBe("<p>ab</p>");
  });
  test("a page past max is passed on in order", async () => {
    const got = await readPage(stream(["<p>aaaa", "bbbb", "cccc</p>"]), 8, 1000);
    expect(got.whole).toBe(false);
    if (!got.whole) expect(await text(got.rest)).toBe("<p>aaaabbbbcccc</p>");
  });
  test("a first chunk that is not markup passes through, even if the stream is short", async () => {
    const got = await readPage(stream(['{"a":1}', "</body>"]), 100, 1000);
    expect(got.whole).toBe(false);
    if (!got.whole) expect(await text(got.rest)).toBe('{"a":1}</body>');
  });
  test("a BOM and leading whitespace before < still count as markup", async () => {
    const bom = Uint8Array.from([0xef, 0xbb, 0xbf, ...enc.encode(" \r\n\t<p>x</p>")]);
    const got = await readPage(stream([bom]), 100, 1000);
    expect(got.whole).toBe(true);
    const spaced = await readPage(stream(["  \n<p>"]), 100, 1000);
    expect(spaced.whole).toBe(true);
  });
  test("a BOM or whitespace followed by something else is not markup", async () => {
    const bom = Uint8Array.from([0xef, 0xbb, 0xbf, ...enc.encode("x<p>")]);
    expect((await readPage(stream([bom]), 100, 1000)).whole).toBe(false);
    expect((await readPage(stream(["  \nplain"]), 100, 1000)).whole).toBe(false);
  });
});
		@@ -841,6 +841,15 @@ file), and if you named an agent, the sentence became issue #1, assigned.
841	841	(`, M the edge owns`). The terminal's pull screen ends its checks row with the
842	842	same words. Nothing here fails a deploy or the check; see
843	843	[`docs/WEBSPEC-PREVIEWS.md`](docs/WEBSPEC-PREVIEWS.md).
	844	- A streamed page gets the pill too. An HTML answer with no
	845	`Content-Length` is read up to 4 MiB for at most 100 ms, and only while its
	846	first chunk starts like markup; if it ends in that window it gets the pill
	847	like a sized page. Otherwise what was read is sent first and the rest of the
	848	stream follows unchanged. The cost: a streamed page that is still going
	849	waits up to those 100 ms before its first byte reaches the browser, so a slow
	850	server-rendered page paints up to 100 ms later (a page sent with a
	851	`Content-Length`, or one that starts with something other than markup, does
	852	not wait). Event streams and WebSockets are untouched.
844	853	- The forge pill acts. The pill the proxy adds to an app's pages
845	854	(`"toolbar": false` in `forge.json` turns it off; to write it in, the proxy asks the app for the page uncompressed, so a page that carries the pill goes out without gzip) links to a forge page,
846	855	`/:owner/:name/toolbar?env=production\|pull-N`, opened in a tab of its own: the
		@@ -37,7 +37,7 @@ import { previewEnv, type EnvName } from "./data.ts";
37	37	import type { AppsRuntime, EnvStatus } from "./runtime.ts";
38	38	import { crewNow, tellLink } from "./crew.ts";
39	39	import { appMeasure } from "./measure.ts";
40		import { rewritable, withToolbar, type PillWhere } from "./toolbar.ts";
	40	import { htmlAnswer, readPage, rewritable, withToolbar, type PillWhere } from "./toolbar.ts";
41	41
42	42	export interface AppTarget {
43	43	readonly app: AppRecord;
		@@ -718,18 +718,32 @@ export async function proxyApp(
718	718	!target.nested &&
719	719	status.toolbar !== false &&
720	720	response.body !== null &&
721		rewritable(out, response.status)
	721	(rewritable(out, response.status) \|\|
	722	(!out.has("content-length") && htmlAnswer(out, response.status)))
722	723	) {
723		let page: Uint8Array<ArrayBuffer> \| undefined;
	724	let bytes: Uint8Array<ArrayBuffer>;
724	725	try {
725		page = withToolbar(
726		new Uint8Array(await response.arrayBuffer()),
727		runtime.options.forgeUrl,
728		pillWhere(runtime, target, vouched?.who),
729		);
	726	if (out.has("content-length")) bytes = new Uint8Array(await response.arrayBuffer());
	727	else {
	728	// A streamed page is held only while it is small, quick and HTML;
	729	// past that it goes on as it came, what was read first.
	730	const read = await readPage(response.body);
	731	if (!read.whole)
	732	return new Response(read.rest, {
	733	status: response.status,
	734	statusText: response.statusText,
	735	headers: out,
	736	});
	737	bytes = read.bytes;
	738	}
730	739	} catch {
731	740	return unavailable(runtime, target, status, "the app stopped answering");
732	741	}
	742	const page = withToolbar(
	743	bytes,
	744	runtime.options.forgeUrl,
	745	pillWhere(runtime, target, vouched?.who),
	746	);
733	747	let sent = page;
734	748	// Whether it is gzipped depends on the client, so every copy says so.
735	749	out.append("vary", "accept-encoding");
		@@ -161,12 +161,15 @@ export const REWRITE_MAX = 4 * 1024 * 1024;
161	161
162	162	const SAFE_CHARSETS = new Set(["utf-8", "utf8", "us-ascii", "iso-8859-1", "latin1", "windows-1252"]);
163	163
	164	/** How long a streamed page may take to show its first bytes and its end. */
	165	export const STREAM_WAIT_MS = 100;
	166
164	167	/**
165		* Whether a response's headers say its body can be rewritten byte-wise:
166		* HTML, not compressed, not an attachment, in a charset where `</body>`
167		* is plain ASCII, and of a length that is declared and bounded.
	168	* Whether a response's headers say its body can be rewritten byte-wise,
	169	* length aside: HTML, not compressed, not an attachment, in a charset where
	170	* `</body>` is plain ASCII.
168	171	*/
169		export function rewritable(headers: Headers, status: number): boolean {
	172	export function htmlAnswer(headers: Headers, status: number): boolean {
170	173	if (status < 200 \|\| status === 204 \|\| status === 205 \|\| status === 206 \|\| status === 304)
171	174	return false;
172	175	// A byte range cannot be rewritten: Content-Range would stop matching.
		@@ -179,6 +182,12 @@ export function rewritable(headers: Headers, status: number): boolean {
179	182	if (encoding !== "" && encoding !== "identity") return false;
180	183	if ((headers.get("content-disposition") ?? "").toLowerCase().startsWith("attachment"))
181	184	return false;
	185	return true;
	186	}
	187
	188	/** An `htmlAnswer` whose length is declared and bounded. */
	189	export function rewritable(headers: Headers, status: number): boolean {
	190	if (!htmlAnswer(headers, status)) return false;
182	191	const length = headers.get("content-length");
183	192	if (length === null \|\| !/^\d{1,9}$/.test(length)) return false;
184	193	return Number(length) <= REWRITE_MAX;
		@@ -235,3 +244,75 @@ export function withToolbar(
235	244	const cut = at < 0 ? buffer.length : at;
236	245	return Buffer.concat([buffer.subarray(0, cut), tag, buffer.subarray(cut)]);
237	246	}
	247
	248	type Read = { done: boolean; value?: Uint8Array \| undefined };
	249
	250	/** What `readPage` found: the whole page, or every byte as a stream. */
	251	export type PageRead =
	252	\| { whole: true; bytes: Uint8Array<ArrayBuffer> }
	253	\| { whole: false; rest: ReadableStream<Uint8Array> };
	254
	255	/** Whether a first chunk looks like markup: optional BOM and space, then `<`. */
	256	function htmlShaped(chunk: Uint8Array): boolean {
	257	let i = 0;
	258	if (chunk[0] === 0xef && chunk[1] === 0xbb && chunk[2] === 0xbf) i = 3;
	259	while (i < chunk.length && [0x20, 0x09, 0x0a, 0x0d].includes(chunk[i]!)) i++;
	260	return chunk[i] === 0x3c;
	261	}
	262
	263	/**
	264	* Read a streamed page of unknown length. If it ends within `max` bytes and
	265	* `wait` milliseconds, and begins like HTML, it is returned whole. Otherwise
	266	* `rest` carries every byte, those already read first, in order.
	267	*/
	268	export async function readPage(
	269	body: ReadableStream<Uint8Array>,
	270	max = REWRITE_MAX,
	271	wait = STREAM_WAIT_MS,
	272	): Promise<PageRead> {
	273	const reader = body.getReader();
	274	const chunks: Uint8Array[] = [];
	275	let size = 0;
	276	let pending: Promise<Read> \| undefined;
	277	let timer: ReturnType<typeof setTimeout> \| undefined;
	278	const late = new Promise<"late">((resolve) => {
	279	timer = setTimeout(resolve, wait, "late");
	280	});
	281	let whole = false;
	282	try {
	283	for (;;) {
	284	pending ??= reader.read();
	285	const got = await Promise.race([pending, late]);
	286	if (got === "late") break;
	287	pending = undefined;
	288	if (got.done) {
	289	whole = true;
	290	break;
	291	}
	292	const first = chunks.length === 0;
	293	chunks.push(got.value!);
	294	size += got.value!.length;
	295	if ((first && !htmlShaped(got.value!)) \|\| size > max) break;
	296	}
	297	} catch (error) {
	298	reader.cancel(error).catch(() => {});
	299	throw error;
	300	} finally {
	301	clearTimeout(timer);
	302	}
	303	if (whole) return { whole: true, bytes: new Uint8Array(Buffer.concat(chunks)) };
	304	return {
	305	whole: false,
	306	rest: new ReadableStream<Uint8Array>({
	307	async pull(controller) {
	308	const next = chunks.shift();
	309	if (next !== undefined) return controller.enqueue(next);
	310	const got = await (pending ?? reader.read());
	311	pending = undefined;
	312	if (got.done) controller.close();
	313	else controller.enqueue(got.value!);
	314	},
	315	cancel: (reason) => reader.cancel(reason),
	316	}),
	317	};
	318	}
		@@ -17,7 +17,7 @@ import { getAppByRepo } from "../src/core/apps.ts";
17	17	import { listRuns } from "../src/core/runs.ts";
18	18	import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
19	19	import { parseManifest } from "../src/apps/manifest.ts";
20		import { REWRITE_MAX, TOOLBAR_PATH, toolbarJs, withToolbar } from "../src/apps/toolbar.ts";
	20	import { REWRITE_MAX, TOOLBAR_PATH, readPage, toolbarJs, withToolbar } from "../src/apps/toolbar.ts";
21	21	import { createUser } from "../src/core/users.ts";
22	22	import { serve, type Servers } from "../src/http/server.ts";
23	23	import { quietLogs } from "./helpers/globals.ts";
		@@ -76,6 +76,46 @@ describe("the forge pill", () => {
76	76	}),
77	77	{ headers: html },
78	78	);
	79	if (path === "/chunked-big") {
	80	const piece = new TextEncoder().encode("<p>" + "b".repeat(1024 * 1024) + "</p>");
	81	return new Response(
	82	new ReadableStream({
	83	start(c) {
	84	c.enqueue(new TextEncoder().encode("<html><body>"));
	85	for (let i = 0; i < 5; i++) c.enqueue(piece);
	86	c.enqueue(new TextEncoder().encode("</body></html>"));
	87	c.close();
	88	},
	89	}),
	90	{ headers: html },
	91	);
	92	}
	93	if (path === "/slow") {
	94	return new Response(
	95	new ReadableStream({
	96	async start(c) {
	97	c.enqueue(new TextEncoder().encode("<html><body>one"));
	98	await Bun.sleep(250);
	99	c.enqueue(new TextEncoder().encode("two</body></html>"));
	100	c.close();
	101	},
	102	}),
	103	{ headers: html },
	104	);
	105	}
	106	if (path === "/notmarkup") {
	107	return new Response(
	108	new ReadableStream({
	109	async start(c) {
	110	c.enqueue(new TextEncoder().encode('{"a":1}'));
	111	await Bun.sleep(10);
	112	c.enqueue(new TextEncoder().encode("</body>"));
	113	c.close();
	114	},
	115	}),
	116	{ headers: html },
	117	);
	118	}
79	119	if (path === "/range")
80	120	return new Response(HTML.slice(0, 20), {
81	121	status: 206,
		@@ -174,7 +214,29 @@ describe("the forge pill", () => {
174	214	expect(gz.headers.get("content-encoding")).toBe("gzip");
175	215	// The client unzips what the app zipped; no tag went in before that.
176	216	expect(await gz.text()).toBe(HTML);
177		expect(await (await at("/stream")).text()).toBe(HTML);
	217	});
	218
	219	test("a chunked page under the limit gets the pill", async () => {
	220	const text = await (await at("/stream")).text();
	221	expect(text).toContain(tag());
	222	expect(text.replace(tag(), "")).toBe(HTML);
	223	});
	224
	225	test("a chunked page over the limit arrives byte-identical with no pill", async () => {
	226	const res = await at("/chunked-big");
	227	const got = Buffer.from(await res.arrayBuffer()).toString();
	228	const piece = "<p>" + "b".repeat(1024 * 1024) + "</p>";
	229	expect(got).toBe("<html><body>" + piece.repeat(5) + "</body></html>");
	230	expect(got).not.toContain(TOOLBAR_PATH);
	231	});
	232
	233	test("a streamed page still open after the wait arrives whole with no pill", async () => {
	234	const got = await (await at("/slow")).text();
	235	expect(got).toBe("<html><body>onetwo</body></html>");
	236	});
	237
	238	test("a streamed text/html answer that does not start like markup is untouched", async () => {
	239	expect(await (await at("/notmarkup")).text()).toBe('{"a":1}</body>');
178	240	});
179	241
180	242	test("a ranged answer, a HEAD, a huge page and a foreign charset pass through", async () => {
		@@ -501,3 +563,56 @@ describe("the opt-out, from forge.json to the proxy", () => {
501	563	expect(html).not.toContain(TOOLBAR_PATH);
502	564	}, 180_000);
503	565	});
	566
	567	describe("readPage", () => {
	568	const enc = new TextEncoder();
	569	const stream = (parts: (string \| Uint8Array \| number)[]) =>
	570	new ReadableStream<Uint8Array>({
	571	async start(c) {
	572	for (const part of parts) {
	573	if (typeof part === "number") await Bun.sleep(part);
	574	else c.enqueue(typeof part === "string" ? enc.encode(part) : part);
	575	}
	576	c.close();
	577	},
	578	});
	579	const text = async (body: ReadableStream<Uint8Array>) => await new Response(body).text();
	580
	581	test("a page that ends in time and in size is returned whole", async () => {
	582	const got = await readPage(stream(["<p>a", "b</p>"]), 100, 1000);
	583	expect(got.whole).toBe(true);
	584	if (got.whole) expect(new TextDecoder().decode(got.bytes)).toBe("<p>ab</p>");
	585	});
	586
	587	test("a page still open after the wait is passed on in order", async () => {
	588	const got = await readPage(stream(["<p>a", 200, "b</p>"]), 100, 30);
	589	expect(got.whole).toBe(false);
	590	if (!got.whole) expect(await text(got.rest)).toBe("<p>ab</p>");
	591	});
	592
	593	test("a page past max is passed on in order", async () => {
	594	const got = await readPage(stream(["<p>aaaa", "bbbb", "cccc</p>"]), 8, 1000);
	595	expect(got.whole).toBe(false);
	596	if (!got.whole) expect(await text(got.rest)).toBe("<p>aaaabbbbcccc</p>");
	597	});
	598
	599	test("a first chunk that is not markup passes through, even if the stream is short", async () => {
	600	const got = await readPage(stream(['{"a":1}', "</body>"]), 100, 1000);
	601	expect(got.whole).toBe(false);
	602	if (!got.whole) expect(await text(got.rest)).toBe('{"a":1}</body>');
	603	});
	604
	605	test("a BOM and leading whitespace before < still count as markup", async () => {
	606	const bom = Uint8Array.from([0xef, 0xbb, 0xbf, ...enc.encode(" \r\n\t<p>x</p>")]);
	607	const got = await readPage(stream([bom]), 100, 1000);
	608	expect(got.whole).toBe(true);
	609	const spaced = await readPage(stream([" \n<p>"]), 100, 1000);
	610	expect(spaced.whole).toBe(true);
	611	});
	612
	613	test("a BOM or whitespace followed by something else is not markup", async () => {
	614	const bom = Uint8Array.from([0xef, 0xbb, 0xbf, ...enc.encode("x<p>")]);
	615	expect((await readPage(stream([bom]), 100, 1000)).whole).toBe(false);
	616	expect((await readPage(stream([" \nplain"]), 100, 1000)).whole).toBe(false);
	617	});
	618	});