2c33c83 · 43041ee · 24420a1 · 7a03c06 · aa15a9d
@@ -72,12 +72,20 @@
QuickTime files are inside, for browsers that refuse `video/quicktime`.
- **In the terminal** an audio or video file is not played; the file pane
  says so, names the format and size, and prints the media route on the
  forge's address (`src/tui/ui/media-note.ts`).
- **A long file:** the page reads at most 1 MB of a file. Audio and video are
  drawn from the media route alone (`STREAMED` in `render/renderers.ts`), so
  past that a file is drawn by its extension (`claimedMedia`) and its bytes
  are confirmed by the media route, which serves a contradicted one as
  `application/octet-stream` and the player shows the note. Under 1 MB, the
  bytes decide first, and a `.mp4` that is text keeps the source view.
- **A long file:** the page reads at most 1 MB of a file's body. Media
  kinds (image, svg, pdf, audio, video, model3d — `MEDIA_KINDS` in
  `render/renderers.ts`) are drawn from the media route alone, so they are
  bounded by `RENDER_CAP`, not by that ceiling: past it the page reads only
  the first `HEAD_BYTES` (`blobHead` in `src/git/read.ts`, which stops
  `cat-file` once they are in) and the size, `fileKind` decides, and the
  renderer gets an empty `body` and the `media` address. A `.pdf` whose
  bytes are text keeps the over-cap note. Where the source gave no bytes (a
  mirrored origin), the extension decides (`claimedMedia`) and the media
  route confirms the bytes, serving a contradicted file as
  `application/octet-stream`, and the player shows the note.
- **Text kinds** (markdown, csv, config, text) and binary keep the page's
  1 MB ceiling: over it the page says "over the render cap" and offers
  Download.
- Over `RENDER_CAP` (500 MB audio, 4 GB video) the page offers Download.
## Adding a renderer
@@ -372,6 +372,44 @@
async function viaReader(
  }
}
/**
 * A blob's first `bytes` bytes, and never one more off the pipe: `cat-file`
 * is stopped once they are in, so a 4 GB file costs what its head does. For
 * a page that must say what a file too large to read is.
 */
export async function blobHead(
  dir: string,
  rev: string,
  path: string,
  bytes: number,
): Promise<Uint8Array | undefined> {
  const spec = `${rev}:${path}`;
  assertBatchSafe(spec);
  const proc = Bun.spawn(["git", "cat-file", "blob", spec], {
    cwd: dir,
    env: gitEnv(),
    stdin: "ignore",
    stdout: "pipe",
    stderr: "ignore",
  });
  const reader = proc.stdout.getReader();
  const head = new Uint8Array(bytes);
  let filled = 0;
  try {
    while (filled < bytes) {
      const next = await reader.read();
      if (next.done || next.value === undefined) break;
      const take = Math.min(next.value.length, bytes - filled);
      head.set(next.value.subarray(0, take), filled);
      filled += take;
    }
  } finally {
    proc.kill("SIGKILL");
  }
  if (filled === 0 && (await proc.exited) !== 0) return undefined;
  return head.subarray(0, filled);
}
function classify(body: Uint8Array, size: number, oid: string): BlobForRender {
  // NUL in the first 8KB is git's own binary heuristic.
  if (body.subarray(0, 8000).includes(0))
@@ -32,6 +32,7 @@
import { diffCommit, type Diff } from "../git/diff.ts";
import { isSafeRef, isSafeTreePath } from "../git/guard.ts";
import {
  blobForRender,
  blobHead,
  commitLog,
  listRefs,
  listTree,
@@ -42,6 +43,7 @@
import {
import { repoDir } from "../git/repos.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import type { Page } from "../origins/types.ts";
import { HEAD_BYTES } from "../web/render/kinds.ts";
import type {
  Blob,
  Branch,
@@ -201,7 +203,10 @@
export function localSource(
      if (!isSafeRef(ref) || !isSafeTreePath(path)) return err("not_found");
      const blob = await blobForRender(dir, ref, path, MAX_BLOB_BYTES);
      if (blob.kind === "missing") return err("not_found");
      return ok(blob);
      if (blob.kind !== "too_large") return ok(blob);
      // Too large to read, but its first bytes still say what it is.
      const head = await blobHead(dir, ref, path, HEAD_BYTES);
      return ok(head === undefined ? blob : { ...blob, head });
    },
    async readme(ref) {
@@ -121,7 +121,15 @@
export type Blob =
      readonly body: Uint8Array;
      readonly size: number;
    }
  | { readonly kind: "too_large"; readonly size: number };
  | {
      readonly kind: "too_large";
      readonly size: number;
      /**
       * Its first bytes (`HEAD_BYTES`), when the source read them: enough to
       * say what the file is, never the file.
       */
      readonly head?: Uint8Array;
    };
/** An issue or a pull, as a list row. */
export interface IssueSummary {
@@ -71,6 +71,7 @@
import { field, readForm } from "../form.ts";
import { html, when, type Html } from "../html.ts";
import { flashTo, fragment, page, redirect } from "../layout.ts";
import { renderMarkdown } from "../markdown.ts";
import { HEAD_BYTES } from "../render/kinds.ts";
import { rendererFor } from "../render/renderers.ts";
import { mediaHref } from "./media.ts";
import { browseBase, localScope, type RepoScope } from "../scope.ts";
@@ -1514,10 +1515,13 @@
export async function pathsJson(ctx: Ctx): Promise<Response> {
  );
}
/** The page's own render ceiling, under the source's fetch ceiling. */
function capped(blob: Blob): Blob {
/**
 * The page's own render ceiling, under the source's fetch ceiling. A body
 * over it is dropped and only its head kept, so a media kind is still drawn.
 */
export function capped(blob: Blob): Blob {
  return blob.kind !== "too_large" && blob.size > MAX_RENDER_BYTES
    ? { kind: "too_large", size: blob.size }
    ? { kind: "too_large", size: blob.size, head: blob.body.slice(0, HEAD_BYTES) }
    : blob;
}
@@ -17,7 +17,7 @@
export interface RenderInput {
  readonly size: number;
  /**
   * The whole body, when the page read it; empty for a file over the page's
   * read ceiling, which only a `STREAMED` kind is drawn from.
   * read ceiling, which only a media kind (`MEDIA_KINDS`) is drawn from.
   */
  readonly body: Uint8Array;
  /** `/:owner/:name/media/:ref/*path`, for `<img>`, `<video>` and `fetch()`. */
@@ -42,23 +42,35 @@
export const RENDERERS: Partial<Readonly<Record<FileKind, Renderer>>> = {
/**
 * Kinds drawn wholly from the media route, never from the body: a file of
 * one of these too large for the page to read is still drawn, by its name,
 * and the media route confirms its bytes before it serves them as media.
 * one of these too large for the page to read is still drawn up to its
 * `RENDER_CAP`, decided by its first bytes (or by its name, when the source
 * gave none), and the media route confirms its bytes before it serves them.
 * Every other kind is bounded by the page's own read ceiling.
 */
const STREAMED: ReadonlySet<FileKind> = new Set<FileKind>(["audio", "video"]);
const MEDIA_KINDS: ReadonlySet<FileKind> = new Set<FileKind>([
  "image",
  "svg",
  "pdf",
  "audio",
  "video",
  "model3d",
]);
/** The renderer for a file and the bytes it is handed, or none: the source view. */
export function rendererFor(
  path: string,
  blob: Blob,
  renderers: Partial<Readonly<Record<FileKind, Renderer>>> = RENDERERS,
): { render: Renderer; body: Uint8Array } | undefined {
  const kind =
    blob.kind === "too_large"
      ? claimedMedia(path)?.kind
      : fileKind(path, blob.body.subarray(0, HEAD_BYTES));
    blob.kind !== "too_large"
      ? fileKind(path, blob.body.subarray(0, HEAD_BYTES))
      : blob.head !== undefined
        ? fileKind(path, blob.head)
        : claimedMedia(path)?.kind;
  if (kind === undefined || blob.size > RENDER_CAP[kind]) return undefined;
  if (blob.kind === "too_large" && !STREAMED.has(kind)) return undefined;
  const render = RENDERERS[kind];
  if (blob.kind === "too_large" && !MEDIA_KINDS.has(kind)) return undefined;
  const render = renderers[kind];
  if (render === undefined) return undefined;
  return { render, body: blob.kind === "too_large" ? NO_BYTES : blob.body };
}
@@ -0,0 +1,192 @@
/**
 * A media file over the page's 1 MB read ceiling still reaches its renderer,
 * up to its kind's `RENDER_CAP` (#7384): its kind is read from its first
 * bytes, never from the whole file. A text kind keeps the 1 MB ceiling.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { blobHead } from "../src/git/read.ts";
import type { Repo } from "../src/core/repos.ts";
import { repoDir } from "../src/git/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { localSource } from "../src/view/local.ts";
import type { Blob } from "../src/view/source.ts";
import { blobBody, capped } from "../src/web/pages/repo.ts";
import { HEAD_BYTES } from "../src/web/render/kinds.ts";
import { rendererFor, type RenderInput, type Renderer } from "../src/web/render/renderers.ts";
import { seedAdmin } from "./helpers/admin.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
const MB = 1_000_000;
const enc = (s: string): Uint8Array => new TextEncoder().encode(s);
/** `size` bytes opening on `head`, the rest filler of the same nature. */
const filled = (head: string, size: number, fill: number): Uint8Array => {
  const out = new Uint8Array(size).fill(fill);
  out.set(enc(head), 0);
  return out;
};
const PDF = filled("%PDF-1.7\n", 3 * MB, 0);
      
        junipercrew
        17d ago
        outdated
      
      Use ~6 MB instead of 50 MB: enough to pass the 5 MB fetch ceiling, far lighter on memory and time.
    
const MARKDOWN = filled("# Notes\n\n", 3 * MB, 0x61);
/** Just over the source's own 5 MB fetch ceiling, so only the head is ever read. */
const BIG_PDF = filled("%PDF-1.7\n", 6 * MB, 0);
const BIG_VIDEO = filled("\0\0\0\x20ftypisom", 6 * MB, 0);
/** A `.mp4` whose bytes are words: its bytes, not its name, decide. */
const WORDS_MP4 = filled("not a video, just words\n", 3 * MB, 0x61);
const blob = (body: Uint8Array): Blob => ({ kind: "binary", body, size: body.length });
/** A renderer that records what it was handed. */
function stub(): { render: Renderer; seen: RenderInput[] } {
  const seen: RenderInput[] = [];
  return { seen, render: (input) => (seen.push(input), { content: "" as never, lines: 0 }) };
}
describe("the page ceiling, by kind", () => {
  test("a 3 MB PDF reaches its renderer with an empty body", () => {
    const pdf = stub();
    const found = rendererFor("doc/big.pdf", capped(blob(PDF)), { pdf: pdf.render });
    expect(found).toBeDefined();
    found!.render({ path: "doc/big.pdf", size: PDF.length, body: found!.body, media: "/m" });
    expect(pdf.seen).toHaveLength(1);
    expect(pdf.seen[0]!.body.length).toBe(0);
    expect(pdf.seen[0]!.media).toBe("/m");
  });
  test("capped keeps only the head of a body it drops", () => {
    const cut = capped(blob(PDF));
    expect(cut.kind).toBe("too_large");
    if (cut.kind !== "too_large") return;
    expect(cut.size).toBe(PDF.length);
    expect(cut.head?.length).toBe(HEAD_BYTES);
  });
  test("a 3 MB Markdown file still shows the over-cap note", () => {
    const md = stub();
    const cut = capped({ kind: "text", body: MARKDOWN, size: MARKDOWN.length });
    expect(rendererFor("notes.md", cut, { markdown: md.render })).toBeUndefined();
    expect(md.seen).toHaveLength(0);
    expect(String(blobBody("/a/r", "main", "notes.md", cut).content)).toContain(
      "over the render cap",
    );
  });
  test("a media name its head contradicts is not drawn", () => {
    const video = stub();
    expect(rendererFor("words.mp4", capped(blob(WORDS_MP4)), { video: video.render })).toBeUndefined();
  });
  test("over its kind's cap, a media file is not drawn", () => {
    const head = PDF.subarray(0, HEAD_BYTES);
    const pdf = stub();
    expect(rendererFor("x.pdf", { kind: "too_large", size: 100 * MB + 1, head }, { pdf: pdf.render })).toBeUndefined();
    expect(rendererFor("x.pdf", { kind: "too_large", size: 100 * MB, head }, { pdf: pdf.render })).toBeDefined();
  });
});
describe("a large blob on the page", () => {
  let dataDir = "";
  let app: App;
  let servers: Servers;
  let base = "";
  let dir = "";
  let repo: Repo;
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-blob-ceiling-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    base = servers.url;
    const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-ceiling-passphrase" });
    if (!a.ok) throw new Error("signup failed");
    const created = await createRepo(app, a.value, { name: "big", visibility: "public" });
    if (!created.ok) throw new Error("repo failed");
    repo = created.value;
    dir = repoDir(app.reposRoot, "alice", "big");
    // Over the commit route's file ceiling, so the files go in by plumbing.
    const made = await commitToBranch(app, a.value, repo, {
      message: "start",
      files: [{ path: "README.md", content: "# big\n" }],
    });
    if (!made.ok) throw new Error("commit failed");
    const git = (args: string[], stdin?: Uint8Array | string): string => {
      const input = typeof stdin === "string" ? enc(stdin) : (stdin ?? "ignore");
      const run = Bun.spawnSync(["git", ...args], {
        cwd: dir,
        stdin: input,
        // The check's machine has no git identity; the commit names its own.
        env: {
          ...process.env,
          GIT_AUTHOR_NAME: "alice",
          GIT_AUTHOR_EMAIL: "[email protected]",
          GIT_COMMITTER_NAME: "alice",
          GIT_COMMITTER_EMAIL: "[email protected]",
        },
      });
      if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`);
      return run.stdout.toString().trim();
    };
    const files: Record<string, Uint8Array> = {
      "big.pdf": BIG_PDF,
      "big.mp4": BIG_VIDEO,
      "words.mp4": WORDS_MP4,
      "notes.md": MARKDOWN,
    };
    const lines = [git(["ls-tree", "main"])];
    for (const [name, body] of Object.entries(files))
      lines.push(`100644 blob ${git(["hash-object", "-w", "--stdin"], body)}\t${name}`);
    const tree = git(["mktree"], lines.join("\n") + "\n");
    const commit = git(["commit-tree", tree, "-p", "main", "-m", "big files"]);
    git(["update-ref", "refs/heads/main", commit]);
  }, 60_000);
  afterAll(async () => {
    await servers?.stop();
      
        junipercrew
        17d ago
        outdated
      
      Wall-clock assertion: drop it (flake). Assert on content instead.
    
    app?.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("blobHead reads the first bytes and no more", async () => {
    const head = await blobHead(dir, "main", "big.pdf", HEAD_BYTES);
    expect(head?.length).toBe(HEAD_BYTES);
    expect(new TextDecoder().decode(head!.subarray(0, 5))).toBe("%PDF-");
    expect(await blobHead(dir, "main", "missing.pdf", HEAD_BYTES)).toBeUndefined();
  });
  test("a video past the source's ceiling is drawn from its head", async () => {
      
        junipercrew
        17d ago
        outdated
      
      Drop the timing bound and assert the response lacks "over the render cap"; status 200 alone passes without the change.
    
    const res = await fetch(`${base}/alice/big/blob/main/big.mp4`);
    const text = await res.text();
    expect(res.status).toBe(200);
    expect(text).toContain("<video");
    expect(text).toContain('src="/alice/big/media/main/big.mp4"');
    expect(text).not.toContain("over the render cap");
    expect(text.length).toBeLessThan(200_000);
  });
  test("a PDF past the source's ceiling comes with its head, and reaches its renderer", async () => {
    const source = localSource(app, repo);
    const read = await source.blob("main", "big.pdf");
    expect(read.ok).toBe(true);
    if (!read.ok || read.value.kind !== "too_large") throw new Error("expected too_large");
    expect(read.value.size).toBe(BIG_PDF.length);
    expect(read.value.head?.length).toBe(HEAD_BYTES);
    const pdf = stub();
    const found = rendererFor("big.pdf", capped(read.value), { pdf: pdf.render });
    expect(found?.body.length).toBe(0);
  });
  test("a 3 MB .mp4 of words, and a 3 MB Markdown file, keep the over-cap note", async () => {
    for (const path of ["words.mp4", "notes.md"]) {
      const text = await (await fetch(`${base}/alice/big/blob/main/${path}`)).text();
      expect(text).toContain("over the render cap");
      expect(text).not.toContain("<video");
    }
  });
});
@@ -25,7 +25,7 @@
const PIPING: Record<string, { sites: string[]; reason: string }> = {
  "src/crew/workdir.ts": { sites: ["stderr"], reason: "stderr read to its end alongside the exit; stdout ignored" },
  "src/git/env.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end, finally clears the timer" },
  "src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" },
  "src/git/read.ts": { sites: ["stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, and by blobStream until its window ends or the reader hangs up (then killed); stderr ignored" },
  "src/git/read.ts": { sites: ["stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed); stderr ignored" },
  "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" },
  "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" },
  "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
		@@ -72,12 +72,20 @@ QuickTime files are inside, for browsers that refuse `video/quicktime`.
72	72	- In the terminal an audio or video file is not played; the file pane
73	73	says so, names the format and size, and prints the media route on the
74	74	forge's address (`src/tui/ui/media-note.ts`).
75		- A long file: the page reads at most 1 MB of a file. Audio and video are
76		drawn from the media route alone (`STREAMED` in `render/renderers.ts`), so
77		past that a file is drawn by its extension (`claimedMedia`) and its bytes
78		are confirmed by the media route, which serves a contradicted one as
79		`application/octet-stream` and the player shows the note. Under 1 MB, the
80		bytes decide first, and a `.mp4` that is text keeps the source view.
	75	- A long file: the page reads at most 1 MB of a file's body. Media
	76	kinds (image, svg, pdf, audio, video, model3d — `MEDIA_KINDS` in
	77	`render/renderers.ts`) are drawn from the media route alone, so they are
	78	bounded by `RENDER_CAP`, not by that ceiling: past it the page reads only
	79	the first `HEAD_BYTES` (`blobHead` in `src/git/read.ts`, which stops
	80	`cat-file` once they are in) and the size, `fileKind` decides, and the
	81	renderer gets an empty `body` and the `media` address. A `.pdf` whose
	82	bytes are text keeps the over-cap note. Where the source gave no bytes (a
	83	mirrored origin), the extension decides (`claimedMedia`) and the media
	84	route confirms the bytes, serving a contradicted file as
	85	`application/octet-stream`, and the player shows the note.
	86	- Text kinds (markdown, csv, config, text) and binary keep the page's
	87	1 MB ceiling: over it the page says "over the render cap" and offers
	88	Download.
81	89	- Over `RENDER_CAP` (500 MB audio, 4 GB video) the page offers Download.
82	90
83	91	## Adding a renderer
		@@ -372,6 +372,44 @@ async function viaReader(
372	372	}
373	373	}
374	374
	375	/**
	376	* A blob's first `bytes` bytes, and never one more off the pipe: `cat-file`
	377	* is stopped once they are in, so a 4 GB file costs what its head does. For
	378	* a page that must say what a file too large to read is.
	379	*/
	380	export async function blobHead(
	381	dir: string,
	382	rev: string,
	383	path: string,
	384	bytes: number,
	385	): Promise<Uint8Array \| undefined> {
	386	const spec = `${rev}:${path}`;
	387	assertBatchSafe(spec);
	388	const proc = Bun.spawn(["git", "cat-file", "blob", spec], {
	389	cwd: dir,
	390	env: gitEnv(),
	391	stdin: "ignore",
	392	stdout: "pipe",
	393	stderr: "ignore",
	394	});
	395	const reader = proc.stdout.getReader();
	396	const head = new Uint8Array(bytes);
	397	let filled = 0;
	398	try {
	399	while (filled < bytes) {
	400	const next = await reader.read();
	401	if (next.done \|\| next.value === undefined) break;
	402	const take = Math.min(next.value.length, bytes - filled);
	403	head.set(next.value.subarray(0, take), filled);
	404	filled += take;
	405	}
	406	} finally {
	407	proc.kill("SIGKILL");
	408	}
	409	if (filled === 0 && (await proc.exited) !== 0) return undefined;
	410	return head.subarray(0, filled);
	411	}
	412
375	413	function classify(body: Uint8Array, size: number, oid: string): BlobForRender {
376	414	// NUL in the first 8KB is git's own binary heuristic.
377	415	if (body.subarray(0, 8000).includes(0))
		@@ -32,6 +32,7 @@ import { diffCommit, type Diff } from "../git/diff.ts";
32	32	import { isSafeRef, isSafeTreePath } from "../git/guard.ts";
33	33	import {
34	34	blobForRender,
	35	blobHead,
35	36	commitLog,
36	37	listRefs,
37	38	listTree,
		@@ -42,6 +43,7 @@ import {
42	43	import { repoDir } from "../git/repos.ts";
43	44	import { err, ok, type Result } from "../kernel/result.ts";
44	45	import type { Page } from "../origins/types.ts";
	46	import { HEAD_BYTES } from "../web/render/kinds.ts";
45	47	import type {
46	48	Blob,
47	49	Branch,
		@@ -201,7 +203,10 @@ export function localSource(
201	203	if (!isSafeRef(ref) \|\| !isSafeTreePath(path)) return err("not_found");
202	204	const blob = await blobForRender(dir, ref, path, MAX_BLOB_BYTES);
203	205	if (blob.kind === "missing") return err("not_found");
204		return ok(blob);
	206	if (blob.kind !== "too_large") return ok(blob);
	207	// Too large to read, but its first bytes still say what it is.
	208	const head = await blobHead(dir, ref, path, HEAD_BYTES);
	209	return ok(head === undefined ? blob : { ...blob, head });
205	210	},
206	211
207	212	async readme(ref) {
		@@ -121,7 +121,15 @@ export type Blob =
121	121	readonly body: Uint8Array;
122	122	readonly size: number;
123	123	}
124		\| { readonly kind: "too_large"; readonly size: number };
	124	\| {
	125	readonly kind: "too_large";
	126	readonly size: number;
	127	/**
	128	* Its first bytes (`HEAD_BYTES`), when the source read them: enough to
	129	* say what the file is, never the file.
	130	*/
	131	readonly head?: Uint8Array;
	132	};
125	133
126	134	/** An issue or a pull, as a list row. */
127	135	export interface IssueSummary {
		@@ -71,6 +71,7 @@ import { field, readForm } from "../form.ts";
71	71	import { html, when, type Html } from "../html.ts";
72	72	import { flashTo, fragment, page, redirect } from "../layout.ts";
73	73	import { renderMarkdown } from "../markdown.ts";
	74	import { HEAD_BYTES } from "../render/kinds.ts";
74	75	import { rendererFor } from "../render/renderers.ts";
75	76	import { mediaHref } from "./media.ts";
76	77	import { browseBase, localScope, type RepoScope } from "../scope.ts";
		@@ -1514,10 +1515,13 @@ export async function pathsJson(ctx: Ctx): Promise<Response> {
1514	1515	);
1515	1516	}
1516	1517
1517		/** The page's own render ceiling, under the source's fetch ceiling. */
1518		function capped(blob: Blob): Blob {
	1518	/**
	1519	* The page's own render ceiling, under the source's fetch ceiling. A body
	1520	* over it is dropped and only its head kept, so a media kind is still drawn.
	1521	*/
	1522	export function capped(blob: Blob): Blob {
1519	1523	return blob.kind !== "too_large" && blob.size > MAX_RENDER_BYTES
1520		? { kind: "too_large", size: blob.size }
	1524	? { kind: "too_large", size: blob.size, head: blob.body.slice(0, HEAD_BYTES) }
1521	1525	: blob;
1522	1526	}
1523	1527
		@@ -17,7 +17,7 @@ export interface RenderInput {
17	17	readonly size: number;
18	18	/**
19	19	* The whole body, when the page read it; empty for a file over the page's
20		* read ceiling, which only a `STREAMED` kind is drawn from.
	20	* read ceiling, which only a media kind (`MEDIA_KINDS`) is drawn from.
21	21	*/
22	22	readonly body: Uint8Array;
23	23	/** `/:owner/:name/media/:ref/path`, for `<img>`, `<video>` and `fetch()`. /
		@@ -42,23 +42,35 @@ export const RENDERERS: Partial<Readonly<Record<FileKind, Renderer>>> = {
42	42
43	43	/**
44	44	* Kinds drawn wholly from the media route, never from the body: a file of
45		* one of these too large for the page to read is still drawn, by its name,
46		* and the media route confirms its bytes before it serves them as media.
	45	* one of these too large for the page to read is still drawn up to its
	46	* `RENDER_CAP`, decided by its first bytes (or by its name, when the source
	47	* gave none), and the media route confirms its bytes before it serves them.
	48	* Every other kind is bounded by the page's own read ceiling.
47	49	*/
48		const STREAMED: ReadonlySet<FileKind> = new Set<FileKind>(["audio", "video"]);
	50	const MEDIA_KINDS: ReadonlySet<FileKind> = new Set<FileKind>([
	51	"image",
	52	"svg",
	53	"pdf",
	54	"audio",
	55	"video",
	56	"model3d",
	57	]);
49	58
50	59	/** The renderer for a file and the bytes it is handed, or none: the source view. */
51	60	export function rendererFor(
52	61	path: string,
53	62	blob: Blob,
	63	renderers: Partial<Readonly<Record<FileKind, Renderer>>> = RENDERERS,
54	64	): { render: Renderer; body: Uint8Array } \| undefined {
55	65	const kind =
56		blob.kind === "too_large"
57		? claimedMedia(path)?.kind
58		: fileKind(path, blob.body.subarray(0, HEAD_BYTES));
	66	blob.kind !== "too_large"
	67	? fileKind(path, blob.body.subarray(0, HEAD_BYTES))
	68	: blob.head !== undefined
	69	? fileKind(path, blob.head)
	70	: claimedMedia(path)?.kind;
59	71	if (kind === undefined \|\| blob.size > RENDER_CAP[kind]) return undefined;
60		if (blob.kind === "too_large" && !STREAMED.has(kind)) return undefined;
61		const render = RENDERERS[kind];
	72	if (blob.kind === "too_large" && !MEDIA_KINDS.has(kind)) return undefined;
	73	const render = renderers[kind];
62	74	if (render === undefined) return undefined;
63	75	return { render, body: blob.kind === "too_large" ? NO_BYTES : blob.body };
64	76	}
		@@ -0,0 +1,192 @@
	1	/**
	2	* A media file over the page's 1 MB read ceiling still reaches its renderer,
	3	* up to its kind's `RENDER_CAP` (#7384): its kind is read from its first
	4	* bytes, never from the whole file. A text kind keeps the 1 MB ceiling.
	5	*/
	6
	7	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	8	import { mkdtemp, rm } from "node:fs/promises";
	9	import { tmpdir } from "node:os";
	10	import { join } from "node:path";
	11	import { createApp, type App } from "../src/app.ts";
	12	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	13	import { blobHead } from "../src/git/read.ts";
	14	import type { Repo } from "../src/core/repos.ts";
	15	import { repoDir } from "../src/git/repos.ts";
	16	import { serve, type Servers } from "../src/http/server.ts";
	17	import { localSource } from "../src/view/local.ts";
	18	import type { Blob } from "../src/view/source.ts";
	19	import { blobBody, capped } from "../src/web/pages/repo.ts";
	20	import { HEAD_BYTES } from "../src/web/render/kinds.ts";
	21	import { rendererFor, type RenderInput, type Renderer } from "../src/web/render/renderers.ts";
	22	import { seedAdmin } from "./helpers/admin.ts";
	23	import { quietLogs } from "./helpers/globals.ts";
	24
	25	quietLogs();
	26
	27	const MB = 1_000_000;
	28	const enc = (s: string): Uint8Array => new TextEncoder().encode(s);
	29	/** `size` bytes opening on `head`, the rest filler of the same nature. */
	30	const filled = (head: string, size: number, fill: number): Uint8Array => {
	31	const out = new Uint8Array(size).fill(fill);
	32	out.set(enc(head), 0);
	33	return out;
	34	};
	35
	36	const PDF = filled("%PDF-1.7\n", 3 * MB, 0);
		junipercrew 17d ago outdated Use ~6 MB instead of 50 MB: enough to pass the 5 MB fetch ceiling, far lighter on memory and time.
	37	const MARKDOWN = filled("# Notes\n\n", 3 * MB, 0x61);
	38	/** Just over the source's own 5 MB fetch ceiling, so only the head is ever read. */
	39	const BIG_PDF = filled("%PDF-1.7\n", 6 * MB, 0);
	40	const BIG_VIDEO = filled("\0\0\0\x20ftypisom", 6 * MB, 0);
	41	/** A `.mp4` whose bytes are words: its bytes, not its name, decide. */
	42	const WORDS_MP4 = filled("not a video, just words\n", 3 * MB, 0x61);
	43
	44	const blob = (body: Uint8Array): Blob => ({ kind: "binary", body, size: body.length });
	45
	46	/** A renderer that records what it was handed. */
	47	function stub(): { render: Renderer; seen: RenderInput[] } {
	48	const seen: RenderInput[] = [];
	49	return { seen, render: (input) => (seen.push(input), { content: "" as never, lines: 0 }) };
	50	}
	51
	52	describe("the page ceiling, by kind", () => {
	53	test("a 3 MB PDF reaches its renderer with an empty body", () => {
	54	const pdf = stub();
	55	const found = rendererFor("doc/big.pdf", capped(blob(PDF)), { pdf: pdf.render });
	56	expect(found).toBeDefined();
	57	found!.render({ path: "doc/big.pdf", size: PDF.length, body: found!.body, media: "/m" });
	58	expect(pdf.seen).toHaveLength(1);
	59	expect(pdf.seen[0]!.body.length).toBe(0);
	60	expect(pdf.seen[0]!.media).toBe("/m");
	61	});
	62
	63	test("capped keeps only the head of a body it drops", () => {
	64	const cut = capped(blob(PDF));
	65	expect(cut.kind).toBe("too_large");
	66	if (cut.kind !== "too_large") return;
	67	expect(cut.size).toBe(PDF.length);
	68	expect(cut.head?.length).toBe(HEAD_BYTES);
	69	});
	70
	71	test("a 3 MB Markdown file still shows the over-cap note", () => {
	72	const md = stub();
	73	const cut = capped({ kind: "text", body: MARKDOWN, size: MARKDOWN.length });
	74	expect(rendererFor("notes.md", cut, { markdown: md.render })).toBeUndefined();
	75	expect(md.seen).toHaveLength(0);
	76	expect(String(blobBody("/a/r", "main", "notes.md", cut).content)).toContain(
	77	"over the render cap",
	78	);
	79	});
	80
	81	test("a media name its head contradicts is not drawn", () => {
	82	const video = stub();
	83	expect(rendererFor("words.mp4", capped(blob(WORDS_MP4)), { video: video.render })).toBeUndefined();
	84	});
	85
	86	test("over its kind's cap, a media file is not drawn", () => {
	87	const head = PDF.subarray(0, HEAD_BYTES);
	88	const pdf = stub();
	89	expect(rendererFor("x.pdf", { kind: "too_large", size: 100 * MB + 1, head }, { pdf: pdf.render })).toBeUndefined();
	90	expect(rendererFor("x.pdf", { kind: "too_large", size: 100 * MB, head }, { pdf: pdf.render })).toBeDefined();
	91	});
	92	});
	93
	94	describe("a large blob on the page", () => {
	95	let dataDir = "";
	96	let app: App;
	97	let servers: Servers;
	98	let base = "";
	99	let dir = "";
	100	let repo: Repo;
	101
	102	beforeAll(async () => {
	103	dataDir = await mkdtemp(join(tmpdir(), "forge-blob-ceiling-"));
	104	app = createApp({ dataDir, port: 0 });
	105	servers = await serve(app);
	106	base = servers.url;
	107	const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-ceiling-passphrase" });
	108	if (!a.ok) throw new Error("signup failed");
	109	const created = await createRepo(app, a.value, { name: "big", visibility: "public" });
	110	if (!created.ok) throw new Error("repo failed");
	111	repo = created.value;
	112	dir = repoDir(app.reposRoot, "alice", "big");
	113	// Over the commit route's file ceiling, so the files go in by plumbing.
	114	const made = await commitToBranch(app, a.value, repo, {
	115	message: "start",
	116	files: [{ path: "README.md", content: "# big\n" }],
	117	});
	118	if (!made.ok) throw new Error("commit failed");
	119	const git = (args: string[], stdin?: Uint8Array \| string): string => {
	120	const input = typeof stdin === "string" ? enc(stdin) : (stdin ?? "ignore");
	121	const run = Bun.spawnSync(["git", ...args], {
	122	cwd: dir,
	123	stdin: input,
	124	// The check's machine has no git identity; the commit names its own.
	125	env: {
	126	...process.env,
	127	GIT_AUTHOR_NAME: "alice",
	128	GIT_AUTHOR_EMAIL: "[email protected]",
	129	GIT_COMMITTER_NAME: "alice",
	130	GIT_COMMITTER_EMAIL: "[email protected]",
	131	},
	132	});
	133	if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`);
	134	return run.stdout.toString().trim();
	135	};
	136	const files: Record<string, Uint8Array> = {
	137	"big.pdf": BIG_PDF,
	138	"big.mp4": BIG_VIDEO,
	139	"words.mp4": WORDS_MP4,
	140	"notes.md": MARKDOWN,
	141	};
	142	const lines = [git(["ls-tree", "main"])];
	143	for (const [name, body] of Object.entries(files))
	144	lines.push(`100644 blob ${git(["hash-object", "-w", "--stdin"], body)}\t${name}`);
	145	const tree = git(["mktree"], lines.join("\n") + "\n");
	146	const commit = git(["commit-tree", tree, "-p", "main", "-m", "big files"]);
	147	git(["update-ref", "refs/heads/main", commit]);
	148	}, 60_000);
	149
	150	afterAll(async () => {
	151	await servers?.stop();
		junipercrew 17d ago outdated Wall-clock assertion: drop it (flake). Assert on content instead.
	152	app?.close();
	153	await rm(dataDir, { recursive: true, force: true });
	154	});
	155
	156	test("blobHead reads the first bytes and no more", async () => {
	157	const head = await blobHead(dir, "main", "big.pdf", HEAD_BYTES);
	158	expect(head?.length).toBe(HEAD_BYTES);
	159	expect(new TextDecoder().decode(head!.subarray(0, 5))).toBe("%PDF-");
	160	expect(await blobHead(dir, "main", "missing.pdf", HEAD_BYTES)).toBeUndefined();
	161	});
	162
	163	test("a video past the source's ceiling is drawn from its head", async () => {
		junipercrew 17d ago outdated Drop the timing bound and assert the response lacks "over the render cap"; status 200 alone passes without the change.
	164	const res = await fetch(`${base}/alice/big/blob/main/big.mp4`);
	165	const text = await res.text();
	166	expect(res.status).toBe(200);
	167	expect(text).toContain("<video");
	168	expect(text).toContain('src="/alice/big/media/main/big.mp4"');
	169	expect(text).not.toContain("over the render cap");
	170	expect(text.length).toBeLessThan(200_000);
	171	});
	172
	173	test("a PDF past the source's ceiling comes with its head, and reaches its renderer", async () => {
	174	const source = localSource(app, repo);
	175	const read = await source.blob("main", "big.pdf");
	176	expect(read.ok).toBe(true);
	177	if (!read.ok \|\| read.value.kind !== "too_large") throw new Error("expected too_large");
	178	expect(read.value.size).toBe(BIG_PDF.length);
	179	expect(read.value.head?.length).toBe(HEAD_BYTES);
	180	const pdf = stub();
	181	const found = rendererFor("big.pdf", capped(read.value), { pdf: pdf.render });
	182	expect(found?.body.length).toBe(0);
	183	});
	184
	185	test("a 3 MB .mp4 of words, and a 3 MB Markdown file, keep the over-cap note", async () => {
	186	for (const path of ["words.mp4", "notes.md"]) {
	187	const text = await (await fetch(`${base}/alice/big/blob/main/${path}`)).text();
	188	expect(text).toContain("over the render cap");
	189	expect(text).not.toContain("<video");
	190	}
	191	});
	192	});
		@@ -25,7 +25,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
25	25	"src/crew/workdir.ts": { sites: ["stderr"], reason: "stderr read to its end alongside the exit; stdout ignored" },
26	26	"src/git/env.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end, finally clears the timer" },
27	27	"src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" },
28		"src/git/read.ts": { sites: ["stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, and by blobStream until its window ends or the reader hangs up (then killed); stderr ignored" },
	28	"src/git/read.ts": { sites: ["stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed); stderr ignored" },
29	29	"src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" },
30	30	"src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" },
31	31	"src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },