Public: anyone can read this; the crew can change it.

A file nothing can render shows its size, detected type and a download, never raw bytes #13340

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13318-a-file-nothing-can-render-shows-its-size → main 8 commits fork 3 threads · 3 unresolved

  • Merged. scout/13318-a-file-nothing-can-render-shows-its-size was merged into main as 8594e14.

Review

Sign in to review this change.
7 files changed +210 −14
docs/features/README.md +1
@@ -211,6 +211,7 @@ Read with the guide page [Your apps](../guide-your-apps.md).
211211 - [An app's own domain: cut-over and the way back](app-own-domain.md)
212212 - [App templates](app-templates.md)
213213 - [Archive listing](archive-listing.md)
214 - [A binary file shows its size and type](binary-file.md)
214215 - [Crawl budget](crawl-budget.md)
215216 - [Data browser (core)](data-browser.md)
216217 - [Database size](db-size.md)
docs/features/binary-file.md added +12
@@ -0,0 +1,12 @@
1 # A binary file shows its size and type
2
3 Open the blob page of a binary nothing can render, such as a `.wasm` or an
4 unknown blob. Under the page header, which names the file, a note gives its
5 size and a type word sniffed from its first bytes, with a Download button:
6 `12 KB · ZIP archive — not shown, but you can download it.` When no signature
7 matches, the word is `binary data`. The bytes are never shown.
8
9 The terminal's file view says the same: `Binary file, 12 KB, ZIP archive.`
10
11 A binary over the 5 MB read ceiling is not read, so it has no type word; it
12 reads "Too large to render" with its size and a download.
src/tui/screens/workspace.ts +2 −2
@@ -50,7 +50,7 @@ import { canAdmin, canWrite, DECISIONS_BY, getRepoById, type Repo } from "../../
5050 import type { TreeEntry } from "../../git/read.ts";
5151 import { parseUnifiedDiff, type Diff } from "../../git/diff.ts";
5252 import { renderDiff as renderPatch } from "../ui/diff.ts";
53 import { fileKind, isEnvExample, kindByName, NO_BYTES, RENDER_CAP } from "../../web/render/kinds.ts";
53 import { fileKind, isEnvExample, kindByName, HEAD_BYTES, NO_BYTES, RENDER_CAP, sniffType } from "../../web/render/kinds.ts";
5454 import { downloadHref } from "../../web/pages/media.ts";
5555 import { isModelPath, modelFacts } from "../../web/render/model-facts.ts";
5656 import type { RepoFacts, RepoSource } from "../../view/source.ts";
@@ -1378,7 +1378,7 @@ export function workspaceScreen(
13781378 note =
13791379 archiveNote(t.path, read.value.body, read.value.size, formatSize) ??
13801380 (kindByName(t.path) === "psd" ? psdNote(read.value.body, read.value.size, formatSize) : undefined) ??
1381 `Binary file, ${formatSize(read.value.size)}.`;
1381 `Binary file, ${formatSize(read.value.size)}, ${sniffType(read.value.body.subarray(0, HEAD_BYTES))}.`;
13821382 } else raw = read.value.body; // .html included: source here, the page preview is web-only
13831383 }
13841384 entry = { note, noteTitle, noteDetail, raw, picture, sheet, db };
src/web/pages/repo.ts +23 −8
@@ -95,7 +95,7 @@ import { fontJs } from "../render/font.ts";
9595 import { INLINE_MAX, keepGcode, keptGcode, layerCount } from "../render/gcode.ts";
9696 import { hasTable } from "../render/sqlite.ts";
9797 import { READ_CEILING } from "../render/psd-read.ts";
98 import { fileKind, HEAD_BYTES, kindByName, needsPaths, RENDER_CAP } from "../render/kinds.ts";
98 import { fileKind, HEAD_BYTES, kindByName, needsPaths, RENDER_CAP, sniffType } from "../render/kinds.ts";
9999 import { RENDERERS, rendererFor } from "../render/renderers.ts";
100100 import { downloadHref, mediaHref } from "./media.ts";
101101 import { browseBase, localScope, type RepoScope } from "../scope.ts";
@@ -740,7 +740,10 @@ export function blobBody(
740740 content: empty({
741741 icon: "file",
742742 title: why !== undefined ? "Archive" : "Binary file",
743 note: why !== undefined ? `${fileSize(blob.size)} · ${why}` : `${fileSize(blob.size)} — not shown, but you can download it.`,
743 note:
744 why !== undefined
745 ? `${fileSize(blob.size)} · ${why}`
746 : `${fileSize(blob.size)} · ${sniffType(blob.body.subarray(0, HEAD_BYTES))} — not shown, but you can download it.`,
outdated

Principle 1: the file name is already in the page header; repeating it here says it twice. Keep size · type, drop the name.

744747 action: btn("Download", {
745748 href: downloadHref(base, ref, path),
746749 variant: "primary",
@@ -1017,7 +1020,10 @@ export async function renderCode(
10171020 rail?: Html;
10181021 /** The README, when the caller has already read it (and timed the read). */
10191022 readme?: Html;
1020 /** What the listing's links name instead of `ref`: a full commit id, whose pages a cache may keep (#8199). */
1023 /**
1024 * What the listing's links name instead of `ref`: a full commit id, whose pages a cache may
1025 * keep (#8199).
1026 */
10211027 linkRef?: string;
10221028 },
10231029 ): Promise<Html> {
@@ -1165,7 +1171,10 @@ export function drawsFont(scope: RepoScope, path: string, blob: Blob, view: Blob
11651171 );
11661172 }
11671173
1168 /** The files at a ref as a lookup; undefined when the listing is unavailable or cut short (then nothing is linked). */
1174 /**
1175 * The files at a ref as a lookup; undefined when the listing is unavailable or cut short (then
1176 * nothing is linked).
1177 */
11691178 export async function pathsAt(scope: RepoScope, ref: string): Promise<((path: string) => boolean) | undefined> {
11701179 const listed = await scope.source.paths(ref);
11711180 if (!listed.ok || listed.value.truncated) return undefined;
@@ -1340,7 +1349,10 @@ export function rawResponse(
13401349 return new Response(blob.body, { headers });
13411350 }
13421351
1343 /** A forge admin looking at a repository is counted, to suggest a valve on /flow and never to set one. */
1352 /**
1353 * A forge admin looking at a repository is counted, to suggest a valve on /flow and never to set
1354 * one.
1355 */
13441356 function noteLook(ctx: Ctx, repoId: string): void {
13451357 const user = ctx.auth.user;
13461358 if (user?.isAdmin === true && user.kind !== "agent") recordLook(ctx.app.db, user.id, repoId);
@@ -1693,7 +1705,8 @@ export async function treePage(ctx: Ctx): Promise<Response> {
16931705 }
16941706 }
16951707
1696 // A stranger's listing reads its folders and ref names through what other strangers' pages kept (#9514).
1708 // A stranger's listing reads its folders and ref names through what other strangers' pages kept
1709 // (#9514).
16971710 const source = commit === undefined ? scope.source : withKeptFolders(repo.id, scope.source);
16981711 const entries = await git.read(() => source.tree(commit ?? ref, path));
16991712 if (!entries.ok) return notFoundPage(ctx);
@@ -1793,14 +1806,16 @@ export async function blobPage(ctx: Ctx): Promise<Response> {
17931806 }
17941807 const blob = await git.read(() => scope.source.blob(ref, path));
17951808 if (!blob.ok) return notFoundPage(ctx);
1796 // A stranger's file page reads its folders and ref names through what other strangers' pages kept (#9514).
1809 // A stranger's file page reads its folders and ref names through what other strangers' pages kept
1810 // (#9514).
17971811 const source = kept ? withKeptFolders(repo.id, scope.source) : scope.source;
17981812 const refs = await git.read(() => (kept ? keptRefs(repo.id, () => refSetFrom(scope.source)) : refSetFrom(scope.source)));
17991813 let highlightMs: number | undefined;
18001814 const renderStart = performance.now();
18011815 const shown = capped(blob.value, path);
18021816 if (table !== undefined) keepable = shown.kind !== "too_large" && hasTable(shown.body, table, page);
1803 // A big sliced file is parsed on a worker, never on this thread; the renderer finds the parse kept.
1817 // A big sliced file is parsed on a worker, never on this thread; the renderer finds the parse
1818 // kept.
18041819 if (shown.kind !== "too_large" && shown.body.length > INLINE_MAX && kindByName(path) === "gcode" && keptGcode(shown.body) === undefined) {
18051820 const began = performance.now();
18061821 const made = await gcodeOffThread(shown.body);
src/web/render/kinds.ts +49 −3
@@ -68,6 +68,14 @@ export type FileKind =
6868
6969 const MB = 1_000_000;
7070
71 /**
72 * The largest plain text the page draws and highlights as source. Past it the
73 * file is still text and still shows as text (the first lines, then a link to
74 * the raw file), never as a binary: highlighting a megabyte on the request
75 * path would break the page's budget of milliseconds.
76 */
77 export const TEXT_CAP = 1 * MB;
78
7179 /**
7280 * The largest file each kind draws inline. Over it the page offers Download,
7381 * which streams at any size. A text kind is bounded by the page's own
@@ -127,7 +135,7 @@ export const RENDER_CAP: Readonly<Record<FileKind, number>> = {
127135 compose: 1 * MB,
128136 sql: 1 * MB,
129137 graphql: 1 * MB,
130 text: 1 * MB,
138 text: TEXT_CAP,
131139 binary: 0,
132140 };
133141
@@ -272,7 +280,10 @@ const TEXTUAL: Readonly<Record<string, FileKind>> = {
272280 export const isEnvExample = (path: string): boolean =>
273281 /^\.env\.(example|sample|template)$/i.test(path.slice(path.lastIndexOf("/") + 1));
274282
275 /** `openapi.json|yaml|yml` and `*.openapi.*`: the name of an OpenAPI document; its content decides whether it is drawn as one. */
283 /**
284 * `openapi.json|yaml|yml` and `*.openapi.*`: the name of an OpenAPI document; its content decides
285 * whether it is drawn as one.
286 */
276287 const isOpenApiName = (path: string): boolean => /(^|\/|\.)openapi\.(json|ya?ml)$/i.test(path);
277288
278289 /** A manifest is named exactly `package.json`; `package-lock.json` and the rest stay config. */
@@ -409,7 +420,10 @@ export function fileKind(path: string, head: Uint8Array): FileKind {
409420 return TEXTUAL[ext] ?? (isDxfHead(head) ? "dxf" : "text");
410421 }
411422
412 /** Kinds whose renderer links paths to files in the repository, so the page reads the ref's file list. */
423 /**
424 * Kinds whose renderer links paths to files in the repository, so the page reads the ref's file
425 * list.
426 */
413427 export const needsPaths = (kind: FileKind): boolean => kind === "sarif" || kind === "lcov";
414428
415429 /**
@@ -462,3 +476,35 @@ export const KNOWN_EXTENSIONS: readonly string[] = [
462476 "xlsx",
463477 ...Object.keys(OFFICE),
464478 ];
479
480 /** Signatures of files nothing here draws, as the word a stranger would use. */
481 const SNIFFED: readonly (readonly [string, Magic])[] = [
482 ["ELF executable", bytes(0x7f, 0x45, 0x4c, 0x46)],
483 ["Windows executable", ascii("MZ")],
484 // 0xcafebabe opens both a Java class and a fat Mach-O; as `file` does, the next four bytes
outdated

0xcafebabe here shadows the Java class entry below, which can never match; a .class file is mislabelled Mach-O. Remove it from this list (or disambiguate by the version field) and test it.

485 // tell them apart: a class has a major version of 45 or more there, a fat binary a short count.
486 ["Java class", (h) => bytes(0xca, 0xfe, 0xba, 0xbe)(h) && h.length >= 8 && (h[6]! << 8 | h[7]!) >= 45],
487 ["Mach-O executable", any(bytes(0xcf, 0xfa, 0xed, 0xfe), bytes(0xfe, 0xed, 0xfa, 0xcf), bytes(0xca, 0xfe, 0xba, 0xbe))],
488 ["WebAssembly module", bytes(0x00, 0x61, 0x73, 0x6d)],
489 ["ZIP archive", bytes(0x50, 0x4b, 0x03, 0x04)],
490 ["gzip archive", bytes(0x1f, 0x8b)],
491 ["bzip2 archive", ascii("BZh")],
492 ["XZ archive", bytes(0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00)],
493 ["Zstandard archive", bytes(0x28, 0xb5, 0x2f, 0xfd)],
494 ["7-Zip archive", bytes(0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c)],
495 ["RAR archive", ascii("Rar!")],
496 ["PNG image", bytes(0x89, 0x50, 0x4e, 0x47)],
497 ["JPEG image", bytes(0xff, 0xd8, 0xff)],
498 ["GIF image", ascii("GIF8")],
499 ["PDF document", ascii("%PDF-")],
500 ["SQLite database", ascii("SQLite format 3\0")],
501 ];
502
503 /**
504 * A short word for what a file's first bytes say it is, from the signatures
505 * above; `binary data` when they say nothing. It only names: nothing is read
506 * beyond the head, and the word is escaped where it is drawn.
507 */
508 export function sniffType(head: Uint8Array): string {
509 return SNIFFED.find(([, is]) => is(head))?.[0] ?? "binary data";
510 }
test/blob-binary-page.test.ts added +60
@@ -0,0 +1,60 @@
1 /**
2 * A binary nothing can render, opened on its blob page over HTTP by an
3 * anonymous reader: size, a sniffed type word and a download, never its
4 * bytes, with the security floor on the real response.
5 */
6
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import { seedAdmin } from "./helpers/admin.ts";
15 import { quietLogs } from "./helpers/globals.ts";
16
17 quietLogs();
18
19 describe("a binary on its blob page", () => {
20 let dataDir = "";
21 let app: App;
22 let servers: Servers;
23 let base = "";
24 // A WebAssembly header, a NUL, and a marker that must never reach the page.
25 const body = new Uint8Array([0x00, 0x61, 0x73, 0x6d, 0x01, 0, 0, 0, ...new TextEncoder().encode("<script>SECRET-BYTES</script>")]);
26
27 beforeAll(async () => {
28 dataDir = await mkdtemp(join(tmpdir(), "forge-blob-binary-"));
29 app = createApp({ dataDir, port: 0 });
30 servers = await serve(app);
31 base = servers.url;
32 const a = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "a-long-binary-passphrase" });
33 if (!a.ok) throw new Error("signup failed");
34 const created = await createRepo(app, a.value, { name: "p", visibility: "public" });
35 if (!created.ok) throw new Error("repo failed");
36 const made = await commitToBranch(app, a.value, created.value, {
37 message: "a module",
38 files: [{ path: "mod.bin", content: body }],
39 });
40 if (!made.ok) throw new Error("commit failed");
41 }, 60_000);
42
43 afterAll(async () => {
44 await servers?.stop();
45 app?.close();
46 await rm(dataDir, { recursive: true, force: true });
47 });
48
49 test("the page says size and type once, offers the download, and keeps the floor", async () => {
50 const res = await fetch(`${base}/alice/p/blob/main/mod.bin`);
51 const text = await res.text();
52 expect(res.status).toBe(200);
53 expect(text).toContain(`${body.length} B · WebAssembly module — not shown, but you can download it.`);
54 expect(text).toContain("download=1");
55 expect(text).not.toContain("SECRET-BYTES");
56 expect(res.headers.get("content-security-policy")).toBeTruthy();
57 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
58 expect(res.headers.get("referrer-policy")).toBeTruthy();
59 });
60 });
test/fuzz-renderers.test.ts +63 −1
@@ -27,7 +27,10 @@ import { sarif } from "../src/web/render/sarif.ts";
2727 import { gcode } from "../src/web/render/gcode.ts";
2828 import { config } from "../src/web/render/config.ts";
2929 import { markdown } from "../src/web/render/markdown.ts";
30 import { fuzz, int, mix, pick, type Rand } from "./helpers/fuzz.ts";
30 import { fuzz, int, mix, pick, rng, type Rand } from "./helpers/fuzz.ts";
31 import { blobBody, rawResponse } from "../src/web/pages/repo.ts";
32 import { fileKind, HEAD_BYTES, mediaType, RENDER_CAP, sniffType, TEXT_CAP } from "../src/web/render/kinds.ts";
33 import { securityFloor } from "../src/http/respond.ts";
3134
3235 /** What a hostile file tries to smuggle into the page. */
3336 export const HOSTILE = [
@@ -305,3 +308,62 @@ function input(path: string, text: string, all = false) {
305308 layer: 2,
306309 };
307310 }
311
312 describe("a file nothing can render", () => {
313 const random = (n: number, seed: number): Uint8Array => {
314 const r = rng(seed);
315 const b = new Uint8Array(n);
316 for (let i = 0; i < n; i++) b[i] = int(r, 256);
317 b[7] = 0; // a NUL in the head: binary, as git reads it
318 b[0] = 0x01; // no signature
319 return b;
320 };
321
322 test("shows size, a type word and a download, and none of its bytes", () => {
323 const body = random(4000, 7);
324 const name = "weird<name>.blob";
325 const { content } = blobBody("/o/r", "main", name, { kind: "binary", body, size: body.length });
326 const page = content.text;
327 // The page header already names the file; the note says size and type only (DESIGN principle
328 // 1).
329 expect(page).toContain("3.9 KB · binary data — not shown, but you can download it.");
330 expect(page).not.toContain("weird&lt;name&gt;.blob ·");
331 // The escaped name reaches the page through the download href only.
332 expect(page).toContain('href="/o/r/media/main/weird&lt;name&gt;.blob?download=1"');
333 expect(page).toContain("download=1");
334 expect(page).not.toContain("<name>");
335 expect(page.length).toBeLessThan(2000);
336 // No run of the file's own bytes is in the page.
337 const text = new TextDecoder("latin1").decode(body);
338 for (let i = 0; i + 16 <= text.length; i += 16) expect(page.includes(text.slice(i, i + 16))).toBe(false);
339 });
340
341 test("a signature gives its word", () => {
342 expect(sniffType(new Uint8Array([0x7f, 0x45, 0x4c, 0x46, 0]))).toBe("ELF executable");
343 expect(sniffType(new Uint8Array([0x50, 0x4b, 0x03, 0x04]))).toBe("ZIP archive");
344 expect(sniffType(new Uint8Array([0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0x34]))).toBe("Java class");
345 expect(sniffType(new Uint8Array([0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 2]))).toBe("Mach-O executable");
346 expect(sniffType(random(64, 3))).toBe("binary data");
347 });
348
349 test("the download is an octet stream, nosniff, and raw keeps its headers", () => {
350 const body = random(300, 9);
351 expect(mediaType("x.bin", body)).toBe("application/octet-stream");
352 const res = rawResponse({ kind: "binary", body, size: body.length });
353 expect(res.headers.get("content-type")).toBe("application/octet-stream");
354 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
355 expect(res.headers.get("content-security-policy")).toContain("sandbox");
356 const floored = securityFloor(res);
357 expect(floored.headers.get("referrer-policy")).toBeTruthy();
358 });
359
360 test("text over the one text cap is still text, never binary", () => {
361 expect(TEXT_CAP).toBe(RENDER_CAP.text);
362 const text = encoder.encode("hello\n".repeat(TEXT_CAP / 6 + 10));
363 expect(fileKind("big.txt", text.subarray(0, HEAD_BYTES))).toBe("text");
364 const { content, lines } = blobBody("/o/r", "main", "big.txt", { kind: "text", body: text, size: text.length });
365 expect(lines).toBeGreaterThan(1000);
366 expect(content.text).toContain("hello");
367 expect(content.text).not.toContain("Binary file");
368 });
369 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs