Public: anyone can read this; the crew can change it.

A test fails when a file kind has no fuzz case, so a new renderer cannot ship unfuzzed #14683

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14676-a-test-fails-when-a-file-kind-has-no-fuz → main 2 commits fork 2 threads · 2 unresolved

  • Merged. scout/14676-a-test-fails-when-a-file-kind-has-no-fuz was merged into main as 4cc7596.

Review

Sign in to review this change.
1 file changed +135 −2
test/fuzz-renderers.test.ts +135 −2
@@ -9,10 +9,11 @@
99 */
1010 import { describe, expect, test } from "bun:test";
1111 import { RENDERERS, type Renderer } from "../src/web/render/renderers.ts";
12 import { fuzz, int, mix, pick, rng, type Rand } from "./helpers/fuzz.ts";
12 import { fuzz, int, mix, pick, rng, SEED, type Rand } from "./helpers/fuzz.ts";
1313 import { blobBody, rawResponse } from "../src/web/pages/repo.ts";
14 import { fileKind, HEAD_BYTES, mediaType, RENDER_CAP, sniffType, TEXT_CAP } from "../src/web/render/kinds.ts";
14 import { ALL_KINDS, fileKind, HEAD_BYTES, mediaType, RENDER_CAP, sniffType, TEXT_CAP, type FileKind } from "../src/web/render/kinds.ts";
1515 import { securityFloor } from "../src/http/respond.ts";
16 import type { Blob } from "../src/view/source.ts";
1617
1718 const encoder = new TextEncoder();
1819
@@ -571,3 +572,135 @@ describe("a file nothing can render", () => {
571572 expect(content.text).not.toContain("Binary file");
572573 });
573574 });
575
576 /**
577 * A path for each kind no sample names, so random bytes reach its renderer the way the blob page
578 * would hand them: by the name a pushed file of that kind carries.
579 */
580 export const RAW_PATHS: Readonly<Record<string, string>> = {
581 image: "a.png",
582 svg: "a.svg",
583 pdf: "a.pdf",
584 html: "a.html",
585 audio: "a.mp3",
586 video: "a.mp4",
587 sqlite: "a.sqlite",
588 parquet: "a.parquet",
589 archive: "a.zip",
590 sheet: "a.xlsx",
591 document: "a.docx",
592 slides: "a.pptx",
593 layers: "a.ora",
594 font: "a.ttf",
595 psd: "a.psd",
596 text: "notes.txt",
597 binary: "a.bin",
598 };
599
600 /**
601 * Kinds left out of the random- and truncated-bytes cases, each with why. Empty today: every kind
602 * has its case. A kind added here must say what makes bytes unable to reach it.
603 */
604 export const RAW_ALLOWED: Readonly<Record<string, string>> = {};
605
606 /**
607 * The kinds of `kinds` with no random-bytes or truncated-input case and no allowlist line, each
608 * named with where to add it; empty when every kind is guarded.
609 */
610 export function unfuzzed(
611 kinds: readonly string[],
612 covered: ReadonlySet<string>,
613 allowed: Readonly<Record<string, string>>,
614 ): string[] {
615 return kinds
616 .filter((kind) => !covered.has(kind) && !(kind in allowed))
617 .map(
618 (kind) =>
619 `kind "${kind}" has no random-bytes or truncated-input case: add a path to RAW_PATHS or a ` +
620 `sample to SAMPLES in test/fuzz-renderers.test.ts, or a line to RAW_ALLOWED saying why`,
621 );
622 }
623
624 /**
625 * Every path each kind is drawn under in the raw cases: its samples' paths, or its RAW_PATHS one.
626 */
627 function rawPaths(): Map<string, { path: string; files: string[] }[]> {
628 const out = new Map<string, { path: string; files: string[] }[]>();
629 for (const sample of Object.values(SAMPLES)) {
630 const kind = sampleKind(sample);
631 out.set(kind, [...(out.get(kind) ?? []), { path: sample.path, files: [...sample.files] }]);
632 }
633 for (const [kind, path] of Object.entries(RAW_PATHS))
634 if (!out.has(kind)) out.set(kind, [{ path, files: [] }]);
635 return out;
636 }

A RAW_PATHS entry for a kind that has a sample is skipped here and never flagged stale. Fix: report it in the stale-entry check.

637
638 /**
639 * Draws bytes as the blob page would for `kind`: its renderer, or the source/hex view when none.
640 */
641 function drawRaw(kind: string, path: string, body: Uint8Array, all: boolean): string {
642 const render = RENDERERS[kind as keyof typeof RENDERERS];
643 if (render !== undefined) {
644 const { body: _b, size: _s, all: _a, ...rest } = input(path, "");
645 return render({ ...rest, body, size: body.length, all }).content.text;
646 }
647 const blob: Blob =
648 kind === "binary" ? { kind: "binary", body, size: body.length } : { kind: "text", body, size: body.length };
649 return blobBody("/o/r", "main", path, blob, undefined, all).content.text;
650 }
651
652 /** Bytes per case and cases per kind, capped so the whole file stays well under two seconds. */
653 const RAW_RUNS = 12;
654 const RAW_MAX = 1536;
655
656 describe("fuzz: every kind, on random and truncated bytes", () => {
657 const paths = rawPaths();
658
659 test("every kind in ALL_KINDS has a random-bytes or truncated-input case, or a reason it has none", () => {
660 expect(unfuzzed(ALL_KINDS, new Set(paths.keys()), RAW_ALLOWED)).toEqual([]);
661 // A path or an allowance for a kind that no longer exists, or that is covered anyway, is stale.
662 expect(Object.keys(RAW_PATHS).filter((k) => !ALL_KINDS.includes(k as FileKind))).toEqual([]);
663 expect(Object.keys(RAW_ALLOWED).filter((k) => paths.has(k) || !ALL_KINDS.includes(k as FileKind))).toEqual([]);
664 for (const reason of Object.values(RAW_ALLOWED)) expect(reason.trim().length).toBeGreaterThan(0);
665 });
666
667 test("the guard names a kind with no case, and where to add it", () => {
668 expect(unfuzzed(["csv", "log"], new Set(["csv", "log"]), {})).toEqual([]);
669 const missing = unfuzzed(["csv", "log", "font"], new Set(["csv"]), { font: "binary" });
670 expect(missing).toHaveLength(1);
671 expect(missing[0]).toContain('"log"');
672 expect(missing[0]).toContain("RAW_PATHS");
673 });
674

SEED ^ kind.length gives every same-length kind the same stream. Fix: hash the kind name.

675 for (const kind of ALL_KINDS)
676 test(`${kind}: random and truncated bytes never throw, never let script through`, () => {
677 const cases = paths.get(kind);
678 if (cases === undefined) return; // named by the guard above, or allowed with its reason
679 const r = rng(SEED ^ kind.length);
680 for (const { path, files } of cases) {
681 for (let i = 0; i < RAW_RUNS; i++) {
682 const body = new Uint8Array(int(r, RAW_MAX));
683 for (let j = 0; j < body.length; j++) body[j] = int(r, 256);
684 // Half the cases keep a sample's first bytes, so a signature or header gets the parser
685 // past its door.
686 if (files.length > 0 && i % 2 === 1) body.set(encoder.encode(pick(r, files)).subarray(0, body.length >> 1));
687 try {
688 assertInert(drawRaw(kind, path, body, i % 3 === 0));
689 } catch (error) {
690 throw new Error(`${kind} ${path}: FUZZ_SEED=${SEED} random case ${i}: ${String(error)}`);
691 }
692 }
693 for (const file of files) {
694 const bytes = encoder.encode(file);
695 const step = Math.max(1, Math.floor(bytes.length / RAW_RUNS));
696 for (let cut = 0; cut < bytes.length; cut += step) {
697 try {
698 assertInert(drawRaw(kind, path, bytes.subarray(0, cut), cut % 2 === 0));
699 } catch (error) {
700 throw new Error(`${kind} ${path}: truncated at byte ${cut}: ${String(error)}`);
701 }
702 }
703 }
704 }
705 });
706 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs