Public: anyone can read this; the crew can change it.

A .wav file shows its waveform, drawn on the server beside the player #14465

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14450-a-wav-file-shows-its-waveform-drawn-on-t → main 6 commits fork 4 threads · 4 unresolved

  • Merged. mason/14450-a-wav-file-shows-its-waveform-drawn-on-t was merged into main as d5b83cd.

Review

Sign in to review this change.
11 files changed +304 −5
docs/design/areas/code.md +2
@@ -18,3 +18,5 @@ An HTML file's preview (`render/html_preview.ts`): `.html-frame` is a bare sandb
1818 - `.front-matter` — a Markdown file's leading `---` block (`frontTable` in `render/markdown.ts`): a `dl.status-rows` of `.status-row` facts above the prose, keys in `--muted-fg`, keys and values wrapping anywhere so a long value never scrolls the page at 360px.
1919
2020 - `.hexdump` — the last-resort view of a binary file (`render/hexdump.ts`): a `pre.card` of offset, sixteen hex bytes and their ASCII in `--mono` 12.5px, scrolling sideways inside its own box so the page never does at 390px.
21
22 - `.waveform` — a PCM WAV's peaks above its player (`render/audio.ts`): a `figure` with its browser margins zeroed and 12px under it, an SVG 96px tall on `--muted` with `--muted-fg` strokes, and a caption line whose height (1.5em) is reserved so nothing shifts; the caption wraps anywhere, so it never scrolls the page at 360px.
docs/features/file-media.md +6
@@ -93,6 +93,12 @@ the media route and the registry's type (`render/audio.ts`,
9393 seeks. A `.mov` also offers itself as `video/mp4`, which is what most
9494 QuickTime files are inside, for browsers that refuse `video/quicktime`.
9595
96 - **WAV waveform:** a PCM `.wav` (8, 16 or 24-bit) up to 8 MB (`WAVE_CAP`, about 47 s of 44.1 kHz stereo 16-bit; the source reads a `.wav` that far, where other media kinds get no body past 1 MB) draws
97 above the player as an SVG of at most 800 peak columns, with duration, sample
98 rate and channels as text (`render/waveform.ts`, drawn by `render/audio.ts`).
99 No script. A file that is malformed, not integer PCM or over the cap, and every
100 other audio format, keeps the plain player. The terminal's file pane names the
101 same facts in its audio note (`tui/ui/media-note.ts`).
96102 - **Header:** size and container (`3.2 MB · WebM`) in place of a line count.
97103 - **Video** is the pane's width at its own aspect ratio, at most 80vh tall.
98104 - **A format the browser cannot decode** fires `error`; the shell script
src/tui/screens/workspace.ts +1 −1
@@ -1528,7 +1528,7 @@ export function workspaceScreen(
15281528 const media =
15291529 read.value.kind === "text"
15301530 ? undefined
1531 : mediaNote(t.path, formatSize(read.value.size), mediaWeb(), ref);
1531 : mediaNote(t.path, formatSize(read.value.size), mediaWeb(), ref, read.value.kind === "binary" ? read.value.body : undefined);
15321532 const tail =
15331533 read.value.kind === "too_large" && read.value.tail !== undefined
15341534 ? tailOf(read.value.tail, read.value.size)
src/tui/ui/media-note.ts +9 −1
@@ -5,6 +5,7 @@
55 */
66
77 import { claimedMedia } from "../../web/render/kinds.ts";
8 import { readWaveform, waveFacts } from "../../web/render/waveform.ts";
89
910 export function mediaNote(
1011 path: string,
@@ -12,10 +13,17 @@ export function mediaNote(
1213 /** `https://forge…/owner/name`, or undefined for a mirror the forge does not serve. */
1314 web: string | undefined,
1415 ref: string,
16 /**
17 * The file's bytes when the screen holds them: a PCM WAV then names its duration, rate, channels
18 * and depth.
19 */
20 body?: Uint8Array,
1521 ): string | undefined {
1622 const media = claimedMedia(path);
1723 if (media === undefined || (media.kind !== "audio" && media.kind !== "video")) return undefined;
18 const what = `${media.kind === "audio" ? "Audio" : "Video"}, ${media.name}, ${size} — not played in the terminal.`;
24 const wave = media.kind === "audio" && body !== undefined && /\.wav$/i.test(path) ? readWaveform(body) : undefined;
25 const facts = wave === undefined ? "" : ` ${waveFacts(wave)}.`;
26 const what = `${media.kind === "audio" ? "Audio" : "Video"}, ${media.name}, ${size} — not played in the terminal.${facts}`;
1927 return web === undefined
2028 ? what
2129 : `${what} Play it in a browser: ${web}/media/${encodeURIComponent(ref)}/${path.split("/").map(encodeURIComponent).join("/")}`;
src/view/local.ts +2 −1
@@ -9,6 +9,7 @@
99 */
1010
1111 import { STL_LIMIT } from "../web/render/stl.ts";
12 import { WAVE_CAP } from "../web/render/waveform.ts";
1213 import { pullProposalAt } from "../core/proposal_token.ts";
1314 import { mayView, sealedScope } from "../core/sealed.ts";
1415 import type { App } from "../app.ts";
@@ -70,7 +71,7 @@ const MAX_BLOB_BYTES = 5_000_000;
7071 * up to the size its facts are read.
7172 */
7273 const blobCeiling = (path: string): number =>
73 /\.(gcode|gco)$/i.test(path) ? RENDER_CAP.gcode : /\.stl$/i.test(path) ? STL_LIMIT : MAX_BLOB_BYTES;
74 /\.(gcode|gco)$/i.test(path) ? RENDER_CAP.gcode : /\.stl$/i.test(path) ? STL_LIMIT : /\.wav$/i.test(path) ? WAVE_CAP : MAX_BLOB_BYTES;
7475 const COMMIT_PAGE = 50;
7576
7677 export function localSource(
src/web/pages/repo.ts +4
@@ -8,6 +8,7 @@
88 * path segment and can never smuggle a path of its own.
99 */
1010
11 import { WAVE_CAP } from "../render/waveform.ts";
1112 import { repoLine } from "../describe.ts";
1213 import { recordLook } from "../../flow/valve_store.ts";
1314 import { PINNED_PUBLIC, immutableWhenAddressed, mayShare } from "../../http/addressed.ts";
@@ -2039,6 +2040,9 @@ export function capped(blob: Blob, path = ""): Blob {
20392040 return blob;
20402041 if (blob.kind !== "too_large" && blob.size <= READ_CEILING && fileKind(path, blob.body.subarray(0, HEAD_BYTES)) === "psd")
20412042 return blob;
2043 // A WAV is read whole up to `WAVE_CAP`, because its waveform is drawn from the samples.
2044 if (blob.kind !== "too_large" && blob.size <= WAVE_CAP && fileKind(path, blob.body.subarray(0, HEAD_BYTES)) === "audio" && /\.wav$/i.test(path))
2045 return blob;
20422046 // A table file's body is already bounded by its prefix; the renderer draws a window of it.
20432047 if (blob.kind !== "too_large" && blob.body.length <= CSV_PREFIX_BYTES && isTablePath(path)) return blob;
20442048 // A big JSON file's body is already its prefix; the renderer outlines it.
src/web/render/audio.ts +30 −2
@@ -1,8 +1,36 @@
1 /** An audio file plays inline: `<audio controls preload="metadata">`. */
1 /**
2 * An audio file plays inline: `<audio controls preload="metadata">`. A PCM WAV the page holds
3 * (up to `WAVE_CAP`) also shows its waveform above the player, drawn here as an SVG of peak
4 * columns with its duration, sample rate and channels as the caption; the SVG is hidden from
5 * screen readers so the facts are read once. No script, no external asset.
6 */
27
8 import { html } from "../html.ts";
39 import { player } from "./player.ts";
410 import type { Rendered, RenderInput } from "./renderers.ts";
11 import { readWaveform, waveFacts, type Waveform } from "./waveform.ts";
12
13 const HEIGHT = 96;
14
15 function draw(w: Waveform) {
16 const mid = HEIGHT / 2;
17 const path = w.peaks
18 .map((p, i) => {
19 const half = Math.max(0.5, p * (mid - 1));
20 return `M${i + 0.5} ${(mid - half).toFixed(1)}V${(mid + half).toFixed(1)}`;
21 })
22 .join("");
23 const facts = waveFacts(w);
24 return html`<figure class="waveform">
outdated

Principle 6/3: <figure class="waveform"> has no CSS. Use tokens and zero the default figure margin.

outdated

Principle 1: nothing explains itself twice. The aria-label restates the figcaption's facts, so they are read twice. Drop role/aria-label and set aria-hidden="true" on the svg; the figcaption carries the text.

25 <svg viewBox="0 0 ${w.peaks.length} ${HEIGHT}" width="100%" height="${HEIGHT}" preserveAspectRatio="none" aria-hidden="true" focusable="false"><path d="${path}" stroke="currentColor" stroke-width="1" fill="none" /></svg>
26 <figcaption class="muted small">${facts}</figcaption>
27 </figure>`;
28 }
529
630 export function audio(input: RenderInput): Rendered {
7 return player("audio", input);
31 const plain = player("audio", input);
32 if (!/\.wav$/i.test(input.path)) return plain;
33 const wave = readWaveform(input.body);
outdated

Dead defence: readWaveform returns undefined for bad input and bounds every read, so this try/catch (and the length===0 check above, which readWaveform already rejects) is unreachable. Remove both.

34 if (wave === undefined) return plain;
35 return { ...plain, content: html`${draw(wave)}${plain.content}` };
836 }
src/web/render/waveform.ts added +95
@@ -0,0 +1,95 @@
1 /**
2 * The facts and the peaks of a PCM WAV file, read from bytes the page already holds.
3 *
4 * The file is untrusted: `WAVE_CAP` bounds the size before anything is parsed, the chunk walk
5 * is bounded by the bytes present, and the sample scan visits each sample once at most, so every
6 * loop is bounded by the cap. Anything that is not 8, 16 or 24-bit integer PCM returns
7 * undefined and the caller keeps the plain player.
8 */
9
10 /**
11 * The largest WAV the page reads to draw a waveform (about 47 s of 44.1 kHz stereo 16-bit): the
12 * source raises its read ceiling for a `.wav` to this (`blobCeiling` in `view/local.ts`), so it is
13 * reachable. A bigger one keeps the plain player.
14 */
15 export const WAVE_CAP = 8_000_000;
16 /** The most peak columns drawn. */
outdated

WAVE_CAP is unreachable while the page reads at most ~1 MB. Align the cap to the real read limit or extend the read.

17 export const MAX_COLUMNS = 800;
18
19 export interface Waveform {
20 readonly seconds: number;
21 readonly sampleRate: number;
22 readonly channels: number;
23 readonly bits: 8 | 16 | 24;
24 /** One 0..1 peak (loudest sample, any channel) per column. */
25 readonly peaks: readonly number[];
26 }
27
28 const tag = (b: Uint8Array, at: number): string => String.fromCharCode(b[at]!, b[at + 1]!, b[at + 2]!, b[at + 3]!);
29
30 export function readWaveform(body: Uint8Array): Waveform | undefined {
31 if (body.length < 44 || body.length > WAVE_CAP) return undefined;
32 if (tag(body, 0) !== "RIFF" || tag(body, 8) !== "WAVE") return undefined;
33 const v = new DataView(body.buffer, body.byteOffset, body.byteLength);
34 let format: { channels: number; rate: number; bits: number; align: number } | undefined;
35 let at = 12;
36 while (at + 8 <= body.length) {
37 const id = tag(body, at);
38 const size = v.getUint32(at + 4, true);
39 const start = at + 8;
40 if (id === "fmt " && size >= 16 && start + 16 <= body.length) {
41 let code = v.getUint16(start, true);
42 // WAVE_FORMAT_EXTENSIBLE names its real format in the first two bytes of the sub-format.
43 if (code === 0xfffe && size >= 26 && start + 26 <= body.length) code = v.getUint16(start + 24, true);
44 if (code !== 1) return undefined;
45 format = {
46 channels: v.getUint16(start + 2, true),
47 rate: v.getUint32(start + 4, true),
48 align: v.getUint16(start + 12, true),
49 bits: v.getUint16(start + 14, true),
50 };
51 } else if (id === "data") {
52 if (format === undefined) return undefined;
53 const { channels, rate, bits, align } = format;
54 if (bits !== 8 && bits !== 16 && bits !== 24) return undefined;
55 const width = bits / 8;
56 if (channels < 1 || channels > 32 || rate < 1 || rate > 1_000_000 || align !== width * channels) return undefined;
57 const length = Math.min(size, body.length - start);
58 const frames = Math.floor(length / align);
59 if (frames < 1) return undefined;
60 const columns = Math.min(MAX_COLUMNS, frames);
61 const peaks: number[] = [];
62 const full = 2 ** (bits - 1);
63 for (let c = 0; c < columns; c++) {
64 const from = Math.floor((c * frames) / columns);
65 const to = Math.floor(((c + 1) * frames) / columns);
66 let peak = 0;
67 for (let f = from; f < to; f++) {
68 for (let ch = 0; ch < channels; ch++) {
69 const o = start + f * align + ch * width;
70 const s = bits === 8 ? body[o]! - 128 : bits === 16 ? v.getInt16(o, true) : (body[o]! | (body[o + 1]! << 8) | (body[o + 2]! << 16)) << 8 >> 8;
71 const a = Math.abs(s);
72 if (a > peak) peak = a;
73 }
74 }
75 peaks.push(Math.min(1, peak / full));
76 }
77 return { seconds: frames / rate, sampleRate: rate, channels, bits, peaks };
78 }
79 // Chunks are word-aligned.
80 at = start + size + (size & 1);
81 }
82 return undefined;
83 }
84
85 /**
86 * The facts as one line: `0:01.0 · 8000 Hz · mono · 16-bit`. Numbers only, nothing from the file as
87 * text.
88 */
89 export function waveFacts(w: Waveform): string {
90 const total = Math.round(w.seconds * 10) / 10;
91 const m = Math.floor(total / 60);
92 const clock = `${m}:${(total - m * 60).toFixed(1).padStart(4, "0")}`;
93 const channels = w.channels === 1 ? "mono" : w.channels === 2 ? "stereo" : `${w.channels} channels`;
94 return `${clock} · ${w.sampleRate} Hz · ${channels} · ${w.bits}-bit`;
95 }
src/web/style/content.ts +3
@@ -577,6 +577,9 @@ img.img-box, .prose img.img-box, .tell-picture.img-box { display: block; width:
577577 /* An audio or video file, played inline (render/player.ts). The video is as
578578 wide as the pane and no wider; height follows its own aspect ratio. */
579579 .media-pane .media-player { display: block; width: 100%; max-width: 100%; }
580 .media-pane .waveform { margin: 0 0 12px; }
581 .waveform svg { display: block; width: 100%; height: 96px; color: var(--muted-fg); background: var(--muted); border-radius: var(--radius-sm); }
582 .waveform figcaption { margin-top: 4px; min-height: 1.5em; overflow-wrap: anywhere; }
580583 .media-pane video.media-player { height: auto; max-height: 80vh; background: var(--muted); border-radius: var(--radius-sm); }
581584
582585 /* An HTML file's page in a sandboxed frame: one hairline, a fixed height, so nothing shifts. */
test/blob-media-ceiling.test.ts +7
@@ -60,6 +60,13 @@ describe("the page ceiling, by kind", () => {
6060 expect(pdf.seen[0]!.media).toBe("/m");
6161 });
6262
63 test("a 3 MB WAV keeps its body for the waveform; a 9 MB one and a 3 MB MP3 do not", () => {
64 const wav = (n: number) => filled("RIFF\0\0\0\0WAVEfmt ", n, 0);
65 expect(capped(blob(wav(3 * MB)), "a.wav").kind).toBe("binary");
66 expect(capped(blob(wav(9 * MB)), "a.wav").kind).toBe("too_large");
67 expect(capped(blob(filled("ID3", 3 * MB, 0)), "a.mp3").kind).toBe("too_large");
68 });
69
6370 test("capped keeps only the head of a body it drops", () => {
6471 const cut = capped(blob(PDF));
6572 expect(cut.kind).toBe("too_large");
test/render-waveform.test.ts added +145
@@ -0,0 +1,145 @@
1 /** A PCM WAV draws its waveform beside the player; anything else keeps the plain player. */

Add tests for the extensible format, odd chunk padding, and 24-bit negative samples.

2
3 import { describe, expect, test } from "bun:test";
4 import { audio } from "../src/web/render/audio.ts";
5 import { mediaNote } from "../src/tui/ui/media-note.ts";
6 import { MAX_COLUMNS, readWaveform, WAVE_CAP } from "../src/web/render/waveform.ts";
7
8 function wav(opts: { bits?: number; channels?: number; rate?: number; frames?: number; code?: number; sample?: (f: number) => number }): Uint8Array {
9 const { bits = 16, channels = 1, rate = 8000, frames = 8000, code = 1, sample = (f) => Math.sin(f / 20) } = opts;
10 const width = bits / 8;
11 const data = frames * channels * width;
12 const out = new Uint8Array(44 + data);
13 const v = new DataView(out.buffer);
14 out.set(new TextEncoder().encode("RIFF"), 0);
15 v.setUint32(4, 36 + data, true);
16 out.set(new TextEncoder().encode("WAVEfmt "), 8);
17 v.setUint32(16, 16, true);
18 v.setUint16(20, code, true);
19 v.setUint16(22, channels, true);
20 v.setUint32(24, rate, true);
21 v.setUint32(28, rate * channels * width, true);
22 v.setUint16(32, channels * width, true);
23 v.setUint16(34, bits, true);
24 out.set(new TextEncoder().encode("data"), 36);
25 v.setUint32(40, data, true);
26 for (let f = 0; f < frames; f++)
27 for (let c = 0; c < channels; c++) {
28 const s = sample(f);
29 const o = 44 + (f * channels + c) * width;
30 if (bits === 8) out[o] = Math.round(s * 127) + 128;
31 else if (bits === 16) v.setInt16(o, Math.round(s * 32767), true);
32 else {
33 const n = Math.round(s * 8388607);
34 out[o] = n & 255;
35 out[o + 1] = (n >> 8) & 255;
36 out[o + 2] = (n >> 16) & 255;
37 }
38 }
39 return out;
40 }
41
42 const input = (path: string, body: Uint8Array) => ({ path, size: body.length, body, media: "/m/x.wav", download: "/d/x.wav" });
43
44 describe("waveform", () => {
45 test.each([8, 16, 24])("reads %i-bit PCM", (bits) => {
46 const w = readWaveform(wav({ bits, channels: 2 }))!;
47 expect(w.bits).toBe(bits);
48 expect(w.channels).toBe(2);
49 expect(w.seconds).toBeCloseTo(1);
50 expect(w.peaks.length).toBe(MAX_COLUMNS);
51 expect(Math.max(...w.peaks)).toBeGreaterThan(0.9);
52 });
53
54 test("draws an SVG and the facts above the player", () => {
55 const html = String(audio(input("a.wav", wav({}))).content);
56 expect(html).toContain("<svg");
57 expect(html).toContain("8000 Hz");
58 expect(html).toContain("mono");
59 expect(html.indexOf("<svg")).toBeLessThan(html.indexOf("<audio"));
60 });
61
62 test("the caption alone carries the facts: the SVG is hidden from screen readers", () => {
63 const html = String(audio(input("a.wav", wav({}))).content);
64 expect(html).toContain('aria-hidden="true"');
65 expect(html).not.toContain("aria-label");
66 expect(html.split("8000 Hz").length - 1).toBe(1);
67 });
68
69 test("an empty or truncated .wav plays without a waveform", () => {
70 expect(String(audio(input("a.wav", new Uint8Array(0))).content)).not.toContain("waveform");
71 expect(String(audio(input("a.wav", wav({}).slice(0, 30))).content)).not.toContain("waveform");
72 });
73
74 test("24-bit samples sign-extend: full-scale negative peaks at 1", () => {
75 const body = wav({ bits: 24, frames: 4, sample: () => -1 });
76 // -1 * 8388607 = -8388607 (0x800001): the top bit of the third byte is set.
77 expect(readWaveform(body)!.peaks).toEqual([8388607 / 8388608, 8388607 / 8388608, 8388607 / 8388608, 8388607 / 8388608]);
78 const min = wav({ bits: 24, frames: 1 });
79 min.set([0, 0, 0x80], 44);
80 expect(readWaveform(min)!.peaks).toEqual([1]);
81 const quiet = wav({ bits: 24, frames: 1 });
82 quiet.set([0xff, 0xff, 0xff], 44); // -1 of 8388608
83 expect(readWaveform(quiet)!.peaks[0]).toBeCloseTo(1 / 8388608, 10);
84 });
85
86 test("WAVE_FORMAT_EXTENSIBLE with a PCM sub-format reads; with another does not", () => {
87 const base = wav({ frames: 100 });
88 const ext = (sub: number) => {
89 // The fmt chunk grows from 16 to 40 bytes: cbSize, valid bits, mask, then the GUID.
90 const out = new Uint8Array(base.length + 24);
91 out.set(base.subarray(0, 36), 0);
92 out.set(base.subarray(36), 60);
93 const v = new DataView(out.buffer);
94 v.setUint32(16, 40, true);
95 v.setUint16(20, 0xfffe, true);
96 v.setUint16(36, 22, true);
97 v.setUint16(44, sub, true); // first two bytes of the sub-format GUID
98 return out;
99 };
100 expect(readWaveform(ext(1))!.peaks.length).toBe(100);
101 expect(readWaveform(ext(3))).toBeUndefined();
102 });
103
104 test("an odd-size chunk is padded to a word before the next", () => {
105 const base = wav({ frames: 50 });
106 const odd = new Uint8Array(base.length + 8 + 3 + 1);
107 odd.set(base.subarray(0, 36), 0);
108 odd.set(new TextEncoder().encode("junk"), 36);
109 new DataView(odd.buffer).setUint32(40, 3, true);
110 odd.set(base.subarray(36), 36 + 8 + 4); // 3 bytes + 1 pad byte
111 expect(readWaveform(odd)!.peaks.length).toBe(50);
112 });
113
114 test("the terminal's note names the facts, and only for a WAV it holds", () => {
115 const note = mediaNote("a.wav", "16 K", undefined, "main", wav({}));
116 expect(note).toContain("0:01.0 · 8000 Hz · mono · 16-bit");
117 expect(note).toContain("not played in the terminal");
118 expect(mediaNote("a.wav", "16 K", undefined, "main")).toBe("Audio, WAV, 16 K — not played in the terminal.");
119 expect(mediaNote("a.wav", "1 K", undefined, "main", new Uint8Array(20))).toBe("Audio, WAV, 1 K — not played in the terminal.");
120 });
121
122 test("a short file has one column per frame", () => {
123 expect(readWaveform(wav({ frames: 10 }))!.peaks.length).toBe(10);
124 });
125
126 test("malformed, non-PCM, truncated and oversized files keep the plain player", () => {
127 const plain = (b: Uint8Array) => {
128 const html = String(audio(input("a.wav", b)).content);
129 return html.includes("<audio") && !html.includes("waveform");
130 };
131 expect(plain(wav({ code: 3, bits: 16 }))).toBe(true);
132 expect(plain(wav({ bits: 32 }))).toBe(true);
133 expect(plain(new Uint8Array(10))).toBe(true);
134 expect(plain(wav({}).subarray(0, 40))).toBe(true);
135 expect(readWaveform(new Uint8Array(WAVE_CAP + 1))).toBeUndefined();
136 const lying = wav({ frames: 100 });
137 new DataView(lying.buffer).setUint32(40, 0xffffffff, true);
138 expect(readWaveform(lying)!.peaks.length).toBe(100);
139 expect(plain(new Uint8Array(0))).toBe(true);
140 });
141
142 test("other audio keeps the plain player", () => {
143 expect(String(audio(input("a.mp3", wav({}))).content)).not.toContain("waveform");
144 });
145 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs