Public: anyone can read this; the crew can change it.
A .log or .txt file over 5 MB opens its last 1,000 lines with a 'jump to end' link and says how much it left out #13622
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13603-a-log-or-txt-file-over-5-mb-opens-its-la → main 5 commits fork 2 threads · 2 unresolved
12 files
changed +448
−5
docs/features/README.md +1
| @@ -221,6 +221,7 @@ Read with the guide page [Your apps](../guide-your-apps.md). | ||
| 221 | 221 | - [](app-templates.md) |
| 222 | 222 | - [](archive-listing.md) |
| 223 | 223 | - [](binary-file.md) |
| 224 | - [](log-tail.md) | |
| 224 | 225 | - [](crawl-budget.md) |
| 225 | 226 | - [](data-browser.md) |
| 226 | 227 | - [](db-size.md) |
docs/features/log-tail.md added +36
| @@ -0,0 +1,36 @@ | ||
| 1 | # A big .log or .txt opens its last lines | |
| 2 | ||
| 3 | A `.log` or `.txt` file over 5 MB is too large to draw whole. Its page opens the | |
| 4 | last 1,000 lines instead, under one sentence: `Showing the last 1,000 of about | |
| 5 | 80,000 lines; the earlier lines are left out.` A **Jump to end** link (an | |
| 6 | anchor, no script) goes to the last line, and **Download** gives the whole file. | |
| 7 | The count is an estimate from the size and the lines seen. | |
| 8 | ||
| 9 | The rows carry no line numbers: the page knows the lines are the last ones, not | |
| 10 | which numbers they are, and a wrong number in a link is worse than none. | |
| 11 | ||
| 12 | Only `.log` and `.txt` names tail. A log is read from its end; `.md`, `.json` | |
| 13 | and source are read from the top, so a cut of their end says little. They keep | |
| 14 | the "Too large to render" note and a download. | |
| 15 | ||
| 16 | ## How it is read, and what it costs | |
| 17 | ||
| 18 | Git stores a blob whole and has no byte range, so the file is streamed past a | |
| 19 | 512 KB buffer (memory stays that small) and the front is dropped. That takes | |
| 20 | time in proportion to the size, about 0.3 s for 32 MB on the forge's machine. | |
| 21 | So: | |
| 22 | ||
| 23 | - the tail is kept by blob id (an id names its bytes forever, so a hit is never | |
| 24 | stale); a second open is a few milliseconds; | |
| 25 | - at most two scans run at once; the rest wait; | |
| 26 | - over **32 MB** the file is not scanned and reads "Too large to render"; | |
| 27 | - a stream whose length is not the file's size gives no tail. | |
| 28 | ||
| 29 | The partial first line is dropped, so a cut never garbles a character. | |
| 30 | ||
| 31 | ## Where it does not apply | |
| 32 | ||
| 33 | - A mirror browsed on `/origins`, where the file is not in a local repository, | |
| 34 | has no tail: it reads "Too large to render" as before. | |
| 35 | - The terminal's file view shows the same sentence and lines, with the download | |
| 36 | address on the line below. |
src/git/read.ts +112
| @@ -764,3 +764,115 @@ export async function blobStream( | ||
| 764 | 764 | }, |
| 765 | 765 | }); |
| 766 | 766 | } |
| 767 | ||
| 768 | /** The most a tail read streams past: beyond it the file is only offered for download. */ | |
| 769 | export const TAIL_SCAN_MAX = 32 * 1024 * 1024; | |
| 770 | /** Scans that run at once; the rest wait their turn. */ | |
| 771 | const TAIL_SCANS_AT_ONCE = 2; | |
| 772 | /** Tails kept, by blob id: an id names its bytes forever, so a hit is never stale. */ | |
| 773 | const TAIL_KEPT = 16; | |
| 774 | class TailKeep { | |
| 775 | // Content-addressed (the key holds the blob id), so no forge can see another's state. | |
| 776 | private readonly held = new Map<string, Promise<Uint8Array | undefined>>(); | |
| 777 | get(key: string): Promise<Uint8Array | undefined> | undefined { | |
| 778 | return this.held.get(key); | |
| 779 | } | |
| 780 | keep(key: string, scan: Promise<Uint8Array | undefined>): void { | |
| 781 | this.held.set(key, scan); | |
| 782 | if (this.held.size > TAIL_KEPT) this.held.delete(this.held.keys().next().value!); | |
| 783 | } | |
| 784 | drop(key: string): void { | |
| 785 | this.held.delete(key); | |
| 786 | } | |
| 787 | } | |
| 788 | const tailsKept = new TailKeep(); | |
| 789 | let tailScans = 0; | |
| 790 | const tailWaiting: Array<() => void> = []; | |
| 791 | ||
| 792 | async function inTurn<T>(work: () => Promise<T>): Promise<T> { | |
| 793 | if (tailScans >= TAIL_SCANS_AT_ONCE) await new Promise<void>((go) => tailWaiting.push(go)); | |
| 794 | tailScans++; | |
| 795 | try { | |
| 796 | return await work(); | |
| 797 | } finally { | |
| 798 | tailScans--; | |
| 799 | tailWaiting.shift()?.(); | |
| 800 | } | |
| 801 | } | |
| 802 | ||
| 803 | /** | |
| 804 | * A blob's last `bytes` bytes. Git stores a blob whole, so there is no byte | |
| 805 | * range to seek to: the stream is read and dropped. That costs time in | |
| 806 | * proportion to the size, so the answer is kept by blob id and at most | |
| 807 | * `TAIL_SCANS_AT_ONCE` scans run together. A file over `TAIL_SCAN_MAX`, or one | |
| 808 | * whose stream is not `size` bytes long, has no tail (`undefined`). | |
| 809 | */ | |
| 810 | export async function blobTail( | |
| 811 | dir: string, | |
| 812 | rev: string, | |
| 813 | path: string, | |
| 814 | bytes: number, | |
| 815 | size: number, | |
| 816 | ): Promise<Uint8Array | undefined> { | |
| 817 | if (size > TAIL_SCAN_MAX || size <= bytes) return undefined; | |
| 818 | const spec = `${rev}:${path}`; | |
| 819 | assertBatchSafe(spec); | |
| 820 | const hold = readerHold(dir); | |
| 821 | if (hold !== undefined && !(await namesOnlyOwn(hold, [rev]))) return undefined; | |
| 822 | const named = await runGit(["rev-parse", "--verify", spec], { cwd: dir }); | |
| 823 | if (!named.ok) return undefined; | |
| 824 | const key = `${dir}\0${named.value.trim()}\0${bytes}`; | |
| 825 | const kept = tailsKept.get(key); | |
| 826 | if (kept !== undefined) return kept; | |
| 827 | const scan = inTurn(() => scanTail(dir, spec, bytes, size)); | |
| 828 | tailsKept.keep(key, scan); | |
| 829 | const out = await scan; | |
| 830 | if (out === undefined) tailsKept.drop(key); | |
| 831 | return out; | |
| 832 | } | |
| 833 | ||
| 834 | async function scanTail(dir: string, spec: string, bytes: number, size: number): Promise<Uint8Array | undefined> { | |
| 835 | let proc: Bun.Subprocess<"ignore", "pipe", "ignore">; | |
| 836 | try { | |
| 837 | proc = Bun.spawn(["git", "cat-file", "blob", spec], { | |
| 838 | cwd: dir, | |
| 839 | env: gitEnv(), | |
| 840 | stdin: "ignore", | |
| 841 | stdout: "pipe", | |
| 842 | stderr: "ignore", | |
| 843 | }); | |
| 844 | } catch { | |
| 845 | return undefined; | |
| 846 | } | |
| 847 | const reader = proc.stdout.getReader(); | |
| 848 | const ring = new Uint8Array(bytes); | |
| 849 | let seen = 0; | |
| 850 | try { | |
| 851 | for (;;) { | |
| 852 | const next = await reader.read(); | |
| 853 | if (next.done) break; | |
| 854 | let chunk = next.value; | |
| 855 | if (chunk.length >= bytes) { | |
| 856 | ring.set(chunk.subarray(chunk.length - bytes)); | |
| 857 | seen += chunk.length; | |
| 858 | continue; | |
| 859 | } | |
| 860 | const at = seen % bytes; | |
| 861 | const room = bytes - at; | |
| 862 | if (chunk.length <= room) ring.set(chunk, at); | |
| 863 | else { | |
| 864 | ring.set(chunk.subarray(0, room), at); | |
| 865 | ring.set(chunk.subarray(room), 0); | |
| 866 | } | |
| 867 | seen += chunk.length; | |
| 868 | } | |
| 869 | } finally { | |
| 870 | proc.kill("SIGKILL"); | |
| 871 | } | |
| 872 | if (seen !== size) return undefined; | |
| 873 | const start = seen % bytes; | |
| 874 | const out = new Uint8Array(bytes); | |
| 875 | out.set(ring.subarray(start), 0); | |
| 876 | out.set(ring.subarray(0, start), bytes - start); | |
| 877 | return out; | |
| 878 | } |
src/tui/screens/workspace.ts +8 −1
| @@ -51,6 +51,7 @@ import type { TreeEntry } from "../../git/read.ts"; | ||
| 51 | 51 | import { parseUnifiedDiff, type Diff } from "../../git/diff.ts"; |
| 52 | 52 | import { renderDiff as renderPatch } from "../ui/diff.ts"; |
| 53 | 53 | import { fileKind, isEnvExample, kindByName, HEAD_BYTES, NO_BYTES, RENDER_CAP, sniffType } from "../../web/render/kinds.ts"; |
| 54 | import { tailOf, tailSentence } from "../../web/log_tail.ts"; | |
| 54 | 55 | import { downloadHref } from "../../web/pages/media.ts"; |
| 55 | 56 | import { isModelPath, modelFacts } from "../../web/render/model-facts.ts"; |
| 56 | 57 | import type { RepoFacts, RepoSource } from "../../view/source.ts"; |
| @@ -1396,10 +1397,16 @@ export function workspaceScreen( | ||
| 1396 | 1397 | read.value.kind === "text" |
| 1397 | 1398 | ? undefined |
| 1398 | 1399 | : mediaNote(t.path, formatSize(read.value.size), mediaWeb(), ref); |
| 1400 | const tail = | |
| 1401 | read.value.kind === "too_large" && read.value.tail !== undefined | |
| 1402 | ? tailOf(read.value.tail, read.value.size) | |
| 1403 | : undefined; | |
| 1399 | 1404 | if (read.value.kind !== "text" && isModelPath(t.path)) |
| 1400 | 1405 | model(read.value.kind === "binary" ? read.value.body : null, read.value.size); |
| 1401 | 1406 | else if (media !== undefined) note = media; |
| 1402 | else if (read.value.kind === "too_large") | |
| 1407 | else if (tail !== undefined) { | |
| 1408 | raw = new TextEncoder().encode(`${tailSentence(tail)}\n${downloadAt(t.path)}\n\n${tail.lines.join("\n")}\n`); | |
| 1409 | } else if (read.value.kind === "too_large") | |
| 1403 | 1410 | note = `${formatSize(read.value.size)} — too large to display.`; |
| 1404 | 1411 | else if (read.value.kind === "binary") { |
| 1405 | 1412 | // A binary blob whose *name* says picture is the one binary we do |
src/view/local.ts +4 −1
| @@ -35,6 +35,7 @@ import { pathsAtTree } from "../git/path_listing.ts"; | ||
| 35 | 35 | import { |
| 36 | 36 | blobForRender, |
| 37 | 37 | blobHead, |
| 38 | blobTail, | |
| 38 | 39 | commitLog, |
| 39 | 40 | listRefs, |
| 40 | 41 | listTree, |
| @@ -44,6 +45,7 @@ import { | ||
| 44 | 45 | import { repoDir } from "../git/repos.ts"; |
| 45 | 46 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 46 | 47 | import type { Page } from "../origins/types.ts"; |
| 48 | import { tailable, TAIL_BYTES } from "../web/log_tail.ts"; | |
| 47 | 49 | import { HEAD_BYTES, RENDER_CAP } from "../web/render/kinds.ts"; |
| 48 | 50 | import type { |
| 49 | 51 | Blob, |
| @@ -213,7 +215,8 @@ export function localSource( | ||
| 213 | 215 | if (blob.kind !== "too_large") return ok(blob); |
| 214 | 216 | // Too large to read, but its first bytes still say what it is. |
| 215 | 217 | const head = await blobHead(dir, ref, path, HEAD_BYTES); |
| 216 | return ok(head === undefined ? blob : { ...blob, head }); | |
| 218 | const tail = tailable(path) ? await blobTail(dir, ref, path, TAIL_BYTES, blob.size) : undefined; | |
| 219 | return ok({ ...blob, ...(head === undefined ? {} : { head }), ...(tail === undefined ? {} : { tail }) }); | |
| 217 | 220 | }, |
| 218 | 221 | |
| 219 | 222 | async readme(ref) { |
src/view/source.ts +2
| @@ -129,6 +129,8 @@ export type Blob = | ||
| 129 | 129 | * say what the file is, never the file. |
| 130 | 130 | */ |
| 131 | 131 | readonly head?: Uint8Array; |
| 132 | /** A `.log` or `.txt`'s last bytes (`TAIL_BYTES`), read by streaming past the rest. */ | |
| 133 | readonly tail?: Uint8Array; | |
| 132 | 134 | }; |
| 133 | 135 | |
| 134 | 136 | /** An issue or a pull, as a list row. */ |
src/web/code_rows.ts +10 −1
| @@ -4,7 +4,7 @@ | ||
| 4 | 4 | * for a cold one, from this one function, so the markup is the same either way. |
| 5 | 5 | */ |
| 6 | 6 | import { countLines, highlightPrefix } from "./code.ts"; |
| 7 | import { html, type Html } from "./html.ts"; | |
| 7 | import { html, type Html, raw } from "./html.ts"; | |
| 8 | 8 | |
| 9 | 9 | /** What a file page draws of a text file. */ |
| 10 | 10 | export interface CodeRows { |
| @@ -23,6 +23,15 @@ export function rowsMarkup(lines: readonly Html[]): string { | ||
| 23 | 23 | .join(""); |
| 24 | 24 | } |
| 25 | 25 | |
| 26 | /** | |
| 27 | * Rows with no line numbers, for lines whose true position is unknown; the last row is `#tail-end`. | |
| 28 | */ | |
| 29 | export function plainRowsMarkup(lines: readonly Html[]): string { | |
| 30 | return lines | |
| 31 | .map((line, i) => html`<tr${raw(i === lines.length - 1 ? ' id="tail-end"' : "")}><td><pre>${line}</pre></td></tr>`.text) | |
| 32 | .join(""); | |
| 33 | } | |
| 34 | ||
| 26 | 35 | /** The first `limit` lines of `text` highlighted for `path`, as rows. */ |
| 27 | 36 | export function codeRows(text: string, path: string, limit: number): CodeRows { |
| 28 | 37 | const lines = highlightPrefix(text, path, limit); |
src/web/log_tail.ts added +39
| @@ -0,0 +1,39 @@ | ||
| 1 | /** | |
| 2 | * The last lines of a file too large to open: plain bytes in, plain data out. | |
| 3 | * The bytes are a file's final stretch, so they begin mid-line, and may begin | |
| 4 | * mid-character; the partial first line is dropped, never shown. | |
| 5 | */ | |
| 6 | ||
| 7 | /** Lines a tail view draws. */ | |
| 8 | export const TAIL_LINES = 1_000; | |
| 9 | /** Bytes read from the end of the file: enough for 1,000 ordinary log lines. */ | |
| 10 | export const TAIL_BYTES = 512 * 1024; | |
| 11 | ||
| 12 | /** | |
| 13 | * Only these names open as a tail: a log is read from its end, while `.md`, | |
| 14 | * `.json` and source are read from the top and do not make sense cut. | |
| 15 | */ | |
| 16 | export const tailable = (path: string): boolean => /\.(log|txt)$/i.test(path); | |
| 17 | ||
| 18 | export interface Tail { | |
| 19 | readonly lines: readonly string[]; | |
| 20 | /** The file's line count, estimated from the lines seen and the size. */ | |
| 21 | readonly about: number; | |
| 22 | } | |
| 23 | ||
| 24 | /** The last `TAIL_LINES` whole lines of `bytes`, the end of a file `size` bytes long. */ | |
| 25 | export function tailOf(bytes: Uint8Array, size: number): Tail | undefined { | |
| 26 | const first = bytes.indexOf(10); | |
| 27 | if (first === -1) return undefined; | |
| 28 | const whole = bytes.subarray(first + 1); | |
| 29 | const text = new TextDecoder().decode(whole).replace(/\n$/, ""); | |
| 30 | if (text === "") return undefined; | |
| 31 | const all = text.split("\n"); | |
| 32 | const lines = all.slice(-TAIL_LINES); | |
| 33 | const perLine = whole.length / all.length; | |
| 34 | return { lines, about: Math.max(lines.length, Math.round(size / perLine)) }; | |
| 35 | } | |
| 36 | ||
| 37 | /** "showing the last 1,000 of about 80,000 lines". */ | |
| 38 | export const tailSentence = (t: Tail): string => | |
| 39 | `Showing the last ${t.lines.length.toLocaleString("en-US")} of about ${t.about.toLocaleString("en-US")} lines; the earlier lines are left out.`; |
src/web/pages/repo.ts +11 −1
| @@ -80,6 +80,7 @@ import { | ||
| 80 | 80 | } from "../chrome.ts"; |
| 81 | 81 | import { codeJs, liveRegionsJs } from "../client.ts"; |
| 82 | 82 | import { countLines, highlightPrefix } from "../code.ts"; |
| 83 | import { tailOf, tailSentence } from "../log_tail.ts"; | |
| 83 | 84 | import { latestWave } from "../../story/waves.ts"; |
| 84 | 85 | import { notePhase, takeSessionTime, timePhase, timePhaseAsync } from "../../core/route_timing.ts"; |
| 85 | 86 | import { liveClientJs, liveMount } from "../live.ts"; |
| @@ -91,7 +92,7 @@ import { field, readForm } from "../form.ts"; | ||
| 91 | 92 | import { html, raw, when, type Html } from "../html.ts"; |
| 92 | 93 | import { templateCard } from "../template_card.ts"; |
| 93 | 94 | import { creditLine } from "../credit_line.ts"; |
| 94 | import { type CodeRows, rowsMarkup } from "../code_rows.ts"; | |
| 95 | import { type CodeRows, plainRowsMarkup, rowsMarkup } from "../code_rows.ts"; | |
| 95 | 96 | import { gcodeOffThread, rowsOffThread } from "../code_rows_pool.ts"; |
| 96 | 97 | import { flashTo, fragment, page, redirect } from "../layout.ts"; |
| 97 | 98 | import { renderMarkdown, type MarkdownRepo } from "../markdown.ts"; |
| @@ -764,6 +765,15 @@ export function blobBody( | ||
| 764 | 765 | }), |
| 765 | 766 | }; |
| 766 | 767 | } |
| 768 | const tail = blob.kind === "too_large" && blob.tail !== undefined ? tailOf(blob.tail, blob.size) : undefined; | |
| 769 | if (blob.kind === "too_large" && tail !== undefined) { | |
| 770 | const rows = plainRowsMarkup(tail.lines.map((l) => html`${l}`)); | |
| 771 | return { | |
| 772 | lines: tail.lines.length, | |
| 773 | content: html`<p class="nb-cap" data-tail>${tailSentence(tail)} <a href="#tail-end">Jump to end</a> · <a href="${downloadHref(base, ref, path)}">Download all ${fileSize(blob.size)}</a></p> | |
| 774 | <div class="card code"><table>${raw(rows)}</table></div>`, | |
| 775 | }; | |
| 776 | } | |
| 767 | 777 | if (blob.kind === "too_large") { |
| 768 | 778 | return { |
| 769 | 779 | lines: 0, |
test/log-tail-doors.test.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** | |
| 2 | * A big .log's tail reaches both doors: the source's `blob`, and the terminal's file view (#13603). | |
| 3 | */ | |
| 4 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 5 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 6 | import { tmpdir } from "node:os"; | |
| 7 | import { join } from "node:path"; | |
| 8 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { repoDir } from "../src/git/repos.ts"; | |
| 10 | import { createApp, type App as Forge } from "../src/app.ts"; | |
| 11 | import type { User } from "../src/core/users.ts"; | |
| 12 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 13 | import type { TuiIdentity } from "../src/tui/session.ts"; | |
| 14 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 15 | import { localSource } from "../src/view/local.ts"; | |
| 16 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 17 | import { quietLogs } from "./helpers/globals.ts"; | |
| 18 | import { settleApp, settleUntil } from "./helpers/tui.ts"; | |
| 19 | ||
| 20 | quietLogs(); | |
| 21 | ||
| 22 | const LINES = 90_000; | |
| 23 | const big = Array.from({ length: LINES }, (_, i) => `INFO request ${i} done ${"x".repeat(60)}`).join("\n") + "\n"; | |
| 24 | ||
| 25 | let dataDir = ""; | |
| 26 | let forge: Forge; | |
| 27 | let alice: User; | |
| 28 | let repo: Awaited<ReturnType<typeof createRepo>> extends infer R ? (R extends { ok: true; value: infer V } ? V : never) : never; | |
| 29 | ||
| 30 | beforeAll(async () => { | |
| 31 | dataDir = await mkdtemp(join(tmpdir(), "forge-tail-doors-")); | |
| 32 | forge = createApp({ dataDir, port: 0 }); | |
| 33 | const a = await seedAdmin(forge, { username: "alice", email: "[email protected]", password: "a-long-passphrase" }); | |
| 34 | if (!a.ok) throw new Error("signup failed"); | |
| 35 | alice = a.value; | |
| 36 | const made = await createRepo(forge, alice, { name: "tails", visibility: "public", description: "files" }); | |
| 37 | if (!made.ok) throw new Error("repo failed"); | |
| 38 | repo = made.value; | |
| 39 | // Over the commit route's file ceiling, so the files go in by plumbing. | |
| 40 | const made2 = await commitToBranch(forge, alice, made.value, { message: "start", files: [{ path: "README.md", content: "# t\n" }] }); | |
| 41 | if (!made2.ok) throw new Error("commit failed"); | |
| 42 | const dir = repoDir(forge.reposRoot, "alice", "tails"); | |
| 43 | const git = (args: string[], stdin?: Uint8Array | string): string => { | |
| 44 | const input = typeof stdin === "string" ? new TextEncoder().encode(stdin) : (stdin ?? "ignore"); | |
| 45 | const run = Bun.spawnSync(["git", ...args], { | |
| 46 | cwd: dir, | |
| 47 | stdin: input, | |
| 48 | env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "[email protected]" }, | |
| 49 | }); | |
| 50 | if (run.exitCode !== 0) throw new Error(`git ${args[0]}: ${run.stderr.toString()}`); | |
| 51 | return run.stdout.toString().trim(); | |
| 52 | }; | |
| 53 | const body = new TextEncoder().encode(big); | |
| 54 | const oid = git(["hash-object", "-w", "--stdin"], body); | |
| 55 | const tree = git(["mktree"], `${git(["ls-tree", "main"])}\n100644 blob ${oid}\tbig.log\n100644 blob ${oid}\tbig.ts\n`); | |
| 56 | git(["update-ref", "refs/heads/main", git(["commit-tree", tree, "-p", "main", "-m", "big"])]); | |
| 57 | }, 60_000); | |
| 58 | afterAll(async () => { | |
| 59 | forge.close(); | |
| 60 | await rm(dataDir, { recursive: true, force: true }); | |
| 61 | }); | |
| 62 | ||
| 63 | describe("the source", () => { | |
| 64 | test("a too-large .log carries its tail", async () => { | |
| 65 | const read = await localSource(forge, repo).blob("main", "big.log"); | |
| 66 | expect(read.ok).toBe(true); | |
| 67 | if (!read.ok) return; | |
| 68 | expect(read.value.kind).toBe("too_large"); | |
| 69 | expect(read.value.kind === "too_large" && read.value.tail !== undefined).toBe(true); | |
| 70 | }); | |
| 71 | ||
| 72 | test("a too-large .ts carries none", async () => { | |
| 73 | const read = await localSource(forge, repo).blob("main", "big.ts"); | |
| 74 | expect(read.ok).toBe(true); | |
| 75 | if (!read.ok) return; | |
| 76 | expect(read.value.kind).toBe("too_large"); | |
| 77 | expect(read.value.kind === "too_large" && read.value.tail).toBeUndefined(); | |
| 78 | }); | |
| 79 | }); | |
| 80 | ||
| 81 | describe("the terminal", () => { | |
| 82 | const identity = (): TuiIdentity => ({ sshUsername: "alice", remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: alice }); | |
| 83 | ||
| 84 | async function view(file: string): Promise<string> { | |
| 85 | const terminal = headlessTerminal(140, 40); | |
| 86 | const { app } = createForgeApp(forge, identity(), { terminal, exitHooks: false }); | |
| 87 | app.start(); | |
| 88 | const send = (key: string): void => app.dispatch({ type: "key", key, ctrl: false, alt: false, shift: false }); | |
| 89 | const press = async (...keys: string[]): Promise<void> => { | |
| 90 | for (const k of keys) send(k); | |
| 91 | await settleApp(app); | |
| 92 | }; | |
| 93 | try { | |
| 94 | await settleApp(app); | |
| 95 | await press("/"); | |
| 96 | for (const ch of "tails") send(ch); | |
| 97 | await press("enter", "enter"); | |
| 98 | await settleUntil(app, "alice/tails"); | |
| 99 | await press("t"); | |
| 100 | for (const ch of file) send(ch); | |
| 101 | await press("enter"); | |
| 102 | await settleApp(app); | |
| 103 | return app.screen.toText(); | |
| 104 | } finally { | |
| 105 | app.stop(); | |
| 106 | } | |
| 107 | } | |
| 108 | ||
| 109 | test("a too-large .log draws the sentence, a way to download, and its last lines", async () => { | |
| 110 | const text = await view("big.log"); | |
| 111 | expect(text).toContain("Showing the last 1,000 of about"); | |
| 112 | expect(text).toContain("the earlier lines are left out"); | |
| 113 | expect(text).toContain("Download"); | |
| 114 | expect(text).toContain("INFO request"); | |
| 115 | }); | |
| 116 | ||
| 117 | test("a too-large .ts draws the too-large note and no tail", async () => { | |
| 118 | const text = await view("big.ts"); | |
| 119 | expect(text).toContain("too large to display"); | |
| 120 | expect(text).not.toContain("Showing the last"); | |
| 121 | }); | |
| 122 | }); |
test/log-tail-view.test.ts added +102
| @@ -0,0 +1,102 @@ | ||
| 1 | /** A .log or .txt over 5 MB opens its last 1,000 lines, and says what it left out (#13603). */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { blobTail } from "../src/git/read.ts"; | |
| 7 | import { TAIL_BYTES, TAIL_LINES, tailOf, tailable } from "../src/web/log_tail.ts"; | |
| 8 | import { blobBody } from "../src/web/pages/repo.ts"; | |
| 9 | ||
| 10 | const LINES = 120_000; | |
| 11 | // "é" is two bytes: some cut will land inside one. | |
| 12 | const line = (i: number): string => `2026-11-15 12:00:00 INFO café request ${i} <b>ok</b>`; | |
| 13 | const text = Array.from({ length: LINES }, (_, i) => line(i)).join("\n") + "\n"; | |
| 14 | const bytes = new TextEncoder().encode(text); | |
| 15 | ||
| 16 | let dir = ""; | |
| 17 | beforeAll(async () => { | |
| 18 | dir = await mkdtemp(join(tmpdir(), "forge-tail-")); | |
| 19 | const git = (args: string[], stdin?: Uint8Array): string => { | |
| 20 | const r = Bun.spawnSync(["git", ...args], { cwd: dir, stdin: stdin ?? "ignore" }); | |
| 21 | if (r.exitCode !== 0) throw new Error(r.stderr.toString()); | |
| 22 | return r.stdout.toString().trim(); | |
| 23 | }; | |
| 24 | git(["init", "-q", "--bare", "."]); | |
| 25 | const oid = git(["hash-object", "-w", "--stdin"], bytes); | |
| 26 | const tree = git(["mktree"], new TextEncoder().encode(`100644 blob ${oid}\tbig.log\n`)); | |
| 27 | const env = { GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" }; | |
| 28 | const c = Bun.spawnSync(["git", "commit-tree", tree, "-m", "x"], { cwd: dir, env: { ...process.env, ...env } }); | |
| 29 | git(["update-ref", "refs/heads/main", c.stdout.toString().trim()]); | |
| 30 | }); | |
| 31 | afterAll(() => rm(dir, { recursive: true, force: true })); | |
| 32 | ||
| 33 | describe("log tail", () => { | |
| 34 | test("the fixture is over 5 MB", () => { | |
| 35 | expect(bytes.length).toBeGreaterThan(5_000_000); | |
| 36 | }); | |
| 37 | ||
| 38 | test("the tail is read by streaming, holding only its own bytes", async () => { | |
| 39 | const tail = await blobTail(dir, "main", "big.log", TAIL_BYTES, bytes.length); | |
| 40 | expect(tail?.length).toBe(TAIL_BYTES); | |
| 41 | expect(Buffer.from(tail!).equals(Buffer.from(bytes.subarray(bytes.length - TAIL_BYTES)))).toBe(true); | |
| 42 | }); | |
| 43 | ||
| 44 | test("the tail is kept by blob id: a second read is the same bytes, not another scan", async () => { | |
| 45 | const first = await blobTail(dir, "main", "big.log", TAIL_BYTES, bytes.length); | |
| 46 | const again = await blobTail(dir, "main", "big.log", TAIL_BYTES, bytes.length); | |
| 47 | expect(again).toBe(first); | |
| 48 | }); | |
| 49 | ||
| 50 | test("several reads at once all get the tail", async () => { | |
| 51 | const reads = await Promise.all( | |
| 52 | Array.from({ length: 6 }, () => blobTail(dir, "main", "big.log", TAIL_BYTES + 1, bytes.length)), | |
| 53 | ); | |
| 54 | expect(reads.every((r) => r?.length === TAIL_BYTES + 1)).toBe(true); | |
| 55 | }); | |
| 56 | ||
| 57 | test("a stream that is not the size named has no tail", async () => { | |
| 58 | expect(await blobTail(dir, "main", "big.log", 1_000, bytes.length - 1)).toBeUndefined(); | |
| 59 | }); | |
| 60 | ||
| 61 | test("a file too big to scan has no tail", async () => { | |
| 62 | expect(await blobTail(dir, "main", "big.log", TAIL_BYTES, 65 * 1024 * 1024)).toBeUndefined(); | |
| 63 | }); | |
| 64 | ||
| 65 | test("the last 1,000 whole lines, the partial first line dropped", async () => { | |
| 66 | const tail = (await blobTail(dir, "main", "big.log", TAIL_BYTES, bytes.length))!; | |
| 67 | const t = tailOf(tail, bytes.length)!; | |
| 68 | expect(t.lines).toHaveLength(TAIL_LINES); | |
| 69 | expect(t.lines.at(-1)).toBe(line(LINES - 1)); | |
| 70 | expect(t.lines[0]).toBe(line(LINES - TAIL_LINES)); | |
| 71 | expect(t.lines.join("")).not.toContain("�"); | |
| 72 | // A cut inside a character costs the partial line, never a garbled one. | |
| 73 | const cut = tailOf(bytes.subarray(bytes.length - 5_000 - 1), bytes.length)!; | |
| 74 | expect(cut.lines.every((l) => l.startsWith("2026-"))).toBe(true); | |
| 75 | expect(Math.abs(t.about - LINES) / LINES).toBeLessThan(0.02); | |
| 76 | }); | |
| 77 | ||
| 78 | test("the page says what it left out, links the end by anchor, and escapes", async () => { | |
| 79 | const tail = (await blobTail(dir, "main", "big.log", TAIL_BYTES, bytes.length))!; | |
| 80 | const page = String( | |
| 81 | blobBody("/a/r", "main", "big.log", { kind: "too_large", size: bytes.length, tail }).content, | |
| 82 | ); | |
| 83 | expect(page).toMatch(/Showing the last 1,000 of about 1\d\d,\d{3} lines; the earlier lines are left out\./); | |
| 84 | expect(page).toContain('href="#tail-end"'); | |
| 85 | expect(page).toContain('id="tail-end"'); | |
| 86 | // The rows' true positions are unknown, so none is numbered or linkable as a line. | |
| 87 | expect(page).not.toContain('id="L1"'); | |
| 88 | expect(page).not.toContain('class="ln"'); | |
| 89 | expect(page).not.toContain("<script"); | |
| 90 | expect(page).not.toContain("<b>ok</b>"); | |
| 91 | expect(page).toContain("<b>ok</b>"); | |
| 92 | }); | |
| 93 | ||
| 94 | test("a file with no tail keeps the too-large note; only .log and .txt tail", () => { | |
| 95 | const page = String(blobBody("/a/r", "main", "big.log", { kind: "too_large", size: 9_000_000 }).content); | |
| 96 | expect(page).toContain("Too large to render"); | |
| 97 | expect(tailable("a/x.LOG")).toBe(true); | |
| 98 | expect(tailable("n.txt")).toBe(true); | |
| 99 | expect(tailable("x.ts")).toBe(false); | |
| 100 | expect(tailable("x.md")).toBe(false); | |
| 101 | }); | |
| 102 | }); |
test/spawn-pipes.test.ts +1 −1
| @@ -34,7 +34,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 34 | 34 | "src/git/combined.ts": { sites: ["stdout"], reason: "spawnSync buffers rev-parse stdout; stderr ignored" }, |
| 35 | 35 | "src/git/local.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "stdout and stderr both read to the end" }, |
| 36 | 36 | "src/git/own_refs.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "each git stdout is read to the end (three as text alongside exited, the rev-list walk chunk by chunk); stderr ignored; a timer kills a walk that hangs" }, |
| 37 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed); stderr ignored" }, | |
| 37 | "src/git/read.ts": { sites: ["stdout", "stdout", "stdout", "stdout"], reason: "cat-file stdout is read by the long-lived reader, by blobStream until its window ends or the reader hangs up, and by blobHead until its bytes are in (then killed), and by blobTail until the blob ends, keeping a ring of its last bytes; stderr ignored" }, | |
| 38 | 38 | "src/git/smart-http.ts": { sites: ["stdout"], reason: "stdout is a pull-pipe FIFO (the response body, read on demand); stderr ignored" }, |
| 39 | 39 | "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout (a pull-pipe FIFO) and stderr into the channel" }, |
| 40 | 40 | "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" }, |
new above
offline
Tail is chosen by file name only. Fine, but principle 8/plain: nothing tells a reader on the page why .md or .json do not tail. Keep it, and document it.