Public: anyone can read this; the crew can change it.
A repository's home draws its README's first 64 KB and links to the rest: grokbot/forge's home under 150 KB #6209
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6201-a-repository-s-home-draws-its-readme-s-f → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +126
−4
src/web/pages/repo.ts +39 −4
| @@ -465,6 +465,32 @@ function readmeKey(source: RepoSource, entry: TreeEntry): string | null { | ||
| 465 | 465 | return `${repo}:${entry.oid}:${entry.name}`; |
| 466 | 466 | } |
| 467 | 467 | |
| 468 | /** How much README source the repository home draws before linking the rest. */ | |
| 469 | export const README_HOME_BYTES = 64 * 1024; | |
| 470 | ||
| 471 | /** | |
| 472 | * The README source up to `limit` bytes, cut between top-level blocks: at the | |
| 473 | * last blank line outside a fenced code block, so no fence is ever split. | |
| 474 | * Null when the whole text fits. | |
| 475 | */ | |
| 476 | export function cutAtBlock(text: string, limit: number): string | null { | |
| 477 | const bytes = new TextEncoder().encode(text); | |
| 478 | if (bytes.length <= limit) return null; | |
| 479 | const head = new TextDecoder().decode(bytes.subarray(0, limit)); | |
| 480 | const lines = head.split("\n"); | |
| 481 | lines.pop(); // the last line may be cut mid-way | |
| 482 | let fence = ""; | |
| 483 | let cut = -1; | |
| 484 | for (let i = 0; i < lines.length; i++) { | |
| 485 | const m = /^ {0,3}(`{3,}|~{3,})/.exec(lines[i]!); | |
| 486 | if (m) { | |
| 487 | if (fence === "") fence = m[1]!; | |
| 488 | else if (m[1]![0] === fence[0] && m[1]!.length >= fence.length) fence = ""; | |
| 489 | } else if (fence === "" && lines[i]!.trim() === "") cut = i; | |
| 490 | } | |
| 491 | return cut < 0 ? "" : lines.slice(0, cut).join("\n"); | |
| 492 | } | |
| 493 | ||
| 468 | 494 | /** Empties the README cache; for tests. */ |
| 469 | 495 | export function clearReadmeCache(): void { |
| 470 | 496 | readmeCache.clear(); |
| @@ -482,13 +508,16 @@ export async function readmeFor( | ||
| 482 | 508 | ref: string, |
| 483 | 509 | entries: readonly TreeEntry[], |
| 484 | 510 | prefix: string, |
| 511 | /** The blob base of the whole README; when given, a long one is cut and links there. */ | |
| 512 | whole?: string, | |
| 485 | 513 | ): Promise<Html> { |
| 486 | 514 | const readme = entries.find( |
| 487 | 515 | (e) => |
| 488 | 516 | e.type === "blob" && /^readme(\.md|\.markdown|\.txt)?$/i.test(e.name), |
| 489 | 517 | ); |
| 490 | 518 | if (!readme) return html``; |
| 491 | const key = readmeKey(source, readme); | |
| 519 | const baseKey = readmeKey(source, readme); | |
| 520 | const key = baseKey === null || whole === undefined ? baseKey : `${baseKey}:${whole}`; | |
| 492 | 521 | const cached = key === null ? undefined : readmeCache.get(key); |
| 493 | 522 | if (key !== null && cached) { |
| 494 | 523 | readmeCache.delete(key); |
| @@ -499,11 +528,17 @@ export async function readmeFor( | ||
| 499 | 528 | const blob = await source.blob(ref, at); |
| 500 | 529 | if (!blob.ok || blob.value.kind !== "text") return html``; |
| 501 | 530 | if (blob.value.size > MAX_RENDER_BYTES) return html``; |
| 502 | const text = new TextDecoder().decode(blob.value.body); | |
| 531 | const full = new TextDecoder().decode(blob.value.body); | |
| 532 | const cut = whole === undefined ? null : cutAtBlock(full, README_HOME_BYTES); | |
| 533 | const text = cut ?? full; | |
| 503 | 534 | const rendered = /\.(md|markdown)$/i.test(readme.name) |
| 504 | 535 | ? renderMarkdown(text) |
| 505 | 536 | : html`<pre>${text}</pre>`; |
| 506 | const made = card(html`<div class="prose">${rendered}</div>`, { | |
| 537 | const more = | |
| 538 | cut === null | |
| 539 | ? html`` | |
| 540 | : html`<p><a href="${`${whole}/${at.split("/").map(encodeURIComponent).join("/")}`}">Read the whole README</a></p>`; | |
| 541 | const made = card(html`<div class="prose">${rendered}${more}</div>`, { | |
| 507 | 542 | pad: false, |
| 508 | 543 | head: html`${icon("file")} |
| 509 | 544 | <h2>${readme.name}</h2>`, |
| @@ -1144,7 +1179,7 @@ async function codeHome(ctx: Ctx, repo: Repo): Promise<Response> { | ||
| 1144 | 1179 | const tree = entries.ok ? entries.value : []; |
| 1145 | 1180 | |
| 1146 | 1181 | const readme = await timePhaseAsync(ctx.request, "readme", () => |
| 1147 | readmeFor(scope.source, ref, tree, ""), | |
| 1182 | readmeFor(scope.source, ref, tree, "", `${scope.base}/blob/${encodeURIComponent(ref)}`), | |
| 1148 | 1183 | ); |
| 1149 | 1184 | const following = await timePhaseAsync(ctx.request, "following", () => followingOf(ctx, repo)); |
| 1150 | 1185 | const latest = await timePhaseAsync(ctx.request, "wave", () => |
test/readme-home-cut.test.ts added +87
| @@ -0,0 +1,87 @@ | ||
| 1 | /** A repository home draws the first 64 KB of a long README and links to the rest. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { createSession } from "../src/core/sessions.ts"; | |
| 9 | import { createUser } from "../src/core/users.ts"; | |
| 10 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 11 | import { cutAtBlock } from "../src/web/pages/repo.ts"; | |
| 12 | import { quietLogs } from "./helpers/globals.ts"; | |
| 13 | ||
| 14 | quietLogs(); | |
| 15 | ||
| 16 | let dataDir: string; | |
| 17 | let app: App; | |
| 18 | let servers: Servers; | |
| 19 | let cookie = ""; | |
| 20 | const cells = process.env["FORGE_CELLS"]; | |
| 21 | ||
| 22 | const section = (i: number) => | |
| 23 | `## Section ${i}\n\nSome prose with *em* and \`code\` for section ${i}, ${"and more words ".repeat(60)}\n\n\`\`\`sh\necho ${i}\n\n\necho again\n\`\`\`\n\n`; | |
| 24 | const BIG = `# Manual\n\n${Array.from({ length: 220 }, (_, i) => section(i)).join("")}`; | |
| 25 | ||
| 26 | beforeAll(async () => { | |
| 27 | process.env["FORGE_CELLS"] = "off"; | |
| 28 | dataDir = await mkdtemp(join(tmpdir(), "forge-readme-cut-")); | |
| 29 | app = createApp({ dataDir, port: 0 }); | |
| 30 | servers = await serve(app); | |
| 31 | const made = await createUser(app.db, { | |
| 32 | origin: "seed", | |
| 33 | trusted: true, | |
| 34 | username: "alice", | |
| 35 | email: "[email protected]", | |
| 36 | password: "a-good-passphrase", | |
| 37 | adminIfFirst: true, | |
| 38 | }); | |
| 39 | if (!made.ok) throw new Error(String(made.error)); | |
| 40 | const alice = made.value; | |
| 41 | cookie = `forge_session=${createSession(app.db, alice.id).token}`; | |
| 42 | for (const [name, content] of [ | |
| 43 | ["big", BIG], | |
| 44 | ["small", `# Small\n\n${"words ".repeat(300)}\n`], | |
| 45 | ] as const) { | |
| 46 | const repo = await createRepo(app, alice, { name, visibility: "public" }); | |
| 47 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 48 | const done = await commitToBranch(app, alice, repo.value, { | |
| 49 | message: "readme", | |
| 50 | files: [{ path: "README.md", content }], | |
| 51 | }); | |
| 52 | if (!done.ok) throw new Error(JSON.stringify(done.error)); | |
| 53 | } | |
| 54 | }); | |
| 55 | ||
| 56 | afterAll(async () => { | |
| 57 | await servers.stop(); | |
| 58 | app.close(); | |
| 59 | if (cells === undefined) delete process.env["FORGE_CELLS"]; | |
| 60 | else process.env["FORGE_CELLS"] = cells; | |
| 61 | await rm(dataDir, { recursive: true, force: true }); | |
| 62 | }); | |
| 63 | ||
| 64 | test("a 200 KB README is cut and the home links to the whole of it", async () => { | |
| 65 | expect(BIG.length).toBeGreaterThan(200_000); | |
| 66 | const res = await fetch(`${servers.url}/alice/big`, { headers: { cookie } }); | |
| 67 | const body = await res.text(); | |
| 68 | expect(res.status).toBe(200); | |
| 69 | expect(body.length).toBeLessThan(150_000); | |
| 70 | expect(body).toContain("Section 1<"); | |
| 71 | expect(body).not.toContain("Section 219"); | |
| 72 | expect(body).toMatch(/<a href="\/alice\/big\/blob\/[^"]+\/README\.md">Read the whole README<\/a>/); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a 2 KB README renders whole with no link", async () => { | |
| 76 | const res = await fetch(`${servers.url}/alice/small`, { headers: { cookie } }); | |
| 77 | const body = await res.text(); | |
| 78 | expect(body).toContain("Small"); | |
| 79 | expect(body).not.toContain("Read the whole README"); | |
| 80 | }); | |
| 81 | ||
| 82 | test("the cut falls between blocks, never inside a fence", () => { | |
| 83 | const cut = cutAtBlock(BIG, 64 * 1024)!; | |
| 84 | expect(new TextEncoder().encode(cut).length).toBeLessThanOrEqual(64 * 1024); | |
| 85 | expect((cut.match(/^```/gm) ?? []).length % 2).toBe(0); | |
| 86 | expect(cutAtBlock("short", 64 * 1024)).toBeNull(); | |
| 87 | }); |
new above
offline
When no blank line exists, this returns an empty string and the card shows nothing but the link. Fall back to the last line boundary outside a fence.