Public: anyone can read this; the crew can change it.
/llms.txt folds public forks under their root, counting only forks a stranger can see #7036
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/7032-llms-txt-folds-public-forks-under-their → main 1 commit fork 1 thread · 1 unresolved
3 files
changed +148
−16
src/core/listings.ts +41 −14
| @@ -13,10 +13,11 @@ import { listReposVisibleTo } from "./repos.ts"; | ||
| 13 | 13 | const LISTING_MS = 60_000; |
| 14 | 14 | export const SITEMAP_ROWS = 5000; |
| 15 | 15 | export const LLMS_ROWS = 200; |
| 16 | export const LLMS_LINES = 50; | |
| 16 | 17 | |
| 17 | 18 | interface Kept { |
| 18 | 19 | readonly sitemap: Fresh<readonly SitemapRepo[]>; |
| 19 | readonly llms: Fresh<readonly string[]>; | |
| 20 | readonly llms: Fresh<LlmsListing>; | |
| 20 | 21 | } |
| 21 | 22 | const kept = new WeakMap<Database, Kept>(); |
| 22 | 23 | |
| @@ -75,19 +76,45 @@ export function sitemapRepos(db: Database): readonly SitemapRepo[] { | ||
| 75 | 76 | }); |
| 76 | 77 | } |
| 77 | 78 | |
| 78 | /** `owner/name` of the newest public repositories, at most `LLMS_ROWS`. */ | |
| 79 | export function llmsRepos(db: Database): readonly string[] { | |
| 80 | return holders(db).llms.get("", () => { | |
| 81 | listingBuilds.llms++; | |
| 82 | return names(db, LLMS_ROWS); | |
| 83 | }); | |
| 79 | /** One line of llms.txt: a repository and how many visible forks fold under it. */ | |
| 80 | export interface LlmsLine { | |
| 81 | readonly path: string; | |
| 82 | readonly forks: number; | |
| 84 | 83 | } |
| 85 | 84 | |
| 86 | function names(db: Database, limit: number): string[] { | |
| 87 | return listReposVisibleTo(db, undefined, { | |
| 88 | limit, | |
| 89 | withoutArchived: true, | |
| 90 | }).map( | |
| 91 | (r) => `${r.ownerName}/${r.name}`, | |
| 92 | ); | |
| 85 | /** The lines shown, and whether more public repositories exist than they name. */ | |
| 86 | export interface LlmsListing { | |
| 87 | readonly lines: readonly LlmsLine[]; | |
| 88 | readonly more: boolean; | |
| 89 | } | |
| 90 | ||
| 91 | /** | |
| 92 | * The newest public repositories as at most `LLMS_LINES` lines. A fork whose | |
| 93 | * parent is itself in the visible set folds into the parent's line; a fork | |
| 94 | * whose parent is not (private, archived, suspended) stands as itself. The | |
| 95 | * grouping reads only the visible set, so nothing outside it is named or counted. | |
| 96 | */ | |
| 97 | export function llmsRepos(db: Database): LlmsListing { | |
| 98 | return holders(db).llms.get("", () => { | |
| 99 | listingBuilds.llms++; | |
| 100 | const repos = listReposVisibleTo(db, undefined, { | |
| 101 | limit: LLMS_ROWS, | |
| 102 | withoutArchived: true, | |
| 103 | }); | |
| 104 | const ids = new Set(repos.map((r) => r.id)); | |
| 105 | const forks = new Map<string, number>(); | |
| 106 | for (const r of repos) | |
| 107 | if (r.forkParentId !== null && ids.has(r.forkParentId)) | |
| 108 | forks.set(r.forkParentId, (forks.get(r.forkParentId) ?? 0) + 1); | |
| 109 | const lines = repos | |
| 110 | .filter((r) => r.forkParentId === null || !ids.has(r.forkParentId)) | |
| 111 | .map((r) => ({ | |
| 112 | path: `${r.ownerName}/${r.name}`, | |
| 113 | forks: forks.get(r.id) ?? 0, | |
| 114 | })); | |
| 115 | return { | |
| 116 | lines: lines.slice(0, LLMS_LINES), | |
| 117 | more: lines.length > LLMS_LINES || repos.length >= LLMS_ROWS, | |
| 118 | }; | |
| 119 | }); | |
| 93 | 120 | } |
src/web/standards.ts +13 −2
| @@ -363,8 +363,19 @@ export function llmsTxt(ctx: Ctx): Response { | ||
| 363 | 363 | // Names and addresses only: a description is whatever its owner |
| 364 | 364 | // wrote, and an agent reading this file must not meet anyone's words |
| 365 | 365 | // as the forge's own. |
| 366 | ...(repos.length > 0 | |
| 367 | ? repos.map((r) => `- [${r}](${o}/${r})`) | |
| 366 | ...(repos.lines.length > 0 | |
| 367 | ? [ | |
| 368 | ...repos.lines.map( | |
| 369 | (r) => | |
| 370 | `- [${r.path}](${o}/${r.path})` + | |
| 371 | (r.forks > 0 | |
| 372 | ? ` · ${r.forks} ${r.forks === 1 ? "fork" : "forks"}` | |
| 373 | : ""), | |
| 374 | ), | |
| 375 | ...(repos.more | |
| 376 | ? [`- More: ${o}/api/v1/explore`] | |
| 377 | : []), | |
| 378 | ] | |
| 368 | 379 | : ["None yet."]), |
| 369 | 380 | "", |
| 370 | 381 | ].join("\n"), |
test/llms-forks.test.ts added +94
| @@ -0,0 +1,94 @@ | ||
| 1 | /** llms.txt folds public forks under their root, counting only what a stranger can see. */ | |
| 2 | import { expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { suspendAccount } from "../src/actions/accounts.ts"; | |
| 8 | import { createRepo, forkRepo, updateRepoSettings } from "../src/actions/repos.ts"; | |
| 9 | import { forgetListings } from "../src/core/listings.ts"; | |
| 10 | import { getRepo } from "../src/core/repos.ts"; | |
| 11 | import { createUser } from "../src/core/users.ts"; | |
| 12 | import { serve } from "../src/http/server.ts"; | |
| 13 | import { promote } from "./helpers/members.ts"; | |
| 14 | ||
| 15 | test("one line with the visible forks counted; a fork of a hidden root stands alone and names no root", async () => { | |
| 16 | const dataDir = await mkdtemp(join(tmpdir(), "forge-llms-forks-")); | |
| 17 | const app = createApp({ dataDir, port: 0 }); | |
| 18 | const servers = await serve(app); | |
| 19 | try { | |
| 20 | const person = async (username: string, adminIfFirst = false) => { | |
| 21 | const made = await createUser(app.db, { | |
| 22 | origin: "seed", trusted: true, username, | |
| 23 | email: `${username}@f.dev`, password: "a-good-passphrase", adminIfFirst, | |
| 24 | }); | |
| 25 | if (!made.ok) throw new Error(made.error); | |
| 26 | promote(app.db, made.value); | |
| 27 | return made.value; | |
| 28 | }; | |
| 29 | const admin = await person("olive", true); | |
| 30 | const alice = await person("alice"); | |
| 31 | const sam = await person("sam"); | |
| 32 | const forkers = [await person("f1"), await person("f2"), await person("f3"), await person("f4")]; | |
| 33 | const make = async (who: typeof alice, name: string) => { | |
| 34 | const r = await createRepo(app, who, { name, visibility: "public" }); | |
| 35 | if (!r.ok) throw new Error(String(r.error)); | |
| 36 | }; | |
| 37 | await make(alice, "root"); | |
| 38 | await make(sam, "tools"); | |
| 39 | const fork = async (who: typeof alice, from: string) => { | |
| 40 | const [o, n] = from.split("/"); | |
| 41 | const r = await forkRepo(app, who, getRepo(app.db, o!, n!)!); | |
| 42 | if (!r.ok) throw new Error(JSON.stringify(r)); | |
| 43 | return r.value; | |
| 44 | }; | |
| 45 | for (const f of forkers.slice(0, 3)) await fork(f, "alice/root"); | |
| 46 | const hidden = await fork(forkers[3]!, "alice/root"); | |
| 47 | const priv = await updateRepoSettings(app, forkers[3]!, getRepo(app.db, "f4", "root")!, { visibility: "private" }); | |
| 48 | expect(priv.ok).toBe(true); | |
| 49 | void hidden; | |
| 50 | const rita = await person("rita"); | |
| 51 | await fork(rita, "sam/tools"); | |
| 52 | expect((await suspendAccount(app, admin, "sam")).ok).toBe(true); | |
| 53 | forgetListings(app.db); | |
| 54 | ||
| 55 | const body = await fetch(`${servers.url}/llms.txt`).then((r) => r.text()); | |
| 56 | const section = body.slice(body.indexOf("## Public repositories")); | |
| 57 | expect(section).toContain(`- [alice/root](${servers.url}/alice/root) · 3 forks\n`); | |
| 58 | expect(section).not.toContain("f1/root"); | |
| 59 | expect(section).not.toContain("f4/root"); | |
| 60 | expect(section).toContain("- [rita/tools]"); | |
| 61 | expect(section).not.toContain("sam/tools"); | |
| 62 | expect(section).not.toContain("sam"); | |
| 63 | } finally { | |
| 64 | servers.stop(); | |
| 65 | app.close(); | |
| 66 | await rm(dataDir, { recursive: true, force: true }); | |
| 67 | } | |
| 68 | }); | |
| 69 | ||
| 70 | test("the section is capped at 50 lines and ends with the explore link", async () => { | |
| 71 | const dataDir = await mkdtemp(join(tmpdir(), "forge-llms-cap-")); | |
| 72 | const app = createApp({ dataDir, port: 0 }); | |
| 73 | const servers = await serve(app); | |
| 74 | try { | |
| 75 | const made = await createUser(app.db, { | |
| 76 | origin: "seed", trusted: true, username: "alice", | |
| 77 | email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true, | |
| 78 | }); | |
| 79 | if (!made.ok) throw new Error(made.error); | |
| 80 | promote(app.db, made.value); | |
| 81 | for (let i = 0; i < 52; i++) { | |
| 82 | const r = await createRepo(app, made.value, { name: `r${i}`, visibility: "public" }); | |
| 83 | if (!r.ok) throw new Error(String(r.error)); | |
| 84 | } | |
| 85 | const body = await fetch(`${servers.url}/llms.txt`).then((r) => r.text()); | |
| 86 | const section = body.slice(body.indexOf("## Public repositories")); | |
| 87 | expect(section.match(/^- \[alice\//gm)?.length).toBe(50); | |
| 88 | expect(section).toContain(`- More: ${servers.url}/api/v1/explore\n`); | |
| 89 | } finally { | |
| 90 | servers.stop(); | |
| 91 | app.close(); | |
| 92 | await rm(dataDir, { recursive: true, force: true }); | |
| 93 | } | |
| 94 | }); |
new above
offline
hiddenis assigned and then discarded withvoid. Callawait fork(forkers[3]!, "alice/root");without the binding and delete this line, or leave it for the next pull that touches this file.