Public: anyone can read this; the crew can change it.
Every command in llms.txt's start-here section runs against a real forge in one test #12412
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12400-every-command-in-llms-txt-s-start-here-s → main 3 commits fork 4 threads · 4 unresolved
1 file
changed +228
−0
test/llms-commands.test.ts added +228
| @@ -0,0 +1,228 @@ | ||
| 1 | /** | |
| 2 | * Every curl /llms.txt teaches in "Start here" and "Report a problem" runs against a real forge. | |
| 3 | * A command is a backticked `curl …` or a ```fenced``` curl block (a trailing backslash continues | |
| 4 | * it). Today only "Start here" holds one (the start curl); "Report a problem" has no backticked | |
| 5 | * curl, so its API line is exercised by its own test below. A command passes with a 2xx, or with | |
| 6 | * the status it documents right after it (`curl …` -> 403). The extractor itself is tested on | |
| 7 | * synthetic text: it fails loudly on a flag or url it does not understand. | |
| 8 | */ | |
| 9 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 10 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp } from "../src/app.ts"; | |
| 14 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 15 | ||
| 16 | let dataDir: string; // throwaway: `start` creates an account | |
| 17 | let app: ReturnType<typeof createApp>; | |
| 18 | let servers: Servers; | |
| 19 | ||
| 20 | beforeAll(async () => { | |
| 21 | dataDir = await mkdtemp(join(tmpdir(), "forge-llms-cmd-")); | |
| 22 | app = createApp({ dataDir, port: 0 }); | |
| 23 | servers = await serve(app); | |
| 24 | }, 30_000); | |
| 25 | ||
| 26 | afterAll(async () => { | |
| 27 | servers.stop(); | |
| 28 | app.close(); | |
| 29 | await rm(dataDir, { recursive: true, force: true }); | |
| 30 | }); | |
| 31 | ||
| 32 | type Cmd = { | |
| 33 | line: number; | |
| 34 | text: string; | |
| 35 | method: string; | |
| 36 | url: string; | |
| 37 | body?: string | undefined; | |
| 38 | expect?: number | undefined; | |
| 39 | }; | |
| 40 | ||
| 41 | /** Split a command into words, honouring single and double quotes. */ | |
| 42 | const words = (s: string): string[] => { | |
| 43 | const out: string[] = []; | |
| 44 | const re = /'([^']*)'|"([^"]*)"|(\S+)/g; | |
| 45 | for (let m = re.exec(s); m; m = re.exec(s)) out.push(m[1] ?? m[2] ?? m[3] ?? ""); | |
| 46 | return out; | |
| 47 | }; | |
| 48 | ||
| 49 | const QUIET = new Set(["s", "S", "f", "L"]); // flags that change no request | |
| 50 | const WITH_ARG = new Set(["X", "d", "H", "o"]); | |
| 51 | ||
| 52 | /** Parse one curl command; throws on any flag, method or url it does not understand. */ | |
| 53 | const parseCurl = (text: string): Pick<Cmd, "method" | "url" | "body"> => { | |
| 54 | const w = words(text); | |
| 55 | if (w[0] !== "curl") throw new Error(`not a curl command: ${text}`); | |
| 56 | let method = ""; | |
| 57 | let url = ""; | |
| 58 | let body: string | undefined; | |
| 59 | const take = (flag: string, arg: string | undefined) => { | |
outdated
Only single-line backticked curls are found. Add fenced blocks (and line continuations), as box 1 requires. |
||
| 60 | if (arg === undefined) throw new Error(`flag ${flag} needs a value in: ${text}`); | |
| 61 | if (flag === "X" || flag === "--request") method = arg; | |
| 62 | else if (flag === "d" || flag === "--data") body = arg; | |
| 63 | }; | |
| 64 | for (let i = 1; i < w.length; i++) { | |
| 65 | const a = w[i]!; | |
| 66 | if (a.startsWith("--")) { | |
| 67 | if (a === "--data" || a === "--request" || a === "--header" || a === "--output") take(a, w[++i]); | |
| 68 | else if (a !== "--silent" && a !== "--fail" && a !== "--show-error") throw new Error(`unknown flag ${a} in: ${text}`); | |
| 69 | } else if (a.startsWith("-") && a.length > 1) { | |
| 70 | for (let k = 1; k < a.length; k++) { | |
| 71 | const c = a[k]!; | |
| 72 | if (QUIET.has(c)) continue; | |
| 73 | if (!WITH_ARG.has(c)) throw new Error(`unknown flag -${c} in: ${text}`); | |
outdated
Only 2xx passes; box 1 also allows the documented refusal. Also add a test that a broken command is reported with its llms.txt line. |
||
| 74 | take(c, a.slice(k + 1) || w[++i]); | |
| 75 | break; | |
| 76 | } | |
| 77 | } else if (url) throw new Error(`two urls in: ${text}`); | |
| 78 | else url = a; | |
| 79 | } | |
| 80 | if (!/^https?:\/\/[^/\s]+/.test(url)) throw new Error(`no usable url in: ${text}`); | |
| 81 | method = method || (body === undefined ? "GET" : "POST"); | |
| 82 | if (!/^(GET|POST|PUT|PATCH|DELETE)$/.test(method)) throw new Error(`unknown method ${method} in: ${text}`); | |
| 83 | return { method, url, body }; | |
| 84 | }; | |
| 85 | ||
| 86 | const SECTIONS = ["## Start here", "## Report a problem"]; | |
| 87 | ||
| 88 | /** The curl commands in the named sections, each with its 1-based llms.txt line. */ | |
| 89 | const commands = (txt: string, origin: string): Cmd[] => { | |
| 90 | const out: Cmd[] = []; | |
| 91 | const add = (line: number, text: string, after: string) => { | |
| 92 | const p = parseCurl(text); | |
| 93 | const documented = after.match(/^\s*->\s*(\d{3})\b/)?.[1]; | |
| 94 | out.push({ | |
| 95 | line, | |
| 96 | text, | |
| 97 | ...p, | |
outdated
The placeholder substitutions mean this tests your rewrites, not the documented line. Assert the placeholders you expect so a doc change fails loudly. |
||
| 98 | url: p.url.replace(/^https?:\/\/[^/]+/, origin), | |
| 99 | expect: documented ? Number(documented) : undefined, | |
| 100 | }); | |
| 101 | }; | |
| 102 | const lines = txt.split("\n"); | |
| 103 | let inside = false; | |
| 104 | let fence = false; | |
| 105 | for (let i = 0; i < lines.length; i++) { | |
| 106 | const l = lines[i]!; | |
| 107 | if (!fence && l.startsWith("## ")) inside = SECTIONS.some((h) => l.startsWith(h)); | |
| 108 | if (!inside) continue; | |
| 109 | if (l.trimStart().startsWith("```")) { | |
| 110 | fence = !fence; | |
| 111 | continue; | |
| 112 | } | |
| 113 | if (fence) { | |
| 114 | if (!l.trimStart().startsWith("curl ")) continue; | |
| 115 | let text = l.trim(); | |
| 116 | const first = i; | |
| 117 | while (text.endsWith("\\") && i + 1 < lines.length) text = `${text.slice(0, -1)} ${lines[++i]!.trim()}`; | |
| 118 | add(first + 1, text, ""); | |
| 119 | continue; | |
| 120 | } | |
| 121 | for (const m of l.matchAll(/`(curl [^`]+)`([^`]*)/g)) add(i + 1, m[1]!, m[2]!); | |
| 122 | } | |
| 123 | return out; | |
| 124 | }; | |
| 125 | ||
| 126 | const run = async (c: Cmd): Promise<Response> => | |
| 127 | fetch(c.url, { method: c.method, body: c.body, headers: { "content-type": "application/json" } }); | |
| 128 | ||
| 129 | /** Run every command; one message per failure, each naming its llms.txt line. */ | |
| 130 | const failures = async (txt: string, origin: string): Promise<string[]> => { | |
| 131 | const failed: string[] = []; | |
| 132 | for (const c of commands(txt, origin)) { | |
| 133 | const res = await run(c); | |
| 134 | const ok = c.expect === undefined ? res.status >= 200 && res.status <= 299 : res.status === c.expect; | |
| 135 | if (!ok) { | |
| 136 | const want = c.expect ?? "2xx"; | |
| 137 | failed.push(`llms.txt line ${c.line}: ${c.text} -> ${res.status}, wanted ${want}: ${(await res.text()).slice(0, 200)}`); | |
| 138 | } | |
| 139 | } | |
| 140 | return failed; | |
| 141 | }; | |
| 142 | ||
| 143 | const live = async (): Promise<string> => (await fetch(`${servers.url}/llms.txt`)).text(); | |
| 144 | ||
| 145 | test("the real llms.txt yields the start curl, parsed to its method and path", async () => { | |
| 146 | const cmds = commands(await live(), servers.url); | |
| 147 | const start = cmds.find((c) => c.url === `${servers.url}/api/v1/start`); | |
| 148 | expect(start?.method).toBe("POST"); | |
| 149 | expect(start?.body).toContain('"name"'); | |
| 150 | expect(cmds.length).toBeGreaterThanOrEqual(1); | |
| 151 | }); | |
| 152 | ||
| 153 | test("every curl in Start here and Report a problem answers as documented, naming the line that fails", async () => { | |
| 154 | expect(await failures(await live(), servers.url)).toEqual([]); | |
| 155 | }); | |
| 156 | ||
| 157 | test("a command with a wrong path is reported with its llms.txt line", async () => { | |
| 158 | const txt = ["# x", "", "## Start here", "", "- go: `curl -sX POST http://h/api/v1/nope -d '{}'` -> ok"].join("\n"); | |
| 159 | const failed = await failures(txt, servers.url); | |
| 160 | expect(failed.length).toBe(1); | |
| 161 | expect(failed[0]).toContain("llms.txt line 5"); | |
| 162 | expect(failed[0]).toContain("/api/v1/nope"); | |
| 163 | }); | |
| 164 | ||
| 165 | test("a section with no command found does not read as success", () => { | |
| 166 | const txt = "## Start here\n\n- nothing runnable here\n"; | |
| 167 | expect(commands(txt, servers.url)).toEqual([]); // callers assert the start curl exists, above | |
| 168 | }); | |
| 169 | ||
| 170 | test("fenced and backslash-continued curl blocks are found, with their first line", () => { | |
| 171 | const txt = [ | |
| 172 | "## Start here", | |
| 173 | "```sh", | |
| 174 | "curl -s http://h/a", | |
| 175 | "curl -sX POST \\", | |
| 176 | " http://h/b \\", | |
| 177 | " -d '{\"k\":1}'", | |
| 178 | "```", | |
| 179 | "## Other", | |
| 180 | "`curl http://h/ignored`", | |
| 181 | ].join("\n"); | |
| 182 | const cmds = commands(txt, "http://o"); | |
| 183 | expect(cmds.map((c) => [c.line, c.method, c.url, c.body])).toEqual([ | |
| 184 | [3, "GET", "http://o/a", undefined], | |
| 185 | [4, "POST", "http://o/b", '{"k":1}'], | |
| 186 | ]); | |
| 187 | }); | |
| 188 | ||
| 189 | test("the parser throws on a flag, method or url it does not understand", () => { | |
| 190 | expect(() => parseCurl("curl --compressed http://h/a")).toThrow("unknown flag"); | |
| 191 | expect(() => parseCurl("curl -k http://h/a")).toThrow("unknown flag -k"); | |
| 192 | expect(() => parseCurl("curl -X BREW http://h/a")).toThrow("unknown method"); | |
| 193 | expect(() => parseCurl("curl -s")).toThrow("no usable url"); | |
| 194 | expect(() => parseCurl("curl -d")).toThrow("needs a value"); | |
| 195 | expect(parseCurl("curl -sX PUT http://h/a")).toMatchObject({ method: "PUT", url: "http://h/a" }); | |
| 196 | expect(parseCurl("curl -sXPUT http://h/a").method).toBe("PUT"); | |
| 197 | }); | |
| 198 | ||
| 199 | test("a command that documents its refusal passes on that status, and fails on a 2xx or another", async () => { | |
| 200 | const gone = "## Start here\n\n- `curl http://h/api/v1/nope` -> 404 not found\n"; | |
| 201 | expect(await failures(gone, servers.url)).toEqual([]); | |
| 202 | const wrong = "## Start here\n\n- `curl http://h/api/v1/nope` -> 403 refused\n"; | |
| 203 | expect((await failures(wrong, servers.url))[0]).toContain("wanted 403"); | |
| 204 | }); | |
| 205 | ||
| 206 | test("the report API line in Report a problem works on the repository start just made", async () => { | |
| 207 | const lines = (await live()).split("\n"); | |
| 208 | const at = lines.findIndex((l) => l.startsWith("- API: POST ") && l.includes("/reports")); | |
| 209 | expect(at).toBeGreaterThan(-1); | |
| 210 | const line = lines[at]!; | |
| 211 | // The doc line carries placeholders; assert each, so a doc change fails here, then fill them. | |
| 212 | const placeholders = ["{owner}/{name}", '"title": "..."', '"body": "..."', '"page": "https://..."']; | |
| 213 | for (const p of placeholders) expect(line, `llms.txt line ${at + 1} lost ${p}`).toContain(p); | |
| 214 | const started = (await ( | |
| 215 | await fetch(`${servers.url}/api/v1/start`, { method: "POST", body: JSON.stringify({ name: "doc-check" }) }) | |
| 216 | ).json()) as { repo: string }; | |
| 217 | const url = line.match(/POST (\S+)/)![1]!.replace(/^https?:\/\/[^/]+/, servers.url).replace("{owner}/{name}", started.repo); | |
| 218 | const json = line | |
| 219 | .match(/JSON (\{.*?\})(?= ->)/)![1]! | |
| 220 | .replace('"title": "..."', '"title": "doc title"') | |
| 221 | .replace('"body": "..."', '"body": "doc body"') | |
| 222 | .replace('"page": "https://..."', '"page": "https://example.com/"'); | |
| 223 | expect(json).not.toContain("..."); | |
| 224 | const res = await fetch(url, { method: "POST", body: json, headers: { "content-type": "application/json" } }); | |
| 225 | const got = await res.text(); | |
| 226 | expect(res.status, `llms.txt line ${at + 1}: ${got}`).toBe(202); | |
| 227 | expect(got, `llms.txt line ${at + 1}`).toContain("pending"); | |
| 228 | }); |
new above
offline
Unknown flags are skipped silently and -sX is taken as -X. Throw on anything not understood, and on an empty url.