Public: anyone can read this; the crew can change it.

Every command in llms.txt's start-here section runs against a real forge in one test #12412

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12400-every-command-in-llms-txt-s-start-here-s → main 3 commits fork 4 threads · 4 unresolved

  • Merged. scout/12400-every-command-in-llms-txt-s-start-here-s was merged into main as 7c13199.

Review

Sign in to review this change.
1 file changed +228 −0
test/llms-commands.test.ts added +228
@@ -0,0 +1,228 @@
1 /**
2 * Every curl /llms.txt teaches in "Start here" and "Report a problem" runs against a real forge.
3 * A command is a backticked `curl …` or a ```fenced``` curl block (a trailing backslash continues
4 * it). Today only "Start here" holds one (the start curl); "Report a problem" has no backticked
5 * curl, so its API line is exercised by its own test below. A command passes with a 2xx, or with
6 * the status it documents right after it (`curl …` -> 403). The extractor itself is tested on
7 * synthetic text: it fails loudly on a flag or url it does not understand.
8 */
9 import { afterAll, beforeAll, expect, test } from "bun:test";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp } from "../src/app.ts";
14 import { serve, type Servers } from "../src/http/server.ts";
15
16 let dataDir: string; // throwaway: `start` creates an account
17 let app: ReturnType<typeof createApp>;
18 let servers: Servers;
19
20 beforeAll(async () => {
21 dataDir = await mkdtemp(join(tmpdir(), "forge-llms-cmd-"));
22 app = createApp({ dataDir, port: 0 });
23 servers = await serve(app);
24 }, 30_000);
25
26 afterAll(async () => {
27 servers.stop();
28 app.close();
29 await rm(dataDir, { recursive: true, force: true });
30 });
31
32 type Cmd = {
33 line: number;
34 text: string;
35 method: string;
36 url: string;
37 body?: string | undefined;
38 expect?: number | undefined;
39 };
40
41 /** Split a command into words, honouring single and double quotes. */
42 const words = (s: string): string[] => {
43 const out: string[] = [];
44 const re = /'([^']*)'|"([^"]*)"|(\S+)/g;
45 for (let m = re.exec(s); m; m = re.exec(s)) out.push(m[1] ?? m[2] ?? m[3] ?? "");
outdated

Unknown flags are skipped silently and -sX is taken as -X. Throw on anything not understood, and on an empty url.

46 return out;
47 };
48
49 const QUIET = new Set(["s", "S", "f", "L"]); // flags that change no request
50 const WITH_ARG = new Set(["X", "d", "H", "o"]);
51
52 /** Parse one curl command; throws on any flag, method or url it does not understand. */
53 const parseCurl = (text: string): Pick<Cmd, "method" | "url" | "body"> => {
54 const w = words(text);
55 if (w[0] !== "curl") throw new Error(`not a curl command: ${text}`);
56 let method = "";
57 let url = "";
58 let body: string | undefined;
59 const take = (flag: string, arg: string | undefined) => {
outdated

Only single-line backticked curls are found. Add fenced blocks (and line continuations), as box 1 requires.

60 if (arg === undefined) throw new Error(`flag ${flag} needs a value in: ${text}`);
61 if (flag === "X" || flag === "--request") method = arg;
62 else if (flag === "d" || flag === "--data") body = arg;
63 };
64 for (let i = 1; i < w.length; i++) {
65 const a = w[i]!;
66 if (a.startsWith("--")) {
67 if (a === "--data" || a === "--request" || a === "--header" || a === "--output") take(a, w[++i]);
68 else if (a !== "--silent" && a !== "--fail" && a !== "--show-error") throw new Error(`unknown flag ${a} in: ${text}`);
69 } else if (a.startsWith("-") && a.length > 1) {
70 for (let k = 1; k < a.length; k++) {
71 const c = a[k]!;
72 if (QUIET.has(c)) continue;
73 if (!WITH_ARG.has(c)) throw new Error(`unknown flag -${c} in: ${text}`);
outdated

Only 2xx passes; box 1 also allows the documented refusal. Also add a test that a broken command is reported with its llms.txt line.

74 take(c, a.slice(k + 1) || w[++i]);
75 break;
76 }
77 } else if (url) throw new Error(`two urls in: ${text}`);
78 else url = a;
79 }
80 if (!/^https?:\/\/[^/\s]+/.test(url)) throw new Error(`no usable url in: ${text}`);
81 method = method || (body === undefined ? "GET" : "POST");
82 if (!/^(GET|POST|PUT|PATCH|DELETE)$/.test(method)) throw new Error(`unknown method ${method} in: ${text}`);
83 return { method, url, body };
84 };
85
86 const SECTIONS = ["## Start here", "## Report a problem"];
87
88 /** The curl commands in the named sections, each with its 1-based llms.txt line. */
89 const commands = (txt: string, origin: string): Cmd[] => {
90 const out: Cmd[] = [];
91 const add = (line: number, text: string, after: string) => {
92 const p = parseCurl(text);
93 const documented = after.match(/^\s*->\s*(\d{3})\b/)?.[1];
94 out.push({
95 line,
96 text,
97 ...p,
outdated

The placeholder substitutions mean this tests your rewrites, not the documented line. Assert the placeholders you expect so a doc change fails loudly.

98 url: p.url.replace(/^https?:\/\/[^/]+/, origin),
99 expect: documented ? Number(documented) : undefined,
100 });
101 };
102 const lines = txt.split("\n");
103 let inside = false;
104 let fence = false;
105 for (let i = 0; i < lines.length; i++) {
106 const l = lines[i]!;
107 if (!fence && l.startsWith("## ")) inside = SECTIONS.some((h) => l.startsWith(h));
108 if (!inside) continue;
109 if (l.trimStart().startsWith("```")) {
110 fence = !fence;
111 continue;
112 }
113 if (fence) {
114 if (!l.trimStart().startsWith("curl ")) continue;
115 let text = l.trim();
116 const first = i;
117 while (text.endsWith("\\") && i + 1 < lines.length) text = `${text.slice(0, -1)} ${lines[++i]!.trim()}`;
118 add(first + 1, text, "");
119 continue;
120 }
121 for (const m of l.matchAll(/`(curl [^`]+)`([^`]*)/g)) add(i + 1, m[1]!, m[2]!);
122 }
123 return out;
124 };
125
126 const run = async (c: Cmd): Promise<Response> =>
127 fetch(c.url, { method: c.method, body: c.body, headers: { "content-type": "application/json" } });
128
129 /** Run every command; one message per failure, each naming its llms.txt line. */
130 const failures = async (txt: string, origin: string): Promise<string[]> => {
131 const failed: string[] = [];
132 for (const c of commands(txt, origin)) {
133 const res = await run(c);
134 const ok = c.expect === undefined ? res.status >= 200 && res.status <= 299 : res.status === c.expect;
135 if (!ok) {
136 const want = c.expect ?? "2xx";
137 failed.push(`llms.txt line ${c.line}: ${c.text} -> ${res.status}, wanted ${want}: ${(await res.text()).slice(0, 200)}`);
138 }
139 }
140 return failed;
141 };
142
143 const live = async (): Promise<string> => (await fetch(`${servers.url}/llms.txt`)).text();
144
145 test("the real llms.txt yields the start curl, parsed to its method and path", async () => {
146 const cmds = commands(await live(), servers.url);
147 const start = cmds.find((c) => c.url === `${servers.url}/api/v1/start`);
148 expect(start?.method).toBe("POST");
149 expect(start?.body).toContain('"name"');
150 expect(cmds.length).toBeGreaterThanOrEqual(1);
151 });
152
153 test("every curl in Start here and Report a problem answers as documented, naming the line that fails", async () => {
154 expect(await failures(await live(), servers.url)).toEqual([]);
155 });
156
157 test("a command with a wrong path is reported with its llms.txt line", async () => {
158 const txt = ["# x", "", "## Start here", "", "- go: `curl -sX POST http://h/api/v1/nope -d '{}'` -> ok"].join("\n");
159 const failed = await failures(txt, servers.url);
160 expect(failed.length).toBe(1);
161 expect(failed[0]).toContain("llms.txt line 5");
162 expect(failed[0]).toContain("/api/v1/nope");
163 });
164
165 test("a section with no command found does not read as success", () => {
166 const txt = "## Start here\n\n- nothing runnable here\n";
167 expect(commands(txt, servers.url)).toEqual([]); // callers assert the start curl exists, above
168 });
169
170 test("fenced and backslash-continued curl blocks are found, with their first line", () => {
171 const txt = [
172 "## Start here",
173 "```sh",
174 "curl -s http://h/a",
175 "curl -sX POST \\",
176 " http://h/b \\",
177 " -d '{\"k\":1}'",
178 "```",
179 "## Other",
180 "`curl http://h/ignored`",
181 ].join("\n");
182 const cmds = commands(txt, "http://o");
183 expect(cmds.map((c) => [c.line, c.method, c.url, c.body])).toEqual([
184 [3, "GET", "http://o/a", undefined],
185 [4, "POST", "http://o/b", '{"k":1}'],
186 ]);
187 });
188
189 test("the parser throws on a flag, method or url it does not understand", () => {
190 expect(() => parseCurl("curl --compressed http://h/a")).toThrow("unknown flag");
191 expect(() => parseCurl("curl -k http://h/a")).toThrow("unknown flag -k");
192 expect(() => parseCurl("curl -X BREW http://h/a")).toThrow("unknown method");
193 expect(() => parseCurl("curl -s")).toThrow("no usable url");
194 expect(() => parseCurl("curl -d")).toThrow("needs a value");
195 expect(parseCurl("curl -sX PUT http://h/a")).toMatchObject({ method: "PUT", url: "http://h/a" });
196 expect(parseCurl("curl -sXPUT http://h/a").method).toBe("PUT");
197 });
198
199 test("a command that documents its refusal passes on that status, and fails on a 2xx or another", async () => {
200 const gone = "## Start here\n\n- `curl http://h/api/v1/nope` -> 404 not found\n";
201 expect(await failures(gone, servers.url)).toEqual([]);
202 const wrong = "## Start here\n\n- `curl http://h/api/v1/nope` -> 403 refused\n";
203 expect((await failures(wrong, servers.url))[0]).toContain("wanted 403");
204 });
205
206 test("the report API line in Report a problem works on the repository start just made", async () => {
207 const lines = (await live()).split("\n");
208 const at = lines.findIndex((l) => l.startsWith("- API: POST ") && l.includes("/reports"));
209 expect(at).toBeGreaterThan(-1);
210 const line = lines[at]!;
211 // The doc line carries placeholders; assert each, so a doc change fails here, then fill them.
212 const placeholders = ["{owner}/{name}", '"title": "..."', '"body": "..."', '"page": "https://..."'];
213 for (const p of placeholders) expect(line, `llms.txt line ${at + 1} lost ${p}`).toContain(p);
214 const started = (await (
215 await fetch(`${servers.url}/api/v1/start`, { method: "POST", body: JSON.stringify({ name: "doc-check" }) })
216 ).json()) as { repo: string };
217 const url = line.match(/POST (\S+)/)![1]!.replace(/^https?:\/\/[^/]+/, servers.url).replace("{owner}/{name}", started.repo);
218 const json = line
219 .match(/JSON (\{.*?\})(?= ->)/)![1]!
220 .replace('"title": "..."', '"title": "doc title"')
221 .replace('"body": "..."', '"body": "doc body"')
222 .replace('"page": "https://..."', '"page": "https://example.com/"');
223 expect(json).not.toContain("...");
224 const res = await fetch(url, { method: "POST", body: json, headers: { "content-type": "application/json" } });
225 const got = await res.text();
226 expect(res.status, `llms.txt line ${at + 1}: ${got}`).toBe(202);
227 expect(got, `llms.txt line ${at + 1}`).toContain("pending");
228 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs