Public: anyone can read this; the crew can change it.

One script says whether v1.0.0 is proven on production: each v1 list item's probe, pass or fail, read-only #12303

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12297-one-script-says-whether-v1-0-0-is-proven → main 6 commits fork 4 threads · 4 unresolved

  • Merged. scout/12297-one-script-says-whether-v1-0-0-is-proven was merged into main as ec4445f.

Review

Sign in to review this change.
3 files changed +419 −0
docs/features/v1-proven.md added +22
@@ -0,0 +1,22 @@
1 # Is v1.0.0 proven?
2
3 ```sh
4 bun scripts/v1-proven.ts https://your-forge.example [--schedules owner/name]
5 ```
6
7 One read-only probe (GET only) for each item on the v1 list, one line each:
8
9 - `pass`: a stranger's request read the evidence.
10 - `fail`: the evidence is missing or wrong; the line says why.
11 - `unproven`: no public read can show the claim, so it is not counted as a pass.
12 Today: #11140 (the free plan sleeps rather than is deleted), #11141 (CPU
13 slowing, the first change landing, lapsed-payment sleep: only the gated page
14 is seen) and #11595 when the `--schedules` repository (default
15 `trevato/forge`) is not readable to a stranger. Point `--schedules` at a
16 public repository to prove it.
17
18 Exit 0 only when every probe passes; 1 when any fails; 3 when none fails but
19 some are unproven; 2 on bad usage. Owner-only pages are probed as "there and
20 gated". The schedule probe compares the last run with the period inferred from
21 the last run and the next one: the API exposes the cron, not the period. It
22 reports and never tags; `scripts/cut-release.ts` does that.
scripts/v1-proven.ts added +259
@@ -0,0 +1,259 @@
1 /**
2 * `bun scripts/v1-proven.ts <base-url> [--schedules owner/name]` (#12297):
3 * says whether v1.0.0 is proven on a running forge. One read-only probe per
4 * item on the v1 list, one line each: `pass`, `fail` or `unproven` with the
5 * reason. Exit 0 only when every probe passes, 1 when any fails, 3 when none
6 * fails but some is `unproven` (no public read can show it). It only sends GET: no sign-up, no payment, no write. It
7 * does not tag; `scripts/cut-release.ts` does that.
8 *
9 * A probe is the strongest evidence a stranger's request can read; what only
10 * a signed-in owner can see is probed as "the page exists and is gated".
11 */
12
13 export type Fetch = (url: string, init?: RequestInit) => Promise<Response>;
14 export interface Probe {
15 readonly issue: number;
16 readonly item: string;
17 /** A string is a pass; `unproven(why)` is "seen what a stranger can, not the claim". */
18 readonly run: (get: Get, opts: Options) => Promise<string | Unproven>;
19 }
20 export interface Options {
21 readonly schedules: string;
22 readonly now: number;
23 }
24 export class Unproven {
25 constructor(readonly why: string) {}
26 }
27 export const unproven = (why: string): Unproven => new Unproven(why);
28 export type Status = "pass" | "fail" | "unproven";
29 export interface Result {
30 readonly issue: number;
31 readonly item: string;
32 readonly status: Status;
33 readonly detail: string;
34 }
35 interface Got {
36 readonly status: number;
37 readonly headers: Headers;
38 readonly text: string;
39 readonly location: string | null;
40 }
41 type Get = (path: string) => Promise<Got>;
42
43 class Fail extends Error {}
44 const fail = (why: string): never => {
45 throw new Fail(why);
46 };
47 const expect = (cond: boolean, why: string): void => {
48 if (!cond) fail(why);
49 };
50
51 async function bodyOk(get: Get, path: string, status = 200): Promise<Got> {
52 const got = await get(path);
53 expect(got.status === status, `${path} answers ${got.status}, not ${status}`);
54 return got;
55 }
56
57 /** The page exists and is gated: a redirect to sign-in, never a 404 or a 5xx. */
58 async function gated(get: Get, path: string): Promise<string> {
59 const got = await get(path);
60 expect(got.status === 303 || got.status === 302, `${path} answers ${got.status}, not a redirect to sign-in`);
61 expect((got.location ?? "").includes("/signin"), `${path} redirects to ${got.location ?? "nowhere"}, not sign-in`);
62 return `${path} is there and asks for sign-in`;
63 }
64
65 const SECURITY_HEADERS: ReadonlyArray<readonly [string, RegExp | null]> = [
66 ["content-security-policy", /default-src/],
67 ["x-content-type-options", /^nosniff$/i],
68 ["referrer-policy", /./],
69 ["strict-transport-security", /max-age=\d+/],
70 ["permissions-policy", /./],
71 ["cross-origin-opener-policy", /./],
72 ];
73
74 export const probes: readonly Probe[] = [
75 {
76 issue: 11541,
77 item: "the front door answers with a way to start a forge",
78 run: async (get) => {
79 const got = await bodyOk(get, "/");
80 expect((got.headers.get("content-type") ?? "").startsWith("text/html"), "/ is not HTML");
81 expect(/<form[^>]*method="post"/i.test(got.text), "/ has no form to start a forge");
82 return "/ answers 200 with a start form";
83 },
84 },
85 {
86 issue: 11539,
87 item: "the forge is alive",
88 run: async (get) => {
89 const got = await bodyOk(get, "/healthz");
90 expect(got.text.trim() === "ok", `/healthz says "${got.text.trim().slice(0, 40)}", not ok`);
91 return "/healthz is ok";
92 },
93 },
94 {
95 issue: 11158,
96 item: "a stranger is told how to start, in words, from the link",
97 run: async (get) => {
98 const got = await bodyOk(get, "/llms.txt");
99 expect(/start my forge/i.test(got.text), "/llms.txt does not say how to start a forge");
100 return "/llms.txt says how to start";
101 },
102 },
103 {
104 issue: 11573,
105 item: "the forge is ready, not draining",
106 run: async (get) => {
107 const got = await bodyOk(get, "/readyz");
108 expect(got.text.trim() === "ok", `/readyz says "${got.text.trim().slice(0, 40)}", not ok`);
109 return "/readyz is ok";
110 },
111 },
112 {
113 issue: 10304,
114 item: "production reports a release version",
115 run: async (get) => {
116 const got = await bodyOk(get, "/api/v1/version");
outdated

Item 11140 claims the free plan sleeps rather than is deleted, but the probe only reads a description string. Print unproven for what has no public read rather than pass.

117 const v = (JSON.parse(got.text) as { version?: unknown }).version;
118 expect(typeof v === "string" && /^\d+\.\d+\.\d+$/.test(v), "/api/v1/version has no x.y.z version");
119 return `version ${String(v)}`;
120 },
121 },
122 {
123 issue: 11507,
124 item: "production names the commit it runs, so it can be told from main",
125 run: async (get) => {
126 const got = await bodyOk(get, "/llms.txt");
127 const m = /Running \S+ at ([0-9a-f]{7,40})/.exec(got.text);
128 expect(m !== null, "/llms.txt does not name the running commit");
outdated

Probe reads trevato/forge schedules, which answers 404 to a stranger on production. Default is guaranteed red; use a public source or report the unreadable case distinctly.

129 return `running ${m?.[1] ?? ""}`;
130 },
131 },
132 {
133 issue: 11138,
134 item: "/pricing sells a forge: a checkout form",
135 run: async (get) => {
136 const got = await bodyOk(get, "/pricing");
137 expect(/<form[^>]*action="\/pricing\/order"/i.test(got.text), "/pricing has no order form");
138 return "/pricing answers with the order form";
139 },
140 },
141 {
142 issue: 11140,
143 item: "plans are listed, and the free one sleeps rather than being deleted",
144 run: async (get) => {
145 const got = await bodyOk(get, "/pricing");
146 const m = /<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i.exec(got.text);
147 expect(m !== null, "/pricing lists no plans");
148 const offers = (JSON.parse(m?.[1] ?? "{}") as { offers?: { description?: string }[] }).offers ?? [];
149 expect(offers.length >= 2, `/pricing lists ${offers.length} plans, not two or more`);
150 expect(offers.some((o) => /sleeps/i.test(o.description ?? "")), "no plan says it sleeps when idle");
151 return unproven(`${offers.length} plans are listed and one says it sleeps when idle; that the free plan sleeps rather than is deleted has no public read`);
152 },
153 },
154 {
155 issue: 11141,
156 item: "an owner's plan and billing page exists",
157 run: async (get) =>
158 unproven(`${await gated(get, "/settings/billing")}; CPU slowing, the first change landing and lapsed-payment sleep have no public read`),
159 },
160 {
161 issue: 11515,
162 item: "an owner's crew settings (their own key and budgets) exist",
163 run: (get) => gated(get, "/settings/crew"),
164 },
165 {
166 issue: 11595,
167 item: "a schedule's last run is under its period",
168 run: async (get, opts) => {
169 const path = `/api/v1/repos/${opts.schedules}/schedules`;
170 const got = await get(path);
171 if (got.status === 401 || got.status === 403 || got.status === 404)
172 return unproven(`${path} answers ${got.status} to a stranger: ${opts.schedules} is not publicly readable; point --schedules at a public repository`);
173 expect(got.status === 200, `${path} answers ${got.status}, not 200`);
174 const items = (JSON.parse(got.text) as { items?: Row[] }).items ?? [];
175 expect(items.length > 0, `${opts.schedules} declares no schedule`);
176 const late: string[] = [];
177 for (const s of items) {
178 if (s.paused) continue;
179 if (!s.last) {
180 late.push(`${s.name} never ran`);
181 continue;
182 }
183 if (s.last.status !== "ok") late.push(`${s.name} last run ${s.last.status}`);
184 else if (s.nextAt !== null && opts.now > s.nextAt + 5 * 60_000) late.push(`${s.name} is past its next run`);
185 else if (s.nextAt !== null && opts.now - s.last.at > s.nextAt - s.last.at + 5 * 60_000)
186 late.push(`${s.name} last ran longer ago than its period`);
187 }
188 expect(late.length === 0, late.join("; "));
189 return `${items.length} schedule(s), each last run within its period`;
190 },
191 },
192 {
193 issue: 10441,
194 item: "the security headers are on the page and on a machine answer",
195 run: async (get) => {
196 for (const path of ["/", "/healthz"]) {
197 const got = await get(path);
198 for (const [name, shape] of SECURITY_HEADERS) {
199 const v = got.headers.get(name);
200 expect(v !== null, `${path} lacks ${name}`);
201 expect(shape === null || shape.test(v ?? ""), `${path} has a bad ${name}: ${v ?? ""}`);
202 }
203 }
204 return "CSP, nosniff, referrer-policy, HSTS, permissions-policy, COOP on / and /healthz";
205 },
206 },
207 ];
208
209 interface Row {
210 name: string;
211 paused: boolean;
212 nextAt: number | null;
213 last: { at: number; status: string } | null;
214 }
215
216 /** Runs every probe against `base` with GET only; a probe that throws is a fail. */
217 export async function prove(base: string, fetcher: Fetch, opts: Partial<Options> = {}): Promise<Result[]> {
218 const root = base.replace(/\/+$/, "");
219 const o: Options = { schedules: opts.schedules ?? "trevato/forge", now: opts.now ?? Date.now() };
220 const get: Get = async (path) => {
221 const res = await fetcher(root + path, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(15_000) });
222 return { status: res.status, headers: res.headers, text: await res.text(), location: res.headers.get("location") };
223 };
224 const out: Result[] = [];
225 for (const p of probes) {
226 try {
227 const r = await p.run(get, o);
228 out.push(r instanceof Unproven ? { issue: p.issue, item: p.item, status: "unproven", detail: r.why } : { issue: p.issue, item: p.item, status: "pass", detail: r });
229 } catch (e) {
230 out.push({ issue: p.issue, item: p.item, status: "fail", detail: e instanceof Fail ? e.message : `probe error: ${String(e)}` });
231 }
232 }
233 return out;
234 }
235
236 export const line = (r: Result): string => `${r.status} #${r.issue} ${r.item}: ${r.detail}`;
237
238 if (import.meta.main) {
239 const args = process.argv.slice(2);
240 const i = args.indexOf("--schedules");
241 const schedules = i >= 0 ? args.splice(i, 2)[1] : undefined;
242 const base = args[0];
243 if (!base || !/^https?:\/\/[^/\s]+/.test(base) || (schedules !== undefined && !/^[\w.-]+\/[\w.-]+$/.test(schedules))) {
244 console.error("usage: bun scripts/v1-proven.ts <base-url> [--schedules owner/name]");
245 process.exit(2);
246 }
247 const results = await prove(base, fetch, schedules === undefined ? {} : { schedules });
248 for (const r of results) console.log(line(r));
249 const bad = results.filter((r) => r.status === "fail").length;
250 const open = results.filter((r) => r.status === "unproven").length;
251 console.log(
252 bad > 0
253 ? `v1.0.0 is not proven: ${bad} of ${results.length} fail, ${open} unproven`
254 : open > 0
255 ? `v1.0.0 is not proven: none fail, ${open} of ${results.length} unproven (no public read)`
256 : "v1.0.0 is proven: every probe passes",
257 );
258 process.exit(bad > 0 ? 1 : open > 0 ? 3 : 0);
259 }
test/v1-proven.test.ts added +138
@@ -0,0 +1,138 @@
1 import { describe, expect, test } from "bun:test";
2 import { line, probes, prove, type Fetch } from "../scripts/v1-proven.ts";
3
4 const NOW = Date.parse("2026-11-14T00:00:00Z");
5 const HEADERS = {
6 "content-security-policy": "default-src 'none'",
7 "x-content-type-options": "nosniff",
8 "referrer-policy": "same-origin",
9 "strict-transport-security": "max-age=31536000",
10 "permissions-policy": "camera=()",
11 "cross-origin-opener-policy": "same-origin",
12 };
13 const ld = JSON.stringify({ offers: [{ description: "A small forge that sleeps when idle" }, { description: "Small, awake" }] });
14 const sched = (last: unknown, nextAt = NOW + 3_600_000) =>
15 JSON.stringify({ items: [{ name: "nightly", paused: false, nextAt, last }] });
16
17 type Over = Record<string, () => Response>;
18 function stub(over: Over = {}, seen: string[] = []): Fetch {
19 const routes: Over = {
20 "/": () => new Response('<form method="post" action="/start"></form>', { headers: { ...HEADERS, "content-type": "text/html" } }),
21 "/healthz": () => new Response("ok", { headers: HEADERS }),
22 "/readyz": () => new Response("ok"),
23 "/llms.txt": () => new Response("Running dev at 058b6cf\nPress Start my forge."),
24 "/api/v1/version": () => Response.json({ version: "1.0.0" }),
25 "/pricing": () =>
26 new Response(`<form method="post" action="/pricing/order"></form><script type="application/ld+json">${ld}</script>`),
27 "/settings/billing": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fbilling" } }),
28 "/settings/crew": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fcrew" } }),
29 "/api/v1/repos/trevato/forge/schedules": () =>
30 new Response(sched({ at: NOW - 3_600_000, status: "ok" })),
31 ...over,
32 };
33 return async (url, init) => {
34 seen.push(`${init?.method ?? "GET"} ${new URL(url).pathname}`);
35 const r = routes[new URL(url).pathname];
36 return r ? r() : new Response("no", { status: 404 });
37 };
38 }
39
40 describe("v1-proven", () => {
41 test("every list item has a probe, and all pass on a healthy forge", async () => {
42 const results = await prove("https://f.test/", stub(), { now: NOW });
43 expect(results.map((r) => r.issue).sort()).toEqual(
44 [11541, 11539, 11158, 11573, 10304, 11507, 11138, 11140, 11141, 11515, 11595, 10441].sort(),
45 );
46 expect(probes.length).toBe(12);
47 const status = (n: number) => results.find((r) => r.issue === n)?.status;
48 // What no public read can show is unproven, never a pass.
49 expect(results.filter((r) => r.status === "unproven").map((r) => r.issue).sort()).toEqual([11140, 11141]);
50 expect(results.filter((r) => r.status === "fail")).toEqual([]);
51 expect(status(11595)).toBe("pass");
52 expect(line(results[0]!)).toStartWith("pass #11541");
53 });
54
55 test("only reads: every request is a GET", async () => {
56 const seen: string[] = [];
57 await prove("https://f.test", stub({}, seen), { now: NOW });
58 expect(seen.length).toBeGreaterThan(10);
59 expect(seen.every((s) => s.startsWith("GET "))).toBe(true);
60 });
61
62 // [issue, what breaks, every issue that fails with it]: some share a page.
63 const breaks: [number, Over, number[]][] = [
64 [11541, { "/": () => new Response("<p>hi</p>", { headers: { "content-type": "text/html" } }) }, [10441, 11541]],
65 [11539, { "/healthz": () => new Response("down", { status: 503 }) }, [10441, 11539]],
66 [11158, { "/llms.txt": () => new Response("Running dev at 058b6cf") }, [11158]],
67 [11573, { "/readyz": () => new Response("draining", { status: 503 }) }, [11573]],
68 [10304, { "/api/v1/version": () => Response.json({ version: "dev" }) }, [10304]],
69 [11507, { "/llms.txt": () => new Response("Start my forge") }, [11507]],
70 [11138, { "/pricing": () => new Response("<p>none</p>") }, [11138, 11140]],
71 [11140, { "/pricing": () => new Response('<form method="post" action="/pricing/order"></form><script type="application/ld+json">{"offers":[]}</script>') }, [11140]],
72 [11141, { "/settings/billing": () => new Response("x", { status: 404 }) }, [11141]],
73 [11515, { "/settings/crew": () => new Response("x", { status: 500 }) }, [11515]],
74 [11595, { "/api/v1/repos/trevato/forge/schedules": () => new Response(sched({ at: NOW - 3 * 86_400_000, status: "ok" }, NOW - 86_400_000)) }, [11595]],
75 [10441, { "/healthz": () => new Response("ok") }, [10441]],
76 ];
77 for (const [issue, over, expected] of breaks)
outdated

toContain(issue) does not prove 'only it' fails. Assert the exact failed set.

78 test(`#${issue} fails, and only it, when its evidence is gone`, async () => {
79 const results = await prove("https://f.test", stub(over), { now: NOW });
80 const failed = results.filter((r) => r.status === "fail").map((r) => r.issue);
81 expect(failed.sort()).toEqual([...expected].sort());
82 expect(results.find((r) => r.issue === issue)?.detail.length).toBeGreaterThan(0);
83 expect(line(results.find((r) => r.issue === issue)!)).toStartWith(`fail #${issue}`);
84 });
85
86 test("a failed last run, a never-run schedule and a network error are fails", async () => {
87 const dead = await prove("https://f.test", stub({ "/api/v1/repos/trevato/forge/schedules": () => new Response("no", { status: 404 }) }), { now: NOW });
88 expect(dead.find((x) => x.issue === 11595)?.status).toBe("unproven");
89 expect(line(dead.find((x) => x.issue === 11595)!)).toStartWith("unproven #11595");
90 for (const body of [sched({ at: NOW - 1000, status: "failed" }), sched(null)]) {
91 const r = await prove("https://f.test", stub({ "/api/v1/repos/trevato/forge/schedules": () => new Response(body) }), { now: NOW });
92 expect(r.find((x) => x.issue === 11595)?.status).toBe("fail");
93 }
94 const down: Fetch = async () => {
95 throw new Error("refused");
outdated

Title says it exits non-zero on a fail, but only bad usage (2) is tested. Run against a failing local stub and assert exit 1.

96 };
97 const all = await prove("https://f.test", down, { now: NOW });
98 expect(all.every((r) => r.status === "fail")).toBe(true);
99 });
100
101 const run = async (answer: Fetch) => {
102 const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: (req) => answer(req.url) });
103 try {
104 const p = Bun.spawn(["bun", "scripts/v1-proven.ts", `http://127.0.0.1:${server.port}`, ], { stdout: "pipe", stderr: "pipe" });
105 const out = await new Response(p.stdout).text();
106 return { code: await p.exited, out };
107 } finally {
108 server.stop(true);
109 }
110 };
111
112 test("the script exits 1 against a failing server and prints the summary", async () => {
113 const { code, out } = await run(async () => new Response("down", { status: 503 }));
114 expect(code).toBe(1);
115 expect(out).toContain("fail #11539");
116 expect(out).toContain("v1.0.0 is not proven: 12 of 12 fail, 0 unproven");
117 });
118
119 test("the script exits 3, not 0, when nothing fails but some items are unproven", async () => {
120 // The script reads the real clock, so the schedule is fresh;
121 // only the two no-public-read items are unproven.
122 const { code, out } = await run(
123 stub({
124 "/api/v1/repos/trevato/forge/schedules": () =>
125 new Response(JSON.stringify({ items: [{ name: "nightly", paused: false, nextAt: Date.now() + 3_600_000, last: { at: Date.now() - 3_600_000, status: "ok" } }] })),
126 }),
127 );
128 expect(out).not.toContain("fail #");
129 expect(code).toBe(3);
130 expect(out).toContain("unproven #11140");
131 expect(out).toContain("v1.0.0 is not proven: none fail, 2 of 12 unproven");
132 });
133
134 test("the script exits 2 on bad usage", () => {
135 const usage = Bun.spawnSync(["bun", "scripts/v1-proven.ts"], { stderr: "pipe" });
136 expect(usage.exitCode).toBe(2);
137 });
138 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs