Public: anyone can read this; the crew can change it.

The v1 proof's "production reports a release version" probe cannot fail: it reads the API's version constant, and the planner acted on the result #13085

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13042-the-v1-proof-s-production-reports-a-rele → main 3 commits fork

  • Merged. mason/13042-the-v1-proof-s-production-reports-a-rele was merged into main as a23c17d.

Review

Sign in to review this change.
4 files changed +46 −16
docs/features/v1-proven.md +7
@@ -14,6 +14,13 @@ One read-only probe (GET only) for each item on the v1 list, one line each:
1414 is seen), and #11595 only against a forge whose health answer predates
1515 #12512.
1616
17 Two probes read the release the forge runs, from `version` on
18 `GET /api/v1/health` (never `/api/v1/version`, which is the API's own constant
19 and says 1.0.0 on every release). #10304 says it: `release 0.7.604`. #13042
20 fails until that release is 1.0.0 or later: `the release is 0.7.604, not 1.0.0`.
21 Only the `v1.0.0` tag running on production passes it; the proof alone is not
22 the owner's word that v1 is done.
23
1724 Exit 0 only when every probe passes; 1 when any fails; 3 when none fails but
1825 some are unproven; 2 on bad usage. Owner-only pages are probed as "there and
1926 gated". The schedule probe reads `schedules` on the public `GET /api/v1/health`:
src/core/v1_proof.ts +20 −5
@@ -39,6 +39,14 @@ async function bodyOk(get: Get, path: string, status = 200): Promise<Got> {
3939 return got;
4040 }
4141
42 /** The release the forge runs, as `/api/v1/health` says it: `x.y.z`, or the probe fails. */
43 async function release(get: Get): Promise<string> {
44 const got = await bodyOk(get, "/api/v1/health");
45 const v = (JSON.parse(got.text) as { version?: unknown }).version;
46 expect(typeof v === "string" && /^\d+\.\d+\.\d+$/.test(v), "/api/v1/health names no x.y.z release");
47 return String(v);
48 }
49
4250 /** The page exists and is gated: a redirect to sign-in, never a 404 or a 5xx. */
4351 async function gated(get: Get, path: string): Promise<string> {
4452 const got = await get(path);
@@ -96,12 +104,19 @@ export const probes: readonly Probe[] = [
96104 },
97105 {
98106 issue: 10304,
99 item: "production reports a release version",
107 item: "production reports the release it runs",
108 // The release the forge runs (`/api/v1/health`), not `/api/v1/version`:
109 // that is the API's own constant.
110 run: async (get) => `release ${await release(get)}`,
111 },
112 {
113 issue: 13042,
114 item: "the release production runs is 1.0.0 or later",
100115 run: async (get) => {
101 const got = await bodyOk(get, "/api/v1/version");
102 const v = (JSON.parse(got.text) as { version?: unknown }).version;
103 expect(typeof v === "string" && /^\d+\.\d+\.\d+$/.test(v), "/api/v1/version has no x.y.z version");
104 return `version ${String(v)}`;
116 const v = await release(get);
117 const [maj = 0, min = 0, pat = 0] = v.split(".").map(Number);
118 expect(maj >= 1, `the release is ${v}, not 1.0.0`);
119 return `release ${v} (${maj}.${min}.${pat}) is 1.0.0 or later`;
105120 },
106121 },
107122 {
test/security-floor-paths.test.ts +3 −1
@@ -212,6 +212,8 @@ test("`bun scripts/v1-proven.ts`'s proof passes 20 times in a row against a boot
212212 const failed: string[] = [];
213213 for (let run = 0; run < 20; run++)
214214 for (const r of await prove(forge.servers.url, fetch))
215 if (r.status !== "pass") failed.push(`run ${run}: ${line(r)}`);
215 // A test forge is built from no release, so the two probes that read the release
216 // (#10304, #13042) have nothing to read; every other probe must pass.
217 if (r.status !== "pass" && r.issue !== 10304 && r.issue !== 13042) failed.push(`run ${run}: ${line(r)}`);
216218 expect(failed).toEqual([]);
217219 }, 60_000);
test/v1-proven.test.ts +16 −10
@@ -24,8 +24,8 @@ const ldWith = (additionalProperty: unknown[]) => JSON.stringify({ offers: [{ na
2424 const pricing = (json: string) => () => new Response(`<form method="post" action="/pricing/order"></form><script type="application/ld+json">${json}</script>${rule(7)}`);
2525 const rule = (days: number, words = days) =>
2626 `<p data-lapse="sleep" data-grace-days="${days}">If a payment lapses, you have ${words} days of grace. After that your forge sleeps and keeps its data.</p>`;
27 const health = (schedules?: unknown) =>
28 Response.json({ ok: true, version: "1.0.0", commit: "058b6cf", draining: false, ...(schedules === undefined ? {} : { schedules }) });
27 const health = (schedules?: unknown, ver = "1.0.0") =>
28 Response.json({ ok: true, version: ver, commit: "058b6cf", draining: false, ...(schedules === undefined ? {} : { schedules }) });
2929
3030 type Over = Record<string, () => Response>;
3131 function stub(over: Over = {}, seen: string[] = []): Fetch {
@@ -34,7 +34,6 @@ function stub(over: Over = {}, seen: string[] = []): Fetch {
3434 "/healthz": () => new Response("ok", { headers: HEADERS }),
3535 "/readyz": () => new Response("ok"),
3636 "/llms.txt": () => new Response("Running dev at 058b6cf\nPress Start my forge."),
37 "/api/v1/version": () => Response.json({ version: "1.0.0" }),
3837 "/pricing": () =>
3938 new Response(`<form method="post" action="/pricing/order"></form><script type="application/ld+json">${ld}</script>${rule(7)}`),
4039 "/api/v1/plans": () => Response.json({ items: [], graceDays: 7, lapse: "sleep" }),
@@ -54,9 +53,9 @@ describe("v1-proven", () => {
5453 test("every list item has a probe, and all pass on a healthy forge", async () => {
5554 const results = await prove("https://f.test/", stub());
5655 expect(results.map((r) => r.issue).sort()).toEqual(
57 [11541, 11539, 11158, 11573, 10304, 11507, 11138, 11140, 11141, 11515, 11595, 10441].sort(),
56 [11541, 11539, 11158, 11573, 10304, 13042, 11507, 11138, 11140, 11141, 11515, 11595, 10441].sort(),
5857 );
59 expect(probes.length).toBe(12);
58 expect(probes.length).toBe(13);
6059 const status = (n: number) => results.find((r) => r.issue === n)?.status;
6160 // What no public read can show is unproven, never a pass.
6261 expect(results.filter((r) => r.status === "unproven").map((r) => r.issue).sort()).toEqual([]);
@@ -65,6 +64,12 @@ describe("v1-proven", () => {
6564 expect(line(results[0]!)).toStartWith("pass #11541");
6665 });
6766
67 test("production at 0.7.604 reports its release and is not proven; the API's own constant does not pass it", async () => {
68 const results = await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") }));
69 expect(line(results.find((r) => r.issue === 10304)!)).toBe("pass #10304 production reports the release it runs: release 0.7.604");
70 expect(line(results.find((r) => r.issue === 13042)!)).toBe("fail #13042 the release production runs is 1.0.0 or later: the release is 0.7.604, not 1.0.0");
71 });
72
6873 test("only reads: every request is a GET", async () => {
6974 const seen: string[] = [];
7075 await prove("https://f.test", stub({}, seen));
@@ -78,7 +83,8 @@ describe("v1-proven", () => {
7883 [11539, { "/healthz": () => new Response("down", { status: 503 }) }, [10441, 11539]],
7984 [11158, { "/llms.txt": () => new Response("Running dev at 058b6cf") }, [11158]],
8085 [11573, { "/readyz": () => new Response("draining", { status: 503 }) }, [11573]],
81 [10304, { "/api/v1/version": () => Response.json({ version: "dev" }) }, [10304]],
86 [10304, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "dev") }, [10304, 13042]],
87 [13042, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") }, [13042]],
8288 [11507, { "/llms.txt": () => new Response("Start my forge") }, [11507]],
8389 [11138, { "/pricing": () => new Response("<p>none</p>") }, [11138, 11140, 11141]],
8490 [11140, { "/pricing": () => new Response('<form method="post" action="/pricing/order"></form><script type="application/ld+json">{"offers":[]}</script>') }, [11140, 11141]],
@@ -106,8 +112,8 @@ describe("v1-proven", () => {
106112 test("the schedule probe fails naming the oldest age, is unproven without the field, and a network error is a fail", async () => {
107113 const late = (await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 2, oldestSeconds: 3 * 3600 + 720 }) }))).find((x) => x.issue === 11595)!;
108114 expect(line(late)).toBe("fail #11595 no schedule is past its period: 2 schedule(s) overdue, the oldest by 3 h 12 min");
109 for (const old of [health(), health({ overdue: "none" })]) {
110 const r = (await prove("https://f.test", stub({ "/api/v1/health": () => old }))).find((x) => x.issue === 11595)!;
115 for (const old of [() => health(), () => health({ overdue: "none" })]) {
116 const r = (await prove("https://f.test", stub({ "/api/v1/health": old }))).find((x) => x.issue === 11595)!;
111117 expect(r.status).toBe("unproven");
112118 expect(line(r)).toStartWith("unproven #11595");
113119 }
@@ -137,7 +143,7 @@ describe("v1-proven", () => {
137143 const { code, out } = await run(async () => new Response("down", { status: 503 }));
138144 expect(code).toBe(1);
139145 expect(out).toContain("fail #11539");
140 expect(out).toContain("v1.0.0 is not proven: 12 of 12 fail, 0 unproven");
146 expect(out).toContain("v1.0.0 is not proven: 13 of 13 fail, 0 unproven");
141147 });
142148
143149 test("the script exits 3, not 0, when nothing fails but some items are unproven", async () => {
@@ -145,7 +151,7 @@ describe("v1-proven", () => {
145151 expect(out).not.toContain("fail #");
146152 expect(code).toBe(3);
147153 expect(out).toContain("unproven #11595");
148 expect(out).toContain("v1.0.0 is not proven: none fail, 1 of 12 unproven");
154 expect(out).toContain("v1.0.0 is not proven: none fail, 1 of 13 unproven");
149155 });
150156
151157 test("the script exits 2 on bad usage", () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs