Public: anyone can read this; the crew can change it.

The v1 proof's schedule probe is unproven when no schedule exists, and passes only after one has run #13269

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13252-the-v1-proof-s-schedule-probe-is-unprove → main 3 commits fork

  • Merged. scout/13252-the-v1-proof-s-schedule-probe-is-unprove was merged into main as fad64fe.

Review

Sign in to review this change.
7 files changed +97 −19
src/actions/health.ts +2 −2
@@ -24,7 +24,7 @@ import type { App } from "../app.ts";
2424 import { appsRuntime } from "../apps/runtime.ts";
2525 import { listApps } from "../core/apps.ts";
2626 import { ledgerNeed } from "../core/ledger.ts";
27 import { overdueSchedules } from "../core/timers.ts";
27 import { overdueSchedules, type ScheduleCounts } from "../core/timers.ts";
2828 import type { User } from "../core/users.ts";
2929 import { forgeHealth, rolloutLine, type ForgeHealth } from "../kernel/guard.ts";
3030 import {
@@ -122,7 +122,7 @@ export interface Parts {
122122
123123 /** The forge's overdue schedules as counts, on the health answer (#12512). */
124124 export interface ScheduleHealth {
125 readonly schedules: { readonly overdue: number; readonly oldestSeconds: number };
125 readonly schedules: ScheduleCounts;
126126 }
127127
128128 /**
src/contract/openapi/routes/getApiV1Health.ts +1 −1
@@ -4,7 +4,7 @@ export const route: Route = {
44 method: "get",
55 path: "/api/v1/health",
66 summary:
7 "How the forge is, uncached; anyone may ask, a stranger is told little, the release probe and a site admin the whole of it. Every answer carries schedules {overdue, oldestSeconds}: how many unpaused schedules are more than five minutes past due and the age in seconds of the most overdue (0 when none), counts only, naming no app or repository",
7 "How the forge is, uncached; anyone may ask, a stranger is told little, the release probe and a site admin the whole of it. Every answer carries schedules {overdue, oldestSeconds, total, ran}: how many unpaused schedules are more than five minutes past due, the age in seconds of the most overdue (0 when none), how many unpaused schedules there are and how many of them have a recorded run, counts only, naming no app or repository",
88 auth: false,
99 role: "anonymous",
1010 reply: { ok: "boolean", draining: "boolean", schedules: "object" },
src/core/timers.ts +33 −6
@@ -108,19 +108,43 @@ export function dueTimers(db: Database, now: number, limit = 50): Timer[] {
108108 */
109109 export const OVERDUE_AFTER_MS = 5 * 60_000;
110110
111 /** The forge's schedules as counts, on the health answer (#12512, #13252). */
112 export interface ScheduleCounts {
113 /** Unpaused schedules due more than `OVERDUE_AFTER_MS` ago. */
114 readonly overdue: number;
115 /** The age in whole seconds of the most overdue (0 when none). */
116 readonly oldestSeconds: number;
117 /** Unpaused schedules in all. */
118 readonly total: number;
119 /** Of those, how many have at least one recorded run. */
120 readonly ran: number;
121 }
122
111123 /**
112 * The forge's overdue schedules, counts only (#12512): how many unpaused schedules are due more
113 * than `OVERDUE_AFTER_MS` ago, and the age in whole seconds of the most overdue (0 when none).
114 * No app, repository or name: a stranger may read it from the health answer.
124 * The forge's schedules, counts only (#12512): how many unpaused schedules there are, how many
125 * of them have ever run, how many are due more than `OVERDUE_AFTER_MS` ago, and the age in whole
126 * seconds of the most overdue (0 when none). No app, repository or name: a stranger may read it
127 * from the health answer.
115128 */
116 export function overdueSchedules(db: Database, now: number): { overdue: number; oldestSeconds: number } {
129 export function overdueSchedules(db: Database, now: number): ScheduleCounts {
117130 const r = db
118131 .query<{ n: number; oldest: number | null }, [number]>(
119132 "SELECT COUNT(*) AS n, MIN(due_at) AS oldest FROM timer WHERE kind = 'schedule' AND paused = 0 AND due_at < ?1",
120133 )
121134 .get(now - OVERDUE_AFTER_MS);
135 const all = db
136 .query<{ total: number; ran: number | null }, []>(
137 `SELECT COUNT(*) AS total, SUM(EXISTS (SELECT 1 FROM timer_run r WHERE r.app_id = t.app_id AND r.name = t.name AND r.env = t.env)) AS ran
138 FROM timer t WHERE t.kind = 'schedule' AND t.paused = 0`,
139 )
140 .get();
122141 const n = r?.n ?? 0;
123 return { overdue: n, oldestSeconds: n === 0 || r?.oldest == null ? 0 : Math.floor((now - r.oldest) / 1000) };
142 return {
143 overdue: n,
144 oldestSeconds: n === 0 || r?.oldest == null ? 0 : Math.floor((now - r.oldest) / 1000),
145 total: all?.total ?? 0,
146 ran: all?.ran ?? 0,
147 };
124148 }
125149
126150 export function getTimer(db: Database, id: number): Timer | undefined {
@@ -258,7 +282,10 @@ export function runsOf(db: Database, appId: string, name?: string, limit = 50):
258282 return rows.map(toRun);
259283 }
260284
261 /** The newest run of each of the app's schedules whose latest run did not go (for the owner's ask). One index walk per schedule. */
285 /**
286 * The newest run of each of the app's schedules whose latest run did not go (for the owner's ask).
287 * One index walk per schedule.
288 */
262289 export function failingNow(db: Database, appId: string): TimerRun[] {
263290 const out: TimerRun[] = [];
264291 for (const t of scheduleTimers(db, appId)) {
src/core/v1_proof.ts +10 −3
@@ -202,14 +202,21 @@ export const probes: readonly Probe[] = [
202202 {
203203 issue: 11595,
204204 item: "no schedule is past its period",
205 // The forge-wide count on the public health answer (#12512): no repository named.
205 // The forge-wide counts on the public health answer (#12512): no repository named. Zero
206 // schedules, or none that has run, proves nothing about schedules running while it sleeps
207 // (#13252).
206208 run: async (get) => {
207209 const got = await bodyOk(get, "/api/v1/health");
208 const s = (JSON.parse(got.text) as { schedules?: { overdue?: unknown; oldestSeconds?: unknown } }).schedules;
210 type Counts = { overdue?: unknown; oldestSeconds?: unknown; total?: unknown; ran?: unknown };
211 const s = (JSON.parse(got.text) as { schedules?: Counts }).schedules;
209212 if (s === undefined || typeof s.overdue !== "number" || typeof s.oldestSeconds !== "number")
210213 return unproven("/api/v1/health carries no schedules count: this forge predates #12512");
211214 expect(s.overdue === 0, `${s.overdue} schedule(s) overdue, the oldest by ${ageWords(s.oldestSeconds)}`);
212 return "/api/v1/health: no schedule is overdue";
215 if (typeof s.total !== "number" || typeof s.ran !== "number")
216 return unproven("/api/v1/health does not count its schedules: this forge predates #13252");
217 if (s.total === 0) return unproven("this forge has no schedule, so none has been seen to run");
218 if (s.ran === 0) return unproven(`${s.total} schedule(s), none of which has run yet`);
219 return `/api/v1/health: ${s.total} schedule(s), ${s.ran} with a recorded run, none overdue`;
213220 },
214221 },
215222 {
test/health-schedules.test.ts +20 −3
@@ -11,7 +11,7 @@ import { createApp, type App } from "../src/app.ts";
1111 import { createApp as createAppRecord } from "../src/core/apps.ts";
1212 import { createRepo } from "../src/core/repos.ts";
1313 import { createUser } from "../src/core/users.ts";
14 import { OVERDUE_AFTER_MS, overdueSchedules, setPaused, scheduleTimer, syncSchedules } from "../src/core/timers.ts";
14 import { OVERDUE_AFTER_MS, overdueSchedules, recordRun, setPaused, scheduleTimer, syncSchedules } from "../src/core/timers.ts";
1515 import { serve, type Servers } from "../src/http/server.ts";
1616
1717 let dataDir: string;
@@ -56,8 +56,8 @@ const strangerHealth = async () => {
5656 };
5757
5858 test("no schedule: none overdue", async () => {
59 expect(overdueSchedules(app.db, Date.now())).toEqual({ overdue: 0, oldestSeconds: 0 });
60 expect(JSON.parse(await strangerHealth()).schedules).toEqual({ overdue: 0, oldestSeconds: 0 });
59 expect(overdueSchedules(app.db, Date.now())).toEqual({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 });
60 expect(JSON.parse(await strangerHealth()).schedules).toEqual({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 });
6161 });
6262
6363 test("a schedule due inside the loop's slack is not overdue; one past it is, with its age", async () => {
@@ -86,3 +86,20 @@ test("a paused schedule is not overdue", async () => {
8686 setPaused(app.db, scheduleTimer(app.db, appId, "weekly", "production")!.id, true, now);
8787 expect(overdueSchedules(app.db, now).overdue).toBe(before - 1);
8888 });
89
90 test("total counts unpaused schedules and ran those with a recorded run (#13252)", async () => {
91 const now = Date.now();
92 const before = overdueSchedules(app.db, now);
93 const appId = await arm("ran-paper", "hourly", now);
94 const armed = overdueSchedules(app.db, now);
95 expect(armed.total).toBe(before.total + 1);
96 expect(armed.ran).toBe(before.ran);
97 recordRun(app.db, {
98 appId, env: "production", name: "hourly", idemKey: "k1", scheduledAt: now, startedAt: now, finishedAt: now,
99 attempts: 1, status: "ok", httpStatus: 200, response: "ok", error: null, lateMs: 0, lateReason: null, durationMs: 3,
100 });
101 expect(overdueSchedules(app.db, now).ran).toBe(before.ran + 1);
102 const body = JSON.parse(await strangerHealth()) as { schedules: { total: number; ran: number } };
103 expect(body.schedules.total).toBe(armed.total);
104 expect(body.schedules.ran).toBe(before.ran + 1);
105 });
test/security-floor-paths.test.ts +13
@@ -18,7 +18,9 @@ import { afterAll, beforeAll, expect, test } from "bun:test";
1818 import { newApp } from "../src/actions/apps.ts";
1919 import type { EnvStatus } from "../src/apps/runtime.ts";
2020 import type { AppRecord } from "../src/core/apps.ts";
21 import { recordRun, syncSchedules } from "../src/core/timers.ts";
2122 import { line, prove } from "../src/core/v1_proof.ts";
23 import { parseSchedules } from "../src/apps/schedule_spec.ts";
2224 import {
2325 fetchAsHost,
2426 startAppsForge,
@@ -209,6 +211,17 @@ test("the proxy's 304 of an app's page keeps the headers the app's 200 left with
209211 });
210212
211213 test("`bun scripts/v1-proven.ts`'s proof passes 20 times in a row against a booted forge", async () => {
214 // The schedule probe (#11595) is unproven on a forge with no schedule that has run (#13252),
215 // so this one gets an app with an hourly schedule and one recorded run, armed for later.
216 const app = await appNamed("proof-schedules", "public");
217 const specs = parseSchedules([{ name: "hourly", cron: "0 * * * *", path: "/jobs/run" }]);
218 if (!specs.ok) throw new Error(specs.error);
219 const now = Date.now();
220 syncSchedules(forge.app.db, app.id, "production", specs.value, "c1", now);
221 recordRun(forge.app.db, {
222 appId: app.id, env: "production", name: "hourly", idemKey: "k1", scheduledAt: now, startedAt: now, finishedAt: now,
223 attempts: 1, status: "ok", httpStatus: 200, response: "ok", error: null, lateMs: 0, lateReason: null, durationMs: 3,
224 });
212225 const failed: string[] = [];
213226 for (let run = 0; run < 20; run++)
214227 for (const r of await prove(forge.servers.url, fetch))
test/v1-proven.test.ts +18 −4
@@ -27,6 +27,7 @@ const rule = (days: number, words = days) =>
2727 const health = (schedules?: unknown, ver = "1.0.0") =>
2828 Response.json({ ok: true, version: ver, commit: "058b6cf", draining: false, ...(schedules === undefined ? {} : { schedules }) });
2929
30 const FRESH = { overdue: 0, oldestSeconds: 0, total: 1, ran: 1 };
3031 const FEED = '<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>Changelog</title><entry><id>x</id></entry></feed>';
3132 type Over = Record<string, () => Response>;
3233 function stub(over: Over = {}, seen: string[] = []): Fetch {
@@ -40,7 +41,7 @@ function stub(over: Over = {}, seen: string[] = []): Fetch {
4041 "/api/v1/plans": () => Response.json({ items: [], graceDays: 7, lapse: "sleep" }),
4142 "/settings/billing": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fbilling" } }),
4243 "/settings/crew": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fcrew" } }),
43 "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }),
44 "/api/v1/health": () => health(FRESH),
4445 "/changelog.atom": () => new Response(FEED),
4546 ...over,
4647 };
@@ -67,7 +68,7 @@ describe("v1-proven", () => {
6768 });
6869
6970 test("production at 0.7.604 reports its release and is not proven; the API's own constant does not pass it", async () => {
70 const results = await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") }));
71 const results = await prove("https://f.test", stub({ "/api/v1/health": () => health(FRESH, "0.7.604") }));
7172 expect(line(results.find((r) => r.issue === 10304)!)).toBe("pass #10304 production reports the release it runs: release 0.7.604");
7273 expect(line(results.find((r) => r.issue === 13042)!)).toBe("fail #13042 the release production runs is 1.0.0 or later: the release is 0.7.604, not 1.0.0");
7374 });
@@ -85,8 +86,8 @@ describe("v1-proven", () => {
8586 [11539, { "/healthz": () => new Response("down", { status: 503 }) }, [10441, 11539]],
8687 [11158, { "/llms.txt": () => new Response("Running dev at 058b6cf") }, [11158]],
8788 [11573, { "/readyz": () => new Response("draining", { status: 503 }) }, [11573]],
88 [10304, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "dev") }, [10304, 13042]],
89 [13042, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") }, [13042]],
89 [10304, { "/api/v1/health": () => health(FRESH, "dev") }, [10304, 13042]],
90 [13042, { "/api/v1/health": () => health(FRESH, "0.7.604") }, [13042]],
9091 [11507, { "/llms.txt": () => new Response("Start my forge") }, [11507]],
9192 [11138, { "/pricing": () => new Response("<p>none</p>") }, [11138, 11140, 11141]],
9293 [11140, { "/pricing": () => new Response('<form method="post" action="/pricing/order"></form><script type="application/ld+json">{"offers":[]}</script>') }, [11140, 11141]],
@@ -120,6 +121,19 @@ describe("v1-proven", () => {
120121 expect(r.seen!.ms).toBeGreaterThanOrEqual(0);
121122 });
122123
124 test("the schedule probe is unproven with no schedule, passes with one fresh that ran, and fails with one overdue (#13252)", async () => {
125 const at = async (schedules: unknown) =>
126 (await prove("https://f.test", stub({ "/api/v1/health": () => health(schedules) }))).find((x) => x.issue === 11595)!;
127 const none = await at({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 });
128 expect(none.status).toBe("unproven");
129 expect(line(none)).toBe("unproven #11595 no schedule is past its period: this forge has no schedule, so none has been seen to run");
130 expect((await at(FRESH)).status).toBe("pass");
131 expect((await at({ overdue: 1, oldestSeconds: 600, total: 1, ran: 1 })).status).toBe("fail");
132 // A schedule that has never run proves nothing yet; a forge that does not count them is older.
133 expect((await at({ overdue: 0, oldestSeconds: 0, total: 1, ran: 0 })).status).toBe("unproven");
134 expect((await at({ overdue: 0, oldestSeconds: 0 })).status).toBe("unproven");
135 });
136
123137 test("the schedule probe fails naming the oldest age, is unproven without the field, and a network error is a fail", async () => {
124138 const late = (await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 2, oldestSeconds: 3 * 3600 + 720 }) }))).find((x) => x.issue === 11595)!;
125139 expect(line(late)).toBe("fail #11595 no schedule is past its period: 2 schedule(s) overdue, the oldest by 3 h 12 min");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs