Public: anyone can read this; the crew can change it.
The v1 proof's schedule probe is unproven when no schedule exists, and passes only after one has run #13269
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13252-the-v1-proof-s-schedule-probe-is-unprove → main 3 commits fork
7 files
changed +97
−19
src/actions/health.ts +2 −2
| @@ -24,7 +24,7 @@ import type { App } from "../app.ts"; | ||
| 24 | 24 | import { appsRuntime } from "../apps/runtime.ts"; |
| 25 | 25 | import { listApps } from "../core/apps.ts"; |
| 26 | 26 | import { ledgerNeed } from "../core/ledger.ts"; |
| 27 | import { overdueSchedules } from "../core/timers.ts"; | |
| 27 | import { overdueSchedules, type ScheduleCounts } from "../core/timers.ts"; | |
| 28 | 28 | import type { User } from "../core/users.ts"; |
| 29 | 29 | import { forgeHealth, rolloutLine, type ForgeHealth } from "../kernel/guard.ts"; |
| 30 | 30 | import { |
| @@ -122,7 +122,7 @@ export interface Parts { | ||
| 122 | 122 | |
| 123 | 123 | /** The forge's overdue schedules as counts, on the health answer (#12512). */ |
| 124 | 124 | export interface ScheduleHealth { |
| 125 | readonly schedules: { readonly overdue: number; readonly oldestSeconds: number }; | |
| 125 | readonly schedules: ScheduleCounts; | |
| 126 | 126 | } |
| 127 | 127 | |
| 128 | 128 | /** |
src/contract/openapi/routes/getApiV1Health.ts +1 −1
| @@ -4,7 +4,7 @@ export const route: Route = { | ||
| 4 | 4 | method: "get", |
| 5 | 5 | path: "/api/v1/health", |
| 6 | 6 | summary: |
| 7 | "How the forge is, uncached; anyone may ask, a stranger is told little, the release probe and a site admin the whole of it. Every answer carries schedules {overdue, oldestSeconds}: how many unpaused schedules are more than five minutes past due and the age in seconds of the most overdue (0 when none), counts only, naming no app or repository", | |
| 7 | "How the forge is, uncached; anyone may ask, a stranger is told little, the release probe and a site admin the whole of it. Every answer carries schedules {overdue, oldestSeconds, total, ran}: how many unpaused schedules are more than five minutes past due, the age in seconds of the most overdue (0 when none), how many unpaused schedules there are and how many of them have a recorded run, counts only, naming no app or repository", | |
| 8 | 8 | auth: false, |
| 9 | 9 | role: "anonymous", |
| 10 | 10 | reply: { ok: "boolean", draining: "boolean", schedules: "object" }, |
src/core/timers.ts +33 −6
| @@ -108,19 +108,43 @@ export function dueTimers(db: Database, now: number, limit = 50): Timer[] { | ||
| 108 | 108 | */ |
| 109 | 109 | export const OVERDUE_AFTER_MS = 5 * 60_000; |
| 110 | 110 | |
| 111 | /** The forge's schedules as counts, on the health answer (#12512, #13252). */ | |
| 112 | export interface ScheduleCounts { | |
| 113 | /** Unpaused schedules due more than `OVERDUE_AFTER_MS` ago. */ | |
| 114 | readonly overdue: number; | |
| 115 | /** The age in whole seconds of the most overdue (0 when none). */ | |
| 116 | readonly oldestSeconds: number; | |
| 117 | /** Unpaused schedules in all. */ | |
| 118 | readonly total: number; | |
| 119 | /** Of those, how many have at least one recorded run. */ | |
| 120 | readonly ran: number; | |
| 121 | } | |
| 122 | ||
| 111 | 123 | /** |
| 112 | * The forge's overdue schedules, counts only (#12512): how many unpaused schedules are due more | |
| 113 | * than `OVERDUE_AFTER_MS` ago, and the age in whole seconds of the most overdue (0 when none). | |
| 114 | * No app, repository or name: a stranger may read it from the health answer. | |
| 124 | * The forge's schedules, counts only (#12512): how many unpaused schedules there are, how many | |
| 125 | * of them have ever run, how many are due more than `OVERDUE_AFTER_MS` ago, and the age in whole | |
| 126 | * seconds of the most overdue (0 when none). No app, repository or name: a stranger may read it | |
| 127 | * from the health answer. | |
| 115 | 128 | */ |
| 116 | export function overdueSchedules(db: Database, now: number): { overdue: number; oldestSeconds: number } { | |
| 129 | export function overdueSchedules(db: Database, now: number): ScheduleCounts { | |
| 117 | 130 | const r = db |
| 118 | 131 | .query<{ n: number; oldest: number | null }, [number]>( |
| 119 | 132 | "SELECT COUNT(*) AS n, MIN(due_at) AS oldest FROM timer WHERE kind = 'schedule' AND paused = 0 AND due_at < ?1", |
| 120 | 133 | ) |
| 121 | 134 | .get(now - OVERDUE_AFTER_MS); |
| 135 | const all = db | |
| 136 | .query<{ total: number; ran: number | null }, []>( | |
| 137 | `SELECT COUNT(*) AS total, SUM(EXISTS (SELECT 1 FROM timer_run r WHERE r.app_id = t.app_id AND r.name = t.name AND r.env = t.env)) AS ran | |
| 138 | FROM timer t WHERE t.kind = 'schedule' AND t.paused = 0`, | |
| 139 | ) | |
| 140 | .get(); | |
| 122 | 141 | const n = r?.n ?? 0; |
| 123 | return { overdue: n, oldestSeconds: n === 0 || r?.oldest == null ? 0 : Math.floor((now - r.oldest) / 1000) }; | |
| 142 | return { | |
| 143 | overdue: n, | |
| 144 | oldestSeconds: n === 0 || r?.oldest == null ? 0 : Math.floor((now - r.oldest) / 1000), | |
| 145 | total: all?.total ?? 0, | |
| 146 | ran: all?.ran ?? 0, | |
| 147 | }; | |
| 124 | 148 | } |
| 125 | 149 | |
| 126 | 150 | export function getTimer(db: Database, id: number): Timer | undefined { |
| @@ -258,7 +282,10 @@ export function runsOf(db: Database, appId: string, name?: string, limit = 50): | ||
| 258 | 282 | return rows.map(toRun); |
| 259 | 283 | } |
| 260 | 284 | |
| 261 | /** The newest run of each of the app's schedules whose latest run did not go (for the owner's ask). One index walk per schedule. */ | |
| 285 | /** | |
| 286 | * The newest run of each of the app's schedules whose latest run did not go (for the owner's ask). | |
| 287 | * One index walk per schedule. | |
| 288 | */ | |
| 262 | 289 | export function failingNow(db: Database, appId: string): TimerRun[] { |
| 263 | 290 | const out: TimerRun[] = []; |
| 264 | 291 | for (const t of scheduleTimers(db, appId)) { |
src/core/v1_proof.ts +10 −3
| @@ -202,14 +202,21 @@ export const probes: readonly Probe[] = [ | ||
| 202 | 202 | { |
| 203 | 203 | issue: 11595, |
| 204 | 204 | item: "no schedule is past its period", |
| 205 | // The forge-wide count on the public health answer (#12512): no repository named. | |
| 205 | // The forge-wide counts on the public health answer (#12512): no repository named. Zero | |
| 206 | // schedules, or none that has run, proves nothing about schedules running while it sleeps | |
| 207 | // (#13252). | |
| 206 | 208 | run: async (get) => { |
| 207 | 209 | const got = await bodyOk(get, "/api/v1/health"); |
| 208 | const s = (JSON.parse(got.text) as { schedules?: { overdue?: unknown; oldestSeconds?: unknown } }).schedules; | |
| 210 | type Counts = { overdue?: unknown; oldestSeconds?: unknown; total?: unknown; ran?: unknown }; | |
| 211 | const s = (JSON.parse(got.text) as { schedules?: Counts }).schedules; | |
| 209 | 212 | if (s === undefined || typeof s.overdue !== "number" || typeof s.oldestSeconds !== "number") |
| 210 | 213 | return unproven("/api/v1/health carries no schedules count: this forge predates #12512"); |
| 211 | 214 | expect(s.overdue === 0, `${s.overdue} schedule(s) overdue, the oldest by ${ageWords(s.oldestSeconds)}`); |
| 212 | return "/api/v1/health: no schedule is overdue"; | |
| 215 | if (typeof s.total !== "number" || typeof s.ran !== "number") | |
| 216 | return unproven("/api/v1/health does not count its schedules: this forge predates #13252"); | |
| 217 | if (s.total === 0) return unproven("this forge has no schedule, so none has been seen to run"); | |
| 218 | if (s.ran === 0) return unproven(`${s.total} schedule(s), none of which has run yet`); | |
| 219 | return `/api/v1/health: ${s.total} schedule(s), ${s.ran} with a recorded run, none overdue`; | |
| 213 | 220 | }, |
| 214 | 221 | }, |
| 215 | 222 | { |
test/health-schedules.test.ts +20 −3
| @@ -11,7 +11,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 11 | 11 | import { createApp as createAppRecord } from "../src/core/apps.ts"; |
| 12 | 12 | import { createRepo } from "../src/core/repos.ts"; |
| 13 | 13 | import { createUser } from "../src/core/users.ts"; |
| 14 | import { OVERDUE_AFTER_MS, overdueSchedules, setPaused, scheduleTimer, syncSchedules } from "../src/core/timers.ts"; | |
| 14 | import { OVERDUE_AFTER_MS, overdueSchedules, recordRun, setPaused, scheduleTimer, syncSchedules } from "../src/core/timers.ts"; | |
| 15 | 15 | import { serve, type Servers } from "../src/http/server.ts"; |
| 16 | 16 | |
| 17 | 17 | let dataDir: string; |
| @@ -56,8 +56,8 @@ const strangerHealth = async () => { | ||
| 56 | 56 | }; |
| 57 | 57 | |
| 58 | 58 | test("no schedule: none overdue", async () => { |
| 59 | expect(overdueSchedules(app.db, Date.now())).toEqual({ overdue: 0, oldestSeconds: 0 }); | |
| 60 | expect(JSON.parse(await strangerHealth()).schedules).toEqual({ overdue: 0, oldestSeconds: 0 }); | |
| 59 | expect(overdueSchedules(app.db, Date.now())).toEqual({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 }); | |
| 60 | expect(JSON.parse(await strangerHealth()).schedules).toEqual({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 }); | |
| 61 | 61 | }); |
| 62 | 62 | |
| 63 | 63 | test("a schedule due inside the loop's slack is not overdue; one past it is, with its age", async () => { |
| @@ -86,3 +86,20 @@ test("a paused schedule is not overdue", async () => { | ||
| 86 | 86 | setPaused(app.db, scheduleTimer(app.db, appId, "weekly", "production")!.id, true, now); |
| 87 | 87 | expect(overdueSchedules(app.db, now).overdue).toBe(before - 1); |
| 88 | 88 | }); |
| 89 | ||
| 90 | test("total counts unpaused schedules and ran those with a recorded run (#13252)", async () => { | |
| 91 | const now = Date.now(); | |
| 92 | const before = overdueSchedules(app.db, now); | |
| 93 | const appId = await arm("ran-paper", "hourly", now); | |
| 94 | const armed = overdueSchedules(app.db, now); | |
| 95 | expect(armed.total).toBe(before.total + 1); | |
| 96 | expect(armed.ran).toBe(before.ran); | |
| 97 | recordRun(app.db, { | |
| 98 | appId, env: "production", name: "hourly", idemKey: "k1", scheduledAt: now, startedAt: now, finishedAt: now, | |
| 99 | attempts: 1, status: "ok", httpStatus: 200, response: "ok", error: null, lateMs: 0, lateReason: null, durationMs: 3, | |
| 100 | }); | |
| 101 | expect(overdueSchedules(app.db, now).ran).toBe(before.ran + 1); | |
| 102 | const body = JSON.parse(await strangerHealth()) as { schedules: { total: number; ran: number } }; | |
| 103 | expect(body.schedules.total).toBe(armed.total); | |
| 104 | expect(body.schedules.ran).toBe(before.ran + 1); | |
| 105 | }); |
test/security-floor-paths.test.ts +13
| @@ -18,7 +18,9 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; | ||
| 18 | 18 | import { newApp } from "../src/actions/apps.ts"; |
| 19 | 19 | import type { EnvStatus } from "../src/apps/runtime.ts"; |
| 20 | 20 | import type { AppRecord } from "../src/core/apps.ts"; |
| 21 | import { recordRun, syncSchedules } from "../src/core/timers.ts"; | |
| 21 | 22 | import { line, prove } from "../src/core/v1_proof.ts"; |
| 23 | import { parseSchedules } from "../src/apps/schedule_spec.ts"; | |
| 22 | 24 | import { |
| 23 | 25 | fetchAsHost, |
| 24 | 26 | startAppsForge, |
| @@ -209,6 +211,17 @@ test("the proxy's 304 of an app's page keeps the headers the app's 200 left with | ||
| 209 | 211 | }); |
| 210 | 212 | |
| 211 | 213 | test("`bun scripts/v1-proven.ts`'s proof passes 20 times in a row against a booted forge", async () => { |
| 214 | // The schedule probe (#11595) is unproven on a forge with no schedule that has run (#13252), | |
| 215 | // so this one gets an app with an hourly schedule and one recorded run, armed for later. | |
| 216 | const app = await appNamed("proof-schedules", "public"); | |
| 217 | const specs = parseSchedules([{ name: "hourly", cron: "0 * * * *", path: "/jobs/run" }]); | |
| 218 | if (!specs.ok) throw new Error(specs.error); | |
| 219 | const now = Date.now(); | |
| 220 | syncSchedules(forge.app.db, app.id, "production", specs.value, "c1", now); | |
| 221 | recordRun(forge.app.db, { | |
| 222 | appId: app.id, env: "production", name: "hourly", idemKey: "k1", scheduledAt: now, startedAt: now, finishedAt: now, | |
| 223 | attempts: 1, status: "ok", httpStatus: 200, response: "ok", error: null, lateMs: 0, lateReason: null, durationMs: 3, | |
| 224 | }); | |
| 212 | 225 | const failed: string[] = []; |
| 213 | 226 | for (let run = 0; run < 20; run++) |
| 214 | 227 | for (const r of await prove(forge.servers.url, fetch)) |
test/v1-proven.test.ts +18 −4
| @@ -27,6 +27,7 @@ const rule = (days: number, words = days) => | ||
| 27 | 27 | const health = (schedules?: unknown, ver = "1.0.0") => |
| 28 | 28 | Response.json({ ok: true, version: ver, commit: "058b6cf", draining: false, ...(schedules === undefined ? {} : { schedules }) }); |
| 29 | 29 | |
| 30 | const FRESH = { overdue: 0, oldestSeconds: 0, total: 1, ran: 1 }; | |
| 30 | 31 | const FEED = '<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>Changelog</title><entry><id>x</id></entry></feed>'; |
| 31 | 32 | type Over = Record<string, () => Response>; |
| 32 | 33 | function stub(over: Over = {}, seen: string[] = []): Fetch { |
| @@ -40,7 +41,7 @@ function stub(over: Over = {}, seen: string[] = []): Fetch { | ||
| 40 | 41 | "/api/v1/plans": () => Response.json({ items: [], graceDays: 7, lapse: "sleep" }), |
| 41 | 42 | "/settings/billing": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fbilling" } }), |
| 42 | 43 | "/settings/crew": () => new Response(null, { status: 303, headers: { location: "/signin?next=%2Fsettings%2Fcrew" } }), |
| 43 | "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }), | |
| 44 | "/api/v1/health": () => health(FRESH), | |
| 44 | 45 | "/changelog.atom": () => new Response(FEED), |
| 45 | 46 | ...over, |
| 46 | 47 | }; |
| @@ -67,7 +68,7 @@ describe("v1-proven", () => { | ||
| 67 | 68 | }); |
| 68 | 69 | |
| 69 | 70 | test("production at 0.7.604 reports its release and is not proven; the API's own constant does not pass it", async () => { |
| 70 | const results = await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") })); | |
| 71 | const results = await prove("https://f.test", stub({ "/api/v1/health": () => health(FRESH, "0.7.604") })); | |
| 71 | 72 | expect(line(results.find((r) => r.issue === 10304)!)).toBe("pass #10304 production reports the release it runs: release 0.7.604"); |
| 72 | 73 | expect(line(results.find((r) => r.issue === 13042)!)).toBe("fail #13042 the release production runs is 1.0.0 or later: the release is 0.7.604, not 1.0.0"); |
| 73 | 74 | }); |
| @@ -85,8 +86,8 @@ describe("v1-proven", () => { | ||
| 85 | 86 | [11539, { "/healthz": () => new Response("down", { status: 503 }) }, [10441, 11539]], |
| 86 | 87 | [11158, { "/llms.txt": () => new Response("Running dev at 058b6cf") }, [11158]], |
| 87 | 88 | [11573, { "/readyz": () => new Response("draining", { status: 503 }) }, [11573]], |
| 88 | [10304, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "dev") }, [10304, 13042]], | |
| 89 | [13042, { "/api/v1/health": () => health({ overdue: 0, oldestSeconds: 0 }, "0.7.604") }, [13042]], | |
| 89 | [10304, { "/api/v1/health": () => health(FRESH, "dev") }, [10304, 13042]], | |
| 90 | [13042, { "/api/v1/health": () => health(FRESH, "0.7.604") }, [13042]], | |
| 90 | 91 | [11507, { "/llms.txt": () => new Response("Start my forge") }, [11507]], |
| 91 | 92 | [11138, { "/pricing": () => new Response("<p>none</p>") }, [11138, 11140, 11141]], |
| 92 | 93 | [11140, { "/pricing": () => new Response('<form method="post" action="/pricing/order"></form><script type="application/ld+json">{"offers":[]}</script>') }, [11140, 11141]], |
| @@ -120,6 +121,19 @@ describe("v1-proven", () => { | ||
| 120 | 121 | expect(r.seen!.ms).toBeGreaterThanOrEqual(0); |
| 121 | 122 | }); |
| 122 | 123 | |
| 124 | test("the schedule probe is unproven with no schedule, passes with one fresh that ran, and fails with one overdue (#13252)", async () => { | |
| 125 | const at = async (schedules: unknown) => | |
| 126 | (await prove("https://f.test", stub({ "/api/v1/health": () => health(schedules) }))).find((x) => x.issue === 11595)!; | |
| 127 | const none = await at({ overdue: 0, oldestSeconds: 0, total: 0, ran: 0 }); | |
| 128 | expect(none.status).toBe("unproven"); | |
| 129 | expect(line(none)).toBe("unproven #11595 no schedule is past its period: this forge has no schedule, so none has been seen to run"); | |
| 130 | expect((await at(FRESH)).status).toBe("pass"); | |
| 131 | expect((await at({ overdue: 1, oldestSeconds: 600, total: 1, ran: 1 })).status).toBe("fail"); | |
| 132 | // A schedule that has never run proves nothing yet; a forge that does not count them is older. | |
| 133 | expect((await at({ overdue: 0, oldestSeconds: 0, total: 1, ran: 0 })).status).toBe("unproven"); | |
| 134 | expect((await at({ overdue: 0, oldestSeconds: 0 })).status).toBe("unproven"); | |
| 135 | }); | |
| 136 | ||
| 123 | 137 | test("the schedule probe fails naming the oldest age, is unproven without the field, and a network error is a fail", async () => { |
| 124 | 138 | const late = (await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 2, oldestSeconds: 3 * 3600 + 720 }) }))).find((x) => x.issue === 11595)!; |
| 125 | 139 | expect(line(late)).toBe("fail #11595 no schedule is past its period: 2 schedule(s) overdue, the oldest by 3 h 12 min"); |
new above
offline