Public: anyone can read this; the crew can change it.
A reach alarm is not raised for a child the host just restarted, and clears when it answers #14032
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13757-a-reach-alarm-is-not-raised-for-a-child → main 3 commits fork
2 files
changed +97
−2
src/spawn/reach.ts +25 −1
| @@ -27,6 +27,8 @@ import { isSlug, slugFor } from "../core/apps.ts"; | ||
| 27 | 27 | import type { FiredAlarm } from "../flow/alarms.ts"; |
| 28 | 28 | import { childPublicUrl, parentOrigin } from "./placement.ts"; |
| 29 | 29 | import { uncleanInFirstMinutes } from "../kernel/boots.ts"; |
| 30 | import { lastStop, stoppedMark } from "./stopped.ts"; | |
| 31 | import { parseRecord, recordPath } from "./cell_record.ts"; | |
| 30 | 32 | import { during } from "../perf/jobs.ts"; |
| 31 | 33 | |
| 32 | 34 | export const APP_HEADER = "x-forge-app"; |
| @@ -319,6 +321,27 @@ export async function settleChild(app: App, name: string): Promise<void> { | ||
| 319 | 321 | } |
| 320 | 322 | } |
| 321 | 323 | |
| 324 | /** When the cell record in a child's directory says its cell started, if it has one. */ | |
| 325 | function startedOf(dir: string): number | undefined { | |
| 326 | try { | |
| 327 | return parseRecord(JSON.parse(readFileSync(recordPath(dir), "utf8")))?.startedAt; | |
| 328 | } catch { | |
| 329 | return undefined; | |
| 330 | } | |
| 331 | } | |
| 332 | ||
| 333 | /** | |
| 334 | * Whether the host stopped or (re)started this child's cell within the settle window: its cell | |
| 335 | * record's start, a stop it noted or marked. Such a child is given the settle window a new one | |
| 336 | * is (#13757): its first failed proof is kept, not raised. | |
| 337 | */ | |
| 338 | export function restartedRecently(app: App, child: Pick<Child, "bodyId">, now = Date.now()): boolean { | |
| 339 | const window = optionsOf(app).settleMs ?? SETTLE_MS; | |
| 340 | const dir = childDirOf(app, child.bodyId); | |
| 341 | const times = [lastStop(app, child.bodyId, now)?.at, stoppedMark(dir)?.at, startedOf(dir)]; | |
| 342 | return times.some((at) => at !== undefined && now - at >= 0 && now - at < window); | |
| 343 | } | |
| 344 | ||
| 322 | 345 | /** |
| 323 | 346 | * One pass over every finished child: prove, confirm a failure once, keep, alarm on a change. |
| 324 | 347 | * A child `asleep` is passed over and keeps its last proof: asking its address wakes it, and |
| @@ -337,7 +360,8 @@ export async function sweepReach(app: App, asleep: (name: string) => boolean = ( | ||
| 337 | 360 | await Bun.sleep(confirmMs); |
| 338 | 361 | proved = await proveChild(app, child); |
| 339 | 362 | } |
| 340 | record(app, proved, true); | |
| 363 | // The settle check reads two files: only a failure needs it. | |
| 364 | record(app, proved, proved.ok || !restartedRecently(app, child)); | |
| 341 | 365 | } catch (error) { |
| 342 | 366 | app.log.error("could not prove a child's reach", { name: child.name, error: String(error) }); |
| 343 | 367 | } |
test/spawn-reach.test.ts +72 −1
| @@ -5,7 +5,7 @@ | ||
| 5 | 5 | * came back, and alarms once on a change. |
| 6 | 6 | */ |
| 7 | 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 8 | import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | |
| 8 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 9 | 9 | import { tmpdir } from "node:os"; |
| 10 | 10 | import { join } from "node:path"; |
| 11 | 11 | import { createApp, type App } from "../src/app.ts"; |
| @@ -14,6 +14,7 @@ import { clientRows, runningWords, useChildFetch } from "../src/spawn/clients.ts | ||
| 14 | 14 | import { Database } from "bun:sqlite"; |
| 15 | 15 | import { childDirOf } from "../src/actions/spawn.ts"; |
| 16 | 16 | import { appUrlOf, slugsOf, proveApp, proveHome, reachAlarms, reachOf, reachWords, sweepReach, useReach } from "../src/spawn/reach.ts"; |
| 17 | import { noteStop, cancelRestarts, lastStop, markStopped } from "../src/spawn/stopped.ts"; | |
| 17 | 18 | import { quietLogs } from "./helpers/globals.ts"; |
| 18 | 19 | |
| 19 | 20 | quietLogs(); |
| @@ -164,4 +165,74 @@ describe("a forge watches its children's addresses", () => { | ||
| 164 | 165 | expect(asked.filter((u) => /evil|@|%|#|\/\/[^/]*\/[^/]/.test(u.replace(/^https:\/\//, "")) && !u.endsWith("/api/v1/forge"))).toEqual([]); |
| 165 | 166 | expect(asked.every((u) => new URL(u).hostname.endsWith("--kid--forge.test") || u === "https://kid--forge.test/api/v1/forge")).toBe(true); |
| 166 | 167 | }); |
| 168 | ||
| 169 | describe("a child the host just restarted is given a settle window (#13757)", () => { | |
| 170 | const down = () => new Response("not found", { status: 404 }); | |
| 171 | const reset = () => { | |
| 172 | world.set("https://ok-app--kid--forge.test/", () => new Response("ok", { headers: { "x-forge-app": "ok-app" } })); | |
| 173 | rmSync(join(dir, "reach"), { recursive: true, force: true }); | |
| 174 | rmSync(childDirOf(parent, "cell-k") + "/cell.json", { force: true }); | |
| 175 | rmSync(childDirOf(parent, "cell-k") + "/stopped.json", { force: true }); | |
| 176 | cancelRestarts(parent); | |
| 177 | }; | |
| 178 | const raised = () => events("forge.child_unreachable").length; | |
| 179 | ||
| 180 | const startCell = (at: number) => | |
| 181 | writeFileSync( | |
| 182 | join(childDirOf(parent, "cell-k"), "cell.json"), | |
| 183 | JSON.stringify({ bodyId: "cell-k", pid: 1, port: 1, supervisor: "x", startedAt: at, name: "kid", size: "small" }), | |
| 184 | ); | |
| 185 | const answers = () => world.set("https://deal-clock--kid--forge.test/", () => new Response("deal", { headers: { "x-forge-app": "deal-clock" } })); | |
| 186 | const keptQuietly = async () => { | |
| 187 | const before = raised(); | |
| 188 | await sweepReach(parent); | |
| 189 | expect(reachOf(parent, "kid")).toMatchObject({ ok: false, alarmed: false }); | |
| 190 | expect(reachAlarms(parent)).toEqual([]); | |
| 191 | expect(raised()).toBe(before); | |
| 192 | }; | |
| 193 | ||
| 194 | test("a stop the host just noted keeps the failure without an alarm", async () => { | |
| 195 | reset(); | |
| 196 | world.set("https://deal-clock--kid--forge.test/", down); | |
| 197 | noteStop(parent, "cell-k", "kid", "memory", Date.now() - 60_000, 3_600_000); | |
| 198 | await keptQuietly(); | |
| 199 | }); | |
| 200 | ||
| 201 | test("a stop mark alone, with no noted stop (after a host restart), keeps the failure without an alarm", async () => { | |
| 202 | reset(); | |
| 203 | world.set("https://deal-clock--kid--forge.test/", down); | |
| 204 | markStopped(childDirOf(parent, "cell-k"), { at: Date.now() - 60_000, why: "memory" }); | |
| 205 | expect(lastStop(parent, "cell-k")).toBeUndefined(); | |
| 206 | await keptQuietly(); | |
| 207 | }); | |
| 208 | ||
| 209 | test("a cell the host started a minute ago keeps the failure without an alarm", async () => { | |
| 210 | reset(); | |
| 211 | world.set("https://deal-clock--kid--forge.test/", down); | |
| 212 | startCell(Date.now() - 60_000); | |
| 213 | await keptQuietly(); | |
| 214 | }); | |
| 215 | ||
| 216 | test("past the window the same failure alarms once, and clears when the child answers", async () => { | |
| 217 | reset(); | |
| 218 | world.set("https://deal-clock--kid--forge.test/", down); | |
| 219 | startCell(Date.now() - 3_600_000); | |
| 220 | const before = raised(); | |
| 221 | await sweepReach(parent); | |
| 222 | expect(raised()).toBe(before + 1); | |
| 223 | expect(reachAlarms(parent)).toHaveLength(1); | |
| 224 | answers(); | |
| 225 | await sweepReach(parent); | |
| 226 | expect(reachOf(parent, "kid")).toMatchObject({ ok: true, alarmed: false }); | |
| 227 | expect(reachAlarms(parent)).toEqual([]); | |
| 228 | }); | |
| 229 | ||
| 230 | test("a healthy child just restarted is proved ok as any other", async () => { | |
| 231 | reset(); | |
| 232 | answers(); | |
| 233 | startCell(Date.now() - 1_000); | |
| 234 | await sweepReach(parent); | |
| 235 | expect(reachOf(parent, "kid")).toMatchObject({ ok: true, alarmed: false }); | |
| 236 | }); | |
| 237 | }); | |
| 167 | 238 | }); |
new above
offline