Public: anyone can read this; the crew can change it.

A reach alarm is not raised for a child the host just restarted, and clears when it answers #14032

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13757-a-reach-alarm-is-not-raised-for-a-child → main 3 commits fork

  • Merged. mason/13757-a-reach-alarm-is-not-raised-for-a-child was merged into main as 30d06a2.

Review

Sign in to review this change.
2 files changed +97 −2
src/spawn/reach.ts +25 −1
@@ -27,6 +27,8 @@ import { isSlug, slugFor } from "../core/apps.ts";
2727 import type { FiredAlarm } from "../flow/alarms.ts";
2828 import { childPublicUrl, parentOrigin } from "./placement.ts";
2929 import { uncleanInFirstMinutes } from "../kernel/boots.ts";
30 import { lastStop, stoppedMark } from "./stopped.ts";
31 import { parseRecord, recordPath } from "./cell_record.ts";
3032 import { during } from "../perf/jobs.ts";
3133
3234 export const APP_HEADER = "x-forge-app";
@@ -319,6 +321,27 @@ export async function settleChild(app: App, name: string): Promise<void> {
319321 }
320322 }
321323
324 /** When the cell record in a child's directory says its cell started, if it has one. */
325 function startedOf(dir: string): number | undefined {
326 try {
327 return parseRecord(JSON.parse(readFileSync(recordPath(dir), "utf8")))?.startedAt;
328 } catch {
329 return undefined;
330 }
331 }
332
333 /**
334 * Whether the host stopped or (re)started this child's cell within the settle window: its cell
335 * record's start, a stop it noted or marked. Such a child is given the settle window a new one
336 * is (#13757): its first failed proof is kept, not raised.
337 */
338 export function restartedRecently(app: App, child: Pick<Child, "bodyId">, now = Date.now()): boolean {
339 const window = optionsOf(app).settleMs ?? SETTLE_MS;
340 const dir = childDirOf(app, child.bodyId);
341 const times = [lastStop(app, child.bodyId, now)?.at, stoppedMark(dir)?.at, startedOf(dir)];
342 return times.some((at) => at !== undefined && now - at >= 0 && now - at < window);
343 }
344
322345 /**
323346 * One pass over every finished child: prove, confirm a failure once, keep, alarm on a change.
324347 * A child `asleep` is passed over and keeps its last proof: asking its address wakes it, and
@@ -337,7 +360,8 @@ export async function sweepReach(app: App, asleep: (name: string) => boolean = (
337360 await Bun.sleep(confirmMs);
338361 proved = await proveChild(app, child);
339362 }
340 record(app, proved, true);
363 // The settle check reads two files: only a failure needs it.
364 record(app, proved, proved.ok || !restartedRecently(app, child));
341365 } catch (error) {
342366 app.log.error("could not prove a child's reach", { name: child.name, error: String(error) });
343367 }
test/spawn-reach.test.ts +72 −1
@@ -5,7 +5,7 @@
55 * came back, and alarms once on a change.
66 */
77 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
8 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
99 import { tmpdir } from "node:os";
1010 import { join } from "node:path";
1111 import { createApp, type App } from "../src/app.ts";
@@ -14,6 +14,7 @@ import { clientRows, runningWords, useChildFetch } from "../src/spawn/clients.ts
1414 import { Database } from "bun:sqlite";
1515 import { childDirOf } from "../src/actions/spawn.ts";
1616 import { appUrlOf, slugsOf, proveApp, proveHome, reachAlarms, reachOf, reachWords, sweepReach, useReach } from "../src/spawn/reach.ts";
17 import { noteStop, cancelRestarts, lastStop, markStopped } from "../src/spawn/stopped.ts";
1718 import { quietLogs } from "./helpers/globals.ts";
1819
1920 quietLogs();
@@ -164,4 +165,74 @@ describe("a forge watches its children's addresses", () => {
164165 expect(asked.filter((u) => /evil|@|%|#|\/\/[^/]*\/[^/]/.test(u.replace(/^https:\/\//, "")) && !u.endsWith("/api/v1/forge"))).toEqual([]);
165166 expect(asked.every((u) => new URL(u).hostname.endsWith("--kid--forge.test") || u === "https://kid--forge.test/api/v1/forge")).toBe(true);
166167 });
168
169 describe("a child the host just restarted is given a settle window (#13757)", () => {
170 const down = () => new Response("not found", { status: 404 });
171 const reset = () => {
172 world.set("https://ok-app--kid--forge.test/", () => new Response("ok", { headers: { "x-forge-app": "ok-app" } }));
173 rmSync(join(dir, "reach"), { recursive: true, force: true });
174 rmSync(childDirOf(parent, "cell-k") + "/cell.json", { force: true });
175 rmSync(childDirOf(parent, "cell-k") + "/stopped.json", { force: true });
176 cancelRestarts(parent);
177 };
178 const raised = () => events("forge.child_unreachable").length;
179
180 const startCell = (at: number) =>
181 writeFileSync(
182 join(childDirOf(parent, "cell-k"), "cell.json"),
183 JSON.stringify({ bodyId: "cell-k", pid: 1, port: 1, supervisor: "x", startedAt: at, name: "kid", size: "small" }),
184 );
185 const answers = () => world.set("https://deal-clock--kid--forge.test/", () => new Response("deal", { headers: { "x-forge-app": "deal-clock" } }));
186 const keptQuietly = async () => {
187 const before = raised();
188 await sweepReach(parent);
189 expect(reachOf(parent, "kid")).toMatchObject({ ok: false, alarmed: false });
190 expect(reachAlarms(parent)).toEqual([]);
191 expect(raised()).toBe(before);
192 };
193
194 test("a stop the host just noted keeps the failure without an alarm", async () => {
195 reset();
196 world.set("https://deal-clock--kid--forge.test/", down);
197 noteStop(parent, "cell-k", "kid", "memory", Date.now() - 60_000, 3_600_000);
198 await keptQuietly();
199 });
200
201 test("a stop mark alone, with no noted stop (after a host restart), keeps the failure without an alarm", async () => {
202 reset();
203 world.set("https://deal-clock--kid--forge.test/", down);
204 markStopped(childDirOf(parent, "cell-k"), { at: Date.now() - 60_000, why: "memory" });
205 expect(lastStop(parent, "cell-k")).toBeUndefined();
206 await keptQuietly();
207 });
208
209 test("a cell the host started a minute ago keeps the failure without an alarm", async () => {
210 reset();
211 world.set("https://deal-clock--kid--forge.test/", down);
212 startCell(Date.now() - 60_000);
213 await keptQuietly();
214 });
215
216 test("past the window the same failure alarms once, and clears when the child answers", async () => {
217 reset();
218 world.set("https://deal-clock--kid--forge.test/", down);
219 startCell(Date.now() - 3_600_000);
220 const before = raised();
221 await sweepReach(parent);
222 expect(raised()).toBe(before + 1);
223 expect(reachAlarms(parent)).toHaveLength(1);
224 answers();
225 await sweepReach(parent);
226 expect(reachOf(parent, "kid")).toMatchObject({ ok: true, alarmed: false });
227 expect(reachAlarms(parent)).toEqual([]);
228 });
229
230 test("a healthy child just restarted is proved ok as any other", async () => {
231 reset();
232 answers();
233 startCell(Date.now() - 1_000);
234 await sweepReach(parent);
235 expect(reachOf(parent, "kid")).toMatchObject({ ok: true, alarmed: false });
236 });
237 });
167238 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs