Public: anyone can read this; the crew can change it.

The release proof removes its own probe forge in-process, with no admin token to configure #12575

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12573-the-release-proof-removes-its-own-probe → main 6 commits fork 1 thread · 1 unresolved

  • Merged. scout/12573-the-release-proof-removes-its-own-probe was merged into main as 4cc77c2.

Review

Sign in to review this change.
4 files changed +183 −28
src/main.ts +8 −3
@@ -104,6 +104,8 @@ import { vitals } from "./kernel/vitals.ts";
104104 import { markReady, readFootprint, keepFootprint } from "./system/footprint.ts";
105105 import { proveRunning } from "./release/prove.ts";
106106 import { walkJourney } from "./release/journey.ts";
107 import { removeForge } from "./spawn/manage.ts";
108 import { listChildren } from "./actions/spawn.ts";
107109 import { proberKey } from "./release/prober.ts";
108110 import { CHECK_EVERY_MS, setBehind, watchBehind } from "./release/behind.ts";
109111 import { readLandings } from "./release/landings.ts";
@@ -116,8 +118,6 @@ import { keeperEnv, keeperPorts } from "./keeper/wire.ts";
116118 import { noteKeeperAtBoot } from "./keeper/at_boot.ts";
117119 import { snapshotPlace } from "./actions/health.ts";
118120 import { dropServed } from "./spawn/clients.ts";
119 import { removeForge as removeHostedForge } from "./spawn/manage.ts";
120 import { listChildren } from "./actions/spawn.ts";
121121 import { sweepLeftoverProbes } from "./release/leftover_sweep.ts";
122122 import { forgesMemory, memoryOverLine } from "./spawn/memory_fit.ts";
123123 import { keepFloor } from "./apps/cgroup.ts";
@@ -770,7 +770,7 @@ const tickAsks = async (): Promise<void> => {
770770 dataDir: app.config.dataDir,
771771 held: async () =>
772772 new Map((await listChildren(app)).map((c) => [c.name, { createdAt: c.createdAt, ...(c.ownerId === undefined ? {} : { ownerId: c.ownerId }) }])),
773 remove: async (name) => (await removeHostedForge(app, name)).ok,
773 remove: async (name) => (await removeForge(app, name)).ok,
774774 dropServed: () => dropServed(app),
775775 log: app.log,
776776 });
@@ -890,6 +890,11 @@ proveRunning({
890890 journey: walkJourney,
891891 footprint: async () => keepFootprint(await readFootprint(app.config.dataDir)),
892892 proberKey: proberKey(app.hookSecret),
893 // The probe forge is removed here, in-process, by what `POST /forges/:name/remove` runs:
894 // production has no FORGE_TOKEN to ask itself with (#12573). The journey refuses any name
895 // that is not `probe-` and eight hex digits.
896 remove: async (name) => (await removeForge(app, name)).ok,
897 list: async () => (await listChildren(app)).map((c) => ({ name: c.name, begunAt: c.createdAt === null ? null : Date.parse(c.createdAt) })),
893898 // The first app's production host: it must never answer as the forge (#6726).
894899 appHost: (() => {
895900 const first = listApps(app.db)[0];
src/release/journey.ts +78 −20
@@ -18,9 +18,12 @@
1818 * The throwaway forge is removed afterwards, passed or failed: its crew run is
1919 * cancelled first (best effort: the forge's removal stops the whole child and
2020 * everything in it), then `POST /forges/<name>/remove` on the host with
21 * `adminToken` — the one thing the host asks an admin for. Without a token the
22 * forge cannot be removed from here, and `removed` says so; the walk's verdict
23 * is its own.
21 * `adminToken` — the one thing the host asks an admin for. The forge walking its
22 * own public address passes `remove` and `list` in place of a token: the same
23 * removal `POST /forges/:name/remove` runs, called in-process (#12573). With
24 * neither the forge cannot be removed from here, and `removed` says so; the
25 * walk's verdict is its own. Either way only a `probe-` and eight hex digits
26 * name is ever removed.
2427 */
2528 import { FIRST_FIVE_MINUTES } from "../design/first_five_minutes.ts";
2629 import { PROBER_HEADER } from "./prober.ts";
@@ -37,6 +40,17 @@ export interface JourneyOptions {
3740 readonly now?: () => number;
3841 /** A site admin's token on the host: removes the throwaway forge afterwards. */
3942 readonly adminToken?: string;
43 /**
44 * Removes a forge on this very host, in-process; true when it is gone. When given it is used
45 * in place of HTTP and `adminToken`, for the walk's own forge and the sweep. Only probe names
46 * reach it.
47 */
48 readonly remove?: (name: string) => Promise<boolean>;
49 /**
50 * The host's own forges, `begunAt` in wall-clock ms (null when unknown): what the sweep reads
51 * in place of `GET /api/v1/forges`.
52 */
53 readonly list?: () => Promise<readonly ProbeListing[]>;
4054 /**
4155 * This boot's prober key (`release/prober.ts`): sent to the host only, so the forge's own walk
4256 * is not a stranger to its rate limits.
@@ -48,6 +62,18 @@ export interface JourneyOptions {
4862 readonly sleep?: (ms: number) => Promise<void>;
4963 }
5064
65 /** One forge the host knows, as the sweep reads it. */
66 export interface ProbeListing {
67 readonly name: string;
68 readonly begunAt: number | null;
69 }
70
71 /** The in-process hands on the host, when the walker is the host itself. */
72 export interface LocalHands {
73 readonly remove: (name: string) => Promise<boolean>;
74 readonly list?: () => Promise<readonly ProbeListing[]>;
75 }
76
5177 export interface JourneyResult {
5278 readonly ok: boolean;
5379 /** The screens a person was shown, in order, the forge last. */
@@ -75,8 +101,12 @@ export const SWEEP_MAX = 5;
75101 /** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */
76102 export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000, 3000, 10_000];
77103
outdated

This comment still says "an hour" but STALE_PROBE_MS is now 30 minutes. Update it to half an hour.

78 /** A probe forge older than this was left by a run that never finished: an hour. */
79 export const STALE_PROBE_MS = 60 * 60 * 1000;
104 /**
105 * A probe forge older than this was left by a run that never finished: half an hour. One walk
106 * holds its forge under five minutes and a whole proof (three settles, three walks, two
107 * pauses) ends in under twenty-four, so a run in progress is never this old.
108 */
109 export const STALE_PROBE_MS = 30 * 60 * 1000;
80110
81111 /** The longest any one request may take. */
82112 export const REQUEST_TIMEOUT_MS = 10_000;
@@ -101,6 +131,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
101131 const doFetch = options.fetch ?? globalThis.fetch;
102132 const pollMs = options.pollMs ?? 500;
103133 const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
134 const local: LocalHands | undefined = options.remove === undefined ? undefined : { remove: options.remove, ...(options.list ? { list: options.list } : {}) };
104135 const screens: string[] = [];
105136 let forge: string | null = null;
106137 let readyMs: number | null = null;
@@ -264,16 +295,16 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
264295 // Afterwards, passed or failed: stop the crew's run, then remove the forge and its account.
265296 if (forge !== null) {
266297 if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
267 removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep);
298 removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local);
268299 // A forge left running is hidden work on the host: the failure names it (never when there
269 // was no token to remove it with).
270 if (!removed && options.adminToken !== undefined && options.adminToken !== "")
300 // was nothing to remove it with).
301 if (!removed && (local !== undefined || (options.adminToken !== undefined && options.adminToken !== "")))
271302 failure = `${failure === undefined ? "" : `${failure}; `}the probe forge ${forge} could not be removed`;
272303 }
273304 }
274305 // What an interrupted run left: swept after this walk's own cleanup, never its own forge.
275306 // Bounded (`SWEEP_BUDGET_MS`, `SWEEP_MAX`) and never part of the verdict.
276 const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep);
307 const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep, local);
277308 const result = {
278309 ok: failure === undefined,
279310 screens,
@@ -288,14 +319,15 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
288319 }
289320
290321 /**
291 * Removes the `probe-` forges the host lists that were begun more than an hour ago
322 * Removes the `probe-` forges the host lists that were begun more than half an hour ago
292323 * (`STALE_PROBE_MS`): what a run killed or timed out part-way left behind. The scope is
293324 * this proof's own namespace: `probe-` plus eight hex digits is the name `probeName` gives
294 * a throwaway forge, and an hour is far past the proof's own budget, so a run still in
325 * a throwaway forge, and half an hour is past the proof's own budget, so a run still in
295326 * progress is never touched. A forge with no begun-at, a name of any other shape, and
296 * `except` are left alone. Reads `GET /api/v1/forges` (an admin's token). Never throws: what
297 * stopped it comes back in `error`. `at` is the wall clock in ms (the forge's `createdAt` is
298 * wall-clock time, not the walk's monotonic one). Bounded by `SWEEP_BUDGET_MS` and `SWEEP_MAX`.
327 * `except` are left alone. Reads `local.list` when given, else `GET /api/v1/forges` (an
328 * admin's token). Never throws: what stopped it comes back in `error`. `at` is the wall clock
329 * in ms (the forge's `createdAt` is wall-clock time, not the walk's monotonic one). Bounded by
330 * `SWEEP_BUDGET_MS` and `SWEEP_MAX`.
299331 */
300332 export async function sweepProbes(
301333 doFetch: typeof fetch,
@@ -304,10 +336,24 @@ export async function sweepProbes(
304336 at: number,
305337 except: string | null = null,
306338 sleep: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms),
339 local?: LocalHands,
307340 ): Promise<{ gone: string[]; error?: string }> {
308341 const gone: string[] = [];
309 if (token === undefined || token === "") return { gone };
310342 const deadline = AbortSignal.timeout(SWEEP_BUDGET_MS);
343 if (local?.list !== undefined) {
344 try {
345 for (const { name, begunAt } of await local.list()) {
346 if (!PROBE_NAME.test(name) || name === except) continue;
347 if (begunAt === null || !Number.isFinite(begunAt) || at - begunAt < STALE_PROBE_MS) continue;
348 if (gone.length >= SWEEP_MAX || deadline.aborted) break;
349 if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name);
350 }
351 return { gone };
352 } catch (e) {
353 return { gone, error: e instanceof Error ? e.message : "the sweep stopped unexpectedly" };
354 }
355 }
356 if (token === undefined || token === "") return { gone };
311357 try {
312358 const res = await doFetch(new URL("/api/v1/forges", host).href, {
313359 redirect: "manual",
@@ -326,7 +372,7 @@ export async function sweepProbes(
326372 const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN;
327373 if (!Number.isFinite(begun) || at - begun < STALE_PROBE_MS) continue;
328374 if (gone.length >= SWEEP_MAX || deadline.aborted) break;
329 if (await removeForge(doFetch, host, name, token, sleep, [])) gone.push(name);
375 if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name);
330376 }
331377 return { gone };
332378 } catch (e) {
@@ -348,7 +394,11 @@ async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr
348394 } catch {}
349395 }
350396
351 /** Removes the throwaway forge from the host; true when the host said it is gone. */
397 /**
398 * Removes the throwaway forge from the host; true when the host said it is gone. In-process
399 * (`local.remove`) when given, else over HTTP with `token`. A name that is not a probe's is
400 * refused.
401 */
352402 async function removeForge(
353403 doFetch: typeof fetch,
354404 host: URL,
@@ -356,13 +406,21 @@ async function removeForge(
356406 token: string | undefined,
357407 sleep: (ms: number) => Promise<void>,
358408 backoff: readonly number[] = REMOVE_BACKOFF_MS,
409 local?: LocalHands,
359410 ): Promise<boolean> {
360 if (token === undefined || token === "") return false;
361 if (await removeOnce(doFetch, host, name, token)) return true;
411 if (!PROBE_NAME.test(name)) return false;
412 const once =
413 local !== undefined
414 ? () => local.remove(name).catch(() => false)
415 : token === undefined || token === ""
416 ? undefined
417 : () => removeOnce(doFetch, host, name, token);
418 if (once === undefined) return false;
419 if (await once()) return true;
362420 // Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once.
363421 for (const ms of backoff) {
364422 await sleep(ms);
365 if (await removeOnce(doFetch, host, name, token)) return true;
423 if (await once()) return true;
366424 }
367425 return false;
368426 }
src/release/prove.ts +9 −2
@@ -22,7 +22,7 @@
2222
2323 import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
2424 import { join } from "node:path";
25 import type { walkJourney, JourneyResult } from "./journey.ts";
25 import type { walkJourney, JourneyResult, ProbeListing } from "./journey.ts";
2626 import { smoke, type Failure, type Heavy } from "./smoke.ts";
2727 import { UNSTAMPED } from "../kernel/version.ts";
2828 import { ownProbeHeaders } from "../kernel/probe.ts";
@@ -336,6 +336,13 @@ export interface ProveOptions {
336336 * limits know it.
337337 */
338338 readonly proberKey?: string;
339 /**
340 * Removes a probe forge on this host in-process (`spawn/manage.ts`), handed to the journey in
341 * place of a token (#12573).
342 */
343 readonly remove?: (name: string) => Promise<boolean>;
344 /** This host's own forges, for the journey's sweep of leftover probes. */
345 readonly list?: () => Promise<readonly ProbeListing[]>;
339346 }
340347
341348 /** Is a proof of `version` due: never tried, or failed `retryAfter` its failures or more ago? */
@@ -422,7 +429,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
422429 const settled = await settle(o.url, doFetch, now, sleep, o.giveUpMs === undefined ? SETTLE_MS : Math.min(SETTLE_MS, o.giveUpMs));
423430 tries++;
424431 walked = settled
425 ? await o.journey({ base: o.url, fetch: doFetch, ...(o.token ? { adminToken: o.token } : {}), ...(o.proberKey ? { proberKey: o.proberKey } : {}) })
432 ? await o.journey({ base: o.url, fetch: doFetch, ...(o.token ? { adminToken: o.token } : {}), ...(o.proberKey ? { proberKey: o.proberKey } : {}), ...(o.remove ? { remove: o.remove } : {}), ...(o.list ? { list: o.list } : {}) })
426433 : { ok: false, screens: [], readyMs: null, firstActivityMs: null, failure: `the forge had not settled in ${SETTLE_MS / 1000} s after its start`, forge: null, removed: true };
427434 if (walked.forge !== null && !walked.removed) leftover.push(walked.forge);
428435 // A passed walk, a refusal by the sign-up limit, or a forge left behind ends the tries:
test/release-journey.test.ts +88 −3
@@ -19,7 +19,8 @@ import { addSubscription } from "../src/core/subscriptions.ts";
1919 import { createUser } from "../src/core/users.ts";
2020 import { FIRST_FIVE_MINUTES } from "../src/design/first_five_minutes.ts";
2121 import { serve, type Servers } from "../src/http/server.ts";
22 import { REMOVE_BACKOFF_MS, sweepProbes, walkJourney } from "../src/release/journey.ts";
22 import { REMOVE_BACKOFF_MS, STALE_PROBE_MS, sweepProbes, walkJourney } from "../src/release/journey.ts";
23 import { removeForge } from "../src/spawn/manage.ts";
2324 import { useProbe } from "../src/spawn/door.ts";
2425 import { firstBoot } from "../src/spawn/first-boot.ts";
2526 import { quietLogs } from "./helpers/globals.ts";
@@ -204,7 +205,7 @@ test("a walk that blows up half-way (outside any step) still removes its forge",
204205 expect(existsSync(kids.get(name)!.dataDir)).toBe(false);
205206 }, 60_000);
206207
207 test("a run sweeps the probe forges an interrupted run left over an hour ago, and only those", async () => {
208 test("a run sweeps the probe forges an interrupted run left over half an hour ago, and only those", async () => {
208209 const start = async (name: string) => {
209210 const res = await fetch(`${parentServers.url}/forges/start`, { method: "POST", redirect: "manual", headers: { origin: "https://forge.test" }, body: new URLSearchParams({ name }) });
210211 await res.body?.cancel();
@@ -224,7 +225,7 @@ test("a run sweeps the probe forges an interrupted run left over an hour ago, an
224225 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), adminToken, pollMs: 0, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} });
225226 expect(result.swept).toEqual(["probe-aaaaaaaa"]);
226227 expect(await forgeGone("probe-aaaaaaaa")).toBe(true);
227 expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under an hour: may be a run in progress
228 expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under half an hour: may be a run in progress
228229 expect(await childNamed(parent, "keepme-forge")).toBeDefined(); // not a probe's name
229230 for (const n of ["probe-bbbbbbbb", "keepme-forge"])
230231 await fetch(`${parentServers.url}/forges/${n}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
@@ -301,3 +302,87 @@ test("a removal that fails three times is still retried, so a slow host does not
301302 expect(result.failure === undefined || !result.failure.includes("could not be removed")).toBe(true);
302303 await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
303304 }, 60_000);
305
306 /**
307 * The in-process hand `src/main.ts` passes: what `POST /forges/:name/remove` runs, the stand-in
308 * child stopped first.
309 */
310 function localRemove(asked: string[]) {
311 return async (name: string) => {
312 asked.push(name);
313 const live = kids.get(name);
314 if (live !== undefined && !stopped.has(live)) {
315 stopped.add(live);
316 await live.crew.stop();
317 live.servers.stop();
318 live.app.close();
319 }
320 return (await removeForge(parent, name)).ok;
321 };
322 }
323
324 test("with no admin token, an in-process remove takes the probe forge away after a passing walk", async () => {
325 broken = undefined;
326 const asked: string[] = [];
327 let t = 0;
328 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: localRemove(asked), now: () => (t += 10), sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time
329 expect(result.failure).toBeUndefined();
330 expect(result.ok).toBe(true);
331 expect(result.removed).toBe(true);
332 expect(asked).toEqual([result.forge!]);
333 expect(await forgeGone(result.forge!)).toBe(true);
334 }, 60_000);
335
336 test("with no admin token, an in-process remove takes the probe forge away after a failing walk", async () => {
337 broken = "POST /claim";
338 const asked: string[] = [];
339 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: localRemove(asked), sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time
340 broken = undefined;
341 expect(result.ok).toBe(false);
342 expect(result.failure).toBe("the account could not be made: it answered 503");
343 expect(result.removed).toBe(true);
344 expect(await forgeGone(result.forge!)).toBe(true);
345 }, 60_000);
346
347 test("an in-process remove that never succeeds names the forge left running", async () => {
348 let t = 0;
349 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: async () => false, pollMs: 0, now: () => (t += 5_000), sleep: async () => {} });
350 expect(result.removed).toBe(false);
351 expect(result.failure).toContain(`the probe forge ${result.forge} could not be removed`);
352 await removeForge(parent, result.forge!);
353 }, 60_000);
354
355 test("the in-process sweep removes a 31-minute-old probe, and leaves a 29-minute-old one and any other name alone", async () => {
356 expect(STALE_PROBE_MS).toBe(30 * 60 * 1000);
357 const at = Date.now();
358 const min = 60_000;
359 const removed: string[] = [];
360 const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch;
361 const list = async () => [
362 { name: "probe-0000000a", begunAt: at - 31 * min },
363 { name: "probe-0000000b", begunAt: at - 29 * min },
364 { name: "keepme-forge", begunAt: at - 5 * 60 * min },
365 { name: "probe-0000000c-x", begunAt: at - 5 * 60 * min },
366 { name: "probe-0000000d", begunAt: null },
367 ];
368 const remove = async (name: string) => (removed.push(name), true);
369 const out = await sweepProbes(never, new URL(parentServers.url), undefined, at, null, async () => {}, { remove, list });
370 expect(out).toEqual({ gone: ["probe-0000000a"] });
371 expect(removed).toEqual(["probe-0000000a"]);
372 });
373
374 test("the in-process hands are never given a name that is not a probe's, nor the walk's own forge", async () => {
375 const at = Date.now();
376 const removed: string[] = [];
377 const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch;
378 const list = async () => [{ name: "probe-0000000e", begunAt: at - 99 * 60_000 }, { name: "Probe-0000000f", begunAt: 0 }, { name: "alice", begunAt: 0 }];
379 const out = await sweepProbes(never, new URL(parentServers.url), undefined, at, "probe-0000000e", async () => {}, { remove: async (n) => (removed.push(n), true), list });
380 expect(out).toEqual({ gone: [] });
381 expect(removed).toEqual([]);
382 });
383
384 test("an in-process list that throws comes back as the sweep's reason", async () => {
385 const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch;
386 const list = async () => { throw new Error("the ledger is unreadable"); };
387 expect(await sweepProbes(never, new URL(parentServers.url), undefined, Date.now(), null, async () => {}, { remove: async () => true, list })).toEqual({ gone: [], error: "the ledger is unreadable" });
388 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs