5d5042a · 1c5f82b · 97b1728 · 39331b3 · 83ee05e
@@ -55,8 +55,25 @@
export interface JourneyResult {
  readonly forge: string | null;
  /** Whether the throwaway forge was removed afterwards (true when none was made). */
  readonly removed: boolean;
  /** `probe-` forges an interrupted run left (older than `STALE_PROBE_MS`) this run removed. */
  readonly swept?: readonly string[];
  /** Why the sweep stopped early, when it did; the walk's verdict never depends on it. */
  readonly sweepError?: string;
}
/** The sweep as a whole may take this long, however many it finds: it never slows the proof. */
export const SWEEP_BUDGET_MS = 5_000;
/** At most this many stale forges are removed per run; a host with more clears over several. */
export const SWEEP_MAX = 5;
/** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */
export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000];
/** A probe forge older than this was left by a run that never finished: an hour. */
export const STALE_PROBE_MS = 60 * 60 * 1000;
/** What a probe forge is called; its account goes with it (the forge's data is its only home). */
const PROBE_NAME = /^probe-[0-9a-f]{8}$/;
/** The longest any one request may take. */
export const REQUEST_TIMEOUT_MS = 10_000;
@@ -240,17 +257,76 @@
export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
    }
  } catch (e) {
    failure = e instanceof Step ? e.message : "the walk stopped unexpectedly";
  } finally {
    // Afterwards, passed or failed: stop the crew's run, then remove the forge and its account.
    if (forge !== null) {
      if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
      removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep);
    }
  }
  // Afterwards, passed or failed: stop the crew's run, then remove the forge.
  if (forge !== null) {
    if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
    removed = await removeForge(doFetch, hostUrl, forge, options.adminToken);
  }
  const result = { ok: failure === undefined, screens, readyMs, firstActivityMs, forge, removed };
  // What an interrupted run left: swept after this walk's own cleanup, never its own forge.
  // Bounded (`SWEEP_BUDGET_MS`, `SWEEP_MAX`) and never part of the verdict.
  const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep);
  const result = {
    ok: failure === undefined,
    screens,
    readyMs,
    firstActivityMs,
    forge,
    removed,
    swept: sweep.gone,
    ...(sweep.error === undefined ? {} : { sweepError: sweep.error }),
  };
  return failure === undefined ? result : { ...result, failure };
}
/**
 * Removes the `probe-` forges the host lists that were begun more than an hour ago
 * (`STALE_PROBE_MS`): what a run killed or timed out part-way left behind. The scope is
 * this proof's own namespace: `probe-` plus eight hex digits is the name `probeName` gives
 * a throwaway forge, and an hour is far past the proof's own budget, so a run still in
 * progress is never touched. A forge with no begun-at, a name of any other shape, and
 * `except` are left alone. Reads `GET /api/v1/forges` (an admin's token). Never throws: what
 * stopped it comes back in `error`. `at` is the wall clock in ms (the forge's `createdAt` is
 * wall-clock time, not the walk's monotonic one). Bounded by `SWEEP_BUDGET_MS` and `SWEEP_MAX`.
 */
export async function sweepProbes(
  doFetch: typeof fetch,
  host: URL,
  token: string | undefined,
  at: number,
  except: string | null = null,
  sleep: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms),
): Promise<{ gone: string[]; error?: string }> {
  const gone: string[] = [];
  if (token === undefined || token === "") return { gone };
  const deadline = AbortSignal.timeout(SWEEP_BUDGET_MS);
  try {
    const res = await doFetch(new URL("/api/v1/forges", host).href, {
      redirect: "manual",
      signal: deadline,
      headers: { authorization: `Bearer ${token}`, accept: "application/json" },
    });
    if (res.status !== 200) {
      await res.body?.cancel();
      return { gone, error: `the forge list answered ${res.status}` };
    }
    const items = ((await res.json()) as { items?: unknown }).items;
    if (!Array.isArray(items)) return { gone, error: "the forge list had no items" };
    for (const item of items as { name?: unknown; createdAt?: unknown }[]) {
      const { name, createdAt } = item ?? {};
      if (typeof name !== "string" || !PROBE_NAME.test(name) || name === except) continue;
      const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN;
      if (!Number.isFinite(begun) || at - begun < STALE_PROBE_MS) continue;
      if (gone.length >= SWEEP_MAX || deadline.aborted) break;
      if (await removeForge(doFetch, host, name, token, sleep, [])) gone.push(name);
    }
    return { gone };
  } catch (e) {
    return { gone, error: e instanceof Error ? e.message : "the sweep stopped unexpectedly" };
  }
}
/** Cancels each run the forge's home links to; a miss is never an error (the forge's removal ends them anyway). */
async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Promise<void> {
  try {
@@ -266,8 +342,25 @@
async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr
}
/** Removes the throwaway forge from the host; true when the host said it is gone. */
async function removeForge(doFetch: typeof fetch, host: URL, name: string, token: string | undefined): Promise<boolean> {
async function removeForge(
  doFetch: typeof fetch,
  host: URL,
  name: string,
  token: string | undefined,
  sleep: (ms: number) => Promise<void>,
  backoff: readonly number[] = REMOVE_BACKOFF_MS,
): Promise<boolean> {
  if (token === undefined || token === "") return false;
  if (await removeOnce(doFetch, host, name, token)) return true;
  // Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once.
  for (const ms of backoff) {
    await sleep(ms);
    if (await removeOnce(doFetch, host, name, token)) return true;
  }
  return false;
}
async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<boolean> {
  try {
    const url = new URL(`/forges/${name}/remove`, host);
    const res = await doFetch(url.href, {
@@ -13,11 +13,12 @@
import { childDirOf, childNamed } from "../src/actions/spawn.ts";
import { createApp, type App } from "../src/app.ts";
import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
import { createToken } from "../src/core/tokens.ts";
import { useChildFetch } from "../src/spawn/clients.ts";
import { addSubscription } from "../src/core/subscriptions.ts";
import { createUser } from "../src/core/users.ts";
import { FIRST_FIVE_MINUTES } from "../src/design/first_five_minutes.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { walkJourney } from "../src/release/journey.ts";
import { REMOVE_BACKOFF_MS, sweepProbes, walkJourney } from "../src/release/journey.ts";
import { useProbe } from "../src/spawn/door.ts";
import { firstBoot } from "../src/spawn/first-boot.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -28,7 +29,7 @@
let dir = "";
let parent: App;
let parentServers: Servers;
let adminToken = "";
const kids = new Map<string, { app: App; servers: Servers; crew: CrewHandle }>();
const kids = new Map<string, { app: App; servers: Servers; crew: CrewHandle; dataDir: string }>();
/** A step that answers 503, as `METHOD path` on the child. */
let broken: string | undefined;
@@ -36,6 +37,8 @@
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-journey-"));
  parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 });
  useProbe(parent, () => async () => true);
  // The forge list asks each child about itself; none of those asks leaves this machine.
  useChildFetch(parent, async () => new Response("{}", { status: 503 }));
  const alice = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  if (!alice.ok) throw new Error(alice.error);
  adminToken = createToken(parent.db, alice.value.id, "journey", "admin").token;
@@ -76,7 +79,7 @@
sleep 20
  );
  chmodSync(fake, 0o755);
  const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 1, budgetUsd: 1, runTimeoutMs: 30_000, networkRetryScale: 0, tickMs: 100 });
  const k = { app, servers, crew };
  const k = { app, servers, crew, dataDir };
  kids.set(name, k);
  return k;
}
@@ -159,3 +162,101 @@
test("an address that is not http(s) is refused before anything is sent", async
  const result = await walkJourney({ base: "ftp://forge.test", fetch: (() => { throw new Error("no request"); }) as unknown as typeof fetch });
  expect(result).toMatchObject({ ok: false, forge: null, removed: true, failure: "the forge's address is not http or https" });
});
test("a walk that blows up half-way (outside any step) still removes its forge", async () => {
  let calls = 0;
  const log: string[] = [];
  const inner = routed(log);
  // `now` throws once the account exists: nothing in the walk catches it as a step.
  const now = () => {
    if (log.includes("POST /claim")) throw new Error("clock exploded");
    return (calls += 1);
  };
  const result = await walkJourney({ base: parentServers.url, fetch: inner, adminToken, now, sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time
  // The walk does not reject: it says what stopped it, and cleans up all the same.
  expect(result.ok).toBe(false);
  expect(result.failure).toBe("the walk stopped unexpectedly");
  expect(result.removed).toBe(true);
  expect(log).toContain("POST /claim");
  const name = [...kids.keys()].at(-1)!;
  expect(await forgeGone(name)).toBe(true);
  // The account lived in the forge's own data: that directory is gone, and the account with it.
  expect(existsSync(kids.get(name)!.dataDir)).toBe(false);
}, 60_000);
test("a run sweeps the probe forges an interrupted run left over an hour ago, and only those", async () => {
  const start = async (name: string) => {
    const res = await fetch(`${parentServers.url}/forges/start`, { method: "POST", redirect: "manual", headers: { origin: "https://forge.test" }, body: new URLSearchParams({ name, js: "1" }) });
    await res.body?.cancel();
    expect(res.status).toBe(303);
  };
  const age = async (name: string, ms: number) => {
    const file = join(dir, "spawn", `${name}.json`);
    const ledger = JSON.parse(await Bun.file(file).text());
    ledger.createdAt = new Date(Date.now() - ms).toISOString();
    await Bun.write(file, JSON.stringify(ledger));
  };
  await start("probe-aaaaaaaa");
  await start("probe-bbbbbbbb");
  await start("keepme-forge");
  await age("probe-aaaaaaaa", 2 * 60 * 60 * 1000);
  await age("keepme-forge", 5 * 60 * 60 * 1000);
  const result = await walkJourney({ base: parentServers.url, fetch: routed([]), adminToken, pollMs: 0, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} });
  expect(result.swept).toEqual(["probe-aaaaaaaa"]);
  expect(await forgeGone("probe-aaaaaaaa")).toBe(true);
  expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under an hour: may be a run in progress
  expect(await childNamed(parent, "keepme-forge")).toBeDefined(); // not a probe's name
  for (const n of ["probe-bbbbbbbb", "keepme-forge"])
    await fetch(`${parentServers.url}/forges/${n}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
}, 60_000);
test("a removal that fails once is tried again after a pause, and the result says it is gone", async () => {
  const log: string[] = [];
  const inner = routed(log);
  const pauses: number[] = [];
  let failed = 0;
  const flaky = (async (input: string | URL | Request, init?: RequestInit) => {
    if (new URL(String(input)).pathname === "/api/v1/forges") throw new Error("the list is down");
    if (/\/forges\/probe-[0-9a-f]{8}\/remove$/.test(new URL(String(input)).pathname) && failed < 1) {
      failed += 1;
      return new Response("busy", { status: 503 });
    }
    return inner(input, init);
  }) as typeof fetch;
  let t = 0;
  const sleep = async (ms: number) => {
    pauses.push(ms);
    await Bun.sleep(100); // real-timer: the stand-in crew child runs in real time
  };
  const result = await walkJourney({ base: parentServers.url, fetch: flaky, adminToken, pollMs: 100, now: () => (t += 10), sleep });
  // The sweep's list was down: that is reported, and the passing walk is still passing.
  expect(result.failure).toBeUndefined();
  expect(result.ok).toBe(true);
  expect(result.sweepError).toBe("the list is down");
  expect(failed).toBe(1);
  expect(result.removed).toBe(true);
  expect(pauses).toContain(REMOVE_BACKOFF_MS[0]!);
  expect(await forgeGone(result.forge!)).toBe(true);
}, 60_000);
test("a removal that never succeeds waits between its attempts and says it is not gone", async () => {
  const inner = routed([]);
  const pauses: number[] = [];
  const stuck = (async (input: string | URL | Request, init?: RequestInit) =>
    /\/remove$/.test(new URL(String(input)).pathname) && init?.method === "POST" ? new Response("no", { status: 503 }) : inner(input, init)) as typeof fetch;
  let t = 0;
  const result = await walkJourney({ base: parentServers.url, fetch: stuck, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async (ms) => void pauses.push(ms) });
  expect(result.removed).toBe(false);
  expect(pauses).toEqual([...REMOVE_BACKOFF_MS]);
  await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
}, 60_000);
test("a sweep with no admin token does nothing and says nothing is wrong", async () => {
  const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch;
  expect(await sweepProbes(never, new URL(parentServers.url), undefined, Date.now())).toEqual({ gone: [] });
});
test("a sweep that throws comes back as its reason and never throws", async () => {
  const broke = (async () => { throw new Error("the list is down"); }) as unknown as typeof fetch;
  expect(await sweepProbes(broke, new URL(parentServers.url), adminToken, Date.now())).toEqual({ gone: [], error: "the list is down" });
});
		@@ -55,8 +55,25 @@ export interface JourneyResult {
55	55	readonly forge: string \| null;
56	56	/** Whether the throwaway forge was removed afterwards (true when none was made). */
57	57	readonly removed: boolean;
	58	/** `probe-` forges an interrupted run left (older than `STALE_PROBE_MS`) this run removed. */
	59	readonly swept?: readonly string[];
	60	/** Why the sweep stopped early, when it did; the walk's verdict never depends on it. */
	61	readonly sweepError?: string;
58	62	}
59	63
	64	/** The sweep as a whole may take this long, however many it finds: it never slows the proof. */
	65	export const SWEEP_BUDGET_MS = 5_000;
	66	/** At most this many stale forges are removed per run; a host with more clears over several. */
	67	export const SWEEP_MAX = 5;
	68	/** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */
	69	export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000];
	70
	71	/** A probe forge older than this was left by a run that never finished: an hour. */
	72	export const STALE_PROBE_MS = 60 * 60 * 1000;
	73
	74	/** What a probe forge is called; its account goes with it (the forge's data is its only home). */
	75	const PROBE_NAME = /^probe-[0-9a-f]{8}$/;
	76
60	77	/** The longest any one request may take. */
61	78	export const REQUEST_TIMEOUT_MS = 10_000;
62	79
		@@ -240,17 +257,76 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
240	257	}
241	258	} catch (e) {
242	259	failure = e instanceof Step ? e.message : "the walk stopped unexpectedly";
	260	} finally {
	261	// Afterwards, passed or failed: stop the crew's run, then remove the forge and its account.
	262	if (forge !== null) {
	263	if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
	264	removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep);
	265	}
243	266	}
244
245		// Afterwards, passed or failed: stop the crew's run, then remove the forge.
246		if (forge !== null) {
247		if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
248		removed = await removeForge(doFetch, hostUrl, forge, options.adminToken);
249		}
250		const result = { ok: failure === undefined, screens, readyMs, firstActivityMs, forge, removed };
	267	// What an interrupted run left: swept after this walk's own cleanup, never its own forge.
	268	// Bounded (`SWEEP_BUDGET_MS`, `SWEEP_MAX`) and never part of the verdict.
	269	const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep);
	270	const result = {
	271	ok: failure === undefined,
	272	screens,
	273	readyMs,
	274	firstActivityMs,
	275	forge,
	276	removed,
	277	swept: sweep.gone,
	278	...(sweep.error === undefined ? {} : { sweepError: sweep.error }),
	279	};
251	280	return failure === undefined ? result : { ...result, failure };
252	281	}
253	282
	283	/**
	284	* Removes the `probe-` forges the host lists that were begun more than an hour ago
	285	* (`STALE_PROBE_MS`): what a run killed or timed out part-way left behind. The scope is
	286	* this proof's own namespace: `probe-` plus eight hex digits is the name `probeName` gives
	287	* a throwaway forge, and an hour is far past the proof's own budget, so a run still in
	288	* progress is never touched. A forge with no begun-at, a name of any other shape, and
	289	* `except` are left alone. Reads `GET /api/v1/forges` (an admin's token). Never throws: what
	290	* stopped it comes back in `error`. `at` is the wall clock in ms (the forge's `createdAt` is
	291	* wall-clock time, not the walk's monotonic one). Bounded by `SWEEP_BUDGET_MS` and `SWEEP_MAX`.
	292	*/
	293	export async function sweepProbes(
	294	doFetch: typeof fetch,
	295	host: URL,
	296	token: string \| undefined,
	297	at: number,
	298	except: string \| null = null,
	299	sleep: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms),
	300	): Promise<{ gone: string[]; error?: string }> {
	301	const gone: string[] = [];
	302	if (token === undefined \|\| token === "") return { gone };
	303	const deadline = AbortSignal.timeout(SWEEP_BUDGET_MS);
	304	try {
	305	const res = await doFetch(new URL("/api/v1/forges", host).href, {
	306	redirect: "manual",
	307	signal: deadline,
	308	headers: { authorization: `Bearer ${token}`, accept: "application/json" },
	309	});
	310	if (res.status !== 200) {
	311	await res.body?.cancel();
	312	return { gone, error: `the forge list answered ${res.status}` };
	313	}
	314	const items = ((await res.json()) as { items?: unknown }).items;
	315	if (!Array.isArray(items)) return { gone, error: "the forge list had no items" };
	316	for (const item of items as { name?: unknown; createdAt?: unknown }[]) {
	317	const { name, createdAt } = item ?? {};
	318	if (typeof name !== "string" \|\| !PROBE_NAME.test(name) \|\| name === except) continue;
	319	const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN;
	320	if (!Number.isFinite(begun) \|\| at - begun < STALE_PROBE_MS) continue;
	321	if (gone.length >= SWEEP_MAX \|\| deadline.aborted) break;
	322	if (await removeForge(doFetch, host, name, token, sleep, [])) gone.push(name);
	323	}
	324	return { gone };
	325	} catch (e) {
	326	return { gone, error: e instanceof Error ? e.message : "the sweep stopped unexpectedly" };
	327	}
	328	}
	329
254	330	/** Cancels each run the forge's home links to; a miss is never an error (the forge's removal ends them anyway). */
255	331	async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Promise<void> {
256	332	try {
		@@ -266,8 +342,25 @@ async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr
266	342	}
267	343
268	344	/** Removes the throwaway forge from the host; true when the host said it is gone. */
269		async function removeForge(doFetch: typeof fetch, host: URL, name: string, token: string \| undefined): Promise<boolean> {
	345	async function removeForge(
	346	doFetch: typeof fetch,
	347	host: URL,
	348	name: string,
	349	token: string \| undefined,
	350	sleep: (ms: number) => Promise<void>,
	351	backoff: readonly number[] = REMOVE_BACKOFF_MS,
	352	): Promise<boolean> {
270	353	if (token === undefined \|\| token === "") return false;
	354	if (await removeOnce(doFetch, host, name, token)) return true;
	355	// Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once.
	356	for (const ms of backoff) {
	357	await sleep(ms);
	358	if (await removeOnce(doFetch, host, name, token)) return true;
	359	}
	360	return false;
	361	}
	362
	363	async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<boolean> {
271	364	try {
272	365	const url = new URL(`/forges/${name}/remove`, host);
273	366	const res = await doFetch(url.href, {
		@@ -13,11 +13,12 @@ import { childDirOf, childNamed } from "../src/actions/spawn.ts";
13	13	import { createApp, type App } from "../src/app.ts";
14	14	import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
15	15	import { createToken } from "../src/core/tokens.ts";
	16	import { useChildFetch } from "../src/spawn/clients.ts";
16	17	import { addSubscription } from "../src/core/subscriptions.ts";
17	18	import { createUser } from "../src/core/users.ts";
18	19	import { FIRST_FIVE_MINUTES } from "../src/design/first_five_minutes.ts";
19	20	import { serve, type Servers } from "../src/http/server.ts";
20		import { walkJourney } from "../src/release/journey.ts";
	21	import { REMOVE_BACKOFF_MS, sweepProbes, walkJourney } from "../src/release/journey.ts";
21	22	import { useProbe } from "../src/spawn/door.ts";
22	23	import { firstBoot } from "../src/spawn/first-boot.ts";
23	24	import { quietLogs } from "./helpers/globals.ts";
		@@ -28,7 +29,7 @@ let dir = "";
28	29	let parent: App;
29	30	let parentServers: Servers;
30	31	let adminToken = "";
31		const kids = new Map<string, { app: App; servers: Servers; crew: CrewHandle }>();
	32	const kids = new Map<string, { app: App; servers: Servers; crew: CrewHandle; dataDir: string }>();
32	33	/** A step that answers 503, as `METHOD path` on the child. */
33	34	let broken: string \| undefined;
34	35
		@@ -36,6 +37,8 @@ beforeAll(async () => {
36	37	dir = await mkdtemp(join(tmpdir(), "forge-journey-"));
37	38	parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 });
38	39	useProbe(parent, () => async () => true);
	40	// The forge list asks each child about itself; none of those asks leaves this machine.
	41	useChildFetch(parent, async () => new Response("{}", { status: 503 }));
39	42	const alice = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
40	43	if (!alice.ok) throw new Error(alice.error);
41	44	adminToken = createToken(parent.db, alice.value.id, "journey", "admin").token;
		@@ -76,7 +79,7 @@ sleep 20
76	79	);
77	80	chmodSync(fake, 0o755);
78	81	const crew = await startCrew(app, { forgeUrl: servers.url, localUrl: servers.url, binary: fake, concurrency: 1, budgetUsd: 1, runTimeoutMs: 30_000, networkRetryScale: 0, tickMs: 100 });
79		const k = { app, servers, crew };
	82	const k = { app, servers, crew, dataDir };
80	83	kids.set(name, k);
81	84	return k;
82	85	}
		@@ -159,3 +162,101 @@ test("an address that is not http(s) is refused before anything is sent", async
159	162	const result = await walkJourney({ base: "ftp://forge.test", fetch: (() => { throw new Error("no request"); }) as unknown as typeof fetch });
160	163	expect(result).toMatchObject({ ok: false, forge: null, removed: true, failure: "the forge's address is not http or https" });
161	164	});
	165
	166	test("a walk that blows up half-way (outside any step) still removes its forge", async () => {
	167	let calls = 0;
	168	const log: string[] = [];
	169	const inner = routed(log);
	170	// `now` throws once the account exists: nothing in the walk catches it as a step.
	171	const now = () => {
	172	if (log.includes("POST /claim")) throw new Error("clock exploded");
	173	return (calls += 1);
	174	};
	175	const result = await walkJourney({ base: parentServers.url, fetch: inner, adminToken, now, sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time
	176	// The walk does not reject: it says what stopped it, and cleans up all the same.
	177	expect(result.ok).toBe(false);
	178	expect(result.failure).toBe("the walk stopped unexpectedly");
	179	expect(result.removed).toBe(true);
	180	expect(log).toContain("POST /claim");
	181	const name = [...kids.keys()].at(-1)!;
	182	expect(await forgeGone(name)).toBe(true);
	183	// The account lived in the forge's own data: that directory is gone, and the account with it.
	184	expect(existsSync(kids.get(name)!.dataDir)).toBe(false);
	185	}, 60_000);
	186
	187	test("a run sweeps the probe forges an interrupted run left over an hour ago, and only those", async () => {
	188	const start = async (name: string) => {
	189	const res = await fetch(`${parentServers.url}/forges/start`, { method: "POST", redirect: "manual", headers: { origin: "https://forge.test" }, body: new URLSearchParams({ name, js: "1" }) });
	190	await res.body?.cancel();
	191	expect(res.status).toBe(303);
	192	};
	193	const age = async (name: string, ms: number) => {
	194	const file = join(dir, "spawn", `${name}.json`);
	195	const ledger = JSON.parse(await Bun.file(file).text());
	196	ledger.createdAt = new Date(Date.now() - ms).toISOString();
	197	await Bun.write(file, JSON.stringify(ledger));
	198	};
	199	await start("probe-aaaaaaaa");
	200	await start("probe-bbbbbbbb");
	201	await start("keepme-forge");
	202	await age("probe-aaaaaaaa", 2 * 60 * 60 * 1000);
	203	await age("keepme-forge", 5 * 60 * 60 * 1000);
	204	const result = await walkJourney({ base: parentServers.url, fetch: routed([]), adminToken, pollMs: 0, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} });
	205	expect(result.swept).toEqual(["probe-aaaaaaaa"]);
	206	expect(await forgeGone("probe-aaaaaaaa")).toBe(true);
	207	expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under an hour: may be a run in progress
	208	expect(await childNamed(parent, "keepme-forge")).toBeDefined(); // not a probe's name
	209	for (const n of ["probe-bbbbbbbb", "keepme-forge"])
	210	await fetch(`${parentServers.url}/forges/${n}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
	211	}, 60_000);
	212
	213	test("a removal that fails once is tried again after a pause, and the result says it is gone", async () => {
	214	const log: string[] = [];
	215	const inner = routed(log);
	216	const pauses: number[] = [];
	217	let failed = 0;
	218	const flaky = (async (input: string \| URL \| Request, init?: RequestInit) => {
	219	if (new URL(String(input)).pathname === "/api/v1/forges") throw new Error("the list is down");
	220	if (/\/forges\/probe-[0-9a-f]{8}\/remove$/.test(new URL(String(input)).pathname) && failed < 1) {
	221	failed += 1;
	222	return new Response("busy", { status: 503 });
	223	}
	224	return inner(input, init);
	225	}) as typeof fetch;
	226	let t = 0;
	227	const sleep = async (ms: number) => {
	228	pauses.push(ms);
	229	await Bun.sleep(100); // real-timer: the stand-in crew child runs in real time
	230	};
	231	const result = await walkJourney({ base: parentServers.url, fetch: flaky, adminToken, pollMs: 100, now: () => (t += 10), sleep });
	232	// The sweep's list was down: that is reported, and the passing walk is still passing.
	233	expect(result.failure).toBeUndefined();
	234	expect(result.ok).toBe(true);
	235	expect(result.sweepError).toBe("the list is down");
	236	expect(failed).toBe(1);
	237	expect(result.removed).toBe(true);
	238	expect(pauses).toContain(REMOVE_BACKOFF_MS[0]!);
	239	expect(await forgeGone(result.forge!)).toBe(true);
	240	}, 60_000);
	241
	242	test("a removal that never succeeds waits between its attempts and says it is not gone", async () => {
	243	const inner = routed([]);
	244	const pauses: number[] = [];
	245	const stuck = (async (input: string \| URL \| Request, init?: RequestInit) =>
	246	/\/remove$/.test(new URL(String(input)).pathname) && init?.method === "POST" ? new Response("no", { status: 503 }) : inner(input, init)) as typeof fetch;
	247	let t = 0;
	248	const result = await walkJourney({ base: parentServers.url, fetch: stuck, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async (ms) => void pauses.push(ms) });
	249	expect(result.removed).toBe(false);
	250	expect(pauses).toEqual([...REMOVE_BACKOFF_MS]);
	251	await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
	252	}, 60_000);
	253
	254	test("a sweep with no admin token does nothing and says nothing is wrong", async () => {
	255	const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch;
	256	expect(await sweepProbes(never, new URL(parentServers.url), undefined, Date.now())).toEqual({ gone: [] });
	257	});
	258
	259	test("a sweep that throws comes back as its reason and never throws", async () => {
	260	const broke = (async () => { throw new Error("the list is down"); }) as unknown as typeof fetch;
	261	expect(await sweepProbes(broke, new URL(parentServers.url), adminToken, Date.now())).toEqual({ gone: [], error: "the list is down" });
	262	});