Public: anyone can read this; the crew can change it.
The release proof removes its own probe forge in-process, with no admin token to configure #12575
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12573-the-release-proof-removes-its-own-probe → main 6 commits fork 1 thread · 1 unresolved
4 files
changed +183
−28
src/main.ts +8 −3
| @@ -104,6 +104,8 @@ import { vitals } from "./kernel/vitals.ts"; | ||
| 104 | 104 | import { markReady, readFootprint, keepFootprint } from "./system/footprint.ts"; |
| 105 | 105 | import { proveRunning } from "./release/prove.ts"; |
| 106 | 106 | import { walkJourney } from "./release/journey.ts"; |
| 107 | import { removeForge } from "./spawn/manage.ts"; | |
| 108 | import { listChildren } from "./actions/spawn.ts"; | |
| 107 | 109 | import { proberKey } from "./release/prober.ts"; |
| 108 | 110 | import { CHECK_EVERY_MS, setBehind, watchBehind } from "./release/behind.ts"; |
| 109 | 111 | import { readLandings } from "./release/landings.ts"; |
| @@ -116,8 +118,6 @@ import { keeperEnv, keeperPorts } from "./keeper/wire.ts"; | ||
| 116 | 118 | import { noteKeeperAtBoot } from "./keeper/at_boot.ts"; |
| 117 | 119 | import { snapshotPlace } from "./actions/health.ts"; |
| 118 | 120 | import { dropServed } from "./spawn/clients.ts"; |
| 119 | import { removeForge as removeHostedForge } from "./spawn/manage.ts"; | |
| 120 | import { listChildren } from "./actions/spawn.ts"; | |
| 121 | 121 | import { sweepLeftoverProbes } from "./release/leftover_sweep.ts"; |
| 122 | 122 | import { forgesMemory, memoryOverLine } from "./spawn/memory_fit.ts"; |
| 123 | 123 | import { keepFloor } from "./apps/cgroup.ts"; |
| @@ -770,7 +770,7 @@ const tickAsks = async (): Promise<void> => { | ||
| 770 | 770 | dataDir: app.config.dataDir, |
| 771 | 771 | held: async () => |
| 772 | 772 | new Map((await listChildren(app)).map((c) => [c.name, { createdAt: c.createdAt, ...(c.ownerId === undefined ? {} : { ownerId: c.ownerId }) }])), |
| 773 | remove: async (name) => (await removeHostedForge(app, name)).ok, | |
| 773 | remove: async (name) => (await removeForge(app, name)).ok, | |
| 774 | 774 | dropServed: () => dropServed(app), |
| 775 | 775 | log: app.log, |
| 776 | 776 | }); |
| @@ -890,6 +890,11 @@ proveRunning({ | ||
| 890 | 890 | journey: walkJourney, |
| 891 | 891 | footprint: async () => keepFootprint(await readFootprint(app.config.dataDir)), |
| 892 | 892 | proberKey: proberKey(app.hookSecret), |
| 893 | // The probe forge is removed here, in-process, by what `POST /forges/:name/remove` runs: | |
| 894 | // production has no FORGE_TOKEN to ask itself with (#12573). The journey refuses any name | |
| 895 | // that is not `probe-` and eight hex digits. | |
| 896 | remove: async (name) => (await removeForge(app, name)).ok, | |
| 897 | list: async () => (await listChildren(app)).map((c) => ({ name: c.name, begunAt: c.createdAt === null ? null : Date.parse(c.createdAt) })), | |
| 893 | 898 | // The first app's production host: it must never answer as the forge (#6726). |
| 894 | 899 | appHost: (() => { |
| 895 | 900 | const first = listApps(app.db)[0]; |
src/release/journey.ts +78 −20
| @@ -18,9 +18,12 @@ | ||
| 18 | 18 | * The throwaway forge is removed afterwards, passed or failed: its crew run is |
| 19 | 19 | * cancelled first (best effort: the forge's removal stops the whole child and |
| 20 | 20 | * everything in it), then `POST /forges/<name>/remove` on the host with |
| 21 | * `adminToken` — the one thing the host asks an admin for. Without a token the | |
| 22 | * forge cannot be removed from here, and `removed` says so; the walk's verdict | |
| 23 | * is its own. | |
| 21 | * `adminToken` — the one thing the host asks an admin for. The forge walking its | |
| 22 | * own public address passes `remove` and `list` in place of a token: the same | |
| 23 | * removal `POST /forges/:name/remove` runs, called in-process (#12573). With | |
| 24 | * neither the forge cannot be removed from here, and `removed` says so; the | |
| 25 | * walk's verdict is its own. Either way only a `probe-` and eight hex digits | |
| 26 | * name is ever removed. | |
| 24 | 27 | */ |
| 25 | 28 | import { FIRST_FIVE_MINUTES } from "../design/first_five_minutes.ts"; |
| 26 | 29 | import { PROBER_HEADER } from "./prober.ts"; |
| @@ -37,6 +40,17 @@ export interface JourneyOptions { | ||
| 37 | 40 | readonly now?: () => number; |
| 38 | 41 | /** A site admin's token on the host: removes the throwaway forge afterwards. */ |
| 39 | 42 | readonly adminToken?: string; |
| 43 | /** | |
| 44 | * Removes a forge on this very host, in-process; true when it is gone. When given it is used | |
| 45 | * in place of HTTP and `adminToken`, for the walk's own forge and the sweep. Only probe names | |
| 46 | * reach it. | |
| 47 | */ | |
| 48 | readonly remove?: (name: string) => Promise<boolean>; | |
| 49 | /** | |
| 50 | * The host's own forges, `begunAt` in wall-clock ms (null when unknown): what the sweep reads | |
| 51 | * in place of `GET /api/v1/forges`. | |
| 52 | */ | |
| 53 | readonly list?: () => Promise<readonly ProbeListing[]>; | |
| 40 | 54 | /** |
| 41 | 55 | * This boot's prober key (`release/prober.ts`): sent to the host only, so the forge's own walk |
| 42 | 56 | * is not a stranger to its rate limits. |
| @@ -48,6 +62,18 @@ export interface JourneyOptions { | ||
| 48 | 62 | readonly sleep?: (ms: number) => Promise<void>; |
| 49 | 63 | } |
| 50 | 64 | |
| 65 | /** One forge the host knows, as the sweep reads it. */ | |
| 66 | export interface ProbeListing { | |
| 67 | readonly name: string; | |
| 68 | readonly begunAt: number | null; | |
| 69 | } | |
| 70 | ||
| 71 | /** The in-process hands on the host, when the walker is the host itself. */ | |
| 72 | export interface LocalHands { | |
| 73 | readonly remove: (name: string) => Promise<boolean>; | |
| 74 | readonly list?: () => Promise<readonly ProbeListing[]>; | |
| 75 | } | |
| 76 | ||
| 51 | 77 | export interface JourneyResult { |
| 52 | 78 | readonly ok: boolean; |
| 53 | 79 | /** The screens a person was shown, in order, the forge last. */ |
| @@ -75,8 +101,12 @@ export const SWEEP_MAX = 5; | ||
| 75 | 101 | /** Pauses between removal attempts: a transient failure is not hit again within milliseconds. */ |
| 76 | 102 | export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000, 3000, 10_000]; |
| 77 | 103 | |
| 78 | /** A probe forge older than this was left by a run that never finished: an hour. */ | |
| 79 | export const STALE_PROBE_MS = 60 * 60 * 1000; | |
| 104 | /** | |
| 105 | * A probe forge older than this was left by a run that never finished: half an hour. One walk | |
| 106 | * holds its forge under five minutes and a whole proof (three settles, three walks, two | |
| 107 | * pauses) ends in under twenty-four, so a run in progress is never this old. | |
| 108 | */ | |
| 109 | export const STALE_PROBE_MS = 30 * 60 * 1000; | |
| 80 | 110 | |
| 81 | 111 | /** The longest any one request may take. */ |
| 82 | 112 | export const REQUEST_TIMEOUT_MS = 10_000; |
| @@ -101,6 +131,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 101 | 131 | const doFetch = options.fetch ?? globalThis.fetch; |
| 102 | 132 | const pollMs = options.pollMs ?? 500; |
| 103 | 133 | const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms)); |
| 134 | const local: LocalHands | undefined = options.remove === undefined ? undefined : { remove: options.remove, ...(options.list ? { list: options.list } : {}) }; | |
| 104 | 135 | const screens: string[] = []; |
| 105 | 136 | let forge: string | null = null; |
| 106 | 137 | let readyMs: number | null = null; |
| @@ -264,16 +295,16 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 264 | 295 | // Afterwards, passed or failed: stop the crew's run, then remove the forge and its account. |
| 265 | 296 | if (forge !== null) { |
| 266 | 297 | if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie); |
| 267 | removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep); | |
| 298 | removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local); | |
| 268 | 299 | // A forge left running is hidden work on the host: the failure names it (never when there |
| 269 | // was no token to remove it with). | |
| 270 | if (!removed && options.adminToken !== undefined && options.adminToken !== "") | |
| 300 | // was nothing to remove it with). | |
| 301 | if (!removed && (local !== undefined || (options.adminToken !== undefined && options.adminToken !== ""))) | |
| 271 | 302 | failure = `${failure === undefined ? "" : `${failure}; `}the probe forge ${forge} could not be removed`; |
| 272 | 303 | } |
| 273 | 304 | } |
| 274 | 305 | // What an interrupted run left: swept after this walk's own cleanup, never its own forge. |
| 275 | 306 | // Bounded (`SWEEP_BUDGET_MS`, `SWEEP_MAX`) and never part of the verdict. |
| 276 | const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep); | |
| 307 | const sweep = await sweepProbes(doFetch, hostUrl, options.adminToken, Date.now(), forge, sleep, local); | |
| 277 | 308 | const result = { |
| 278 | 309 | ok: failure === undefined, |
| 279 | 310 | screens, |
| @@ -288,14 +319,15 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 288 | 319 | } |
| 289 | 320 | |
| 290 | 321 | /** |
| 291 | * Removes the `probe-` forges the host lists that were begun more than an hour ago | |
| 322 | * Removes the `probe-` forges the host lists that were begun more than half an hour ago | |
| 292 | 323 | * (`STALE_PROBE_MS`): what a run killed or timed out part-way left behind. The scope is |
| 293 | 324 | * this proof's own namespace: `probe-` plus eight hex digits is the name `probeName` gives |
| 294 | * a throwaway forge, and an hour is far past the proof's own budget, so a run still in | |
| 325 | * a throwaway forge, and half an hour is past the proof's own budget, so a run still in | |
| 295 | 326 | * progress is never touched. A forge with no begun-at, a name of any other shape, and |
| 296 | * `except` are left alone. Reads `GET /api/v1/forges` (an admin's token). Never throws: what | |
| 297 | * stopped it comes back in `error`. `at` is the wall clock in ms (the forge's `createdAt` is | |
| 298 | * wall-clock time, not the walk's monotonic one). Bounded by `SWEEP_BUDGET_MS` and `SWEEP_MAX`. | |
| 327 | * `except` are left alone. Reads `local.list` when given, else `GET /api/v1/forges` (an | |
| 328 | * admin's token). Never throws: what stopped it comes back in `error`. `at` is the wall clock | |
| 329 | * in ms (the forge's `createdAt` is wall-clock time, not the walk's monotonic one). Bounded by | |
| 330 | * `SWEEP_BUDGET_MS` and `SWEEP_MAX`. | |
| 299 | 331 | */ |
| 300 | 332 | export async function sweepProbes( |
| 301 | 333 | doFetch: typeof fetch, |
| @@ -304,10 +336,24 @@ export async function sweepProbes( | ||
| 304 | 336 | at: number, |
| 305 | 337 | except: string | null = null, |
| 306 | 338 | sleep: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms), |
| 339 | local?: LocalHands, | |
| 307 | 340 | ): Promise<{ gone: string[]; error?: string }> { |
| 308 | 341 | const gone: string[] = []; |
| 309 | if (token === undefined || token === "") return { gone }; | |
| 310 | 342 | const deadline = AbortSignal.timeout(SWEEP_BUDGET_MS); |
| 343 | if (local?.list !== undefined) { | |
| 344 | try { | |
| 345 | for (const { name, begunAt } of await local.list()) { | |
| 346 | if (!PROBE_NAME.test(name) || name === except) continue; | |
| 347 | if (begunAt === null || !Number.isFinite(begunAt) || at - begunAt < STALE_PROBE_MS) continue; | |
| 348 | if (gone.length >= SWEEP_MAX || deadline.aborted) break; | |
| 349 | if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name); | |
| 350 | } | |
| 351 | return { gone }; | |
| 352 | } catch (e) { | |
| 353 | return { gone, error: e instanceof Error ? e.message : "the sweep stopped unexpectedly" }; | |
| 354 | } | |
| 355 | } | |
| 356 | if (token === undefined || token === "") return { gone }; | |
| 311 | 357 | try { |
| 312 | 358 | const res = await doFetch(new URL("/api/v1/forges", host).href, { |
| 313 | 359 | redirect: "manual", |
| @@ -326,7 +372,7 @@ export async function sweepProbes( | ||
| 326 | 372 | const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN; |
| 327 | 373 | if (!Number.isFinite(begun) || at - begun < STALE_PROBE_MS) continue; |
| 328 | 374 | if (gone.length >= SWEEP_MAX || deadline.aborted) break; |
| 329 | if (await removeForge(doFetch, host, name, token, sleep, [])) gone.push(name); | |
| 375 | if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name); | |
| 330 | 376 | } |
| 331 | 377 | return { gone }; |
| 332 | 378 | } catch (e) { |
| @@ -348,7 +394,11 @@ async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr | ||
| 348 | 394 | } catch {} |
| 349 | 395 | } |
| 350 | 396 | |
| 351 | /** Removes the throwaway forge from the host; true when the host said it is gone. */ | |
| 397 | /** | |
| 398 | * Removes the throwaway forge from the host; true when the host said it is gone. In-process | |
| 399 | * (`local.remove`) when given, else over HTTP with `token`. A name that is not a probe's is | |
| 400 | * refused. | |
| 401 | */ | |
| 352 | 402 | async function removeForge( |
| 353 | 403 | doFetch: typeof fetch, |
| 354 | 404 | host: URL, |
| @@ -356,13 +406,21 @@ async function removeForge( | ||
| 356 | 406 | token: string | undefined, |
| 357 | 407 | sleep: (ms: number) => Promise<void>, |
| 358 | 408 | backoff: readonly number[] = REMOVE_BACKOFF_MS, |
| 409 | local?: LocalHands, | |
| 359 | 410 | ): Promise<boolean> { |
| 360 | if (token === undefined || token === "") return false; | |
| 361 | if (await removeOnce(doFetch, host, name, token)) return true; | |
| 411 | if (!PROBE_NAME.test(name)) return false; | |
| 412 | const once = | |
| 413 | local !== undefined | |
| 414 | ? () => local.remove(name).catch(() => false) | |
| 415 | : token === undefined || token === "" | |
| 416 | ? undefined | |
| 417 | : () => removeOnce(doFetch, host, name, token); | |
| 418 | if (once === undefined) return false; | |
| 419 | if (await once()) return true; | |
| 362 | 420 | // Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once. |
| 363 | 421 | for (const ms of backoff) { |
| 364 | 422 | await sleep(ms); |
| 365 | if (await removeOnce(doFetch, host, name, token)) return true; | |
| 423 | if (await once()) return true; | |
| 366 | 424 | } |
| 367 | 425 | return false; |
| 368 | 426 | } |
src/release/prove.ts +9 −2
| @@ -22,7 +22,7 @@ | ||
| 22 | 22 | |
| 23 | 23 | import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; |
| 24 | 24 | import { join } from "node:path"; |
| 25 | import type { walkJourney, JourneyResult } from "./journey.ts"; | |
| 25 | import type { walkJourney, JourneyResult, ProbeListing } from "./journey.ts"; | |
| 26 | 26 | import { smoke, type Failure, type Heavy } from "./smoke.ts"; |
| 27 | 27 | import { UNSTAMPED } from "../kernel/version.ts"; |
| 28 | 28 | import { ownProbeHeaders } from "../kernel/probe.ts"; |
| @@ -336,6 +336,13 @@ export interface ProveOptions { | ||
| 336 | 336 | * limits know it. |
| 337 | 337 | */ |
| 338 | 338 | readonly proberKey?: string; |
| 339 | /** | |
| 340 | * Removes a probe forge on this host in-process (`spawn/manage.ts`), handed to the journey in | |
| 341 | * place of a token (#12573). | |
| 342 | */ | |
| 343 | readonly remove?: (name: string) => Promise<boolean>; | |
| 344 | /** This host's own forges, for the journey's sweep of leftover probes. */ | |
| 345 | readonly list?: () => Promise<readonly ProbeListing[]>; | |
| 339 | 346 | } |
| 340 | 347 | |
| 341 | 348 | /** Is a proof of `version` due: never tried, or failed `retryAfter` its failures or more ago? */ |
| @@ -422,7 +429,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 422 | 429 | const settled = await settle(o.url, doFetch, now, sleep, o.giveUpMs === undefined ? SETTLE_MS : Math.min(SETTLE_MS, o.giveUpMs)); |
| 423 | 430 | tries++; |
| 424 | 431 | walked = settled |
| 425 | ? await o.journey({ base: o.url, fetch: doFetch, ...(o.token ? { adminToken: o.token } : {}), ...(o.proberKey ? { proberKey: o.proberKey } : {}) }) | |
| 432 | ? await o.journey({ base: o.url, fetch: doFetch, ...(o.token ? { adminToken: o.token } : {}), ...(o.proberKey ? { proberKey: o.proberKey } : {}), ...(o.remove ? { remove: o.remove } : {}), ...(o.list ? { list: o.list } : {}) }) | |
| 426 | 433 | : { ok: false, screens: [], readyMs: null, firstActivityMs: null, failure: `the forge had not settled in ${SETTLE_MS / 1000} s after its start`, forge: null, removed: true }; |
| 427 | 434 | if (walked.forge !== null && !walked.removed) leftover.push(walked.forge); |
| 428 | 435 | // A passed walk, a refusal by the sign-up limit, or a forge left behind ends the tries: |
test/release-journey.test.ts +88 −3
| @@ -19,7 +19,8 @@ import { addSubscription } from "../src/core/subscriptions.ts"; | ||
| 19 | 19 | import { createUser } from "../src/core/users.ts"; |
| 20 | 20 | import { FIRST_FIVE_MINUTES } from "../src/design/first_five_minutes.ts"; |
| 21 | 21 | import { serve, type Servers } from "../src/http/server.ts"; |
| 22 | import { REMOVE_BACKOFF_MS, sweepProbes, walkJourney } from "../src/release/journey.ts"; | |
| 22 | import { REMOVE_BACKOFF_MS, STALE_PROBE_MS, sweepProbes, walkJourney } from "../src/release/journey.ts"; | |
| 23 | import { removeForge } from "../src/spawn/manage.ts"; | |
| 23 | 24 | import { useProbe } from "../src/spawn/door.ts"; |
| 24 | 25 | import { firstBoot } from "../src/spawn/first-boot.ts"; |
| 25 | 26 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -204,7 +205,7 @@ test("a walk that blows up half-way (outside any step) still removes its forge", | ||
| 204 | 205 | expect(existsSync(kids.get(name)!.dataDir)).toBe(false); |
| 205 | 206 | }, 60_000); |
| 206 | 207 | |
| 207 | test("a run sweeps the probe forges an interrupted run left over an hour ago, and only those", async () => { | |
| 208 | test("a run sweeps the probe forges an interrupted run left over half an hour ago, and only those", async () => { | |
| 208 | 209 | const start = async (name: string) => { |
| 209 | 210 | const res = await fetch(`${parentServers.url}/forges/start`, { method: "POST", redirect: "manual", headers: { origin: "https://forge.test" }, body: new URLSearchParams({ name }) }); |
| 210 | 211 | await res.body?.cancel(); |
| @@ -224,7 +225,7 @@ test("a run sweeps the probe forges an interrupted run left over an hour ago, an | ||
| 224 | 225 | const result = await walkJourney({ base: parentServers.url, fetch: routed([]), adminToken, pollMs: 0, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} }); |
| 225 | 226 | expect(result.swept).toEqual(["probe-aaaaaaaa"]); |
| 226 | 227 | expect(await forgeGone("probe-aaaaaaaa")).toBe(true); |
| 227 | expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under an hour: may be a run in progress | |
| 228 | expect(await childNamed(parent, "probe-bbbbbbbb")).toBeDefined(); // under half an hour: may be a run in progress | |
| 228 | 229 | expect(await childNamed(parent, "keepme-forge")).toBeDefined(); // not a probe's name |
| 229 | 230 | for (const n of ["probe-bbbbbbbb", "keepme-forge"]) |
| 230 | 231 | await fetch(`${parentServers.url}/forges/${n}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } }); |
| @@ -301,3 +302,87 @@ test("a removal that fails three times is still retried, so a slow host does not | ||
| 301 | 302 | expect(result.failure === undefined || !result.failure.includes("could not be removed")).toBe(true); |
| 302 | 303 | await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } }); |
| 303 | 304 | }, 60_000); |
| 305 | ||
| 306 | /** | |
| 307 | * The in-process hand `src/main.ts` passes: what `POST /forges/:name/remove` runs, the stand-in | |
| 308 | * child stopped first. | |
| 309 | */ | |
| 310 | function localRemove(asked: string[]) { | |
| 311 | return async (name: string) => { | |
| 312 | asked.push(name); | |
| 313 | const live = kids.get(name); | |
| 314 | if (live !== undefined && !stopped.has(live)) { | |
| 315 | stopped.add(live); | |
| 316 | await live.crew.stop(); | |
| 317 | live.servers.stop(); | |
| 318 | live.app.close(); | |
| 319 | } | |
| 320 | return (await removeForge(parent, name)).ok; | |
| 321 | }; | |
| 322 | } | |
| 323 | ||
| 324 | test("with no admin token, an in-process remove takes the probe forge away after a passing walk", async () => { | |
| 325 | broken = undefined; | |
| 326 | const asked: string[] = []; | |
| 327 | let t = 0; | |
| 328 | const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: localRemove(asked), now: () => (t += 10), sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time | |
| 329 | expect(result.failure).toBeUndefined(); | |
| 330 | expect(result.ok).toBe(true); | |
| 331 | expect(result.removed).toBe(true); | |
| 332 | expect(asked).toEqual([result.forge!]); | |
| 333 | expect(await forgeGone(result.forge!)).toBe(true); | |
| 334 | }, 60_000); | |
| 335 | ||
| 336 | test("with no admin token, an in-process remove takes the probe forge away after a failing walk", async () => { | |
| 337 | broken = "POST /claim"; | |
| 338 | const asked: string[] = []; | |
| 339 | const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: localRemove(asked), sleep: () => Bun.sleep(100), pollMs: 100 }); // real-timer: the stand-in crew child runs in real time | |
| 340 | broken = undefined; | |
| 341 | expect(result.ok).toBe(false); | |
| 342 | expect(result.failure).toBe("the account could not be made: it answered 503"); | |
| 343 | expect(result.removed).toBe(true); | |
| 344 | expect(await forgeGone(result.forge!)).toBe(true); | |
| 345 | }, 60_000); | |
| 346 | ||
| 347 | test("an in-process remove that never succeeds names the forge left running", async () => { | |
| 348 | let t = 0; | |
| 349 | const result = await walkJourney({ base: parentServers.url, fetch: routed([]), remove: async () => false, pollMs: 0, now: () => (t += 5_000), sleep: async () => {} }); | |
| 350 | expect(result.removed).toBe(false); | |
| 351 | expect(result.failure).toContain(`the probe forge ${result.forge} could not be removed`); | |
| 352 | await removeForge(parent, result.forge!); | |
| 353 | }, 60_000); | |
| 354 | ||
| 355 | test("the in-process sweep removes a 31-minute-old probe, and leaves a 29-minute-old one and any other name alone", async () => { | |
| 356 | expect(STALE_PROBE_MS).toBe(30 * 60 * 1000); | |
| 357 | const at = Date.now(); | |
| 358 | const min = 60_000; | |
| 359 | const removed: string[] = []; | |
| 360 | const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch; | |
| 361 | const list = async () => [ | |
| 362 | { name: "probe-0000000a", begunAt: at - 31 * min }, | |
| 363 | { name: "probe-0000000b", begunAt: at - 29 * min }, | |
| 364 | { name: "keepme-forge", begunAt: at - 5 * 60 * min }, | |
| 365 | { name: "probe-0000000c-x", begunAt: at - 5 * 60 * min }, | |
| 366 | { name: "probe-0000000d", begunAt: null }, | |
| 367 | ]; | |
| 368 | const remove = async (name: string) => (removed.push(name), true); | |
| 369 | const out = await sweepProbes(never, new URL(parentServers.url), undefined, at, null, async () => {}, { remove, list }); | |
| 370 | expect(out).toEqual({ gone: ["probe-0000000a"] }); | |
| 371 | expect(removed).toEqual(["probe-0000000a"]); | |
| 372 | }); | |
| 373 | ||
| 374 | test("the in-process hands are never given a name that is not a probe's, nor the walk's own forge", async () => { | |
| 375 | const at = Date.now(); | |
| 376 | const removed: string[] = []; | |
| 377 | const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch; | |
| 378 | const list = async () => [{ name: "probe-0000000e", begunAt: at - 99 * 60_000 }, { name: "Probe-0000000f", begunAt: 0 }, { name: "alice", begunAt: 0 }]; | |
| 379 | const out = await sweepProbes(never, new URL(parentServers.url), undefined, at, "probe-0000000e", async () => {}, { remove: async (n) => (removed.push(n), true), list }); | |
| 380 | expect(out).toEqual({ gone: [] }); | |
| 381 | expect(removed).toEqual([]); | |
| 382 | }); | |
| 383 | ||
| 384 | test("an in-process list that throws comes back as the sweep's reason", async () => { | |
| 385 | const never = (() => { throw new Error("no request"); }) as unknown as typeof fetch; | |
| 386 | const list = async () => { throw new Error("the ledger is unreadable"); }; | |
| 387 | expect(await sweepProbes(never, new URL(parentServers.url), undefined, Date.now(), null, async () => {}, { remove: async () => true, list })).toEqual({ gone: [], error: "the ledger is unreadable" }); | |
| 388 | }); |
new above
offline
This comment still says "an hour" but STALE_PROBE_MS is now 30 minutes. Update it to half an hour.