Public: anyone can read this; the crew can change it.
The release proof's log line and record say removed=true|false and why #12627
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12574-the-release-proof-s-log-line-and-record → main 4 commits fork
4 files
changed +84
−19
src/release/journey.ts +36 −18
| @@ -88,6 +88,8 @@ export interface JourneyResult { | ||
| 88 | 88 | readonly forge: string | null; |
| 89 | 89 | /** Whether the throwaway forge was removed afterwards (true when none was made). */ |
| 90 | 90 | readonly removed: boolean; |
| 91 | /** Why it was not removed, in a plain sentence; set only when `removed` is false. */ | |
| 92 | readonly removeReason?: string; | |
| 91 | 93 | /** `probe-` forges an interrupted run left (older than `STALE_PROBE_MS`) this run removed. */ |
| 92 | 94 | readonly swept?: readonly string[]; |
| 93 | 95 | /** Why the sweep stopped early, when it did; the walk's verdict never depends on it. */ |
| @@ -141,6 +143,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 141 | 143 | let child: URL | undefined; |
| 142 | 144 | let cookie = ""; |
| 143 | 145 | let removed = true; |
| 146 | let removeReason: string | undefined; | |
| 144 | 147 | |
| 145 | 148 | try { |
| 146 | 149 | host = new URL(options.base); |
| @@ -295,7 +298,9 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 295 | 298 | // Afterwards, passed or failed: stop the crew's run, then remove the forge and its account. |
| 296 | 299 | if (forge !== null) { |
| 297 | 300 | if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie); |
| 298 | removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local); | |
| 301 | const removal = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local); | |
| 302 | removed = removal.removed; | |
| 303 | removeReason = removal.reason; | |
| 299 | 304 | // A forge left running is hidden work on the host: the failure names it (never when there |
| 300 | 305 | // was nothing to remove it with). |
| 301 | 306 | if (!removed && (local !== undefined || (options.adminToken !== undefined && options.adminToken !== ""))) |
| @@ -312,6 +317,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul | ||
| 312 | 317 | firstActivityMs, |
| 313 | 318 | forge, |
| 314 | 319 | removed, |
| 320 | ...(removed || removeReason === undefined ? {} : { removeReason }), | |
| 315 | 321 | swept: sweep.gone, |
| 316 | 322 | ...(sweep.error === undefined ? {} : { sweepError: sweep.error }), |
| 317 | 323 | }; |
| @@ -346,7 +352,7 @@ export async function sweepProbes( | ||
| 346 | 352 | if (!PROBE_NAME.test(name) || name === except) continue; |
| 347 | 353 | if (begunAt === null || !Number.isFinite(begunAt) || at - begunAt < STALE_PROBE_MS) continue; |
| 348 | 354 | if (gone.length >= SWEEP_MAX || deadline.aborted) break; |
| 349 | if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name); | |
| 355 | if ((await removeForge(doFetch, host, name, token, sleep, [], local)).removed) gone.push(name); | |
| 350 | 356 | } |
| 351 | 357 | return { gone }; |
| 352 | 358 | } catch (e) { |
| @@ -372,7 +378,7 @@ export async function sweepProbes( | ||
| 372 | 378 | const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN; |
| 373 | 379 | if (!Number.isFinite(begun) || at - begun < STALE_PROBE_MS) continue; |
| 374 | 380 | if (gone.length >= SWEEP_MAX || deadline.aborted) break; |
| 375 | if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name); | |
| 381 | if ((await removeForge(doFetch, host, name, token, sleep, [], local)).removed) gone.push(name); | |
| 376 | 382 | } |
| 377 | 383 | return { gone }; |
| 378 | 384 | } catch (e) { |
| @@ -395,9 +401,9 @@ async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr | ||
| 395 | 401 | } |
| 396 | 402 | |
| 397 | 403 | /** |
| 398 | * Removes the throwaway forge from the host; true when the host said it is gone. In-process | |
| 399 | * (`local.remove`) when given, else over HTTP with `token`. A name that is not a probe's is | |
| 400 | * refused. | |
| 404 | * Removes the throwaway forge from the host; `removed` when the host said it is gone, else | |
| 405 | * `reason` says why not in a plain sentence (the last try's). In-process (`local.remove`) when | |
| 406 | * given, else over HTTP with `token`. A name that is not a probe's is refused. | |
| 401 | 407 | */ |
| 402 | 408 | async function removeForge( |
| 403 | 409 | doFetch: typeof fetch, |
| @@ -407,25 +413,37 @@ async function removeForge( | ||
| 407 | 413 | sleep: (ms: number) => Promise<void>, |
| 408 | 414 | backoff: readonly number[] = REMOVE_BACKOFF_MS, |
| 409 | 415 | local?: LocalHands, |
| 410 | ): Promise<boolean> { | |
| 411 | if (!PROBE_NAME.test(name)) return false; | |
| 412 | const once = | |
| 416 | ): Promise<Removal> { | |
| 417 | if (!PROBE_NAME.test(name)) return { removed: false, reason: "the name is not a probe forge's" }; | |
| 418 | const once: (() => Promise<string | undefined>) | undefined = | |
| 413 | 419 | local !== undefined |
| 414 | ? () => local.remove(name).catch(() => false) | |
| 420 | ? () => | |
| 421 | local.remove(name).then( | |
| 422 | (gone) => (gone ? undefined : "the host could not remove it"), | |
| 423 | () => "the removal failed on the host", | |
| 424 | ) | |
| 415 | 425 | : token === undefined || token === "" |
| 416 | 426 | ? undefined |
| 417 | 427 | : () => removeOnce(doFetch, host, name, token); |
| 418 | if (once === undefined) return false; | |
| 419 | if (await once()) return true; | |
| 428 | if (once === undefined) return { removed: false, reason: "no way to remove it was given" }; | |
| 429 | let reason = await once(); | |
| 420 | 430 | // Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once. |
| 421 | 431 | for (const ms of backoff) { |
| 432 | if (reason === undefined) break; | |
| 422 | 433 | await sleep(ms); |
| 423 | if (await once()) return true; | |
| 434 | reason = await once(); | |
| 424 | 435 | } |
| 425 | return false; | |
| 436 | return reason === undefined ? { removed: true } : { removed: false, reason }; | |
| 426 | 437 | } |
| 427 | 438 | |
| 428 | async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<boolean> { | |
| 439 | /** One removal's outcome: `reason` only when it was not removed. */ | |
| 440 | interface Removal { | |
| 441 | readonly removed: boolean; | |
| 442 | readonly reason?: string; | |
| 443 | } | |
| 444 | ||
| 445 | /** One removal over HTTP; undefined when the host said it is gone, else why not. */ | |
| 446 | async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<string | undefined> { | |
| 429 | 447 | try { |
| 430 | 448 | const url = new URL(`/forges/${name}/remove`, host); |
| 431 | 449 | const res = await doFetch(url.href, { |
| @@ -435,8 +453,8 @@ async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: | ||
| 435 | 453 | headers: { origin: url.origin, authorization: `Bearer ${token}` }, |
| 436 | 454 | }); |
| 437 | 455 | await res.body?.cancel(); |
| 438 | return res.status === 200; | |
| 439 | } catch { | |
| 440 | return false; | |
| 456 | return res.status === 200 ? undefined : `the host said ${res.status} to the removal`; | |
| 457 | } catch (e) { | |
| 458 | return e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError") ? "the removal timed out" : "the removal could not reach the host"; | |
| 441 | 459 | } |
| 442 | 460 | } |
src/release/prove.ts +9 −1
| @@ -86,6 +86,13 @@ export interface Proof { | ||
| 86 | 86 | readonly retryNow?: boolean; |
| 87 | 87 | /** How many walks of the stranger's journey it took (1 when it passed or failed at once). */ |
| 88 | 88 | readonly tries?: number; |
| 89 | /** | |
| 90 | * Whether the last walk's probe forge was removed (true when none was made); set when a walk | |
| 91 | * ran. | |
| 92 | */ | |
| 93 | readonly removed?: boolean; | |
| 94 | /** Why the last walk's probe forge was not removed, in a plain sentence; only then. */ | |
| 95 | readonly removeReason?: string; | |
| 89 | 96 | /** Probe forges the proof could not remove: the needs-you stack lists each with Remove. */ |
| 90 | 97 | readonly leftover?: readonly string[]; |
| 91 | 98 | } |
| @@ -378,6 +385,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 378 | 385 | ...(measured === undefined ? {} : { footprint: measured }), |
| 379 | 386 | ...(walked === undefined ? {} : { journey: { readyMs: walked.readyMs, firstActivityMs: walked.firstActivityMs, screens: walked.screens } }), |
| 380 | 387 | ...(tries === undefined ? {} : { tries }), |
| 388 | ...(walked === undefined ? {} : { removed: walked.removed, ...(walked.removed ? {} : { removeReason: walked.removeReason ?? "no reason was given" }) }), | |
| 381 | 389 | ...(leftover.length === 0 ? {} : { leftover }), |
| 382 | 390 | ...(failure === undefined |
| 383 | 391 | ? {} |
| @@ -551,5 +559,5 @@ export function proveRunning( | ||
| 551 | 559 | schedule: Schedule = unrefTimer, |
| 552 | 560 | prove: (o: ProveOptions) => Promise<Proof | undefined> = proveRelease, |
| 553 | 561 | ): () => void { |
| 554 | return startProving(o, (p) => log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.tries === undefined ? {} : { tries: String(p.tries) }), ...(p.failure ? { failure: p.failure } : {}) }), everyMs, schedule, prove, HANG_MS, log); | |
| 562 | return startProving(o, (p) => log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.tries === undefined ? {} : { tries: String(p.tries) }), ...(p.removed === undefined ? {} : { removed: String(p.removed) }), ...(p.removed === false ? { remove_reason: p.removeReason ?? "no reason was given" } : {}), ...(p.failure ? { failure: p.failure } : {}) }), everyMs, schedule, prove, HANG_MS, log); | |
| 555 | 563 | } |
test/release-journey.test.ts +2
| @@ -173,6 +173,7 @@ test("a forge that is never ready fails in words, with the budget in them", asyn | ||
| 173 | 173 | test("without an admin token the forge cannot be removed from here, and the result says so", async () => { |
| 174 | 174 | const result = await walkJourney({ base: parentServers.url, fetch: routed([]), pollMs: 100, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} }); |
| 175 | 175 | expect(result.removed).toBe(false); |
| 176 | expect(result.removeReason).toBe("no way to remove it was given"); | |
| 176 | 177 | expect(result.forge).not.toBeNull(); |
| 177 | 178 | await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } }); |
| 178 | 179 | }); |
| @@ -268,6 +269,7 @@ test("a removal that never succeeds waits between its attempts and says it is no | ||
| 268 | 269 | let t = 0; |
| 269 | 270 | const result = await walkJourney({ base: parentServers.url, fetch: stuck, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async (ms) => void pauses.push(ms) }); |
| 270 | 271 | expect(result.removed).toBe(false); |
| 272 | expect(result.removeReason).toBe("the host said 503 to the removal"); | |
| 271 | 273 | expect(result.ok).toBe(false); |
| 272 | 274 | expect(result.failure).toContain(`the probe forge ${result.forge} could not be removed`); |
| 273 | 275 | expect(pauses).toEqual([...REMOVE_BACKOFF_MS]); |
test/release-prove.test.ts +37
| @@ -153,6 +153,17 @@ describe("proving a release", () => { | ||
| 153 | 153 | expect(leftoverProbes(dir)).toEqual(["probe-abcd1234"]); |
| 154 | 154 | }); |
| 155 | 155 | |
| 156 | test("the proof records whether the walk's probe forge was removed, and why not (#12574)", async () => { | |
| 157 | const kept = await run("4.4.1", site("4.4.1").fake, { journey: walk({ forge: "probe-abcd1235", removed: false, removeReason: "the removal timed out" }) }); | |
| 158 | // a forge left behind is no part of the walk's verdict | |
| 159 | expect(kept?.ok).toBe(true); | |
| 160 | expect(kept).toMatchObject({ removed: false, removeReason: "the removal timed out" }); | |
| 161 | expect(lastProof(dir)).toMatchObject({ removed: false, removeReason: "the removal timed out" }); | |
| 162 | const gone = await run("4.4.2", site("4.4.2").fake, { journey: walk({}) }); | |
| 163 | expect(gone?.removed).toBe(true); | |
| 164 | expect(gone?.removeReason).toBeUndefined(); | |
| 165 | }); | |
| 166 | ||
| 156 | 167 | test("a passing walk keeps its timings, and they reach the card", async () => { |
| 157 | 168 | const p = await run("4.0.1", site("4.0.1").fake, { journey: walk({}) }); |
| 158 | 169 | expect(p?.ok).toBe(true); |
| @@ -511,6 +522,32 @@ describe("proving a release", () => { | ||
| 511 | 522 | ]); |
| 512 | 523 | }); |
| 513 | 524 | |
| 525 | test("the release proof line says removed=true|false and, when false, remove_reason (#12574)", async () => { | |
| 526 | const lines: [string, unknown][] = []; | |
| 527 | const queued: (() => void)[] = []; | |
| 528 | const results: Proof[] = [ | |
| 529 | { version: "4.6.1", at: "a", ok: true, tries: 1, removed: false, removeReason: "the host said 500 to the removal" }, | |
| 530 | { version: "4.6.1", at: "b", ok: true, tries: 1, removed: true }, | |
| 531 | ]; | |
| 532 | const stop = proveRunning( | |
| 533 | { dataDir: dir, version: "4.6.1", url: "https://forge.test" }, | |
| 534 | { info: (msg, fields) => void lines.push([msg, fields]), warn: () => {} }, | |
| 535 | 60_000, | |
| 536 | (fn) => (queued.push(fn), () => {}), | |
| 537 | async () => results.shift(), | |
| 538 | ); | |
| 539 | for (let i = 0; i < 2; i++) { | |
| 540 | await Bun.sleep(0); // real-timer: lets the queued tick's proof promise settle | |
| 541 | queued.shift()?.(); | |
| 542 | } | |
| 543 | await Bun.sleep(0); // real-timer: lets the last tick's proof promise settle | |
| 544 | stop(); | |
| 545 | expect(lines).toEqual([ | |
| 546 | ["release proof", { version: "4.6.1", ok: "true", tries: "1", removed: "false", remove_reason: "the host said 500 to the removal" }], | |
| 547 | ["release proof", { version: "4.6.1", ok: "true", tries: "1", removed: "true" }], | |
| 548 | ]); | |
| 549 | }); | |
| 550 | ||
| 514 | 551 | test("an unstamped forge proves nothing", async () => { |
| 515 | 552 | const s = site("dev"); |
| 516 | 553 | expect(await run("dev", s.fake)).toBeUndefined(); |
new above
offline