Public: anyone can read this; the crew can change it.

The release proof's log line and record say removed=true|false and why #12627

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12574-the-release-proof-s-log-line-and-record → main 4 commits fork

  • Merged. scout/12574-the-release-proof-s-log-line-and-record was merged into main as 0737e38.

Review

Sign in to review this change.
4 files changed +84 −19
src/release/journey.ts +36 −18
@@ -88,6 +88,8 @@ export interface JourneyResult {
8888 readonly forge: string | null;
8989 /** Whether the throwaway forge was removed afterwards (true when none was made). */
9090 readonly removed: boolean;
91 /** Why it was not removed, in a plain sentence; set only when `removed` is false. */
92 readonly removeReason?: string;
9193 /** `probe-` forges an interrupted run left (older than `STALE_PROBE_MS`) this run removed. */
9294 readonly swept?: readonly string[];
9395 /** Why the sweep stopped early, when it did; the walk's verdict never depends on it. */
@@ -141,6 +143,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
141143 let child: URL | undefined;
142144 let cookie = "";
143145 let removed = true;
146 let removeReason: string | undefined;
144147
145148 try {
146149 host = new URL(options.base);
@@ -295,7 +298,9 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
295298 // Afterwards, passed or failed: stop the crew's run, then remove the forge and its account.
296299 if (forge !== null) {
297300 if (child !== undefined && cookie !== "") await cancelRuns(doFetch, child, cookie);
298 removed = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local);
301 const removal = await removeForge(doFetch, hostUrl, forge, options.adminToken, sleep, REMOVE_BACKOFF_MS, local);
302 removed = removal.removed;
303 removeReason = removal.reason;
299304 // A forge left running is hidden work on the host: the failure names it (never when there
300305 // was nothing to remove it with).
301306 if (!removed && (local !== undefined || (options.adminToken !== undefined && options.adminToken !== "")))
@@ -312,6 +317,7 @@ export async function walkJourney(options: JourneyOptions): Promise<JourneyResul
312317 firstActivityMs,
313318 forge,
314319 removed,
320 ...(removed || removeReason === undefined ? {} : { removeReason }),
315321 swept: sweep.gone,
316322 ...(sweep.error === undefined ? {} : { sweepError: sweep.error }),
317323 };
@@ -346,7 +352,7 @@ export async function sweepProbes(
346352 if (!PROBE_NAME.test(name) || name === except) continue;
347353 if (begunAt === null || !Number.isFinite(begunAt) || at - begunAt < STALE_PROBE_MS) continue;
348354 if (gone.length >= SWEEP_MAX || deadline.aborted) break;
349 if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name);
355 if ((await removeForge(doFetch, host, name, token, sleep, [], local)).removed) gone.push(name);
350356 }
351357 return { gone };
352358 } catch (e) {
@@ -372,7 +378,7 @@ export async function sweepProbes(
372378 const begun = typeof createdAt === "string" ? Date.parse(createdAt) : Number.NaN;
373379 if (!Number.isFinite(begun) || at - begun < STALE_PROBE_MS) continue;
374380 if (gone.length >= SWEEP_MAX || deadline.aborted) break;
375 if (await removeForge(doFetch, host, name, token, sleep, [], local)) gone.push(name);
381 if ((await removeForge(doFetch, host, name, token, sleep, [], local)).removed) gone.push(name);
376382 }
377383 return { gone };
378384 } catch (e) {
@@ -395,9 +401,9 @@ async function cancelRuns(doFetch: typeof fetch, child: URL, cookie: string): Pr
395401 }
396402
397403 /**
398 * Removes the throwaway forge from the host; true when the host said it is gone. In-process
399 * (`local.remove`) when given, else over HTTP with `token`. A name that is not a probe's is
400 * refused.
404 * Removes the throwaway forge from the host; `removed` when the host said it is gone, else
405 * `reason` says why not in a plain sentence (the last try's). In-process (`local.remove`) when
406 * given, else over HTTP with `token`. A name that is not a probe's is refused.
401407 */
402408 async function removeForge(
403409 doFetch: typeof fetch,
@@ -407,25 +413,37 @@ async function removeForge(
407413 sleep: (ms: number) => Promise<void>,
408414 backoff: readonly number[] = REMOVE_BACKOFF_MS,
409415 local?: LocalHands,
410 ): Promise<boolean> {
411 if (!PROBE_NAME.test(name)) return false;
412 const once =
416 ): Promise<Removal> {
417 if (!PROBE_NAME.test(name)) return { removed: false, reason: "the name is not a probe forge's" };
418 const once: (() => Promise<string | undefined>) | undefined =
413419 local !== undefined
414 ? () => local.remove(name).catch(() => false)
420 ? () =>
421 local.remove(name).then(
422 (gone) => (gone ? undefined : "the host could not remove it"),
423 () => "the removal failed on the host",
424 )
415425 : token === undefined || token === ""
416426 ? undefined
417427 : () => removeOnce(doFetch, host, name, token);
418 if (once === undefined) return false;
419 if (await once()) return true;
428 if (once === undefined) return { removed: false, reason: "no way to remove it was given" };
429 let reason = await once();
420430 // Each retry waits first (`backoff`); the sweep passes none, so a stale forge is tried once.
421431 for (const ms of backoff) {
432 if (reason === undefined) break;
422433 await sleep(ms);
423 if (await once()) return true;
434 reason = await once();
424435 }
425 return false;
436 return reason === undefined ? { removed: true } : { removed: false, reason };
426437 }
427438
428 async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<boolean> {
439 /** One removal's outcome: `reason` only when it was not removed. */
440 interface Removal {
441 readonly removed: boolean;
442 readonly reason?: string;
443 }
444
445 /** One removal over HTTP; undefined when the host said it is gone, else why not. */
446 async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token: string): Promise<string | undefined> {
429447 try {
430448 const url = new URL(`/forges/${name}/remove`, host);
431449 const res = await doFetch(url.href, {
@@ -435,8 +453,8 @@ async function removeOnce(doFetch: typeof fetch, host: URL, name: string, token:
435453 headers: { origin: url.origin, authorization: `Bearer ${token}` },
436454 });
437455 await res.body?.cancel();
438 return res.status === 200;
439 } catch {
440 return false;
456 return res.status === 200 ? undefined : `the host said ${res.status} to the removal`;
457 } catch (e) {
458 return e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError") ? "the removal timed out" : "the removal could not reach the host";
441459 }
442460 }
src/release/prove.ts +9 −1
@@ -86,6 +86,13 @@ export interface Proof {
8686 readonly retryNow?: boolean;
8787 /** How many walks of the stranger's journey it took (1 when it passed or failed at once). */
8888 readonly tries?: number;
89 /**
90 * Whether the last walk's probe forge was removed (true when none was made); set when a walk
91 * ran.
92 */
93 readonly removed?: boolean;
94 /** Why the last walk's probe forge was not removed, in a plain sentence; only then. */
95 readonly removeReason?: string;
8996 /** Probe forges the proof could not remove: the needs-you stack lists each with Remove. */
9097 readonly leftover?: readonly string[];
9198 }
@@ -378,6 +385,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
378385 ...(measured === undefined ? {} : { footprint: measured }),
379386 ...(walked === undefined ? {} : { journey: { readyMs: walked.readyMs, firstActivityMs: walked.firstActivityMs, screens: walked.screens } }),
380387 ...(tries === undefined ? {} : { tries }),
388 ...(walked === undefined ? {} : { removed: walked.removed, ...(walked.removed ? {} : { removeReason: walked.removeReason ?? "no reason was given" }) }),
381389 ...(leftover.length === 0 ? {} : { leftover }),
382390 ...(failure === undefined
383391 ? {}
@@ -551,5 +559,5 @@ export function proveRunning(
551559 schedule: Schedule = unrefTimer,
552560 prove: (o: ProveOptions) => Promise<Proof | undefined> = proveRelease,
553561 ): () => void {
554 return startProving(o, (p) => log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.tries === undefined ? {} : { tries: String(p.tries) }), ...(p.failure ? { failure: p.failure } : {}) }), everyMs, schedule, prove, HANG_MS, log);
562 return startProving(o, (p) => log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.tries === undefined ? {} : { tries: String(p.tries) }), ...(p.removed === undefined ? {} : { removed: String(p.removed) }), ...(p.removed === false ? { remove_reason: p.removeReason ?? "no reason was given" } : {}), ...(p.failure ? { failure: p.failure } : {}) }), everyMs, schedule, prove, HANG_MS, log);
555563 }
test/release-journey.test.ts +2
@@ -173,6 +173,7 @@ test("a forge that is never ready fails in words, with the budget in them", asyn
173173 test("without an admin token the forge cannot be removed from here, and the result says so", async () => {
174174 const result = await walkJourney({ base: parentServers.url, fetch: routed([]), pollMs: 100, now: (() => { let t = 0; return () => (t += 5_000); })(), sleep: async () => {} });
175175 expect(result.removed).toBe(false);
176 expect(result.removeReason).toBe("no way to remove it was given");
176177 expect(result.forge).not.toBeNull();
177178 await fetch(`${parentServers.url}/forges/${result.forge}/remove`, { method: "POST", headers: { origin: parentServers.url, authorization: `Bearer ${adminToken}` } });
178179 });
@@ -268,6 +269,7 @@ test("a removal that never succeeds waits between its attempts and says it is no
268269 let t = 0;
269270 const result = await walkJourney({ base: parentServers.url, fetch: stuck, adminToken, pollMs: 0, now: () => (t += 5_000), sleep: async (ms) => void pauses.push(ms) });
270271 expect(result.removed).toBe(false);
272 expect(result.removeReason).toBe("the host said 503 to the removal");
271273 expect(result.ok).toBe(false);
272274 expect(result.failure).toContain(`the probe forge ${result.forge} could not be removed`);
273275 expect(pauses).toEqual([...REMOVE_BACKOFF_MS]);
test/release-prove.test.ts +37
@@ -153,6 +153,17 @@ describe("proving a release", () => {
153153 expect(leftoverProbes(dir)).toEqual(["probe-abcd1234"]);
154154 });
155155
156 test("the proof records whether the walk's probe forge was removed, and why not (#12574)", async () => {
157 const kept = await run("4.4.1", site("4.4.1").fake, { journey: walk({ forge: "probe-abcd1235", removed: false, removeReason: "the removal timed out" }) });
158 // a forge left behind is no part of the walk's verdict
159 expect(kept?.ok).toBe(true);
160 expect(kept).toMatchObject({ removed: false, removeReason: "the removal timed out" });
161 expect(lastProof(dir)).toMatchObject({ removed: false, removeReason: "the removal timed out" });
162 const gone = await run("4.4.2", site("4.4.2").fake, { journey: walk({}) });
163 expect(gone?.removed).toBe(true);
164 expect(gone?.removeReason).toBeUndefined();
165 });
166
156167 test("a passing walk keeps its timings, and they reach the card", async () => {
157168 const p = await run("4.0.1", site("4.0.1").fake, { journey: walk({}) });
158169 expect(p?.ok).toBe(true);
@@ -511,6 +522,32 @@ describe("proving a release", () => {
511522 ]);
512523 });
513524
525 test("the release proof line says removed=true|false and, when false, remove_reason (#12574)", async () => {
526 const lines: [string, unknown][] = [];
527 const queued: (() => void)[] = [];
528 const results: Proof[] = [
529 { version: "4.6.1", at: "a", ok: true, tries: 1, removed: false, removeReason: "the host said 500 to the removal" },
530 { version: "4.6.1", at: "b", ok: true, tries: 1, removed: true },
531 ];
532 const stop = proveRunning(
533 { dataDir: dir, version: "4.6.1", url: "https://forge.test" },
534 { info: (msg, fields) => void lines.push([msg, fields]), warn: () => {} },
535 60_000,
536 (fn) => (queued.push(fn), () => {}),
537 async () => results.shift(),
538 );
539 for (let i = 0; i < 2; i++) {
540 await Bun.sleep(0); // real-timer: lets the queued tick's proof promise settle
541 queued.shift()?.();
542 }
543 await Bun.sleep(0); // real-timer: lets the last tick's proof promise settle
544 stop();
545 expect(lines).toEqual([
546 ["release proof", { version: "4.6.1", ok: "true", tries: "1", removed: "false", remove_reason: "the host said 500 to the removal" }],
547 ["release proof", { version: "4.6.1", ok: "true", tries: "1", removed: "true" }],
548 ]);
549 });
550
514551 test("an unstamped forge proves nothing", async () => {
515552 const s = site("dev");
516553 expect(await run("dev", s.fake)).toBeUndefined();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs