Public: anyone can read this; the crew can change it.

Sentinel: probe:probe-aa6bcd07 was degraded 41 minutes today: The release proof's probe forge probe-aa6bcd07 could not be removed and is still running. #12551

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12550-sentinel-probe-probe-aa6bcd07-was-degrad → main 8 commits fork 2 threads · 2 unresolved

  • Merged. scout/12550-sentinel-probe-probe-aa6bcd07-was-degrad was merged into main as 0b1873c.

Review

Sign in to review this change.
7 files changed +320 −5
src/kernel/vitals.ts +2 −2
@@ -36,7 +36,7 @@ import { lastDrill } from "../snapshot/drills.ts";
3636 import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
3737 import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
3838 import { drillNeed, lastVerdict } from "../snapshot/verdict.ts";
39 import { leftoverProbes, proofNeed, proofNotice } from "../release/prove.ts";
39 import { leftoverProbes, PROBE_NAME, proofNeed, proofNotice } from "../release/prove.ts";
4040 import { running as runningVersion } from "./version.ts";
4141 import { rollbackNeed } from "../release/launch.ts";
4242 import { behindStanding } from "../release/behind.ts";
@@ -657,7 +657,7 @@ export function vitals(input: VitalsInput): Vitals {
657657 // A probe forge the proof could not remove is hidden work on the host: a card each, its
658658 // Remove page one tap away.
659659 for (const name of leftoverProbes(input.dataDir))
660 if (/^probe-[0-9a-f]{8}$/.test(name)) need("todo", `probe:${name}`, `The release proof's probe forge ${name} could not be removed and is still running.`, undefined, `/forges/${name}/remove`);
660 if (PROBE_NAME.test(name)) need("todo", `probe:${name}`, `The release proof's probe forge ${name} could not be removed and is still running.`, undefined, `/forges/${name}/remove`);
661661
662662 // A roll the image's launcher undid is a card until a later version reads ready. Its sentence
663663 // carries the release's own last words (stderr), so anything credential-shaped is masked.
src/main.ts +14
@@ -116,6 +116,10 @@ import { stepWithBeat } from "./keeper/beat.ts";
116116 import { keeperEnv, keeperPorts } from "./keeper/wire.ts";
117117 import { noteKeeperAtBoot } from "./keeper/at_boot.ts";
118118 import { snapshotPlace } from "./actions/health.ts";
119 import { dropServed } from "./spawn/clients.ts";
120 import { removeForge as removeHostedForge } from "./spawn/manage.ts";
121 import { listChildren } from "./actions/spawn.ts";
122 import { sweepLeftoverProbes } from "./release/leftover_sweep.ts";
119123 import { forgesMemory, memoryOverLine } from "./spawn/memory_fit.ts";
120124 import { keepFloor } from "./apps/cgroup.ts";
121125 import { limitValue } from "./settings/limits.ts";
@@ -752,6 +756,16 @@ const tickAsks = async (): Promise<void> => {
752756 app.log.warn("database size reading failed", { error: String(error) });
753757 }
754758 }
759 // A probe forge the release proof could not remove is removed here, before the vitals
760 // read it (#12550).
761 await sweepLeftoverProbes({
762 dataDir: app.config.dataDir,
763 held: async () =>
764 new Map((await listChildren(app)).map((c) => [c.name, { createdAt: c.createdAt, ...(c.ownerId === undefined ? {} : { ownerId: c.ownerId }) }])),
765 remove: async (name) => (await removeHostedForge(app, name)).ok,
766 dropServed: () => dropServed(app),
767 log: app.log,
768 });
755769 // The degraded history rides the same tick; it catches its own errors.
756770 try {
757771 recordDegraded(
src/release/journey.ts +1 −3
@@ -24,6 +24,7 @@
2424 */
2525 import { FIRST_FIVE_MINUTES } from "../design/first_five_minutes.ts";
2626 import { PROBER_HEADER } from "./prober.ts";
27 import { PROBE_NAME } from "./prove.ts";
2728
2829 export interface JourneyOptions {
2930 /** The live forge's address, http(s). */
@@ -77,9 +78,6 @@ export const REMOVE_BACKOFF_MS: readonly number[] = [250, 1000];
7778 /** A probe forge older than this was left by a run that never finished: an hour. */
7879 export const STALE_PROBE_MS = 60 * 60 * 1000;
7980
80 /** What a probe forge is called; its account goes with it (the forge's data is its only home). */
81 const PROBE_NAME = /^probe-[0-9a-f]{8}$/;
82
8381 /** The longest any one request may take. */
outdated

Orphaned doc comment for the removed const; delete it.

8482 export const REQUEST_TIMEOUT_MS = 10_000;
8583
src/release/leftover_sweep.ts added +114
@@ -0,0 +1,114 @@
1 /**
2 * The probe forges the last release proof could not remove are removed here, on the forge's own
3 * tick, through the same hand the Remove page uses (#12550): a probe left running is hidden work
4 * on the host, and waiting for the next release to sweep it kept its card open for hours. Only
5 * names the proof itself recorded, of the proof's own shape (`PROBE_NAME`), are touched, and only
6 * when the forge the host holds by that name is the proof's (`theProofs`): a name anyone may
7 * take once the probe is gone is never proof enough. A name the host no longer holds, or holds
8 * for someone else, is the proof's no longer. Each one gone leaves the proof's record, so its
9 * card closes on the same tick: the record is the one source the vitals read.
10 */
11
12 import type { Log } from "../kernel/log.ts";
13 import { forgetLeftovers, lastProof, PROBE_NAME } from "./prove.ts";
14
15 /** How long one read of the host's forges, or one removal, may take before the tick moves on. */
16 export const SWEEP_STEP_MS = 30_000;
17
18 /** What the host says of a forge it holds: who made it, and when it was begun. */
19 export interface HeldForge {
20 /** The person who made it; absent for a stranger's, as the proof's walk is. */
21 readonly ownerId?: string;
22 /** When it was begun, ISO; null when its ledger does not say. */
23 readonly createdAt: string | null;
24 }
25
26 /**
27 * Whether `forge` is the probe a proof finished at `proofAt` made: a stranger's (the walk signs
28 * up no one before it names the forge), begun no later than the proof that recorded it. A
29 * person's forge, or one begun after the proof (the name taken again once the probe was gone),
30 * is not; neither is one whose begun-at is unknown.
31 */
32 export function theProofs(forge: HeldForge, proofAt: string): boolean {
33 if (forge.ownerId !== undefined) return false;
34 const begun = forge.createdAt === null ? Number.NaN : Date.parse(forge.createdAt);
35 const at = Date.parse(proofAt);
36 return Number.isFinite(begun) && Number.isFinite(at) && begun <= at;
37 }
38
39 export interface LeftoverSweep {
40 readonly dataDir: string;
41 /** The forges the host holds now, by name. */
42 readonly held: () => Promise<ReadonlyMap<string, HeldForge>>;
43 /** Removes one forge; true when it is gone. */
44 readonly remove: (name: string) => Promise<boolean>;
45 /** The bound on each step, ms; `SWEEP_STEP_MS` unless a test gives one. */
46 readonly stepMs?: number;
47 }
48
49 /** `work`, or a rejection once `ms` pass: a hung step cannot stall the tick that carries it. */
50 function bounded<T>(work: Promise<T>, ms: number, what: string): Promise<T> {
outdated

Removes by name shape alone; a user-owned forge named probe-<8hex> would be removed. Verify it is the proof probe before removing. A timed-out removal also keeps running and can be started again next tick; skip names in flight.

51 let timer: ReturnType<typeof setTimeout> | undefined;
52 const late = new Promise<never>((_, reject) => {
53 timer = setTimeout(() => reject(new Error(`${what} took longer than ${ms} ms`)), ms);
54 });
55 return Promise.race([work, late]).finally(() => clearTimeout(timer));
56 }
57
58 /**
59 * Removals begun and not yet settled, by name: one cut off at the bound still runs (nothing can
60 * cancel it), so a later tick skips its name rather than start a second removal of the same forge.
61 */
62 const removing = new Set<string>();
63
64 /**
65 * Removes each leftover probe the host still holds as the proof's; the names gone from the
66 * proof's view (removed, no longer held, or held for someone else, who keeps it). A removal
67 * that fails or runs past the bound keeps its name, as does one still running from an earlier
68 * tick; a host read that fails or runs past the bound throws.
69 */
70 export async function removeLeftoverProbes(o: LeftoverSweep): Promise<string[]> {
71 const proof = lastProof(o.dataDir);
72 const names = (proof?.leftover ?? []).filter((n) => PROBE_NAME.test(n) && !removing.has(n));
73 if (proof === undefined || names.length === 0) return [];
74 const ms = o.stepMs ?? SWEEP_STEP_MS;
75 const held = await bounded(o.held(), ms, "reading the host's forges");
76 const gone: string[] = [];
77 for (const name of names) {
78 const forge = held.get(name);
79 if (forge === undefined || !theProofs(forge, proof.at)) {
80 gone.push(name);
81 continue;
82 }
83 removing.add(name);
84 const work = o.remove(name);
85 work.then(
86 () => removing.delete(name),
87 () => removing.delete(name),
88 );
89 if (await bounded(work, ms, `removing ${name}`).catch(() => false)) gone.push(name);
90 }
91 forgetLeftovers(o.dataDir, gone);
92 return gone;
93 }
94
95 export interface SweepTick extends LeftoverSweep {
96 readonly log: Log;
97 /** Forgets the cached list of served forges, so the next read sees the removal. */
98 readonly dropServed: () => void;
99 }
100
101 /** The ask tick's step: sweep, refresh the served cache when anything went, log. Never throws. */
102 export async function sweepLeftoverProbes(o: SweepTick): Promise<string[]> {
103 try {
104 const gone = await removeLeftoverProbes(o);
105 if (gone.length > 0) {
106 o.dropServed();
107 o.log.info("removed probe forges the release proof left", { names: gone.join(",") });
108 }
109 return gone;
110 } catch (error) {
111 o.log.warn("leftover probe removal failed", { error: String(error) });
112 return [];
113 }
114 }
src/release/prove.ts +16
@@ -276,11 +276,27 @@ export function proofNotice(dataDir: string, running?: string, now = Date.now())
276276 };
277277 }
278278
279 /** A probe forge's name, the only shape the proof makes and removes: `probe-` and 8 hex. */
280 export const PROBE_NAME = /^probe-[0-9a-f]{8}$/;
281
279282 /** Probe forges the last proof could not remove, until a later proof of any version leaves none. */
280283 export function leftoverProbes(dataDir: string): readonly string[] {
281284 return lastProof(dataDir)?.leftover ?? [];
282285 }
283286
287 /**
288 * Drops `gone` from the last proof's leftovers: forges removed since, by hand or by the
289 * tick (#12550).
290 */
291 export function forgetLeftovers(dataDir: string, gone: readonly string[]): void {
292 const p = lastProof(dataDir);
293 if (p?.leftover === undefined || gone.length === 0) return;
294 const left = p.leftover.filter((n) => !gone.includes(n));
295 if (left.length === p.leftover.length) return;
296 const { leftover: _, ...rest } = p;
297 keep(dataDir, left.length === 0 ? rest : { ...rest, leftover: left });
298 }
299
284300 /** Asks for the last failed proof to be tried again at the next tick, whatever its backoff says; false when there is none to try. */
285301 export function proveAgain(dataDir: string): boolean {
286302 const p = lastProof(dataDir);
test/release-leftover-sweep.test.ts added +172
@@ -0,0 +1,172 @@
1 /**
2 * A probe forge the release proof could not remove is removed on the forge's own tick, and its
3 * card closes with it (#12550); only the proof's own names are touched.
4 */
5
6 import { afterEach, beforeEach, expect, test } from "bun:test";
7 import { mkdirSync, writeFileSync } from "node:fs";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { leftoverProbes } from "../src/release/prove.ts";
12 import { type HeldForge, removeLeftoverProbes, sweepLeftoverProbes, theProofs } from "../src/release/leftover_sweep.ts";
13 import type { Log } from "../src/kernel/log.ts";
14
15 let dataDir: string;
16 beforeEach(async () => {
17 dataDir = await mkdtemp(join(tmpdir(), "forge-leftover-sweep-"));
18 });
19 afterEach(async () => {
20 await rm(dataDir, { recursive: true, force: true });
21 });
22
23 const keep = (leftover: string[]) => {
24 mkdirSync(join(dataDir, "release"), { recursive: true });
25 writeFileSync(join(dataDir, "release", "proof.json"), JSON.stringify({ version: "9.9.9", at: "2026-11-14T12:00:00.000Z", ok: false, failure: "x", leftover }));
26 };
27
28 /** The forges the host holds, each the proof's (a stranger's, begun before it), unless given. */
29 const hosts = (names: string[], forge: HeldForge = { createdAt: "2026-11-14T11:58:00.000Z" }) => new Map(names.map((n) => [n, forge]));
30
31 test("a leftover probe the host holds is removed and leaves the proof's record", async () => {
32 keep(["probe-aa6bcd07"]);
33 const removed: string[] = [];
34 const gone = await removeLeftoverProbes({ dataDir, held: async () => hosts(["probe-aa6bcd07", "mine"]), remove: async (n) => (removed.push(n), true) });
35 expect(gone).toEqual(["probe-aa6bcd07"]);
36 expect(removed).toEqual(["probe-aa6bcd07"]);
37 expect(leftoverProbes(dataDir)).toEqual([]);
38 });
39
40 test("one the host no longer holds is forgotten without a removal; one that fails stays", async () => {
41 keep(["probe-00000001", "probe-00000002"]);
42 const removed: string[] = [];
43 const gone = await removeLeftoverProbes({
44 dataDir,
45 held: async () => hosts(["probe-00000002"]),
46 remove: async (n) => {
47 removed.push(n);
48 throw new Error("busy");
49 },
50 });
51 expect(gone).toEqual(["probe-00000001"]);
52 expect(removed).toEqual(["probe-00000002"]);
53 expect(leftoverProbes(dataDir)).toEqual(["probe-00000002"]);
54 });
55
56 test("a name not of the probe's shape is never removed", async () => {
57 keep(["mine"]);
58 const removed: string[] = [];
59 await removeLeftoverProbes({ dataDir, held: async () => hosts(["mine"]), remove: async (n) => (removed.push(n), true) });
60 expect(removed).toEqual([]);
61 expect(leftoverProbes(dataDir)).toEqual(["mine"]);
62 });
63
64 const recording = () => {
65 const lines: string[] = [];
66 const log: Log = {
67 debug: () => {},
68 info: (m) => void lines.push(`info ${m}`),
69 warn: (m, f) => void lines.push(`warn ${m} ${String(f?.error)}`),
70 error: () => {},
71 };
72 return { lines, log };
73 };
74
75 test("the tick's step drops the served cache and logs when a probe went", async () => {
76 keep(["probe-aa6bcd07"]);
77 const { lines, log } = recording();
78 let dropped = 0;
79 const gone = await sweepLeftoverProbes({ dataDir, log, dropServed: () => void dropped++, held: async () => hosts(["probe-aa6bcd07"]), remove: async () => true });
80 expect(gone).toEqual(["probe-aa6bcd07"]);
81 expect(dropped).toBe(1);
82 expect(lines).toEqual(["info removed probe forges the release proof left"]);
83 });
84
85 test("with nothing gone the served cache is kept and nothing is logged", async () => {
86 keep(["probe-aa6bcd07"]);
87 const { lines, log } = recording();
88 let dropped = 0;
89 await sweepLeftoverProbes({ dataDir, log, dropServed: () => void dropped++, held: async () => hosts(["probe-aa6bcd07"]), remove: async () => false });
90 expect(dropped).toBe(0);
91 expect(lines).toEqual([]);
92 });
93
94 test("a host read that fails is logged, and the step does not throw", async () => {
95 keep(["probe-aa6bcd07"]);
96 const { lines, log } = recording();
97 const gone = await sweepLeftoverProbes({ dataDir, log, dropServed: () => {}, held: async () => { throw new Error("host down"); }, remove: async () => true });
98 expect(gone).toEqual([]);
99 expect(lines).toEqual(["warn leftover probe removal failed Error: host down"]);
100 expect(leftoverProbes(dataDir)).toEqual(["probe-aa6bcd07"]);
101 });
102
103 test("a removal that never answers is cut off at the bound and keeps its name; the next goes on", async () => {
104 keep(["probe-00000001", "probe-00000002"]);
105 const { log } = recording();
106 const gone = await sweepLeftoverProbes({
107 dataDir,
108 log,
109 stepMs: 5,
110 dropServed: () => {},
111 held: async () => hosts(["probe-00000001", "probe-00000002"]),
112 remove: (n) => (n === "probe-00000001" ? new Promise<boolean>(() => {}) : Promise.resolve(true)),
113 });
114 expect(gone).toEqual(["probe-00000002"]);
115 expect(leftoverProbes(dataDir)).toEqual(["probe-00000001"]);
116 });
117
118 test("a host read that never answers is cut off at the bound and logged", async () => {
119 keep(["probe-aa6bcd07"]);
120 const { lines, log } = recording();
121 await sweepLeftoverProbes({ dataDir, log, stepMs: 5, dropServed: () => {}, held: () => new Promise<Map<string, HeldForge>>(() => {}), remove: async () => true });
122 expect(lines).toEqual(["warn leftover probe removal failed Error: reading the host's forges took longer than 5 ms"]);
123 });
124
125 test("a held forge of the probe's name that is not the proof's is left running and leaves the record", async () => {
126 keep(["probe-00000101", "probe-00000102", "probe-00000103"]);
127 const removed: string[] = [];
128 const held = new Map<string, HeldForge>([
129 // A person's forge that took the name.
130 ["probe-00000101", { ownerId: "u1", createdAt: "2026-11-14T11:58:00.000Z" }],
131 // A stranger's, begun after the proof: the name taken again once the probe was gone.
132 ["probe-00000102", { createdAt: "2026-11-14T12:30:00.000Z" }],
133 // One whose begun-at is unknown.
134 ["probe-00000103", { createdAt: null }],
135 ]);
136 const gone = await removeLeftoverProbes({ dataDir, held: async () => held, remove: async (n) => (removed.push(n), true) });
137 expect(removed).toEqual([]);
138 expect(gone).toEqual(["probe-00000101", "probe-00000102", "probe-00000103"]);
139 expect(leftoverProbes(dataDir)).toEqual([]);
140 });
141
142 test("theProofs: only a stranger's forge begun no later than the proof", () => {
143 const at = "2026-11-14T12:00:00.000Z";
144 expect(theProofs({ createdAt: "2026-11-14T11:59:00.000Z" }, at)).toBe(true);
145 expect(theProofs({ createdAt: at }, at)).toBe(true);
146 expect(theProofs({ createdAt: "2026-11-14T12:00:01.000Z" }, at)).toBe(false);
147 expect(theProofs({ ownerId: "u1", createdAt: "2026-11-14T11:59:00.000Z" }, at)).toBe(false);
148 expect(theProofs({ createdAt: null }, at)).toBe(false);
149 expect(theProofs({ createdAt: "not a date" }, at)).toBe(false);
150 });
151
152 test("a removal still running past the bound is not started again on the next tick", async () => {
153 keep(["probe-0000000a"]);
154 let calls = 0;
155 let finish: (ok: boolean) => void = () => {};
156 const remove = () => {
157 calls++;
158 return new Promise<boolean>((resolve) => (finish = resolve));
159 };
160 const o = { dataDir, stepMs: 5, held: async () => hosts(["probe-0000000a"]), remove };
161 expect(await removeLeftoverProbes(o)).toEqual([]);
162 expect(await removeLeftoverProbes(o)).toEqual([]);
163 expect(calls).toBe(1);
164 expect(leftoverProbes(dataDir)).toEqual(["probe-0000000a"]);
165 // Once it settles, a later tick may try again.
166 finish(false);
167 // The in-flight mark clears on the settled removal's own callback, a microtask later.
168 await Promise.resolve();
169 await removeLeftoverProbes(o);
170 expect(calls).toBe(2);
171 finish(true);
172 });
test/timers-named.test.ts +1
@@ -119,6 +119,7 @@ const ALLOWED: readonly Allowed[] = [
119119 ["src/apps/toolbar.ts", "resolve"],
120120 ["src/flow/lanes.ts", "() => reject("],
121121 ["src/web/map_snapshot.ts", "() => reject("],
122 ["src/release/leftover_sweep.ts", "() => reject("],
122123 ["src/web/og/raster.ts", "() => reject("],
123124 ["src/looks/thumb.ts", "() => reject("],
124125 ["src/looks/capture.ts", "() => reject("],

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs