Public: anyone can read this; the crew can change it.
The minute's follow tick reads every hosted forge, asleep or not #13827
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13820-the-minute-s-follow-tick-reads-every-hos → main 9 commits fork
4 files
changed +158
−8
src/spawn/clients.ts +72 −7
| @@ -13,12 +13,13 @@ import { liveChildDisk, readMountinfo, type ChildDisk } from "./disk_mount.ts"; | ||
| 13 | 13 | import type { App } from "../app.ts"; |
| 14 | 14 | import { childDirOf, listChildren, statusTokenOf, type Child } from "../actions/spawn.ts"; |
| 15 | 15 | import { timingSafeEqual } from "node:crypto"; |
| 16 | import { statSync } from "node:fs"; | |
| 16 | 17 | import { join } from "node:path"; |
| 17 | 18 | import { sha256 } from "../kernel/hash.ts"; |
| 18 | 19 | import { monthSum, TOO_LARGE, UNREADABLE, type LedgerBroken, type MonthSum } from "../core/ledger.ts"; |
| 19 | 20 | import { creditView, usdWords } from "./credit.ts"; |
| 20 | 21 | import type { ChildAppsSummary } from "./child-apps.ts"; |
| 21 | import { cellsHere, childAwake, hostedChildApps } from "./host.ts"; | |
| 22 | import { cellsHere, childAwake, hostedChildApps, lastTouched } from "./host.ts"; | |
| 22 | 23 | import type { PlacementKind } from "./placement.ts"; |
| 23 | 24 | import { whereOf, type CellsVerdict, type Where } from "./where.ts"; |
| 24 | 25 | import { announceSandboxed } from "./sandboxed.ts"; |
| @@ -307,6 +308,36 @@ function ask(app: App, child: Child): Promise<ForgeState | undefined> { | ||
| 307 | 308 | return asked; |
| 308 | 309 | } |
| 309 | 310 | |
| 311 | /** | |
| 312 | * What reading a sleeping forge's files gave, kept until its ledger or files change or a request or | |
| 313 | * wake reaches it: a forge this host put to sleep cannot have changed them, so the minute's pass | |
| 314 | * reads them once. Its heartbeat is not among them; that is read every pass. | |
| 315 | */ | |
| 316 | interface Sleeper { | |
| 317 | readonly sig: string; | |
| 318 | readonly at: number; | |
| 319 | readonly person: Awaited<ReturnType<typeof childPerson>>; | |
| 320 | readonly links: AppLinkLine[]; | |
| 321 | readonly keep: number | null; | |
| 322 | readonly madeFrom: MadeFrom | null; | |
| 323 | } | |
| 324 | /** | |
| 325 | * The row's own record plus when the files read from its directory last changed (the parent writes | |
| 326 | * `parent.json`). | |
| 327 | */ | |
| 328 | function sigOf(app: App, c: Child): string { | |
| 329 | const dir = childDirOf(app, c.bodyId); | |
| 330 | const stamps = ["parent.json", "forge.db", "keep"].map((f) => { | |
| 331 | try { | |
| 332 | return Math.floor(statSync(join(dir, f)).mtimeMs); | |
| 333 | } catch { | |
| 334 | return 0; | |
| 335 | } | |
| 336 | }); | |
| 337 | return JSON.stringify([c, stamps]); | |
| 338 | } | |
| 339 | const readSleepers = new WeakMap<App, Map<string, Sleeper>>(); | |
| 340 | ||
| 310 | 341 | /** |
| 311 | 342 | * Every child this forge sent, newest first. A child never heard from is |
| 312 | 343 | * asked now (all at once, each for at most two seconds); one heard from |
| @@ -335,11 +366,45 @@ export async function clientRows(app: App, options: { probe?: boolean } = {}): P | ||
| 335 | 366 | return last.state; |
| 336 | 367 | }), |
| 337 | 368 | ); |
| 338 | const beats = await Promise.all(children.map((c) => lastHeard(app, c.name))); | |
| 369 | let kept = readSleepers.get(app); | |
| 370 | if (kept === undefined) readSleepers.set(app, (kept = new Map())); | |
| 371 | const keptNow = kept; | |
| 372 | // A forge that is gone leaves nothing kept. | |
| 373 | const names = new Set(children.map((c) => c.name)); | |
| 374 | for (const name of [...keptNow.keys()]) if (!names.has(name)) keptNow.delete(name); | |
| 375 | const sleeping = children.map((c) => { | |
| 376 | if (!c.done || childAwake(app, c.name)) { | |
| 377 | keptNow.delete(c.name); | |
| 378 | return undefined; | |
| 379 | } | |
| 380 | const have = keptNow.get(c.name); | |
| 381 | return have !== undefined && have.sig === sigOf(app, c) && lastTouched(app, c.bodyId) < have.at ? have : undefined; | |
| 382 | }); | |
| 339 | 383 | // Who did what there, read from each forge's own tables (asleep or awake, on any code), never |
| 340 | // from its word. | |
| 341 | const people = await Promise.all(children.map((c) => (c.done ? childPerson(childDirOf(app, c.bodyId)) : undefined))); | |
| 342 | const links = await Promise.all(children.map((c) => (c.apps.length === 0 ? [] : appLinkLines(app, c)))); | |
| 384 | // from its word. A forge asleep with nothing changed is not read again. | |
| 385 | const reads = await Promise.all( | |
| 386 | children.map(async (c, i) => { | |
| 387 | const have = sleeping[i]; | |
| 388 | if (have !== undefined) return have; | |
| 389 | const at = Date.now(); | |
| 390 | const dir = childDirOf(app, c.bodyId); | |
| 391 | const read: Sleeper = { | |
| 392 | sig: sigOf(app, c), | |
| 393 | at, | |
| 394 | person: c.done ? await childPerson(dir) : undefined, | |
| 395 | links: c.apps.length === 0 ? [] : await appLinkLines(app, c), | |
| 396 | keep: keepBytes(dir) ?? null, | |
| 397 | madeFrom: madeFromOf(dir) ?? null, | |
| 398 | }; | |
| 399 | if (c.done && !childAwake(app, c.name) && lastTouched(app, c.bodyId) < at) keptNow.set(c.name, read); | |
| 400 | return read; | |
| 401 | }), | |
| 402 | ); | |
| 403 | // The heartbeat is read every pass, kept row or not: a beat can arrive for a forge this host | |
| 404 | // thinks asleep, and what the row says of it (last heard, next step) must not go stale. | |
| 405 | const beats = await Promise.all(children.map((c) => lastHeard(app, c.name))); | |
| 406 | const people = reads.map((r) => r.person); | |
| 407 | const links = reads.map((r) => r.links); | |
| 343 | 408 | const notices = readNotices(app); |
| 344 | 409 | const replaced = linksReplaced(app); |
| 345 | 410 | // Asked once, and only when a forge here is not in a cell: what its row says depends on whether |
| @@ -373,7 +438,7 @@ export async function clientRows(app: App, options: { probe?: boolean } = {}): P | ||
| 373 | 438 | claimedAt: people[i]?.claimedAt ?? null, |
| 374 | 439 | billTo: billTo(people[i]?.claimantEmail), |
| 375 | 440 | noticedAt: notices[c.name] ?? null, |
| 376 | keepBytes: keepBytes(childDirOf(app, c.bodyId)) ?? null, | |
| 441 | keepBytes: reads[i]!.keep, | |
| 377 | 442 | awake: childAwake(app, c.name), |
| 378 | 443 | stopped: stoppedWords(app, c), |
| 379 | 444 | running: hostedChildApps(app, c.name) ?? null, |
| @@ -393,7 +458,7 @@ export async function clientRows(app: App, options: { probe?: boolean } = {}): P | ||
| 393 | 458 | link: states[i]?.link ?? null, |
| 394 | 459 | replacedAt: replaced.get(c.name) ?? null, |
| 395 | 460 | // Read from the record this forge wrote when it made the child, never from the child's word. |
| 396 | madeFrom: madeFromOf(childDirOf(app, c.bodyId)) ?? null, | |
| 461 | madeFrom: reads[i]!.madeFrom, | |
| 397 | 462 | nextStep: nextStepOf(claimed[i] ?? null, beats[i]?.beat), |
| 398 | 463 | })), |
| 399 | 464 | now, |
src/spawn/host.ts +16
| @@ -143,7 +143,22 @@ const personWakes = new WeakMap<App, WakeRate>(); | ||
| 143 | 143 | export const gateClock = { now: () => Date.now() }; |
| 144 | 144 | const sweepers = new WeakMap<App, ReturnType<typeof setInterval>>(); |
| 145 | 145 | |
| 146 | const lastTouchedAt = new WeakMap<App, Map<string, number>>(); | |
| 147 | ||
| 148 | /** When a request or a wake last reached this child (ms); 0 when none has since this start. */ | |
| 149 | export function lastTouched(parent: App, bodyId: string): number { | |
| 150 | return lastTouchedAt.get(parent)?.get(bodyId) ?? 0; | |
| 151 | } | |
| 152 | ||
| 153 | /** Drop what is kept of when a request last reached a child: it is gone. */ | |
| 154 | export function forgetTouched(parent: App, bodyId: string): void { | |
| 155 | lastTouchedAt.get(parent)?.delete(bodyId); | |
| 156 | } | |
| 157 | ||
| 146 | 158 | function touch(parent: App, ledger: { bodyId: string; name: string }, now = Date.now()): void { |
| 159 | let stamps = lastTouchedAt.get(parent); | |
| 160 | if (!stamps) lastTouchedAt.set(parent, (stamps = new Map())); | |
| 161 | stamps.set(ledger.bodyId, Date.now()); | |
| 147 | 162 | let m = touched.get(parent); |
| 148 | 163 | if (!m) touched.set(parent, (m = new Map())); |
| 149 | 164 | m.set(ledger.bodyId, { name: ledger.name, at: now }); |
| @@ -452,6 +467,7 @@ export function childAwake(parent: App, name: string): boolean { | ||
| 452 | 467 | |
| 453 | 468 | async function closeOne(parent: App, cache: Map<string, App>, bodyId: string, child: App, name: string): Promise<string> { |
| 454 | 469 | cache.delete(bodyId); |
| 470 | lastTouchedAt.get(parent)?.delete(bodyId); | |
| 455 | 471 | bootOpened.get(parent)?.delete(bodyId); |
| 456 | 472 | touched.get(parent)?.delete(bodyId); |
| 457 | 473 | waking.get(parent)?.delete(bodyId); |
src/spawn/manage.ts +2 −1
| @@ -13,7 +13,7 @@ import { revokeChildFollowTokens } from "../core/proposal_token.ts"; | ||
| 13 | 13 | import { restoreClaim, revokeClaim, spawnedOrg } from "./claim.ts"; |
| 14 | 14 | import { forgetHeard } from "./clients.ts"; |
| 15 | 15 | import { forgetForge } from "../core/child_reach.ts"; |
| 16 | import { noteLedger, sleepChild } from "./host.ts"; | |
| 16 | import { forgetTouched, noteLedger, sleepChild } from "./host.ts"; | |
| 17 | 17 | import { resizeRefusal } from "./quota.ts"; |
| 18 | 18 | import { isSize, SIZE_ORDER, SIZES, type Size } from "./sizes.ts"; |
| 19 | 19 | import { retireBody } from "./door.ts"; |
| @@ -51,6 +51,7 @@ export async function removeForge(app: App, name: string): Promise<Managed> { | ||
| 51 | 51 | for (const dir of ["reach", "beats", "credit"]) await rm(join(app.config.dataDir, dir, `${name}.json`), { force: true }); |
| 52 | 52 | // The count of forges sent on this node is the forges that exist. |
| 53 | 53 | await retireBody(app, child.bodyId); |
| 54 | forgetTouched(app, child.bodyId); | |
| 54 | 55 | app.bus.append({ kind: "forge.removed", payload: { name } }); |
| 55 | 56 | return { ok: true, says: `${name} is removed, with its data` }; |
| 56 | 57 | } |
test/forges-sleeping-read-once.test.ts added +68
| @@ -0,0 +1,68 @@ | ||
| 1 | /** | |
| 2 | * The minute's pass reads a sleeping forge's files once: while its ledger is unchanged and | |
| 3 | * nothing has reached it, the row is the one kept; a changed ledger is read afresh. | |
| 4 | */ | |
| 5 | ||
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdir, mkdtemp, rm, stat, utimes, writeFile } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { clientRows } from "../src/spawn/clients.ts"; | |
| 12 | ||
| 13 | const SHA = "a".repeat(40); | |
| 14 | let dir = ""; | |
| 15 | let app: App; | |
| 16 | ||
| 17 | const record = (repo: string) => JSON.stringify({ template: { repo, tip: SHA } }); | |
| 18 | const ledger = (extra: object = {}) => | |
| 19 | JSON.stringify({ name: "nap", bodyId: "body1", done: true, ...extra }); | |
| 20 | const madeFrom = async () => (await clientRows(app, { probe: false })).find((r) => r.name === "nap")?.madeFrom?.template; | |
| 21 | ||
| 22 | beforeAll(async () => { | |
| 23 | dir = await mkdtemp(join(tmpdir(), "sleeping-")); | |
| 24 | app = createApp({ dataDir: dir, port: 0 }); | |
| 25 | await mkdir(join(dir, "spawn"), { recursive: true }); | |
| 26 | await mkdir(join(dir, "children", "body1"), { recursive: true }); | |
| 27 | await writeFile(join(dir, "spawn", "nap.json"), ledger()); | |
| 28 | await writeFile(join(dir, "children", "body1", "parent.json"), record("a/one")); | |
| 29 | }); | |
| 30 | afterAll(async () => { | |
| 31 | app.close(); | |
| 32 | await rm(dir, { recursive: true, force: true }); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a sleeping forge's files are read once, and again when they or its ledger change", async () => { | |
| 36 | const file = join(dir, "children", "body1", "parent.json"); | |
| 37 | expect(await madeFrom()).toBe("a/one"); | |
| 38 | const was = await stat(file); | |
| 39 | // Changed with its stamp kept: nothing says it moved, so the kept row is served. | |
| 40 | await writeFile(file, record("a/two")); | |
| 41 | await utimes(file, was.atime, was.mtime); | |
| 42 | expect(await madeFrom()).toBe("a/one"); | |
| 43 | // Its stamp moved (the parent writes parent.json): the next pass reads it afresh. | |
| 44 | await utimes(file, new Date(), new Date(was.mtimeMs + 5000)); | |
| 45 | expect(await madeFrom()).toBe("a/two"); | |
| 46 | }); | |
| 47 | ||
| 48 | test("a ledger that changed is read afresh, with the files as they are", async () => { | |
| 49 | const file = join(dir, "children", "body1", "parent.json"); | |
| 50 | const was = await stat(file); | |
| 51 | const now = await madeFrom(); | |
| 52 | // Content moved with its stamp kept: the kept row is served. | |
| 53 | await writeFile(file, record("a/three")); | |
| 54 | await utimes(file, was.atime, was.mtime); | |
| 55 | expect(await madeFrom()).toBe(now); | |
| 56 | // Its ledger changed: read afresh, and the new content shows. | |
| 57 | await writeFile(join(dir, "spawn", "nap.json"), ledger({ forWhom: "someone" })); | |
| 58 | expect(await madeFrom()).toBe("a/three"); | |
| 59 | }); | |
| 60 | ||
| 61 | test("a beat that arrives for a kept forge shows on the next pass", async () => { | |
| 62 | const row = async () => (await clientRows(app, { probe: false })).find((r) => r.name === "nap"); | |
| 63 | expect((await row())?.lastHeartbeatAt).toBeNull(); | |
| 64 | await mkdir(join(dir, "beats"), { recursive: true }); | |
| 65 | const beat = { name: "nap", at: Date.now(), version: "1", openProposals: 0, children: 0 }; | |
| 66 | await writeFile(join(dir, "beats", "nap.json"), JSON.stringify({ receivedAt: Date.now(), beat })); | |
| 67 | expect((await row())?.lastHeartbeatAt).not.toBeNull(); | |
| 68 | }); |
new above
offline