Public: anyone can read this; the crew can change it.
'Running' means reachable: prove every app and new forge from its public address, alarm when it isn't, and carry websockets into cells #7981
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/7959-running-means-reachable-prove-every-app → main 5 commits fork
27 files
changed +978
−15
docs/MCP.md +1 −1
| @@ -90,7 +90,7 @@ for a tunnel and a forge that sleeps. | ||
| 90 | 90 | | `follow_mode` | `{ repo, … }`: choose how a spawned forge follows its parent, `tell-me` or `take-updates`; its `forge` repository's admin only | yes | |
| 91 | 91 | | `following` | where a fork stands against what it follows, and what the crew is doing about it | | |
| 92 | 92 | | `forge_proposals` | `{ repo }`: what this forge proposed to its parent, newest first; `reachable` is false if the parent did not answer | | |
| 93 | | `forges` | an admin's list of the forges sent from here: name, address, whom for, apps, claimed or not, last activity | | | |
| 93 | | `forges` | an admin's list of the forges sent from here: name, address, whom for, apps, claimed or not, last activity, whether each public address answers | | | |
| 94 | 94 | | `forks` | `{ repo }`: `{ parent, forks: [{ repo, ahead, behind, updatedAt }] }`, the forks the caller may see (as `/:owner/:name/forks` lists them), each with its commits ahead of and behind this one; `null` when git cannot say | | |
| 95 | 95 | | `front_page` | `{ pinned?, hidden?, weights?, dryRun? }`: the front page as a stranger sees it, ranked, with numbers; a site admin sets what is pinned (first public one featured), hidden, and what counts (weights of forks, ships, use, fresh, 0 to 10) | yes | |
| 96 | 96 | | `health` | how this forge is: the version and commit it runs, draining or not, the rollout and its incidents | | |
docs/design/areas/lists.md +2
| @@ -14,3 +14,5 @@ trails a palette row. | ||
| 14 | 14 | `src/web/style/content.ts` (`.finder .palette-item .cat`). |
| 15 | 15 | - `.flow-row` — a §5 `.row` link on the flow panel (`flow-panel.ts`: idle pulls and the "and K more" row) with a 44px |
| 16 | 16 | minimum height, so it is a touch target on a 360px phone. Styled in `src/web/style/components.ts` (list rows). |
| 17 | ||
| 18 | The proofs under a forge on `/forges` (#7959) are `row()` `children`, one per public address; the row's trail badge says "reachable" or "not reachable", so the state is a word and not only a colour. No new class. |
docs/features/child-reach.md added +32
| @@ -0,0 +1,32 @@ | ||
| 1 | # "Running" is proven from outside | |
| 2 | ||
| 3 | A child forge's process being up inside its cell says nothing about whether a | |
| 4 | visitor reaches it. The parent asks the way a visitor does (#7959). | |
| 5 | ||
| 6 | - **What is asked.** The child's home (`<child>--<parent>/api/v1/forge`, which | |
| 7 | must answer its own JSON `claimed`) and each public app | |
| 8 | (`<app>--<child>--<parent>/`, which must carry the `x-forge-app: <slug>` header | |
| 9 | the proxy writes on everything it carries back). A status alone proves | |
| 10 | nothing: the platform's wildcard standby page answers 200 for a mistyped | |
| 11 | `<app>--<child>.<apex>`. | |
| 12 | - **Where it shows.** A failing child reads **Running, not reachable: deal-clock | |
| 13 | answered 404, not by the app** on `/forges` (with every address and its | |
| 14 | result under the row), in the terminal's Forges screen, in the MCP `forges` | |
| 15 | tool (`reach`), and in the result of `POST /api/v1/forges` / MCP `spawn` | |
| 16 | (`reach`, right after the child is made). | |
| 17 | - **Alarm.** A new child is re-proved every 3 seconds for 3 minutes; if it never | |
| 18 | answers, `forge.child_unreachable` goes on the outbox and the notifier pages | |
| 19 | the owner (the `unreachable` flow alarm). Every 5 minutes the parent | |
| 20 | re-proves every finished child (a failure is confirmed once before it is | |
| 21 | believed); a change pages once, and `forge.child_reachable` ends it. | |
| 22 | - **State** is `<data>/reach/<child>.json`, derived and rebuilt by the next pass. | |
| 23 | - **Websockets** to a child's home or apps reach its cell: `forwardSocket` | |
| 24 | (`src/spawn/cell.ts`) opens the cell's side first, then answers the visitor. | |
| 25 | - Tests: `test/spawn-reach.test.ts`, `test/spawn-reach-door.test.ts`, | |
| 26 | `test/spawn-cell-socket.test.ts`, and the real-cell `test/spawn-cell-e2e.test.ts` | |
| 27 | (skipped, and says so, where bwrap and socat are missing). | |
| 28 | - **In the terminal:** open the palette (ctrl-k), "Forges you sent". A forge that | |
| 29 | is not reachable reads *Running, not reachable: …*, and under it one line per | |
| 30 | public address with its result (`NOT REACHABLE deal--kid--host/: answered 404, not by the app`). | |
| 31 | - **Slugs are the forge's rule.** A child writes its own database, so a slug that | |
| 32 | is not `[a-z0-9-]` (`evil.example#`, `u@host`) is dropped before any address is built. |
src/apps/proxy.ts +6 −3
| @@ -1097,6 +1097,9 @@ export async function proxyApp( | ||
| 1097 | 1097 | } |
| 1098 | 1098 | const out = new Headers(response.headers); |
| 1099 | 1099 | for (const name of HOP_BY_HOP) out.delete(name); |
| 1100 | // Which app answered: a probe from outside (`spawn/reach.ts`) tells this | |
| 1101 | // from a platform's standby page, which answers 200 for any name. | |
| 1102 | out.set("x-forge-app", target.app.slug); | |
| 1100 | 1103 | // An agent reading a public app's page learns where to report a problem |
| 1101 | 1104 | // without the app having to say so. |
| 1102 | 1105 | if (isHtml(out)) |
| @@ -1282,13 +1285,13 @@ const WS_IDLE_SECONDS = 120; | ||
| 1282 | 1285 | * that merely says `Upgrade: websocket` is an ordinary request, and is |
| 1283 | 1286 | * fetched as one with the hop-by-hop headers stripped, as it always was. |
| 1284 | 1287 | */ |
| 1285 | const isWebSocketHandshake = (request: Request): boolean => | |
| 1288 | export const isWebSocketHandshake = (request: Request): boolean => | |
| 1286 | 1289 | request.method === "GET" && |
| 1287 | 1290 | request.headers.get("upgrade")?.toLowerCase() === "websocket" && |
| 1288 | 1291 | request.headers.has("sec-websocket-key"); |
| 1289 | 1292 | |
| 1290 | 1293 | /** The subprotocols a handshake offers, in the order it offers them. */ |
| 1291 | const offeredProtocols = (request: Request): string[] => | |
| 1294 | export const offeredProtocols = (request: Request): string[] => | |
| 1292 | 1295 | (request.headers.get("sec-websocket-protocol") ?? "") |
| 1293 | 1296 | .split(",") |
| 1294 | 1297 | .map((s) => s.trim()) |
| @@ -1347,7 +1350,7 @@ async function proxyWebSocket( | ||
| 1347 | 1350 | * sentence for the "not running" page: the app was not listening, or it |
| 1348 | 1351 | * answered the handshake with something other than a socket. |
| 1349 | 1352 | */ |
| 1350 | function connectUpstream( | |
| 1353 | export function connectUpstream( | |
| 1351 | 1354 | url: URL, |
| 1352 | 1355 | headers: Record<string, string>, |
| 1353 | 1356 | protocols: string[], |
src/core/event_audience.ts +2
| @@ -159,6 +159,8 @@ export function classifiedAudience(kind: string): EventAudience | undefined { | ||
| 159 | 159 | kind.startsWith("backup.") || |
| 160 | 160 | // The weekly model review's changes and results are the owner's (`actions/model_review.ts`). |
| 161 | 161 | kind.startsWith("crew.model_") || |
| 162 | // A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`). | |
| 163 | kind.startsWith("forge.child_") || | |
| 162 | 164 | // A clog is the whole forge's flow, the owner's to hear (`flow/clog_tick.ts`). |
| 163 | 165 | kind.startsWith("clog.") || |
| 164 | 166 | // What the throughput keeper did to lift a brake is the owner's too (`flow/keeper.ts`). |
src/crew/notify.ts +2 −1
| @@ -41,6 +41,7 @@ import { health } from "../http/health.ts"; | ||
| 41 | 41 | import { forgeVitals } from "../actions/health.ts"; |
| 42 | 42 | import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch } from "./incidents.ts"; |
| 43 | 43 | import { firedAlarms, type FiredAlarm } from "../flow/alarms.ts"; |
| 44 | import { reachAlarms } from "../spawn/reach.ts"; | |
| 44 | 45 | import { loadClog } from "../flow/clog_tick.ts"; |
| 45 | 46 | import { forgeCeiling } from "../flow/ceiling.ts"; |
| 46 | 47 | import { askLinesFor } from "../web/studio/asks.ts"; |
| @@ -257,7 +258,7 @@ export function alarmLines( | ||
| 257 | 258 | fired?: readonly FiredAlarm[], |
| 258 | 259 | ): { id: string; line: string; items: readonly string[]; link: string }[] { |
| 259 | 260 | if (!user.isAdmin || user.kind !== "user") return []; |
| 260 | const alarms = fired ?? firedAlarms(app.db, new Date(now), { ceilingUsd: forgeCeiling(app.config) }); | |
| 261 | const alarms = fired ?? [...firedAlarms(app.db, new Date(now), { ceilingUsd: forgeCeiling(app.config) }), ...reachAlarms(app)]; | |
| 261 | 262 | return alarms.map((a) => { |
| 262 | 263 | const id = `alarm:${a.key}@${a.since}`; |
| 263 | 264 | return { id, line: a.says, items: [id], link: a.href }; |
src/flow/alarms.ts +1 −1
| @@ -17,7 +17,7 @@ import { redMainAlarm } from "./alarm_red_main.ts"; | ||
| 17 | 17 | import { alarmStuck } from "./alarm_stuck.ts"; |
| 18 | 18 | |
| 19 | 19 | export interface FiredAlarm { |
| 20 | readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release"; | |
| 20 | readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release" | "unreachable"; | |
| 21 | 21 | readonly says: string; |
| 22 | 22 | readonly since: string; |
| 23 | 23 | /** A same-origin path to the item the alarm names. */ |
src/http/mcp/tools/forges.ts +1 −1
| @@ -3,7 +3,7 @@ import { good, bad, type Tool } from "../shared.ts"; | ||
| 3 | 3 | |
| 4 | 4 | export const tool: Tool = { |
| 5 | 5 | name: "forges", |
| 6 | doc: "an admin's list of the forges sent from here: name, address, whom for, apps, claimed or not, last activity", | |
| 6 | doc: "an admin's list of the forges sent from here: name, address, whom for, apps, claimed or not, last activity, whether each public address answers", | |
| 7 | 7 | description: |
| 8 | 8 | "Forges sent from here (admin): name, address, apps, claimed, spendTodayUsd, ceilingUsd.", |
| 9 | 9 | inputSchema: { type: "object", properties: {} }, |
src/http/mcp/tools/spawn.ts +3 −1
| @@ -66,10 +66,12 @@ export const tool: Tool = { | ||
| 66 | 66 | const made = door.body["dryRun"] === false; |
| 67 | 67 | const { claimUrl, ...rest } = door.body; |
| 68 | 68 | const reveal = args["reveal"] === true && typeof claimUrl === "string"; |
| 69 | const reach = door.body["reach"] as { ok?: boolean; says?: string } | undefined; | |
| 70 | const reached = reach?.ok === false ? ` ${reach.says}` : ""; | |
| 69 | 71 | return good({ |
| 70 | 72 | says: !made |
| 71 | 73 | ? "This is the plan; nothing was made. Send dryRun: false to spawn." |
| 72 | : `Spawned ${String(door.body["name"])} at ${String(door.body["url"])}. ${reveal ? LINK_REVEALED : LINK_KEPT}`, | |
| 74 | : `Spawned ${String(door.body["name"])} at ${String(door.body["url"])}. ${reveal ? LINK_REVEALED : LINK_KEPT}${reached}`, | |
| 73 | 75 | ...rest, |
| 74 | 76 | ...(reveal ? { claimUrl } : {}), |
| 75 | 77 | }); |
src/http/server.ts +7 −2
| @@ -564,13 +564,14 @@ import { holdAppHost } from "../apps/holding.ts"; | ||
| 564 | 564 | import { |
| 565 | 565 | appWebSocket, |
| 566 | 566 | type AppSocket, |
| 567 | isWebSocketHandshake, | |
| 567 | 568 | proxyApp, |
| 568 | 569 | resolveAppHost, |
| 569 | 570 | } from "../apps/proxy.ts"; |
| 570 | 571 | |
| 571 | 572 | import { stopCellChildren } from "../spawn/cell.ts"; |
| 572 | 573 | import { portInsteadOfSocket, tcpAddress } from "../git/hook-address.ts"; |
| 573 | import { answerFromCell, closeHostedChildren, resolveHostedChild, resolveHostedChildApp } from "../spawn/host.ts"; | |
| 574 | import { answerFromCell, closeHostedChildren, resolveHostedChild, resolveHostedChildApp, socketToCell } from "../spawn/host.ts"; | |
| 574 | 575 | import { getVerifiedDomain } from "../core/domains.ts"; |
| 575 | 576 | |
| 576 | 577 | // ---- auth ---- |
| @@ -2349,9 +2350,13 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2349 | 2350 | // A hosted child's app (`<app>--<child>--<parentHost>`) is the child's |
| 2350 | 2351 | // own runtime's, as the parent's apps are the parent's. |
| 2351 | 2352 | // A child in its own cell answers through this one door, for its pages and its apps. |
| 2352 | const fromCell = async (name?: string): Promise<Response> => { | |
| 2353 | const fromCell = async (name?: string): Promise<Response | undefined> => { | |
| 2353 | 2354 | // The child sees only this forge's relay, so it is told who the client is. |
| 2354 | 2355 | notePeer(request, server.requestIP(request)?.address, trust); |
| 2356 | if (isWebSocketHandshake(request)) { | |
| 2357 | const socket = await socketToCell(app, request, server, name); | |
| 2358 | return socket === undefined ? undefined : securityFloor(socket); | |
| 2359 | } | |
| 2355 | 2360 | armClose(request, () => server.timeout(request, 1)); |
| 2356 | 2361 | const response = await answered(app, request, () => answerFromCell(app, request, name)); |
| 2357 | 2362 | counted(app, request, response, started); |
src/main.ts +4
| @@ -23,6 +23,7 @@ import { bootHostedChildren } from "./spawn/host.ts"; | ||
| 23 | 23 | import { firstBoot } from "./spawn/first-boot.ts"; |
| 24 | 24 | import { startHeartbeat } from "./spawn/heartbeat.ts"; |
| 25 | 25 | import { startUpstreamWatch } from "./spawn/upstream.ts"; |
| 26 | import { startReachWatch } from "./spawn/reach.ts"; | |
| 26 | 27 | import { pruneSessions } from "./core/sessions.ts"; |
| 27 | 28 | import { runDelivery } from "./core/webhooks.ts"; |
| 28 | 29 | import { isHookName, runHookClient } from "./git/hooks.ts"; |
| @@ -483,6 +484,8 @@ await firstBoot(app); | ||
| 483 | 484 | const heartbeatTimer = startHeartbeat(app); |
| 484 | 485 | // ...and looks upstream once an hour (spawn/upstream.ts). |
| 485 | 486 | const upstreamTimer = startUpstreamWatch(app); |
| 487 | // ...and this forge asks every child's public addresses what a visitor would get, every few minutes (spawn/reach.ts). | |
| 488 | const reachTimer = startReachWatch(app); | |
| 486 | 489 | // Prompts from Instagram (docs/APPS.md): once a day, for each repository |
| 487 | 490 | // that connected an account, the top comment on its latest Reel is told to |
| 488 | 491 | // the crew. A minute's clock; nothing leaves the box until a source exists. |
| @@ -749,6 +752,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 749 | 752 | clearInterval(deliveryTimer); |
| 750 | 753 | clearInterval(heartbeatTimer); |
| 751 | 754 | clearInterval(upstreamTimer); |
| 755 | clearInterval(reachTimer); | |
| 752 | 756 | clearInterval(syncTimer); |
| 753 | 757 | clearInterval(staleTimer); |
| 754 | 758 | clearInterval(followTimer); |
src/spawn/cell.ts +35
| @@ -11,6 +11,7 @@ | ||
| 11 | 11 | import { dirname, join } from "node:path"; |
| 12 | 12 | import { clientAddress } from "../http/peer.ts"; |
| 13 | 13 | import type { App } from "../app.ts"; |
| 14 | import { connectUpstream, offeredProtocols, type AppSocket } from "../apps/proxy.ts"; | |
| 14 | 15 | import { cellsAvailable, killTree, openCell, type Cell, type CellSpec, type Limits } from "../apps/contain.ts"; |
| 15 | 16 | import { childEnv, freePort } from "../apps/process.ts"; |
| 16 | 17 | import { childDirOf } from "../actions/spawn.ts"; |
| @@ -312,3 +313,37 @@ export async function forward( | ||
| 312 | 313 | return new Response("this forge is not answering", { status: 502 }); |
| 313 | 314 | } |
| 314 | 315 | } |
| 316 | ||
| 317 | /** | |
| 318 | * Carry a websocket handshake to the child's loopback port: the child's side | |
| 319 | * is opened first, and only then is the visitor's answered, as `proxyApp` | |
| 320 | * does for an app (`apps/proxy.ts`). Frames then go through `appWebSocket`, | |
| 321 | * the forge's one pump. `undefined` means the request became a socket. | |
| 322 | */ | |
| 323 | export async function forwardSocket( | |
| 324 | child: Pick<CellChild, "port">, | |
| 325 | request: Request, | |
| 326 | server: Bun.Server<AppSocket>, | |
| 327 | db: AppSocket["db"], | |
| 328 | clientHeader?: string, | |
| 329 | ): Promise<Response | undefined> { | |
| 330 | const at = new URL(request.url); | |
| 331 | const headers: Record<string, string> = {}; | |
| 332 | const named = clientHeader?.toLowerCase(); | |
| 333 | for (const [k, v] of request.headers) { | |
| 334 | // The client writes its own handshake; the visitor's key, version and extensions end here. | |
| 335 | if (!HOP.has(k) && !WHO.has(k) && k !== named && !k.startsWith("sec-websocket-")) headers[k] = v; | |
| 336 | } | |
| 337 | headers["x-forwarded-for"] = clientAddress(request); | |
| 338 | const url = new URL(`ws://127.0.0.1:${child.port}${at.pathname}${at.search}`); | |
| 339 | const opened = await connectUpstream(url, headers, offeredProtocols(request)); | |
| 340 | if (!opened.ok) return new Response("this forge is not answering", { status: 502 }); | |
| 341 | const upstream = opened.value; | |
| 342 | const upgraded = server.upgrade(request, { | |
| 343 | data: { upstream, userId: undefined, db, port: child.port }, | |
| 344 | ...(upstream.protocol === "" ? {} : { headers: { "sec-websocket-protocol": upstream.protocol } }), | |
| 345 | }); | |
| 346 | if (upgraded) return undefined; | |
| 347 | upstream.close(1000); | |
| 348 | return new Response("bad websocket handshake", { status: 400 }); | |
| 349 | } |
src/spawn/clients.ts +7
| @@ -17,6 +17,7 @@ import type { ChildAppsSummary } from "./child-apps.ts"; | ||
| 17 | 17 | import { hostedChildApps } from "./host.ts"; |
| 18 | 18 | import { hasAdmin, spawnedOrg } from "./claim.ts"; |
| 19 | 19 | import { lastHeard, type Beat } from "./heartbeat.ts"; |
| 20 | import { reachOf, reachWords, type ChildReach } from "./reach.ts"; | |
| 20 | 21 | import type { Size } from "./sizes.ts"; |
| 21 | 22 | import { forgeUse, type ForgeUse } from "./usage.ts"; |
| 22 | 23 | |
| @@ -39,6 +40,8 @@ export interface ClientRow { | ||
| 39 | 40 | readonly lastActivityAt: string | null; |
| 40 | 41 | /** The apps this parent runs for the child; null when it runs none here. */ |
| 41 | 42 | readonly running: ChildAppsSummary | null; |
| 43 | /** What a visitor got from its public addresses when last asked (`spawn/reach.ts`); null before it was. */ | |
| 44 | readonly reach: ChildReach | null; | |
| 42 | 45 | /** Why this forge needs its owner now; null when it does not. */ |
| 43 | 46 | readonly attention: Attention | null; |
| 44 | 47 | /** Dollars the child's crew spent today, from its last heartbeat; null when it has not said. */ |
| @@ -202,6 +205,7 @@ export async function clientRows(app: App): Promise<ClientRow[]> { | ||
| 202 | 205 | claimed: states[i]?.claimed ?? null, |
| 203 | 206 | lastActivityAt: states[i]?.lastActivityAt ?? null, |
| 204 | 207 | running: hostedChildApps(app, c.name) ?? null, |
| 208 | reach: reachOf(app, c.name) ?? null, | |
| 205 | 209 | spendTodayUsd: beats[i]?.beat.spendTodayUsd ?? null, |
| 206 | 210 | ceilingUsd: beats[i]?.beat.ceilingUsd ?? null, |
| 207 | 211 | size: c.size, |
| @@ -236,6 +240,7 @@ export function attentionOf(r: ClientRow, now: number): Attention | null { | ||
| 236 | 240 | const last = r.lastActivityAt === null ? NaN : Date.parse(r.lastActivityAt); |
| 237 | 241 | if (r.claimed === null || (!Number.isNaN(last) && now - last > SILENT_MS)) return "silent"; |
| 238 | 242 | if (r.running !== null && r.running.notRunning.length > 0) return "down"; |
| 243 | if (r.reach !== null && !r.reach.ok) return "down"; | |
| 239 | 244 | const sent = r.sentAt === null ? NaN : Date.parse(r.sentAt); |
| 240 | 245 | if (r.claimed === false && !Number.isNaN(sent) && now - sent > UNCLAIMED_MS) return "unclaimed"; |
| 241 | 246 | return null; |
| @@ -270,6 +275,8 @@ export function claimState(r: ClientRow): keyof typeof CLAIM_WORDS { | ||
| 270 | 275 | |
| 271 | 276 | /** "3 apps running", or which one is not and why; null when none are run here. */ |
| 272 | 277 | export function runningWords(r: ClientRow): string | null { |
| 278 | const unreachable = reachWords(r.reach); | |
| 279 | if (unreachable !== null) return unreachable; | |
| 273 | 280 | const a = r.running; |
| 274 | 281 | if (a === null || a.total === 0) return null; |
| 275 | 282 | const word = `${a.running} ${a.running === 1 ? "app" : "apps"} running`; |
src/spawn/door.ts +15
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | 9 | import { startWatch, probeOf, type Probe } from "./live.ts"; |
| 10 | import { firstProof, reachWords, settleChild } from "./reach.ts"; | |
| 10 | 11 | import { reissueClaim, spawnForge, type SpawnError } from "../actions/spawn.ts"; |
| 11 | 12 | import type { App } from "../app.ts"; |
| 12 | 13 | import { maxChildren } from "../core/capacity.ts"; |
| @@ -314,11 +315,25 @@ export async function spawnDoor( | ||
| 314 | 315 | void bootHostedChild(ctx.app, made.value.name); |
| 315 | 316 | startWatch(ctx.app, { name: made.value.name, startedAt, probe: (probes.get(ctx.app) ?? probeOf)(made.value.url) }); |
| 316 | 317 | } |
| 318 | // Proven from outside before it is called done: the child's home and each | |
| 319 | // app's public address, each with what came back. What does not answer yet | |
| 320 | // is kept proving behind this answer, and is an alarm if it never does. | |
| 321 | const proved = made.value.existed ? undefined : await firstProof(ctx.app, made.value.name); | |
| 322 | if (proved !== undefined) void settleChild(ctx.app, made.value.name); | |
| 317 | 323 | return { |
| 318 | 324 | status: made.value.existed ? 200 : 201, |
| 319 | 325 | body: { |
| 320 | 326 | dryRun: false, |
| 321 | 327 | ...made.value, |
| 328 | ...(proved === undefined | |
| 329 | ? {} | |
| 330 | : { | |
| 331 | reach: { | |
| 332 | ok: proved.ok, | |
| 333 | says: proved.ok ? "Every public address answered." : `${reachWords(proved)}. Still checking; the owner is alarmed if it does not come up in 3 minutes.`, | |
| 334 | proofs: proved.proofs, | |
| 335 | }, | |
| 336 | }), | |
| 322 | 337 | ...(typeof modelKey === "string" ? { keyNote: KEY_BUDGET_NOTE } : {}), |
| 323 | 338 | }, |
| 324 | 339 | }; |
src/spawn/host.ts +22 −2
| @@ -8,7 +8,7 @@ | ||
| 8 | 8 | import { childDirOf } from "../actions/spawn.ts"; |
| 9 | 9 | import { createApp, type App } from "../app.ts"; |
| 10 | 10 | import { appsRuntime, type AppsRuntime } from "../apps/runtime.ts"; |
| 11 | import { resolveAppHost, type AppTarget } from "../apps/proxy.ts"; | |
| 11 | import { resolveAppHost, type AppSocket, type AppTarget } from "../apps/proxy.ts"; | |
| 12 | 12 | import { childAppsSummary, startChildApps, stopChildApps } from "./child-apps.ts"; |
| 13 | 13 | import { childHost } from "./door.ts"; |
| 14 | 14 | import { firstBoot } from "./first-boot.ts"; |
| @@ -22,7 +22,7 @@ import { | ||
| 22 | 22 | type PlacementKind, |
| 23 | 23 | } from "./placement.ts"; |
| 24 | 24 | import { DEFAULT_SIZE, isSize, type Size } from "./sizes.ts"; |
| 25 | import { cellChildren, forward, type CellChild } from "./cell.ts"; | |
| 25 | import { cellChildren, forward, forwardSocket, type CellChild } from "./cell.ts"; | |
| 26 | 26 | import { cellsAvailable } from "../apps/contain.ts"; |
| 27 | 27 | import { readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| 28 | 28 | import { join } from "node:path"; |
| @@ -292,3 +292,23 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 292 | 292 | void settleCell(parent, ledger); |
| 293 | 293 | return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5", "cache-control": "no-store" } }); |
| 294 | 294 | } |
| 295 | ||
| 296 | /** | |
| 297 | * A websocket handshake for a child in its own cell (its home or one of its | |
| 298 | * apps): carried to the cell like any request. `undefined` means it became a | |
| 299 | * socket; while the cell is not up the answer is 503, as for a page. | |
| 300 | */ | |
| 301 | export async function socketToCell( | |
| 302 | parent: App, | |
| 303 | request: Request, | |
| 304 | server: Bun.Server<AppSocket>, | |
| 305 | named?: string, | |
| 306 | ): Promise<Response | undefined> { | |
| 307 | const name = named ?? parseHostedChildName(request.headers.get("host") ?? "", childHost(parent)); | |
| 308 | const ledger = name === undefined ? undefined : readLedgerSync(parent, name); | |
| 309 | if (ledger === undefined || ledger.placement !== "cell") return new Response("not found", { status: 404 }); | |
| 310 | const child = cellChildren(parent).get(ledger.bodyId); | |
| 311 | if (child !== undefined) return forwardSocket(child, request, server, parent.db, parent.config.clientIpHeader); | |
| 312 | void settleCell(parent, ledger); | |
| 313 | return new Response("this forge is starting", { status: 503, headers: { "retry-after": "5", "cache-control": "no-store" } }); | |
| 314 | } |
src/spawn/reach.ts added +299
| @@ -0,0 +1,299 @@ | ||
| 1 | /** | |
| 2 | * "Running" is proven from outside (#7959). A child forge's process being up | |
| 3 | * inside its cell says nothing about whether a visitor reaches it: Starck's | |
| 4 | * pages said "App · running" for days while every public address answered | |
| 5 | * 404. So the parent asks the way a visitor does — its own public address | |
| 6 | * for the child's home and for each app (`<app>--<child>--<parentHost>`) — | |
| 7 | * and keeps what came back, per URL, under `<data>/reach/<child>.json`. | |
| 8 | * | |
| 9 | * A probe asserts the *answer*, not the status: the platform's wildcard | |
| 10 | * standby page answers 200 for a mistyped `<app>--<child>.<apex>`, so a | |
| 11 | * 200 proves nothing. The child's home must answer its own `/api/v1/forge` | |
| 12 | * (a JSON `claimed`), and an app must carry the `x-forge-app: <slug>` the | |
| 13 | * proxy writes on everything it carries back. | |
| 14 | * | |
| 15 | * A child that stays unreachable raises an alarm: `forge.child_unreachable` | |
| 16 | * on the outbox and an `unreachable` flow alarm the notifier pages the | |
| 17 | * owner with (`crew/notify.ts`); `forge.child_reachable` ends it. | |
| 18 | */ | |
| 19 | ||
| 20 | import { Database } from "bun:sqlite"; | |
| 21 | import { mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | |
| 22 | import { join } from "node:path"; | |
| 23 | import { childDirOf, listChildren, type Child } from "../actions/spawn.ts"; | |
| 24 | import type { App } from "../app.ts"; | |
| 25 | import { isSlug, slugFor } from "../core/apps.ts"; | |
| 26 | import type { FiredAlarm } from "../flow/alarms.ts"; | |
| 27 | import { childPublicUrl, parentOrigin } from "./placement.ts"; | |
| 28 | ||
| 29 | export const APP_HEADER = "x-forge-app"; | |
| 30 | ||
| 31 | const PROBE_MS = 3000; | |
| 32 | /** How often the parent re-proves every child; the schedule is `startReachWatch`. */ | |
| 33 | export const REACH_EVERY_MS = 5 * 60_000; | |
| 34 | /** A new child gets this long to come up before its failure is an alarm. */ | |
| 35 | export const SETTLE_MS = 3 * 60_000; | |
| 36 | const SETTLE_POLL_MS = 3000; | |
| 37 | /** A failing sweep is confirmed once more after this, so one dropped request pages nobody. */ | |
| 38 | const CONFIRM_MS = 5000; | |
| 39 | const BATCH = 4; | |
| 40 | ||
| 41 | /** One public address and what came back from it. */ | |
| 42 | export interface Proof { | |
| 43 | /** `home`, or the app's slug. */ | |
| 44 | readonly label: string; | |
| 45 | readonly url: string; | |
| 46 | readonly ok: boolean; | |
| 47 | /** The HTTP status; null when nothing answered. */ | |
| 48 | readonly status: number | null; | |
| 49 | /** One sentence: what was asked and what is wrong, or "answered". */ | |
| 50 | readonly says: string; | |
| 51 | } | |
| 52 | ||
| 53 | export interface ChildReach { | |
| 54 | readonly name: string; | |
| 55 | /** When it was last proved, ISO. */ | |
| 56 | readonly at: string; | |
| 57 | readonly ok: boolean; | |
| 58 | /** When it first failed in the run of failures it is in, ISO; null while it is reachable. */ | |
| 59 | readonly since: string | null; | |
| 60 | /** Whether the owner has been told it is unreachable; false while a new child is still settling. */ | |
| 61 | readonly alarmed: boolean; | |
| 62 | readonly proofs: readonly Proof[]; | |
| 63 | } | |
| 64 | ||
| 65 | type FetchLike = (url: string, init?: RequestInit) => Promise<Response>; | |
| 66 | ||
| 67 | interface Options { | |
| 68 | /** True where there is no public network to prove through (a test suite); an injected `fetch` still works. */ | |
| 69 | readonly off?: boolean; | |
| 70 | readonly fetch?: FetchLike; | |
| 71 | readonly settleMs?: number; | |
| 72 | readonly pollMs?: number; | |
| 73 | readonly confirmMs?: number; | |
| 74 | } | |
| 75 | ||
| 76 | const options = new WeakMap<App, Options>(); | |
| 77 | ||
| 78 | /** Tests swap how public addresses are asked, and how long a settle waits. */ | |
| 79 | export function useReach(app: App, o: Options): void { | |
| 80 | options.set(app, o); | |
| 81 | } | |
| 82 | ||
| 83 | let fallback: Options = {}; | |
| 84 | ||
| 85 | /** What every app without its own `useReach` does; the test preload turns proving off. */ | |
| 86 | export function useReachByDefault(o: Options): void { | |
| 87 | fallback = o; | |
| 88 | } | |
| 89 | ||
| 90 | const optionsOf = (app: App): Options => options.get(app) ?? fallback; | |
| 91 | const off = (app: App): boolean => optionsOf(app).fetch === undefined && optionsOf(app).off === true; | |
| 92 | ||
| 93 | export const homeUrlOf = (child: Pick<Child, "url">): string => `${child.url}/api/v1/forge`; | |
| 94 | ||
| 95 | /** An app of a child, at the address a visitor types: `<app>--<child>--<parentHost>`. */ | |
| 96 | export const appUrlOf = (app: App, child: Pick<Child, "name">, slug: string): string => | |
| 97 | `${childPublicUrl(`${slug}--${child.name}`, parentOrigin(app.config))}/`; | |
| 98 | ||
| 99 | /** | |
| 100 | * The slugs of a child's public apps: those it was given (the repositories | |
| 101 | * in the ledger, which a child makes apps of on its first boot) and any its | |
| 102 | * own database holds since (it sits in this forge's data directory, read-only). | |
| 103 | */ | |
| 104 | export function slugsOf(app: App, child: Child): string[] { | |
| 105 | const slugs = new Set(child.apps.map((ref) => slugFor(ref.split("/")[1] ?? ref))); | |
| 106 | try { | |
| 107 | const db = new Database(join(childDirOf(app, child.bodyId), "forge.db"), { readonly: true }); | |
| 108 | try { | |
| 109 | // The child's database is the child's to write: a slug that is not the forge's slug rule | |
| 110 | // would steer this fetch to another host (`evil.example#`, `u@host`), so it is dropped. | |
| 111 | for (const r of db.query<{ slug: string }, []>("SELECT slug FROM app WHERE access = 'public'").all()) if (typeof r.slug === "string" && isSlug(r.slug)) slugs.add(r.slug); | |
| 112 | } finally { | |
| 113 | db.close(); | |
| 114 | } | |
| 115 | } catch { | |
| 116 | // Not yet a database to read: the ledger's repositories are what is known. | |
| 117 | } | |
| 118 | return [...slugs].sort(); | |
| 119 | } | |
| 120 | ||
| 121 | async function ask(fetcher: FetchLike, url: string): Promise<{ status: number; headers: Headers; body: string } | string> { | |
| 122 | try { | |
| 123 | const res = await fetcher(url, { redirect: "manual", signal: AbortSignal.timeout(PROBE_MS), headers: { accept: "*/*" } }); | |
| 124 | const body = res.headers.get("content-type")?.includes("json") ? await res.text() : (await res.body?.cancel(), ""); | |
| 125 | return { status: res.status, headers: res.headers, body: body.slice(0, 4096) }; | |
| 126 | } catch (error) { | |
| 127 | return String(error instanceof Error ? error.message : error).slice(0, 120); | |
| 128 | } | |
| 129 | } | |
| 130 | ||
| 131 | /** The child's home answers as the forge it is. */ | |
| 132 | export async function proveHome(fetcher: FetchLike, child: Pick<Child, "url">): Promise<Proof> { | |
| 133 | const url = homeUrlOf(child); | |
| 134 | const got = await ask(fetcher, url); | |
| 135 | if (typeof got === "string") return { label: "home", url, ok: false, status: null, says: `no answer (${got})` }; | |
| 136 | let claimed = false; | |
| 137 | try { | |
| 138 | claimed = typeof (JSON.parse(got.body) as { claimed?: unknown }).claimed === "boolean"; | |
| 139 | } catch { | |
| 140 | claimed = false; | |
| 141 | } | |
| 142 | if (got.status === 200 && claimed) return { label: "home", url, ok: true, status: 200, says: "answered 200" }; | |
| 143 | return { label: "home", url, ok: false, status: got.status, says: `answered ${got.status}, not by the forge` }; | |
| 144 | } | |
| 145 | ||
| 146 | /** An app answers as itself: not a 5xx, and carrying the proxy's `x-forge-app` for this slug. */ | |
| 147 | export async function proveApp(fetcher: FetchLike, url: string, slug: string): Promise<Proof> { | |
| 148 | const got = await ask(fetcher, url); | |
| 149 | if (typeof got === "string") return { label: slug, url, ok: false, status: null, says: `no answer (${got})` }; | |
| 150 | if (got.headers.get(APP_HEADER) !== slug) | |
| 151 | return { label: slug, url, ok: false, status: got.status, says: `answered ${got.status}, not by the app` }; | |
| 152 | if (got.status >= 500) return { label: slug, url, ok: false, status: got.status, says: `the app answered ${got.status}` }; | |
| 153 | return { label: slug, url, ok: true, status: got.status, says: `answered ${got.status}` }; | |
| 154 | } | |
| 155 | ||
| 156 | /** Ask every public address of a child, as a visitor does. Nothing is kept. */ | |
| 157 | export async function proveChild(app: App, child: Child): Promise<ChildReach> { | |
| 158 | const fetcher = optionsOf(app).fetch ?? fetch; | |
| 159 | const proofs = await Promise.all([ | |
| 160 | proveHome(fetcher, child), | |
| 161 | ...slugsOf(app, child).map((slug) => proveApp(fetcher, appUrlOf(app, child, slug), slug)), | |
| 162 | ]); | |
| 163 | return { name: child.name, at: new Date().toISOString(), ok: proofs.every((p) => p.ok), since: null, alarmed: false, proofs }; | |
| 164 | } | |
| 165 | ||
| 166 | // ---- what was last proved ---- | |
| 167 | ||
| 168 | const dirOf = (app: App): string => join(app.config.dataDir, "reach"); | |
| 169 | ||
| 170 | /** What was last proved of a child; undefined before it was ever asked. */ | |
| 171 | export function reachOf(app: App, name: string): ChildReach | undefined { | |
| 172 | try { | |
| 173 | const raw = JSON.parse(readFileSync(join(dirOf(app), `${name}.json`), "utf8")) as ChildReach; | |
| 174 | return Array.isArray(raw.proofs) && typeof raw.ok === "boolean" ? raw : undefined; | |
| 175 | } catch { | |
| 176 | return undefined; | |
| 177 | } | |
| 178 | } | |
| 179 | ||
| 180 | function keep(app: App, reach: ChildReach): void { | |
| 181 | try { | |
| 182 | mkdirSync(dirOf(app), { recursive: true }); | |
| 183 | const path = join(dirOf(app), `${reach.name}.json`); | |
| 184 | writeFileSync(`${path}.tmp`, JSON.stringify(reach)); | |
| 185 | renameSync(`${path}.tmp`, path); | |
| 186 | } catch (error) { | |
| 187 | app.log.error("could not keep the reach", { name: reach.name, error: String(error) }); | |
| 188 | } | |
| 189 | } | |
| 190 | ||
| 191 | /** The failing proofs in a sentence a person can read, or null when all are fine. */ | |
| 192 | export function reachWords(reach: ChildReach | null | undefined): string | null { | |
| 193 | if (reach === null || reach === undefined || reach.ok) return null; | |
| 194 | const bad = reach.proofs.filter((p) => !p.ok).map((p) => `${p.label} ${p.says}`); | |
| 195 | return `Running, not reachable: ${bad.join("; ")}`; | |
| 196 | } | |
| 197 | ||
| 198 | /** | |
| 199 | * Keep a proof, carrying `since` from the run of failures it continues, and | |
| 200 | * put the change on the outbox: unreachable once when the owner is first | |
| 201 | * told, reachable once when that ends. `alarm` is false while a new child | |
| 202 | * is still settling. | |
| 203 | */ | |
| 204 | function record(app: App, proved: ChildReach, alarm: boolean): ChildReach { | |
| 205 | const before = reachOf(app, proved.name); | |
| 206 | const since = proved.ok ? null : (before?.ok === false ? before.since : null) ?? proved.at; | |
| 207 | const raise = alarm && !proved.ok && before?.alarmed !== true; | |
| 208 | const end = proved.ok && before?.alarmed === true; | |
| 209 | const reach: ChildReach = { ...proved, since, alarmed: proved.ok ? false : raise || before?.alarmed === true }; | |
| 210 | keep(app, reach); | |
| 211 | if (raise) app.bus.append({ kind: "forge.child_unreachable", payload: { name: reach.name, says: reachWords(reach) ?? "", since } }); | |
| 212 | else if (end) app.bus.append({ kind: "forge.child_reachable", payload: { name: reach.name } }); | |
| 213 | return reach; | |
| 214 | } | |
| 215 | ||
| 216 | /** | |
| 217 | * Prove a child once and say what came back. A new child is the caller's | |
| 218 | * to settle (`settleChild`); a failure here is kept without an alarm. | |
| 219 | */ | |
| 220 | export async function firstProof(app: App, name: string): Promise<ChildReach | undefined> { | |
| 221 | if (off(app)) return undefined; | |
| 222 | const child = (await listChildren(app)).find((c) => c.name === name); | |
| 223 | if (child === undefined || !child.done) return undefined; | |
| 224 | return record(app, await proveChild(app, child), false); | |
| 225 | } | |
| 226 | ||
| 227 | /** | |
| 228 | * Keep proving a new child until all of it answers or `SETTLE_MS` passes; | |
| 229 | * if it never does, that is the alarm. Never throws. | |
| 230 | */ | |
| 231 | export async function settleChild(app: App, name: string): Promise<void> { | |
| 232 | if (off(app)) return; | |
| 233 | const o = optionsOf(app); | |
| 234 | const until = Date.now() + (o?.settleMs ?? SETTLE_MS); | |
| 235 | try { | |
| 236 | for (;;) { | |
| 237 | const child = (await listChildren(app)).find((c) => c.name === name); | |
| 238 | if (child === undefined || app.closing.signal.aborted) return; | |
| 239 | const proved = await proveChild(app, child); | |
| 240 | const last = Date.now() >= until; | |
| 241 | const reach = record(app, proved, last); | |
| 242 | if (reach.ok || last) return; | |
| 243 | await new Promise<void>((done) => { | |
| 244 | const t = setTimeout(done, o?.pollMs ?? SETTLE_POLL_MS); | |
| 245 | app.closing.signal.addEventListener("abort", () => (clearTimeout(t), done()), { once: true }); | |
| 246 | }); | |
| 247 | } | |
| 248 | } catch (error) { | |
| 249 | app.log.error("could not settle a new child's reach", { name, error: String(error) }); | |
| 250 | } | |
| 251 | } | |
| 252 | ||
| 253 | /** One pass over every finished child: prove, confirm a failure once, keep, alarm on a change. */ | |
| 254 | export async function sweepReach(app: App): Promise<void> { | |
| 255 | if (off(app)) return; | |
| 256 | const confirmMs = optionsOf(app).confirmMs ?? CONFIRM_MS; | |
| 257 | const children = (await listChildren(app)).filter((c) => c.done); | |
| 258 | for (let i = 0; i < children.length; i += BATCH) { | |
| 259 | await Promise.all( | |
| 260 | children.slice(i, i + BATCH).map(async (child) => { | |
| 261 | try { | |
| 262 | let proved = await proveChild(app, child); | |
| 263 | if (!proved.ok) { | |
| 264 | await Bun.sleep(confirmMs); | |
| 265 | proved = await proveChild(app, child); | |
| 266 | } | |
| 267 | record(app, proved, true); | |
| 268 | } catch (error) { | |
| 269 | app.log.error("could not prove a child's reach", { name: child.name, error: String(error) }); | |
| 270 | } | |
| 271 | }), | |
| 272 | ); | |
| 273 | } | |
| 274 | } | |
| 275 | ||
| 276 | /** Re-prove every child on a schedule, the first pass soon after boot. Returns the timer to clear. */ | |
| 277 | export function startReachWatch(app: App, everyMs = REACH_EVERY_MS): ReturnType<typeof setInterval> { | |
| 278 | const first = setTimeout(() => void sweepReach(app), 30_000); | |
| 279 | const timer = setInterval(() => void sweepReach(app), everyMs); | |
| 280 | app.closing.signal.addEventListener("abort", () => clearTimeout(first), { once: true }); | |
| 281 | return timer; | |
| 282 | } | |
| 283 | ||
| 284 | /** The children that are unreachable now, as flow alarms the notifier pages the owner with. */ | |
| 285 | export function reachAlarms(app: App): FiredAlarm[] { | |
| 286 | let files: string[] = []; | |
| 287 | try { | |
| 288 | files = readdirSync(dirOf(app)); | |
| 289 | } catch { | |
| 290 | return []; | |
| 291 | } | |
| 292 | const out: FiredAlarm[] = []; | |
| 293 | for (const f of files.filter((f) => f.endsWith(".json")).sort()) { | |
| 294 | const reach = reachOf(app, f.slice(0, -".json".length)); | |
| 295 | if (reach === undefined || reach.ok || !reach.alarmed || reach.since === null) continue; | |
| 296 | out.push({ key: "unreachable", says: `${reach.name} is ${reachWords(reach)?.toLowerCase() ?? "not reachable"}`, since: reach.since, href: "/forges" }); | |
| 297 | } | |
| 298 | return out; | |
| 299 | } |
src/tui/screens/forges.ts +8 −2
| @@ -17,12 +17,18 @@ import { clientRows } from "../../spawn/clients.ts"; | ||
| 17 | 17 | /** [name, "Not reporting · for Maria · Claimed · 3 apps running"] per row; an empty line when none. */ |
| 18 | 18 | export function forgeLines(rows: readonly ClientRow[]): [string, string][] { |
| 19 | 19 | if (rows.length === 0) return [["", "You have not sent a forge yet"]]; |
| 20 | return rows.map((r) => [ | |
| 20 | return rows.flatMap((r): [string, string][] => [[ | |
| 21 | 21 | r.name, |
| 22 | 22 | [r.attention === null ? null : ATTENTION_WORDS[r.attention], r.for === null ? null : `for ${r.for}`, CLAIM_WORDS[claimState(r)], runningWords(r), r.nextStep.words, spendWords(r)?.words ?? null, useLine(r.size, r.use), r.lastHeartbeatAt === null ? "no heartbeat yet" : `heartbeat ${timeAgo(r.lastHeartbeatAt)}`] |
| 23 | 23 | .filter((m): m is string => m !== null) |
| 24 | 24 | .join(" · "), |
| 25 | ]); | |
| 25 | ], ...addressLines(r)]); | |
| 26 | } | |
| 27 | ||
| 28 | /** Under a forge that is running, not reachable: each public address and what it answered (#7959), as /forges lists them. */ | |
| 29 | function addressLines(r: ClientRow): [string, string][] { | |
| 30 | if (r.reach === null || r.reach.ok) return []; | |
| 31 | return r.reach.proofs.map((p): [string, string] => ["", ` ${p.ok ? "reachable" : "NOT REACHABLE"} ${p.url.replace(/^https?:\/\//, "")}: ${p.says}`]); | |
| 26 | 32 | } |
| 27 | 33 | |
| 28 | 34 | /** The lines that fit in `room` rows; when some do not, the last row says how many. */ |
src/web/pages/forges.ts +15 −1
| @@ -24,7 +24,8 @@ import { claimLinkBody } from "./new-forge.ts"; | ||
| 24 | 24 | function runningNote(r: ClientRow): Html | null { |
| 25 | 25 | const words = runningWords(r); |
| 26 | 26 | if (words === null) return null; |
| 27 | return html`<span data-apps="${r.running!.notRunning.length === 0 ? "up" : "down"}" | |
| 27 | const down = (r.running?.notRunning.length ?? 0) > 0 || r.reach?.ok === false; | |
| 28 | return html`<span data-apps="${down ? "down" : "up"}" | |
| 28 | 29 | >${words}</span |
| 29 | 30 | >`; |
| 30 | 31 | } |
| @@ -58,6 +59,19 @@ function forgeRow(r: ClientRow): Html { | ||
| 58 | 59 | title: html`<a href="${r.url}">${r.name}</a>`, |
| 59 | 60 | meta: html`${meta.map((m, i) => html`${i > 0 ? " · " : ""}${m}`)}`, |
| 60 | 61 | data: { forge: r.name, claimed: state }, |
| 62 | ...(r.reach === null | |
| 63 | ? {} | |
| 64 | : { | |
| 65 | // One row per public address; the word in the trail, not a colour, says which failed. | |
| 66 | children: r.reach.proofs.map((p) => | |
| 67 | row({ | |
| 68 | title: html`<a href="${p.url}">${p.url.replace(/^https?:\/\//, "")}</a>`, | |
| 69 | meta: html`${p.says}`, | |
| 70 | trail: badge(p.ok ? "reachable" : "not reachable", p.ok ? { tone: "success" as const } : { tone: "attn" as const }), | |
| 71 | data: { proof: p.ok ? "ok" : "failed" }, | |
| 72 | }), | |
| 73 | ), | |
| 74 | }), | |
| 61 | 75 | trail: html`${r.attention === null |
| 62 | 76 | ? null |
| 63 | 77 | : html`<span data-attention="${r.attention}">${badge(ATTENTION_WORDS[r.attention], { tone: "attn" })}</span>`}${badge(CLAIM_WORDS[state], { |
test/forges-attention.test.ts +1
| @@ -20,6 +20,7 @@ const row = (name: string, o: Partial<ClientRow> = {}): Omit<ClientRow, "attenti | ||
| 20 | 20 | claimed: true, |
| 21 | 21 | lastActivityAt: ago(1), |
| 22 | 22 | running: null, |
| 23 | reach: null, | |
| 23 | 24 | size: "small", |
| 24 | 25 | use: { kind: "shared" }, |
| 25 | 26 | lastHeartbeatAt: null, |
test/helpers/preload.ts +4
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | import { provideCpusForTest } from "../../src/core/capacity.ts"; |
| 7 | 7 | import { provideMirrorOutboundForTest } from "../../src/origins/git-mirror.ts"; |
| 8 | 8 | import { installCheapHash } from "./cheap-hash.ts"; |
| 9 | import { useReachByDefault } from "../../src/spawn/reach.ts"; | |
| 9 | 10 | |
| 10 | 11 | installCheapHash(); |
| 11 | 12 | // Defaults sized from CPUs (#2549) read as on a 4-CPU pod — 4 shards, one |
| @@ -18,3 +19,6 @@ provideMirrorOutboundForTest({ allow: ["127.0.0.1", "::1"] }); | ||
| 18 | 19 | // Loopback is reachable for origin discovery only in dev (#2993); the suite |
| 19 | 20 | // serves its fake forges from this machine. |
| 20 | 21 | process.env.FORGE_DEV = "1"; |
| 22 | ||
| 23 | // A suite has no public network to prove a child through (`spawn/reach.ts`); a test of reach injects its own fetch. | |
| 24 | useReachByDefault({ off: true }); |
test/helpers/real-spawn-e2e.ts added +85
| @@ -0,0 +1,85 @@ | ||
| 1 | /** | |
| 2 | * A real spawn, end to end (#7959), as `test/spawn-cell-e2e.test.ts` drives | |
| 3 | * it. Its own process, for the reason `real-child-forge.ts` is: a check sets | |
| 4 | * `FORGE_CELLS=off` for the forges its tests boot. | |
| 5 | * | |
| 6 | * A parent with the real server makes a child forge in a real cell from one | |
| 7 | * app, then asks `<app>--<child>--<parent>` through that server until the | |
| 8 | * app's own body comes back. Nothing is faked: not the cell, not the child's | |
| 9 | * runtime, not the route. Printed: one JSON line of what was seen. | |
| 10 | */ | |
| 11 | import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; | |
| 12 | import { tmpdir } from "node:os"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { createRepo } from "../../src/actions/repos.ts"; | |
| 15 | import { spawnForge } from "../../src/actions/spawn.ts"; | |
| 16 | import { createApp } from "../../src/app.ts"; | |
| 17 | import { cellsAvailable } from "../../src/apps/contain.ts"; | |
| 18 | import { createUser } from "../../src/core/users.ts"; | |
| 19 | import { repoDir } from "../../src/git/repos.ts"; | |
| 20 | import { commitFiles } from "../../src/git/write.ts"; | |
| 21 | import { serve } from "../../src/http/server.ts"; | |
| 22 | import { cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts"; | |
| 23 | import { bootHostedChild } from "../../src/spawn/host.ts"; | |
| 24 | import { memoryBodies } from "../../src/spawn/placement.ts"; | |
| 25 | ||
| 26 | const cells = await cellsAvailable(); | |
| 27 | if (!cells.ok) { | |
| 28 | console.log(JSON.stringify({ skipped: cells.reason })); | |
| 29 | process.exit(0); | |
| 30 | } | |
| 31 | ||
| 32 | const BODY = "bakery says hi"; | |
| 33 | const dataDir = mkdtempSync(join(tmpdir(), "real-spawn-")); | |
| 34 | mkdirSync(dataDir, { recursive: true }); | |
| 35 | const parent = createApp({ dataDir, url: "https://forge.test", port: 0 }); | |
| 36 | const servers = await serve(parent); | |
| 37 | try { | |
| 38 | const alice = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true }); | |
| 39 | if (!alice.ok) throw new Error("user"); | |
| 40 | const repo = await createRepo(parent, alice.value, { name: "bakery", visibility: "public" }); | |
| 41 | if (!repo.ok) throw new Error("repo"); | |
| 42 | const committed = await commitFiles(repoDir(parent.reposRoot, "alice", "bakery"), { | |
| 43 | branch: "main", | |
| 44 | message: "bakery", | |
| 45 | files: [ | |
| 46 | { path: "forge.json", content: JSON.stringify({ app: { run: "bun run index.ts", health: "/" } }) }, | |
| 47 | { path: "index.ts", content: `Bun.serve({ port: Number(process.env.PORT), fetch() { return new Response(${JSON.stringify(BODY)}) } })\n` }, | |
| 48 | ], | |
| 49 | author: { name: "a", email: "[email protected]" }, | |
| 50 | }); | |
| 51 | if (!committed.ok) throw new Error("commit"); | |
| 52 | const made = await spawnForge( | |
| 53 | parent, | |
| 54 | alice.value, | |
| 55 | { name: "kid", org: "kids", apps: ["alice/bakery"], carryBriefs: false, carrySubscriptions: false }, | |
| 56 | { catalog: [{ ref: "alice/bakery", readable: true, bytes: 10, forgeJsonBytes: 0 }], briefs: [], subscriptions: [] }, | |
| 57 | { | |
| 58 | placement: cellPlacement(parent, memoryBodies(), () => 10), | |
| 59 | request: new Request("https://forge.test/x", { method: "POST", headers: { origin: "https://forge.test" } }), | |
| 60 | url: new URL("https://forge.test/x"), | |
| 61 | }, | |
| 62 | ); | |
| 63 | if (!made.ok) throw new Error(JSON.stringify(made.error)); | |
| 64 | await bootHostedChild(parent, "kid"); | |
| 65 | let status = 0; | |
| 66 | let body = ""; | |
| 67 | let app: string | null = null; | |
| 68 | const until = Date.now() + 150_000; | |
| 69 | while (Date.now() < until) { | |
| 70 | const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined); | |
| 71 | if (res !== undefined) { | |
| 72 | status = res.status; | |
| 73 | body = await res.text(); | |
| 74 | app = res.headers.get("x-forge-app"); | |
| 75 | if (status === 200 && app === "bakery") break; | |
| 76 | } | |
| 77 | await Bun.sleep(1000); | |
| 78 | } | |
| 79 | console.log(JSON.stringify({ placement: made.value.placement, status, body: body.slice(0, 200), app })); | |
| 80 | } finally { | |
| 81 | servers.stop(); | |
| 82 | await stopCellChildren(parent); | |
| 83 | parent.close(); | |
| 84 | rmSync(dataDir, { recursive: true, force: true }); | |
| 85 | } |
test/spawn-cell-e2e.test.ts added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | /** | |
| 2 | * A real spawn, end to end (#7959): a child forge in a real cell, from an | |
| 3 | * app, then `<app>--<child>--<parent>` through the real server answers with | |
| 4 | * the app's own body. `spawn-cell-app-hosts.test.ts` fakes the cell; this | |
| 5 | * does not. It runs where bwrap and socat are (as `spawn-cell.test.ts`'s | |
| 6 | * real child does) and says it skipped where they are not. | |
| 7 | */ | |
| 8 | import { expect, test } from "bun:test"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { portInsteadOfSocket } from "../src/git/hook-address.ts"; | |
| 11 | import { missing } from "./helpers/tools.ts"; | |
| 12 | ||
| 13 | const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !missing("bwrap", "socat"); | |
| 14 | ||
| 15 | test.skipIf(!cellTools)("a child forge spawned into a cell serves its app at <app>--<child>--<parent>", async () => { | |
| 16 | const env: Record<string, string> = {}; | |
| 17 | for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v; | |
| 18 | const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-spawn-e2e.ts")], { env, stdout: "pipe", stderr: "pipe" }); | |
| 19 | const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); | |
| 20 | const line = out.trim().split("\n").pop() ?? ""; | |
| 21 | let seen: Record<string, unknown>; | |
| 22 | try { | |
| 23 | seen = JSON.parse(line) as Record<string, unknown>; | |
| 24 | } catch { | |
| 25 | throw new Error(`the spawn run printed no result: ${out.slice(-800)} ${errText.slice(-800)}`); | |
| 26 | } | |
| 27 | if (typeof seen["skipped"] === "string") { | |
| 28 | console.log(`SKIPPING: this host refuses a cell inside the test (${seen["skipped"]}), so no child was spawned into one`); | |
| 29 | return; | |
| 30 | } | |
| 31 | expect(seen["placement"]).toBe("cell"); | |
| 32 | expect(seen).toMatchObject({ status: 200, app: "bakery", body: "bakery says hi" }); | |
| 33 | }, 200_000); |
test/spawn-cell-socket.test.ts added +106
| @@ -0,0 +1,106 @@ | ||
| 1 | /** | |
| 2 | * A websocket reaches an app in a child's cell (#7959): `forward` is a fetch, | |
| 3 | * which cannot carry an upgrade, so a handshake for the child's home or one | |
| 4 | * of its apps is opened to the cell first and answered second, as `proxyApp` | |
| 5 | * does for an app of this forge. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 13 | import { cellChildren } from "../src/spawn/cell.ts"; | |
| 14 | import { quietLogs } from "./helpers/globals.ts"; | |
| 15 | ||
| 16 | quietLogs(); | |
| 17 | ||
| 18 | let parent: App; | |
| 19 | let servers: Servers; | |
| 20 | let cell: ReturnType<typeof Bun.serve>; | |
| 21 | const seen: { host: string | null; forwarded: string | null; protocols: string | null; path: string }[] = []; | |
| 22 | ||
| 23 | beforeAll(async () => { | |
| 24 | const dataDir = mkdtempSync(join(tmpdir(), "cell-socket-")); | |
| 25 | parent = createApp({ dataDir, url: "https://forge.test", port: 0 }); | |
| 26 | mkdirSync(join(dataDir, "spawn"), { recursive: true }); | |
| 27 | writeFileSync(join(dataDir, "spawn", "kid.json"), JSON.stringify({ name: "kid", bodyId: "cell-k", done: true, placement: "cell", size: "small" })); | |
| 28 | writeFileSync(join(dataDir, "spawn", "down.json"), JSON.stringify({ name: "down", bodyId: "cell-d", done: true, placement: "cell", size: "small" })); | |
| 29 | cell = Bun.serve({ | |
| 30 | port: 0, | |
| 31 | fetch(request, server) { | |
| 32 | const url = new URL(request.url); | |
| 33 | if (url.pathname === "/ws") { | |
| 34 | seen.push({ | |
| 35 | host: request.headers.get("host"), | |
| 36 | forwarded: request.headers.get("x-forwarded-for"), | |
| 37 | protocols: request.headers.get("sec-websocket-protocol"), | |
| 38 | path: url.pathname + url.search, | |
| 39 | }); | |
| 40 | if (server.upgrade(request, { data: null, headers: { "sec-websocket-protocol": "chat" } })) return undefined; | |
| 41 | } | |
| 42 | return new Response("plain", { status: 404 }); | |
| 43 | }, | |
| 44 | websocket: { | |
| 45 | open: (ws) => void ws.send("hello from the cell"), | |
| 46 | message: (ws, m) => void ws.send(`echo ${String(m)}`), | |
| 47 | }, | |
| 48 | }); | |
| 49 | const port = cell.port; | |
| 50 | (cellChildren(parent) as { get: unknown }).get = (id: string) => (id === "cell-k" ? { port } : undefined); | |
| 51 | servers = await serve(parent); | |
| 52 | }); | |
| 53 | ||
| 54 | afterAll(() => { | |
| 55 | servers.stop(); | |
| 56 | void cell.stop(true); | |
| 57 | parent.close(); | |
| 58 | }); | |
| 59 | ||
| 60 | const open = (host: string, path = "/ws?x=1", protocols: string[] = ["chat"]) => | |
| 61 | new Promise<{ socket: WebSocket; frames: string[]; closed: Promise<number> } | string>((resolve) => { | |
| 62 | const socket = new WebSocket(`${servers.url.replace("http", "ws")}${path}`, { protocols, headers: { host } } as never); | |
| 63 | const frames: string[] = []; | |
| 64 | const closed = new Promise<number>((done) => socket.addEventListener("close", (e) => done(e.code))); | |
| 65 | socket.addEventListener("message", (e) => frames.push(String(e.data))); | |
| 66 | socket.addEventListener("open", () => resolve({ socket, frames, closed }), { once: true }); | |
| 67 | socket.addEventListener("close", () => resolve("refused"), { once: true }); | |
| 68 | }); | |
| 69 | ||
| 70 | const until = async (ok: () => boolean) => { | |
| 71 | for (let i = 0; i < 100 && !ok(); i++) await Bun.sleep(20); | |
| 72 | }; | |
| 73 | ||
| 74 | test("an app's socket in a cell: frames go both ways, with the app's own Host and path", async () => { | |
| 75 | const opened = await open("deal--kid--forge.test"); | |
| 76 | if (typeof opened === "string") throw new Error("the socket did not open"); | |
| 77 | await until(() => opened.frames.length > 0); | |
| 78 | expect(opened.frames[0]).toBe("hello from the cell"); | |
| 79 | expect(opened.socket.protocol).toBe("chat"); | |
| 80 | opened.socket.send("ping"); | |
| 81 | await until(() => opened.frames.length > 1); | |
| 82 | expect(opened.frames[1]).toBe("echo ping"); | |
| 83 | opened.socket.close(1000); | |
| 84 | await opened.closed; | |
| 85 | const last = seen.at(-1)!; | |
| 86 | expect(last.host).toBe("deal--kid--forge.test"); | |
| 87 | expect(last.path).toBe("/ws?x=1"); | |
| 88 | expect(last.protocols).toBe("chat"); | |
| 89 | }); | |
| 90 | ||
| 91 | test("the child's own host carries a socket too, and the cell hears the address the forge resolved", async () => { | |
| 92 | const opened = await open("kid--forge.test"); | |
| 93 | if (typeof opened === "string") throw new Error("the socket did not open"); | |
| 94 | await until(() => opened.frames.length > 0); | |
| 95 | expect(opened.frames[0]).toBe("hello from the cell"); | |
| 96 | expect(seen.at(-1)!.host).toBe("kid--forge.test"); | |
| 97 | expect(seen.at(-1)!.forwarded).not.toBeNull(); | |
| 98 | opened.socket.close(1000); | |
| 99 | await opened.closed; | |
| 100 | }); | |
| 101 | ||
| 102 | test("a path the cell does not upgrade is a refusal, and a cell that is not up is not a socket", async () => { | |
| 103 | expect(await open("deal--kid--forge.test", "/nope")).toBe("refused"); | |
| 104 | expect(await open("deal--down--forge.test")).toBe("refused"); | |
| 105 | expect(await open("deal--nobody--forge.test")).toBe("refused"); | |
| 106 | }); |
test/spawn-child-runs-apps.test.ts +2
| @@ -130,6 +130,8 @@ describe("a hosted child's apps run in the parent", () => { | ||
| 130 | 130 | test("<app>--<child>--<parent> answers from the child's app, on the child's own data", async () => { |
| 131 | 131 | const res = await ask("bakery--maria--forge.test"); |
| 132 | 132 | expect(res.status).toBe(200); |
| 133 | // Says which app answered, so a probe from outside tells it from a platform's standby page (spawn/reach.ts). | |
| 134 | expect(res.headers.get("x-forge-app")).toBe("bakery"); | |
| 133 | 135 | const data = await res.text(); |
| 134 | 136 | // The child's directory, never the parent's or another child's. |
| 135 | 137 | expect(data.startsWith(join(childDir, "apps"))).toBe(true); |
test/spawn-reach-door.test.ts added +122
| @@ -0,0 +1,122 @@ | ||
| 1 | /** | |
| 2 | * A new forge is proven before it is called done (#7959): the spawn result | |
| 3 | * lists the child's home and each app's public address with what came back, | |
| 4 | * through the API and the MCP, and what never answers is an alarm. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { createRepo } from "../src/actions/repos.ts"; | |
| 12 | import { createToken } from "../src/core/tokens.ts"; | |
| 13 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 14 | import { alarmLines } from "../src/crew/notify.ts"; | |
| 15 | import { listEvents } from "../src/core/events.ts"; | |
| 16 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 17 | import { ok } from "../src/kernel/result.ts"; | |
| 18 | import { useChildFetch } from "../src/spawn/clients.ts"; | |
| 19 | import { usePlacement, useProbe } from "../src/spawn/door.ts"; | |
| 20 | import { reachAlarms, reachOf, useReach } from "../src/spawn/reach.ts"; | |
| 21 | import { quietLogs } from "./helpers/globals.ts"; | |
| 22 | ||
| 23 | quietLogs(); | |
| 24 | ||
| 25 | let dataDir = ""; | |
| 26 | let app: App; | |
| 27 | let servers: Servers; | |
| 28 | let token = ""; | |
| 29 | /** What the public network answers: the child's home, but a standby page for its app. */ | |
| 30 | const answers = new Map<string, () => Response>(); | |
| 31 | let n = 0; | |
| 32 | ||
| 33 | const spawn = (name: string) => | |
| 34 | fetch(`${servers.url}/api/v1/forges`, { | |
| 35 | method: "POST", | |
| 36 | headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, | |
| 37 | body: JSON.stringify({ name, apps: ["alice/notes"], dryRun: false }), | |
| 38 | }); | |
| 39 | ||
| 40 | beforeAll(async () => { | |
| 41 | dataDir = await mkdtemp(join(tmpdir(), "forge-reach-door-")); | |
| 42 | app = createApp({ dataDir, url: "https://forge.test", port: 0 }); | |
| 43 | servers = await serve(app); | |
| 44 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 45 | if (!a.ok) throw new Error("user"); | |
| 46 | token = createToken(app.db, a.value.id, "rw", "rw").token; | |
| 47 | const repo = await createRepo(app, a.value, { name: "notes" }); | |
| 48 | if (!repo.ok) throw new Error("repo"); | |
| 49 | useProbe(app, () => async () => true); | |
| 50 | usePlacement(app, { | |
| 51 | place: async (plan) => ok({ url: `https://${plan.name}--forge.test`, bodyId: `fake-${++n}` }), | |
| 52 | retire: async () => ok(), | |
| 53 | }); | |
| 54 | useChildFetch(app, async () => Response.json({ claimed: false })); | |
| 55 | useReach(app, { | |
| 56 | settleMs: 0, | |
| 57 | pollMs: 1, | |
| 58 | confirmMs: 0, | |
| 59 | fetch: async (url) => (answers.get(url) ?? (() => new Response("<h1>standby</h1>", { status: 200 })))(), | |
| 60 | }); | |
| 61 | }, 30_000); | |
| 62 | ||
| 63 | afterAll(async () => { | |
| 64 | servers.stop(); | |
| 65 | app.close(); | |
| 66 | await rm(dataDir, { recursive: true, force: true }); | |
| 67 | }); | |
| 68 | ||
| 69 | test("the spawn result lists each address with its result; a standby page is a failure, not green", async () => { | |
| 70 | answers.set("https://kid--forge.test/api/v1/forge", () => Response.json({ claimed: false })); | |
| 71 | const res = await spawn("kid"); | |
| 72 | expect(res.status).toBe(201); | |
| 73 | const made = (await res.json()) as any; | |
| 74 | expect(made.reach.ok).toBe(false); | |
| 75 | expect(made.reach.says).toContain("Running, not reachable: notes answered 200, not by the app"); | |
| 76 | expect(made.reach.proofs).toEqual([ | |
| 77 | { label: "home", url: "https://kid--forge.test/api/v1/forge", ok: true, status: 200, says: "answered 200" }, | |
| 78 | { label: "notes", url: "https://notes--kid--forge.test/", ok: false, status: 200, says: "answered 200, not by the app" }, | |
| 79 | ]); | |
| 80 | // The never-answering app is the alarm, once its time to come up is over. | |
| 81 | for (let i = 0; i < 100 && reachAlarms(app).length === 0; i++) await Bun.sleep(20); | |
| 82 | expect(reachAlarms(app).map((a) => a.key)).toEqual(["unreachable"]); | |
| 83 | expect(listEvents(app.db, { kind: "forge.child_unreachable", limit: 5 })).toHaveLength(1); | |
| 84 | expect(reachOf(app, "kid")?.alarmed).toBe(true); | |
| 85 | }); | |
| 86 | ||
| 87 | test("/forges lists the same results, and the MCP spawn says it", async () => { | |
| 88 | const list = await fetch(`${servers.url}/api/v1/forges`, { headers: { authorization: `Bearer ${token}` } }); | |
| 89 | expect(list.status).toBe(200); | |
| 90 | answers.set("https://lamp--forge.test/api/v1/forge", () => Response.json({ claimed: false })); | |
| 91 | const res = await fetch(`${servers.url}/mcp`, { | |
| 92 | method: "POST", | |
| 93 | headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, | |
| 94 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "spawn", arguments: { name: "lamp", apps: ["alice/notes"], dryRun: false } } }), | |
| 95 | }); | |
| 96 | const text = ((await res.json()) as any).result.content[0].text as string; | |
| 97 | expect(text).toContain("Running, not reachable"); | |
| 98 | const forges = await fetch(`${servers.url}/mcp`, { | |
| 99 | method: "POST", | |
| 100 | headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, | |
| 101 | body: JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: "forges", arguments: {} } }), | |
| 102 | }); | |
| 103 | const body = JSON.parse(((await forges.json()) as any).result.content[0].text) as { forges: { name: string; reach: { ok: boolean } }[] }; | |
| 104 | expect(body.forges.find((f) => f.name === "kid")?.reach.ok).toBe(false); | |
| 105 | }); | |
| 106 | ||
| 107 | test("/forges says Running, not reachable with each address and its result", async () => { | |
| 108 | const page = await fetch(`${servers.url}/forges`, { headers: { authorization: `Bearer ${token}` } }); | |
| 109 | expect(page.status).toBe(200); | |
| 110 | const html = await page.text(); | |
| 111 | expect(html).toContain("Running, not reachable: notes answered 200, not by the app"); | |
| 112 | expect(html).toContain('data-proof="failed"'); | |
| 113 | expect(html).toContain("notes--kid--forge.test/"); | |
| 114 | expect(html).toContain('data-proof="ok"'); | |
| 115 | }); | |
| 116 | ||
| 117 | test("the owner is paged: the notifier's alarm lines carry the unreachable child", async () => { | |
| 118 | const admin = getUserByName(app.db, "alice")!; | |
| 119 | const lines = alarmLines(app, admin); | |
| 120 | expect(lines.map((l) => l.line)).toContainEqual(expect.stringContaining("kid is running, not reachable")); | |
| 121 | expect(lines.find((l) => l.line.startsWith("kid"))?.link).toBe("/forges"); | |
| 122 | }); |
test/spawn-reach.test.ts added +146
| @@ -0,0 +1,146 @@ | ||
| 1 | /** | |
| 2 | * "Running" is proven from outside (#7959): the parent asks a child's public | |
| 3 | * addresses as a visitor does, asserts the answer is the child's and the | |
| 4 | * app's own (a platform's standby page answers 200 for any name), keeps what | |
| 5 | * came back, and alarms once on a change. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 8 | import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { listEvents } from "../src/core/events.ts"; | |
| 13 | import { clientRows, runningWords, useChildFetch } from "../src/spawn/clients.ts"; | |
| 14 | import { Database } from "bun:sqlite"; | |
| 15 | import { childDirOf } from "../src/actions/spawn.ts"; | |
| 16 | import { appUrlOf, slugsOf, proveApp, proveHome, reachAlarms, reachOf, reachWords, sweepReach, useReach } from "../src/spawn/reach.ts"; | |
| 17 | import { quietLogs } from "./helpers/globals.ts"; | |
| 18 | ||
| 19 | quietLogs(); | |
| 20 | ||
| 21 | const standby = () => new Response("<h1>This site is on standby</h1>", { status: 200, headers: { "content-type": "text/html" } }); | |
| 22 | ||
| 23 | describe("a probe asserts the answer, not the status", () => { | |
| 24 | test("the platform's standby page is a 200 and is not the child", async () => { | |
| 25 | expect(standby().status).toBe(200); | |
| 26 | const home = await proveHome(async () => standby(), { url: "https://kid--forge.test" }); | |
| 27 | expect(home).toMatchObject({ ok: false, status: 200, says: "answered 200, not by the forge" }); | |
| 28 | }); | |
| 29 | ||
| 30 | test("the child's home is its own /api/v1/forge", async () => { | |
| 31 | const seen: string[] = []; | |
| 32 | const home = await proveHome(async (url) => (seen.push(url), Response.json({ claimed: false })), { url: "https://kid--forge.test" }); | |
| 33 | expect(seen).toEqual(["https://kid--forge.test/api/v1/forge"]); | |
| 34 | expect(home.ok).toBe(true); | |
| 35 | }); | |
| 36 | ||
| 37 | test("an app is the one that carries its own slug; a 200 without it is not it", async () => { | |
| 38 | const own = await proveApp(async () => new Response("hi", { headers: { "x-forge-app": "deal" } }), "https://deal--kid--forge.test/", "deal"); | |
| 39 | expect(own).toMatchObject({ ok: true, status: 200 }); | |
| 40 | const other = await proveApp(async () => standby(), "https://deal--kid--forge.test/", "deal"); | |
| 41 | expect(other).toMatchObject({ ok: false, status: 200, says: "answered 200, not by the app" }); | |
| 42 | const wrong = await proveApp(async () => new Response("x", { headers: { "x-forge-app": "other" } }), "https://deal--kid--forge.test/", "deal"); | |
| 43 | expect(wrong.ok).toBe(false); | |
| 44 | }); | |
| 45 | ||
| 46 | test("the status is kept: a 404 from the platform, a 500 from the app, no answer at all", async () => { | |
| 47 | const miss = await proveApp(async () => new Response("not found", { status: 404 }), "https://a--k--f.test/", "a"); | |
| 48 | expect(miss).toMatchObject({ ok: false, status: 404, says: "answered 404, not by the app" }); | |
| 49 | const broke = await proveApp(async () => new Response("x", { status: 502, headers: { "x-forge-app": "a" } }), "https://a--k--f.test/", "a"); | |
| 50 | expect(broke).toMatchObject({ ok: false, status: 502, says: "the app answered 502" }); | |
| 51 | const none = await proveApp(async () => Promise.reject(new Error("connection refused")), "https://a--k--f.test/", "a"); | |
| 52 | expect(none).toMatchObject({ ok: false, status: null }); | |
| 53 | expect(none.says).toContain("no answer"); | |
| 54 | }); | |
| 55 | }); | |
| 56 | ||
| 57 | describe("a forge watches its children's addresses", () => { | |
| 58 | let dir = ""; | |
| 59 | let parent: App; | |
| 60 | /** What each public address answers right now; absent is "no answer". */ | |
| 61 | const world = new Map<string, () => Response>(); | |
| 62 | const asked: string[] = []; | |
| 63 | ||
| 64 | const events = (kind: string) => listEvents(parent.db, { kind, limit: 20 }); | |
| 65 | ||
| 66 | beforeAll(() => { | |
| 67 | dir = mkdtempSync(join(tmpdir(), "forge-reach-")); | |
| 68 | parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 }); | |
| 69 | mkdirSync(join(dir, "spawn"), { recursive: true }); | |
| 70 | writeFileSync( | |
| 71 | join(dir, "spawn", "kid.json"), | |
| 72 | JSON.stringify({ name: "kid", bodyId: "cell-k", done: true, placement: "hosted", size: "small", apps: ["alice/deal-clock"], createdAt: "2026-11-01T00:00:00.000Z" }), | |
| 73 | ); | |
| 74 | useChildFetch(parent, async () => Response.json({ claimed: true, lastActivityAt: new Date().toISOString() })); | |
| 75 | useReach(parent, { | |
| 76 | confirmMs: 0, | |
| 77 | fetch: async (url) => { | |
| 78 | asked.push(url); | |
| 79 | const answer = world.get(url); | |
| 80 | if (answer === undefined) throw new Error("ENOTFOUND"); | |
| 81 | return answer(); | |
| 82 | }, | |
| 83 | }); | |
| 84 | }); | |
| 85 | ||
| 86 | afterAll(() => parent.close()); | |
| 87 | ||
| 88 | test("the app's address is <app>--<child>--<parentHost>", () => { | |
| 89 | expect(appUrlOf(parent, { name: "kid" }, "deal-clock")).toBe("https://deal-clock--kid--forge.test/"); | |
| 90 | }); | |
| 91 | ||
| 92 | test("a child whose addresses answer with standby pages is running, not reachable: kept, alarmed once, listed", async () => { | |
| 93 | world.set("https://kid--forge.test/api/v1/forge", () => Response.json({ claimed: true })); | |
| 94 | world.set("https://deal-clock--kid--forge.test/", () => new Response("not found", { status: 404 })); | |
| 95 | await sweepReach(parent); | |
| 96 | await sweepReach(parent); | |
| 97 | const reach = reachOf(parent, "kid")!; | |
| 98 | expect(reach.ok).toBe(false); | |
| 99 | expect(reach.proofs.map((p) => [p.label, p.ok, p.status])).toEqual([["home", true, 200], ["deal-clock", false, 404]]); | |
| 100 | expect(reachWords(reach)).toBe("Running, not reachable: deal-clock answered 404, not by the app"); | |
| 101 | // One page for the standing failure, not one per sweep. | |
| 102 | expect(events("forge.child_unreachable")).toHaveLength(1); | |
| 103 | const [alarm] = reachAlarms(parent); | |
| 104 | expect(alarm).toMatchObject({ key: "unreachable", href: "/forges", since: reach.since! }); | |
| 105 | expect(alarm!.says).toContain("kid is running, not reachable"); | |
| 106 | const [row] = await clientRows(parent); | |
| 107 | expect(row!.reach?.ok).toBe(false); | |
| 108 | expect(row!.attention).toBe("down"); | |
| 109 | expect(runningWords(row!)).toBe("Running, not reachable: deal-clock answered 404, not by the app"); | |
| 110 | }); | |
| 111 | ||
| 112 | test("when it answers again the alarm ends, once", async () => { | |
| 113 | world.set("https://deal-clock--kid--forge.test/", () => new Response("deal", { headers: { "x-forge-app": "deal-clock" } })); | |
| 114 | await sweepReach(parent); | |
| 115 | await sweepReach(parent); | |
| 116 | expect(reachOf(parent, "kid")).toMatchObject({ ok: true, since: null, alarmed: false }); | |
| 117 | expect(reachAlarms(parent)).toEqual([]); | |
| 118 | expect(events("forge.child_reachable")).toHaveLength(1); | |
| 119 | const [row] = await clientRows(parent); | |
| 120 | expect(row!.attention).not.toBe("down"); | |
| 121 | }); | |
| 122 | ||
| 123 | test("one dropped request pages nobody: a failure is confirmed before it is believed", async () => { | |
| 124 | const before = events("forge.child_unreachable").length; | |
| 125 | let calls = 0; | |
| 126 | world.set("https://kid--forge.test/api/v1/forge", () => (++calls === 1 ? new Response("oops", { status: 502 }) : Response.json({ claimed: true }))); | |
| 127 | await sweepReach(parent); | |
| 128 | expect(calls).toBe(2); | |
| 129 | expect(reachOf(parent, "kid")?.ok).toBe(true); | |
| 130 | expect(events("forge.child_unreachable")).toHaveLength(before); | |
| 131 | }); | |
| 132 | ||
| 133 | test("a slug a child writes that is not the forge's slug rule is never asked", async () => { | |
| 134 | const child = { name: "kid", bodyId: "cell-k", done: true, placement: "hosted", size: "small", apps: ["alice/deal-clock"], createdAt: "" } as never; | |
| 135 | mkdirSync(childDirOf(parent, "cell-k"), { recursive: true }); | |
| 136 | const db = new Database(join(childDirOf(parent, "cell-k"), "forge.db")); | |
| 137 | db.run("CREATE TABLE app (slug TEXT, access TEXT)"); | |
| 138 | for (const slug of ["evil.example#", "u@host", "a/b", "UP", "ok-app"]) db.run("INSERT INTO app VALUES (?, 'public')", [slug]); | |
| 139 | db.close(); | |
| 140 | expect(slugsOf(parent, child)).toEqual(["deal-clock", "ok-app"]); | |
| 141 | asked.length = 0; | |
| 142 | await sweepReach(parent); | |
| 143 | expect(asked.filter((u) => /evil|@|%|#|\/\/[^/]*\/[^/]/.test(u.replace(/^https:\/\//, "")) && !u.endsWith("/api/v1/forge"))).toEqual([]); | |
| 144 | expect(asked.every((u) => new URL(u).hostname.endsWith("--kid--forge.test") || u === "https://kid--forge.test/api/v1/forge")).toBe(true); | |
| 145 | }); | |
| 146 | }); |
test/tui-forges.test.ts +17
| @@ -25,6 +25,7 @@ const row = (over: Partial<ClientRow>): ClientRow => ({ | ||
| 25 | 25 | claimed: false, |
| 26 | 26 | lastActivityAt: null, |
| 27 | 27 | running: null, |
| 28 | reach: null, | |
| 28 | 29 | attention: null, |
| 29 | 30 | spendTodayUsd: null, |
| 30 | 31 | ceilingUsd: null, |
| @@ -100,6 +101,22 @@ describe("forges you sent in the terminal", () => { | ||
| 100 | 101 | } |
| 101 | 102 | }); |
| 102 | 103 | |
| 104 | test("a forge running, not reachable says so, with each address and its result", () => { | |
| 105 | const reach = { | |
| 106 | name: "maria", at: "2026-11-02T00:00:00.000Z", ok: false, since: "2026-11-02T00:00:00.000Z", alarmed: true, | |
| 107 | proofs: [ | |
| 108 | { label: "home", url: "https://maria--forge.test/api/v1/forge", ok: true, status: 200, says: "answered 200" }, | |
| 109 | { label: "deal", url: "https://deal--maria--forge.test/", ok: false, status: 404, says: "answered 404, not by the app" }, | |
| 110 | ], | |
| 111 | }; | |
| 112 | const lines = forgeLines([row({ reach })]); | |
| 113 | expect(lines[0]![1]).toContain("Running, not reachable: deal answered 404, not by the app"); | |
| 114 | expect(lines.slice(1).map(([, t]) => t.trim())).toEqual([ | |
| 115 | "reachable maria--forge.test/api/v1/forge: answered 200", | |
| 116 | "NOT REACHABLE deal--maria--forge.test/: answered 404, not by the app", | |
| 117 | ]); | |
| 118 | }); | |
| 119 | ||
| 103 | 120 | test("a row carries the crew's spend today, and says when the ceiling is reached", () => { |
| 104 | 121 | expect(forgeLines([row({ spendTodayUsd: 10, ceilingUsd: 10 })])[0]![1]).toContain("ceiling reached"); |
| 105 | 122 | expect(forgeLines([row({ spendTodayUsd: 3.2, ceilingUsd: 10 })])[0]![1]).toContain("$3.20 of $10 today"); |
new above
offline