Public: anyone can read this; the crew can change it.

A cell that outlives the host keeps its heartbeat: the beat door moves into #13755's cell supervisor #13931

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13906-a-cell-that-outlives-the-host-keeps-its → main 3 commits fork 1 thread · 1 unresolved

  • Merged. scout/13906-a-cell-that-outlives-the-host-keeps-its was merged into main as 5228fa5.

Review

Sign in to review this change.
10 files changed +249 −13
src/apps/contain.ts +7 −4
@@ -394,6 +394,11 @@ export interface CellSpec {
394394 * asked.
395395 */
396396 readonly outlive?: boolean | undefined;
397 /**
398 * The loopback port in `reach` that is the parent's beat door: an outliving cell's supervisor
399 * relays it to the host while the host lives, and answers it itself after (#13906).
400 */
401 readonly door?: number | undefined;
397402 }
398403
399404 /**
@@ -1003,10 +1008,8 @@ export function openCell(
10031008 const hooks = { allowed: spec.allowed, refused: spec.onRefused, undeclared: spec.onUndeclared };
10041009 // A cell that outlives the host has these run by a supervisor of its own; the rest, by the host.
10051010 // Both serve the one `policy`.
1006 const supervised =
1007 spec.outlive === true
1008 ? startSupervisor(mkdtempSync(join(tmpdir(), "fc-ctl-")), { sockets, ...policy }, { ...hooks, log })
1009 : undefined;
1011 const config = spec.door === undefined ? { sockets, ...policy } : { sockets, ...policy, door: spec.door };
1012 const supervised = spec.outlive === true ? startSupervisor(mkdtempSync(join(tmpdir(), "fc-ctl-")), config, { ...hooks, log }) : undefined;
10101013 if (supervised === undefined) {
10111014 opened.push(startEgress(join(sockets, "out.sock"), proxyOptions(policy, hooks), log));
10121015 for (const port of policy.loops)
src/apps/egress.ts +6 −2
@@ -971,7 +971,11 @@ export function startEgress(
971971 */
972972 export function relay(
973973 listen: { tcp: number } | { unix: string },
974 target: { unix: string } | { tcp: number },
974 /**
975 * A `tcp` target may be a function, asked on each connection (the beat door a supervisor relays
976 * to its host while one lives and to its own held door after, #13906).
977 */
978 target: { unix: string } | { tcp: number | (() => number) },
975979 ): Egress {
976980 interface Pair {
977981 peer: Socket<undefined> | undefined;
@@ -993,7 +997,7 @@ export function relay(
993997 const dial =
994998 "unix" in target
995999 ? { unix: target.unix }
996 : { hostname: "127.0.0.1", port: target.tcp };
1000 : { hostname: "127.0.0.1", port: typeof target.tcp === "function" ? target.tcp() : target.tcp };
9971001 Bun.connect<undefined>({
9981002 ...dial,
9991003 socket: {
src/spawn/beat_wire.ts added +5
@@ -0,0 +1,5 @@
1 /**
2 * The header a child's beat carries its signature in: its own module, so a cell's supervisor reads
3 * it without the heartbeat's imports (#13906).
4 */
5 export const SIGNATURE_HEADER = "x-forge-signature";
src/spawn/cell.ts +5 −2
@@ -32,8 +32,10 @@ import { childGate, type ChildGate } from "./child_reach.ts";
3232 * Whether a child's cell outlives this process (`FORGE_CELLS_OUTLIVE=on`; #13755): its proxy and
3333 * relays run in a supervisor of their own, the cell is not tied to its parent, and a stop of this
3434 * process leaves it running. Off by default and undocumented: nothing yet re-adopts or reaps a cell
35 * on start (#13750 step 2), its beat door (`beat-door.ts`) still runs in this process, and its
36 * egress bytes are counted in the supervisor, out of this process's gauges (#13921).
35 * on start (#13750 step 2), its beat door (`beat-door.ts`) is this process's, reached through the
36 * supervisor's relay while this process lives and answered by the supervisor's held door after
37 * (`held_door.ts`, #13906), and its egress bytes are counted in the supervisor, out of this
38 * process's gauges (#13921).
3739 */
3840 export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on";
3941
@@ -189,6 +191,7 @@ export function cellSpecFor(input: {
189191 io: SIZE_IO[input.size],
190192 listen: input.port,
191193 ...(cellsOutlive() ? { outlive: true } : {}),
194 ...(input.parentPort === undefined ? {} : { door: input.parentPort }),
192195 label: input.label,
193196 // Its shell's prompt names the child, never the parent's pod (#10850).
194197 hostname: input.name,
src/spawn/heartbeat.ts +2 −1
@@ -31,7 +31,8 @@ import { forgeCeiling } from "../flow/ceiling.ts";
3131 import { during } from "../perf/jobs.ts";
3232 import { forgeHasNoKey, NO_KEY_HELD } from "../crew/no_key.ts";
3333
34 export const SIGNATURE_HEADER = "x-forge-signature";
34 import { SIGNATURE_HEADER } from "./beat_wire.ts";
35 export { SIGNATURE_HEADER };
3536 /** A child reports this often; one silent for three beats reads as quiet. */
3637 export const BEAT_MS = 10 * 60_000;
3738 export const QUIET_MS = 3 * BEAT_MS;
src/spawn/held_door.ts added +50
@@ -0,0 +1,50 @@
1 /**
2 * The beat door a cell's supervisor answers once its host is gone (#13906): the cell's beat still
3 * answers, and the latest one is held in its control directory for a host that adopts the cell to
4 * check (#13750 step 2). It holds no key, so it checks nothing and vouches for nothing: it answers
5 * `202 { ok: true, held: true }` to a beat that carries a signature and fits, and `503` to
6 * everything else the real door answers (the broker, reach, the lent model), which need the host.
7 */
8
9 import { renameSync, writeFileSync } from "node:fs";
10 import { securityFloor } from "../http/respond.ts";
11 import { readCapped } from "./bounded_read.ts";
12 import { SIGNATURE_HEADER } from "./beat_wire.ts";
13
14 /** As large as a beat the real door reads (`beat-door.ts`). */
15 const MAX_BODY = 1024;
16
17 export interface HeldDoor {
18 readonly port: number;
19 close(): void;
20 }
21
22 /** A beat held for the host, as one JSON line: what the cell sent and when it arrived. */
23 export interface HeldBeat {
24 readonly raw: string;
25 readonly signature: string;
26 readonly receivedAt: number;
27 }
28
29 export function openHeldDoor(path: string, now: () => number = Date.now): HeldDoor {
30 const server = Bun.serve({
31 port: 0,
32 hostname: "127.0.0.1",
33 maxRequestBodySize: MAX_BODY,
34 fetch: async (req) => securityFloor(await answer(path, req, now)),
35 });
36 return { port: server.port!, close: () => server.stop(true) };
37 }
38
39 async function answer(path: string, req: Request, now: () => number): Promise<Response> {
40 if (req.method !== "POST" || new URL(req.url).pathname !== "/api/v1/heartbeat")
41 return new Response("no host holds this door", { status: 503 });
42 const signature = req.headers.get(SIGNATURE_HEADER);
43 if (signature === null || signature.length > 256) return new Response("no such door", { status: 404 });
44 const bytes = await readCapped(req, MAX_BODY);
45 if (bytes === undefined) return new Response("no such door", { status: 404 });
46 const held: HeldBeat = { raw: new TextDecoder().decode(bytes), signature, receivedAt: now() };
47 writeFileSync(`${path}.tmp`, JSON.stringify(held));

Non-blocking: if this write fails (disk full, control directory gone), the request errors out as a 500. Answer 503 so the cell retries on its next beat.

48 renameSync(`${path}.tmp`, path);
49 return Response.json({ ok: true, held: true }, { status: 202 });
50 }
src/spawn/supervisor.ts +40 −1
@@ -11,6 +11,7 @@
1111 * cell.pid the cell's root process (host writes once it is spawned)
1212 * cell.log the cell's stdout and stderr, as a file the cell holds open
1313 * allowed.json egress entries allowed since the cell started (host rewrites when they change)
14 * beat.held the cell's latest beat once no host holds its door (supervisor writes; #13906)
1415 * events.jsonl refusals and undeclared hosts (supervisor appends; the host tails)
1516 *
1617 * It ends when the cell's root process is gone, on SIGTERM, or when no cell appears in a minute.
@@ -21,6 +22,7 @@ import { join } from "node:path";
2122 import { during } from "../perf/jobs.ts";
2223 import { PROXY_VARS, relay, startEgress, type Egress } from "../apps/egress.ts";
2324 import { proxyOptions, type CellPolicy } from "../apps/cell_policy.ts";
25 import { openHeldDoor } from "./held_door.ts";
2426
2527 /** Plain data: what a supervisor serves. */
2628 export interface SupervisorConfig extends CellPolicy {
@@ -28,6 +30,15 @@ export interface SupervisorConfig extends CellPolicy {
2830 readonly sockets: string;
2931 /** Tests: the proxy listens on this loopback port, where a unix socket cannot be bound. */
3032 readonly outPort?: number;
33 /**
34 * The port the cell knows its parent's beat door by (`FORGE_PARENT_LOCAL`, #13906): its loop is
35 * relayed to the host's door while the host lives, and to the supervisor's held door
36 * (`held_door.ts`) from the first time it is seen gone, so the cell's beat still answers. The
37 * door's keys stay in the host; the held door checks nothing and keeps the latest beat.
38 */
39 readonly door?: number;
40 /** Tests: the door's loop listens on this loopback port, where a unix socket cannot be bound. */
41 readonly doorAt?: number;
3142 }
3243
3344 export type SupervisorEvent =
@@ -80,8 +91,28 @@ export const files = (ctl: string) => ({
8091 log: join(ctl, "cell.log"),
8192 allowed: join(ctl, "allowed.json"),
8293 events: join(ctl, "events.jsonl"),
94 held: join(ctl, "beat.held"),
8395 });
8496
97 /**
98 * Where the cell's beat goes, asked on each connection: the host's door while the host that wrote
99 * `host.pid` lives, and the held door from the first time it is seen gone, for good, so a port a
100 * dead host left is never dialled again, whoever holds it or the host's pid now.
101 */
102 export function doorTarget(ctl: string, door: number, held: number): () => number {
103 let gone = false;
104 return () => {
105 if (!gone) {
106 const host = readPid(files(ctl).host);
107 gone = host === undefined || !alive(host);
108 }
109 return gone ? held : door;
110 };
111 }
112
113 /** The loops relayed as they are: every one but the beat door's, which `doorTarget` steers. */
114 export const plainLoops = (config: Pick<SupervisorConfig, "loops" | "door">): number[] => [...new Set(config.loops)].filter((port) => port !== config.door);
115
85116 export function alive(pid: number): boolean {
86117 if (!Number.isInteger(pid) || pid <= 0) return false;
87118 try {
@@ -147,6 +178,8 @@ export function parseConfig(text: string): SupervisorConfig | undefined {
147178 if (!Array.isArray(v["loops"]) || !v["loops"].every(isPort)) return undefined;
148179 if (v["listen"] !== undefined && !isPort(v["listen"])) return undefined;
149180 if (v["outPort"] !== undefined && !isPort(v["outPort"])) return undefined;
181 if (v["door"] !== undefined && !isPort(v["door"])) return undefined;
182 if (v["doorAt"] !== undefined && !isPort(v["doorAt"])) return undefined;
150183 const origins = v["origins"];
151184 if (!Array.isArray(origins)) return undefined;
152185 for (const o of origins as Record<string, unknown>[]) if (typeof o !== "object" || o === null || typeof o["host"] !== "string" || !isPort(o["port"])) return undefined;
@@ -204,7 +237,13 @@ export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT
204237 });
205238 const out = config.outPort === undefined ? join(config.sockets, "out.sock") : { tcp: config.outPort };
206239 opened.push(startEgress(out, policy, (line) => say({ e: "log", line })));
207 for (const port of new Set(config.loops)) opened.push(relay({ unix: join(config.sockets, `loop-${port}.sock`) }, { tcp: port }));
240 for (const port of plainLoops(config)) opened.push(relay({ unix: join(config.sockets, `loop-${port}.sock`) }, { tcp: port }));
241 if (config.door !== undefined) {
242 const held = openHeldDoor(files(ctl).held);
243 opened.push(held);
244 const at = config.doorAt === undefined ? { unix: join(config.sockets, `loop-${config.door}.sock`) } : { tcp: config.doorAt };
245 opened.push(relay(at, { tcp: doorTarget(ctl, config.door, held.port) }));
246 }
208247 if (config.listen !== undefined) opened.push(relay({ tcp: config.listen }, { unix: join(config.sockets, "in.sock") }));
209248
210249 const began = Date.now();
test/cell-outlives-host.test.ts +108 −1
@@ -10,8 +10,9 @@ import { cellPolicy, openCell } from "../src/apps/contain.ts";
1010 import { proxyOptions } from "../src/apps/cell_policy.ts";
1111 import { createApp } from "../src/app.ts";
1212 import { freePort } from "../src/apps/process.ts";
13 import { openHeldDoor } from "../src/spawn/held_door.ts";
1314 import { cellChildren, cellLimits, cellSpecFor, cellsOutlive, spawnInCell, stopCellChildren } from "../src/spawn/cell.ts";
14 import { alive, files, parseConfig, parseEvent, readAllowed, tailLines, type Supervised } from "../src/spawn/supervisor.ts";
15 import { alive, doorTarget, files, parseConfig, parseEvent, plainLoops, readAllowed, tailLines, type Supervised } from "../src/spawn/supervisor.ts";
1516
1617 const killed: number[] = [];
1718 afterEach(() => {
@@ -32,6 +33,15 @@ const until = async (what: () => boolean, ms = 8_000): Promise<boolean> => {
3233 return what();
3334 };
3435
36 const untilAsync = async (what: () => Promise<boolean>, ms = 5_000): Promise<boolean> => {
37 const end = Date.now() + ms;
38 while (Date.now() < end) {
39 if (await what()) return true;
40 await Bun.sleep(50); // real-timer: polling a real supervisor's relay
41 }
42 return what();
43 };
44
3545 const accepts = (port: number): Promise<boolean> =>
3646 new Promise((resolve) => {
3747 Bun.connect({ hostname: "127.0.0.1", port, socket: { open: (s) => (s.end(), resolve(true)), data() {}, error: () => resolve(false), connectError: () => resolve(false) } }).catch(() => resolve(false));
@@ -310,3 +320,100 @@ test("stopping the host leaves an outliving cell running, and stops a default on
310320 expect(await stopsWith(true)).toBe(false);
311321 expect(await stopsWith(false)).toBe(true);
312322 });
323
324 test("kill the host: the cell's beat still answers, from the supervisor's held door, never through the dead host's port", async () => {
325 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
326 const ctl = join(dir, "ctl");
327 const sockets = join(dir, "sockets");
328 const doorAt = freePort();
329 const script = `
330 import { startSupervisor } from ${JSON.stringify(join(import.meta.dir, "../src/spawn/supervisor.ts"))};
331 const door = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("first host") });
332 const s = startSupervisor(${JSON.stringify(ctl)}, { sockets: ${JSON.stringify(sockets)}, hosts: [], origins: [], report: false, loops: [door.port], door: door.port, doorAt: ${doorAt}, outPort: ${freePort()} }, { log: () => {} });
333 const cell = Bun.spawn(["sleep", "300"], { stdout: s.logFd, stderr: s.logFd });
334 cell.unref();
335 s.adopt(cell.pid);
336 console.log(JSON.stringify({ supervisor: s.pid, cell: cell.pid }));
337 setInterval(() => {}, 1000);
338 `;
339 const host = Bun.spawn([process.execPath, "-e", script], { stdout: "pipe", stderr: "inherit", env: { PATH: process.env["PATH"] ?? "", FORGE_TEST_SOCKET: "port" } });
340 killed.push(host.pid);
341 let text = "";
342 for await (const chunk of host.stdout) {
343 text += new TextDecoder().decode(chunk);
344 if (text.includes("\n")) break;
345 }
346 const { supervisor, cell } = JSON.parse(text) as { supervisor: number; cell: number };
347 killed.push(supervisor, cell);
348 const beat = async (): Promise<Response | undefined> => {
349 try {
350 return await fetch(`http://127.0.0.1:${doorAt}/api/v1/heartbeat`, { method: "POST", headers: { "x-forge-signature": "s" }, body: '{"name":"c"}', signal: AbortSignal.timeout(2_000) });
351 } catch {
352 return undefined;
353 }
354 };
355 let first: Response | undefined;
356 expect(await untilAsync(async () => (first = await beat()) !== undefined)).toBe(true);
357 expect(await first?.text()).toBe("first host");
358
359 host.kill("SIGKILL");
360 await host.exited;
361 // No host: the supervisor answers the beat itself and holds it for a host that adopts the cell.
362 let again: Response | undefined;
363 expect(await untilAsync(async () => (again = await beat()) !== undefined)).toBe(true);
364 expect(again?.status).toBe(202);
365 expect(await again?.json()).toEqual({ ok: true, held: true });
366 expect(again?.headers.get("x-content-type-options")).toBe("nosniff");
367 expect(JSON.parse(readFileSync(files(ctl).held, "utf8"))).toMatchObject({ raw: '{"name":"c"}', signature: "s" });
368 expect(alive(supervisor)).toBe(true);
369 process.kill(cell, "SIGKILL");
370 expect(await until(() => !alive(supervisor), 3_000)).toBe(true);
371 });
372
373 test("the held door keeps a signed beat that fits, and answers nothing else", async () => {
374 const path = join(mkdtempSync(join(tmpdir(), "held-")), "beat.held");
375 const door = openHeldDoor(path, () => 7);
376 const at = `http://127.0.0.1:${door.port}`;
377 try {
378 const unsigned = await fetch(`${at}/api/v1/heartbeat`, { method: "POST", body: "{}" });
379 expect(unsigned.status).toBe(404);
380 expect(existsSync(path)).toBe(false);
381 const big = await fetch(`${at}/api/v1/heartbeat`, { method: "POST", headers: { "x-forge-signature": "s" }, body: "x".repeat(2_000) });
382 expect(big.ok).toBe(false);
383 expect(existsSync(path)).toBe(false);
384 const other = await fetch(`${at}/api/v1/reach`, { method: "POST", headers: { "x-forge-signature": "s" }, body: "{}" });
385 expect(other.status).toBe(503);
386 expect(other.headers.get("content-security-policy")).not.toBeNull();
387 const kept = await fetch(`${at}/api/v1/heartbeat`, { method: "POST", headers: { "x-forge-signature": "s" }, body: "{}" });
388 expect(kept.status).toBe(202);
389 expect(JSON.parse(readFileSync(path, "utf8"))).toEqual({ raw: "{}", signature: "s", receivedAt: 7 });
390 } finally {
391 door.close();
392 }
393 });
394
395 test("the beat goes to the host while it lives, and to the held door for good once it is seen gone", () => {
396 const ctl = mkdtempSync(join(tmpdir(), "door-"));
397 const f = files(ctl);
398 writeFileSync(f.host, String(process.pid));
399 const target = doorTarget(ctl, 1111, 2222);
400 expect(target()).toBe(1111);
401 writeFileSync(f.host, "999999999");
402 expect(target()).toBe(2222);
403 // A live pid written after (a recycled pid, say) does not bring the dead host's port back.
404 writeFileSync(f.host, String(process.pid));
405 expect(target()).toBe(2222);
406 expect(doorTarget(join(ctl, "none"), 1111, 2222)()).toBe(2222);
407 });
408
409 test("the beat door's loop is steered, not relayed as it is", () => {
410 expect(plainLoops({ loops: [8080, 9090, 8080], door: 9090 })).toEqual([8080]);
411 expect(plainLoops({ loops: [8080, 9090] })).toEqual([8080, 9090]);
412 });
413
414 test("a door in the config is a port, or the config is refused", () => {
415 const good = { sockets: "/s", hosts: [], origins: [], report: false, loops: [8080], door: 8080 };
416 expect(parseConfig(JSON.stringify(good))).toBeDefined();
417 expect(parseConfig(JSON.stringify({ ...good, door: "8080" }))).toBeUndefined();
418 expect(parseConfig(JSON.stringify({ ...good, doorAt: 70_000 }))).toBeUndefined();
419 });
test/security-floor-doors.test.ts +20
@@ -17,6 +17,7 @@ import { createApp, type App } from "../src/app.ts";
1717 import { securityFloor } from "../src/http/respond.ts";
1818 import { serve, type Servers } from "../src/http/server.ts";
1919 import { openBeatDoor } from "../src/spawn/beat-door.ts";
20 import { openHeldDoor } from "../src/spawn/held_door.ts";
2021 import { quietLogs } from "./helpers/globals.ts";
2122
2223 quietLogs();
@@ -31,6 +32,10 @@ const LISTENERS: Record<string, { count: number; why: string }> = {
3132 why: "the public door (`handle`, walked below) and the internal hook listener, whose fetch is `securityFloor(await answerInternal(...))`",
3233 },
3334 "src/spawn/beat-door.ts": { count: 1, why: "fetch is `securityFloor(await answerBeat(...))`, asked below" },
35 "src/spawn/held_door.ts": {
36 count: 1,
37 why: "the beat door a supervisor holds once its host is gone; fetch is `securityFloor(await answer(...))`, asked below",
38 },
3439 "src/kernel/port.ts": { count: 1, why: "a port probe, stopped before any request can reach it" },
3540 "src/apps/process.ts": { count: 1, why: "a port probe, stopped before any request can reach it" },
3641 "src/actions/spawn.ts": { count: 1, why: "text of a fresh app's index.ts, written into its repository" },
@@ -55,6 +60,8 @@ test("the internal listener and the beat door hand their fetch to securityFloor"
5560 expect(server).toContain("fetch: async (request) => securityFloor(await answerInternal(app, request)),");
5661 const beat = readFileSync(join(ROOT, "src/spawn/beat-door.ts"), "utf8");
5762 expect(beat).toContain("fetch: async (req) => securityFloor(await answerBeat(app, deps, req)),");
63 const held = readFileSync(join(ROOT, "src/spawn/held_door.ts"), "utf8");
64 expect(held).toContain("fetch: async (req) => securityFloor(await answer(path, req, now)),");
5865 });
5966
6067 /**
@@ -159,6 +166,19 @@ test("the beat door floors its refusals", async () => {
159166 }
160167 });
161168
169 test("the held beat door floors its refusals", async () => {
170 const dir = await mkdtemp(join(tmpdir(), "held-door-"));
171 const door = openHeldDoor(join(dir, "beat.json"));
172 try {
173 const res = await fetch(`http://127.0.0.1:${door.port}/nowhere`);
174 expect(res.status).toBe(503);
175 expect(lacks(res)).toEqual([]);
176 } finally {
177 door.close();
178 await rm(dir, { recursive: true, force: true });
179 }
180 });
181
162182 test("securityFloor over an html response or a redirect lays the floor and keeps a page's own policy", () => {
163183 const html = securityFloor(
164184 new Response("<!doctype html>", { headers: { "content-type": "text/html", "content-security-policy": "default-src 'self'" } }),
test/talk-captain.test.ts +6 −2
@@ -167,7 +167,11 @@ describe("the runner", () => {
167167 test("a due routine posts into one Captain thread; a quiet one does not push", async () => {
168168 const sent: string[] = [];
169169 const send = async (url: string, body: string) => (sent.push(`${url} ${body}`), true);
170 const done = await runDue(app, new Date(), send);
170 // Half past the current UTC hour (in the past), so "a minute later" never crosses the
171 // paper's 07:00 UTC slot, as it did for a check that ran at 06:59.
172 const hour = Math.floor(Date.now() / 3_600_000) * 3_600_000;
173 const at = hour + 1_800_000 <= Date.now() ? hour + 1_800_000 : hour - 1_800_000;
174 const done = await runDue(app, new Date(at), send);
171175 expect(done.map((d) => d.kind).sort()).toEqual(["checkin", "friends", "paper"]);
172176 expect(new Set(done.map((d) => d.conversationId)).size).toBe(1);
173177 const turns = listTurns(app.db, owner.id, done[0]!.conversationId, { limit: 50 });
@@ -176,7 +180,7 @@ describe("the runner", () => {
176180 expect(turns.value[0]!.text).toContain("**");
177181 expect(sent).toEqual([]);
178182 // Ran now: nothing is due a minute later, and a person who is not an admin gets none.
179 expect(await runDue(app, new Date(Date.now() + 60_000), send)).toEqual([]);
183 expect(await runDue(app, new Date(at + 60_000), send)).toEqual([]);
180184 expect(listRoutines(app.db, owner.id).every((r) => r.lastAt !== null)).toBe(true);
181185 expect(app.db.query("SELECT COUNT(*) AS n FROM talk_routine WHERE user_id = ?1").get(other.id)).toEqual({ n: 0 });
182186 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs