Public: anyone can read this; the crew can change it.

The SSH terminal survives a release swap without dropping the owner #13270

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13260-the-ssh-terminal-survives-a-release-swap → main 4 commits fork

  • Merged. scout/13260-the-ssh-terminal-survives-a-release-swap was merged into main as 16ed9f2.

Review

Sign in to review this change.
10 files changed +219 −75
docs/REFERENCE.md +1 −1

Diff not shown — a line 4,902 characters wide (limit 2,000). +1 −1

View file
src/main.ts +3 −1
@@ -1037,9 +1037,11 @@ const handOver = drainOnHandover(servers, () => drain("SIGTERM"), close);
10371037 handover.drain = () => {
10381038 if (draining) return;
10391039 draining = true;
1040 // The release bound the SSH port beside this one (SO_REUSEPORT): new
1041 // connections are its own, terminals here are told and closed (#13260).
1042 ssh?.handOver();
10401043 void handOver();
10411044 };
1042 handover.ssh = ssh !== undefined;
10431045 process.on("SIGINT", shutdown);
10441046 process.on("SIGTERM", shutdown);
10451047 // The terminal it ran in went away. Apps that run without a cell lead
src/release/swap.ts +12 −13
@@ -129,12 +129,12 @@ export function drainOnHandover(
129129
130130 /**
131131 * What the running forge does once the release is well: drain, as a SIGTERM
132 * does, but letting go of the port first (`drainOnHandover`). Set at boot,
133 * with whether this process holds the SSH listener (a new process cannot
134 * bind that port while this one does, and a release must not quietly close
135 * the terminal).
132 * does, but letting go of the port first (`drainOnHandover`). Set at boot.
133 * The SSH listener is bound with SO_REUSEPORT too, so the release binds it
134 * beside this process and the drain hands it over with the web port: its
135 * terminals are told the forge is updating (#13260).
136136 */
137 export const handover: { drain?: () => void; ssh?: boolean; migrate?: boolean } = {};
137 export const handover: { drain?: () => void; migrate?: boolean } = {};
138138
139139 const MIGRATE_SYSCTL = "/proc/sys/net/ipv4/tcp_migrate_req";
140140
@@ -183,10 +183,9 @@ let underWay = false;
183183 /**
184184 * Take the one swap this process may make. Refused while another is in its
185185 * budget or once one has succeeded (this process is then draining), so two
186 * landings close together never leave two releases serving; and refused while
187 * this process holds an SSH listener. `tell` says whether the landed pull
188 * should hear of it. Refused too unless the kernel migrates queued connections
189 * between listeners of the port.
186 * landings close together never leave two releases serving. `tell` says
187 * whether the landed pull should hear of it. Refused too unless the kernel
188 * migrates queued connections between listeners of the port.
190189 *
191190 * The release is a child of this process, not detached: the swap is for a forge
192191 * run directly, not under a supervisor (PID 1 of a container, a systemd unit
@@ -201,16 +200,16 @@ export function beginSwap(): { ok: true } | { ok: false; line: string; tell: boo
201200 return { ok: true };
202201 }
203202
204 /** Why this process cannot swap at all (SSH held, queued connections not migrated); undefined when it can. */
203 /** Why this process cannot swap at all (queued connections not migrated); undefined when it can. */
205204 export function swapBlocked(): string | undefined {
206 if (handover.ssh === true)
207 return "this process holds the SSH terminal, which a new process cannot take over yet; restart the forge to run this tree";
208205 if (!migratesQueuedConnections())
209206 return "the kernel would reset connections queued on the old listener when it closes; set net.ipv4.tcp_migrate_req=1 (Linux 5.14+) to swap, or restart the forge to run this tree";
210207 return undefined;
211208 }
212209
213 /** A swap that failed gives the turn back; one that worked keeps it, for this process is leaving. */
210 /**
211 * A swap that failed gives the turn back; one that worked keeps it, for this process is leaving.
212 */
214213 export function endSwap(swapped: boolean): void {
215214 if (!swapped) underWay = false;
216215 }
src/ssh/server.ts +45 −1
@@ -25,6 +25,11 @@ export interface SshServerOptions {
2525 port: number;
2626 /** @default "0.0.0.0" */
2727 hostname?: string;
28 /**
29 * Bind with SO_REUSEPORT, so a release can bind the same port beside this
30 * process and take every new connection once this one hands over (#13260).
31 */
32 reusePort?: boolean;
2833 /** `host_ed25519` and `host_ed25519.pub` are created here if absent. */
2934 hostKeyDir: string;
3035 /** Optional USERAUTH_BANNER text, shown before success or failure. */
@@ -60,6 +65,12 @@ export interface SshServer {
6065 * ended on its own: a release's old process, letting people finish.
6166 */
6267 drain(): Promise<void>;
68 /**
69 * A release listens on this port too (#13260): stop accepting at once, so
70 * every new connection is the release's, and close every terminal with
71 * `HANDOVER_LINE`. A running push or clone is left to `drain`.
72 */
73 handOver(): void;
6374 }
6475
6576 type Socket = Bun.Socket<Connection | undefined>;
@@ -76,6 +87,8 @@ class Connection {
7687
7788 key: SshPublicKey | undefined;
7889 username = "";
90 /** The sessions opened on this connection, so a handover can tell them it is going. */
91 private readonly sessions = new Set<SshSession>();
7992
8093 constructor(
8194 private readonly socket: Socket,
@@ -128,7 +141,10 @@ class Connection {
128141 unimplemented: (seq) => this.transport.unimplemented(seq),
129142 disconnect: (reason, description) =>
130143 this.transport.disconnect(reason, description),
131 onSession: (session) => options.onSession(session),
144 onSession: (session) => {
145 this.sessions.add(session);
146 options.onSession(session);
147 },
132148 // SAFETY: the channel layer only runs after USERAUTH_SUCCESS, which is
133149 // the same moment `key`/`username` are set.
134150 get key(): SshPublicKey {
@@ -256,8 +272,31 @@ class Connection {
256272 "server shutting down",
257273 );
258274 }
275
276 /**
277 * A release took the port (#13260): a connection still running a command
278 * (a push or clone) is left to finish; any other — a terminal, or one not
279 * yet in — is told `line` on its screen and in the DISCONNECT, and closed.
280 * Returns whether it was closed.
281 */
282 handOver(line: string): boolean {
283 const live = [...this.sessions].filter((s) => !s.ended);
284 if (live.some((s) => s.kind === "exec")) return false;
285 for (const session of live) {
286 try {
287 session.write(`\r\n${line}\r\n`);
288 } catch {
289 // Already gone.
290 }
291 }
292 this.transport.disconnect(DISCONNECT.BY_APPLICATION, line);
293 return true;
294 }
259295 }
260296
297 /** What a terminal on a process handing its port to a release is told as it closes (#13260). */
298 export const HANDOVER_LINE = "the forge is updating, reconnecting";
299
261300 export function serveSsh(options: SshServerOptions): Promise<SshServer> {
262301 const hostKey = loadOrCreateHostKey(options.hostKeyDir);
263302 const connections = new Set<Connection>();
@@ -272,6 +311,7 @@ export function serveSsh(options: SshServerOptions): Promise<SshServer> {
272311 const listener = Bun.listen<Connection | undefined>({
273312 hostname: options.hostname ?? "0.0.0.0",
274313 port: options.port,
314 ...(options.reusePort === true ? { reusePort: true } : {}),
275315 socket: {
276316 open(socket) {
277317 // Before a Connection exists, so a refused peer costs no key work.
@@ -322,6 +362,10 @@ export function serveSsh(options: SshServerOptions): Promise<SshServer> {
322362 if (connections.size === 0) return Promise.resolve();
323363 return new Promise((resolve) => drained.push(resolve));
324364 },
365 handOver(): void {
366 listener.stop(false);
367 for (const connection of [...connections]) connection.handOver(HANDOVER_LINE);
368 },
325369 stop(): void {
326370 for (const connection of [...connections]) connection.close();
327371 listener.stop(true);
src/tui/ssh.ts +2
@@ -499,6 +499,8 @@ export async function serveForgeSsh(
499499 const server = await serveSsh({
500500 port: options.port,
501501 ...(options.hostname === undefined ? {} : { hostname: options.hostname }),
502 // A swap binds the release beside this process on the same port (#13260).
503 ...(forge.config.selfSwap === true ? { reusePort: true } : {}),
502504 hostKeyDir: options.hostKeyDir ?? join(forge.config.dataDir, "ssh"),
503505 log: forge.log,
504506 // Guests are welcome: the key is an identity, not a permit.
test/capacity-selfswap.test.ts +4 −15
@@ -1,8 +1,8 @@
11 /**
22 * /capacity and MCP `capacity` say whether this forge can swap itself, and
3 * why not (#8383): ready, the kernel not migrating queued connections, the
4 * SSH terminal held, or FORGE_SELF_SWAP off. `handover` stands in for the
5 * kernel file and the SSH listener.
3 * why not (#8383): ready, the kernel not migrating queued connections, or
4 * FORGE_SELF_SWAP off. `handover` stands in for the kernel file. The SSH
5 * terminal no longer blocks: it is handed over with the port (#13260).
66 */
77 import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
88 import { mkdtemp, rm } from "node:fs/promises";
@@ -34,7 +34,6 @@ beforeAll(async () => {
3434
3535 afterEach(() => {
3636 delete handover.migrate;
37 delete handover.ssh;
3837 });
3938
4039 afterAll(async () => {
@@ -68,9 +67,8 @@ const tool = async (url = servers.url, token = bearer): Promise<{ enabled: boole
6867 };
6968
7069 describe("self-swap on /capacity and MCP capacity", () => {
71 test("ready: enabled, the kernel migrates, no SSH held", async () => {
70 test("ready: enabled, the kernel migrates", async () => {
7271 handover.migrate = true;
73 handover.ssh = false;
7472 expect(await tool()).toEqual({ enabled: true, migrateReq: "1", blocked: null });
7573 const row = await page();
7674 expect(row).toContain("ready");
@@ -89,18 +87,9 @@ describe("self-swap on /capacity and MCP capacity", () => {
8987 expect(row).toMatch(/reads\s*<span class="mono">0</);
9088 });
9189
92 test("SSH held: the terminal is named, whatever the kernel says", async () => {
93 handover.migrate = true;
94 handover.ssh = true;
95 const s = await tool();
96 expect(s.migrateReq).toBe("1");
97 expect(s.blocked).toContain("holds the SSH terminal");
98 expect(await page()).toContain("holds the SSH terminal");
99 });
10090
10191 test("FORGE_SELF_SWAP off is a reason of its own, on the page and the tool", async () => {
10292 handover.migrate = true;
103 handover.ssh = false;
10493 expect(selfSwapState(false).blocked).toContain("FORGE_SELF_SWAP is off");
10594 const dir = await mkdtemp(join(tmpdir(), "forge-selfswap-off-"));
10695 const off = createApp({ dataDir: dir, port: 0 });
test/release-keeper-swap.test.ts +15 −9
@@ -58,7 +58,9 @@ afterAll(() => {
5858 for (const d of dirs) rmSync(d, { recursive: true, force: true });
5959 });
6060
61 /** `migrations` is how many the tree knows: more than any live database, unless a test says fewer. */
61 /**
62 * `migrations` is how many the tree knows: more than any live database, unless a test says fewer.
63 */
6264 function tree(root: string, main = STAND_IN, pkg = "{}", migrations = 9999): void {
6365 mkdirSync(join(root, "src", "core"), { recursive: true });
6466 writeFileSync(join(root, "src", "core", "schema.ts"), `export const migrations = new Array(${migrations}).fill(0);\n`);
@@ -123,7 +125,8 @@ async function hosted(landed: Record<string, { main?: string; pkg?: string; pref
123125 timed: switched(f.app.db, () => 0),
124126 drain: () => {
125127 drained.push(now.commit);
126 // The process that handed over lets go of the port; the one before it, started by a swap, is stopped.
128 // The process that handed over lets go of the port; the one before it, started by a swap, is
129 // stopped.
127130 if (before === undefined) old.stop(true);
128131 else
129132 try {
@@ -175,7 +178,8 @@ describe("the keeper swaps main's tree in; a pod restart comes back on it", () =
175178 const cut = await tick(h.ports);
176179 expect(cut).toContain("cut 0.7.284");
177180 expect(cut).toContain(`promoted to tree/${B}:0.7.284`);
178 // The new process answers on the same port as its own commit and version, beside the draining one.
181 // The new process answers on the same port as its own commit and version, beside the draining
182 // one.
179183 expect(await (async () => { for (let i = 0; i < 40; i++) { const g = await health(h.port); if (g?.commit === B) return g; await Bun.sleep(50); } })()).toMatchObject({ commit: B, version: "0.7.284" });
180184 expect(h.drained).toEqual([A]);
181185 expect(readServing(h.dataDir)?.sha).toBe(B);
@@ -224,7 +228,8 @@ describe("the keeper swaps main's tree in; a pod restart comes back on it", () =
224228 stop = true;
225229 await hammer;
226230 expect(seen.ok).toBeGreaterThan(10);
227 // The promise of 0 refused requests holds where the kernel migrates queued connections (the check runs there).
231 // The promise of 0 refused requests holds where the kernel migrates queued connections (the
232 // check runs there).
228233 if (realMigrate) expect(seen.bad).toBe(0);
229234 }, 60_000);
230235
@@ -240,14 +245,14 @@ describe("the keeper swaps main's tree in; a pod restart comes back on it", () =
240245 expect(dependencyChange(join(h.running))).toBeUndefined();
241246 });
242247
243 test("a process that cannot swap (it holds the SSH terminal) cuts nothing", async () => {
248 test("a process that cannot swap (the kernel would reset queued connections) cuts nothing", async () => {
244249 const h = await hosted({ [B]: {} });
245250 h.f.world.now += 61_000;
246 handover.ssh = true;
251 handover.migrate = false;
247252 try {
248 expect(await tick(h.ports)).toContain("not cut: this process holds the SSH terminal");
253 expect(await tick(h.ports)).toContain("not cut: the kernel would reset connections");
249254 } finally {
250 delete handover.ssh;
255 handover.migrate = true;
251256 }
252257 expect(h.f.world.tags).toHaveLength(0);
253258 });
@@ -309,7 +314,8 @@ describe("the keeper swaps main's tree in; a pod restart comes back on it", () =
309314 await expect(deployer.setImage(`tree/${B}:0.7.284`)).rejects.toThrow(`${B} predates schema ${live}; rolling back would need a down-migration`);
310315 expect(h.drained).toEqual([]);
311316 expect((await health(h.port))?.commit).toBe(A);
312 // A tree with the live schema is not refused by the guard (this one then fails to answer in its short budget, a different line).
317 // A tree with the live schema is not refused by the guard (this one then fails to answer in its
318 // short budget, a different line).
313319 await expect(deployer.setImage(`tree/${C}:0.7.285`)).rejects.not.toThrow("predates schema");
314320 }, 20_000);
315321
test/release-swap.test.ts −27
@@ -307,33 +307,6 @@ describe("on landing", () => {
307307 }
308308 }, 90_000);
309309
310 test("a process holding the SSH terminal refuses the swap and tells the pull", async () => {
311 const port = await freePort();
312 const old = oldForge(port);
313 let drained = 0;
314 handover.drain = () => {
315 drained++;
316 };
317 handover.ssh = true;
318 try {
319 const s = await setup(true, port, STAND_IN);
320 await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
321 expect(drained).toBe(0);
322 expect(await commitAt(port)).toBe(OLD);
323 const told = listComments(s.app.db, s.pull.id).map((c) => c.body);
324 expect(told.some((b) => b.includes("did not swap in") && b.includes("SSH"))).toBe(true);
325 // Refusing took no turn: with the terminal off, the next swap goes.
326 handover.ssh = false;
327 expect(beginSwap().ok).toBe(true);
328 endSwap(false);
329 s.app.close();
330 } finally {
331 delete handover.drain;
332 delete handover.ssh;
333 old.stop(true);
334 }
335 }, 60_000);
336
337310 test("with the switch off, a green landing swaps nothing", async () => {
338311 const port = await freePort();
339312 let drained = 0;
test/ssh-handover.test.ts added +137
@@ -0,0 +1,137 @@
1 /**
2 * The SSH terminal survives a release swap (#13260). Both processes bind the
3 * SSH port with SO_REUSEPORT; the old one's `handOver` stops accepting at
4 * once, so every new connection is the release's and none is refused, and a
5 * terminal still on the old one is told `HANDOVER_LINE` as it closes.
6 *
7 * The port half dials raw TCP and reads the version line, so it runs on any
8 * loopback (the sandbox's FORGE_TEST_SOCKET mode included). The terminal half
9 * drives the real OpenSSH client and skips, by name, where it is missing.
10 */
11
12 import { afterAll, describe, expect, test } from "bun:test";
13 import { mkdtempSync, rmSync } from "node:fs";
14 import { tmpdir } from "node:os";
15 import { join } from "node:path";
16 import { createLog } from "../src/kernel/log.ts";
17 import { HANDOVER_LINE, serveSsh, type SshServer, type SshSession } from "../src/ssh/server.ts";
18 import { quietLogs } from "./helpers/globals.ts";
19 import { missing } from "./helpers/tools.ts";
20
21 quietLogs();
22 const SSH = "/usr/bin/ssh";
23 const SSH_KEYGEN = "/usr/bin/ssh-keygen";
24 const NO_OPENSSH = missing(SSH, SSH_KEYGEN);
25 const log = createLog("ssh-handover-test");
26 const root = mkdtempSync(join(tmpdir(), "forge-ssh-handover-"));
27 const servers: SshServer[] = [];
28
29 afterAll(() => {
30 for (const server of servers) server.stop();
31 rmSync(root, { recursive: true, force: true });
32 });
33
34 function start(name: string, port: number, onSession: (s: SshSession) => void = () => {}): Promise<SshServer> {
35 return serveSsh({
36 port,
37 hostname: "127.0.0.1",
38 reusePort: true,
39 hostKeyDir: join(root, name),
40 serverVersion: `SSH-2.0-${name}`,
41 log,
42 authorize: () => true,
43 onSession,
44 }).then((server) => {
45 servers.push(server);
46 return server;
47 });
48 }
49
50 /** The version line whoever answers `port` sends first; "refused" when nobody does. */
51 async function whoAnswers(port: number): Promise<string> {
52 const line = Promise.withResolvers<string>();
53 let text = "";
54 const socket = await Bun.connect({
55 hostname: "127.0.0.1",
56 port,
57 socket: {
58 data(_s, chunk) {
59 text += new TextDecoder().decode(chunk);
60 const end = text.indexOf("\r\n");
61 if (end >= 0) line.resolve(text.slice(0, end));
62 },
63 close() {
64 line.resolve(text || "closed");
65 },
66 error() {
67 line.resolve("refused");
68 },
69 },
70 }).catch(() => undefined);
71 if (socket === undefined) return "refused";
72 // A silent server is caught by the test's own timeout, not a timer here.
73 const answer = await line.promise;
74 socket.end();
75 return answer;
76 }
77
78 describe("the SSH port is handed to a release", () => {
79 test("the release binds beside the old process, and after handOver every new connection is its own", async () => {
80 const old = await start("old", 0);
81 const release = await start("release", old.port);
82 expect(release.port).toBe(old.port);
83 old.handOver();
84 const answers = await Promise.all(Array.from({ length: 20 }, () => whoAnswers(old.port)));
85 // Not one refused, not one left on the old process.
86 expect(answers.every((a) => a === "SSH-2.0-release")).toBe(true);
87 });
88
89 test("a connection still logging in on the old process is let go at the handover", async () => {
90 const old = await start("old-login", 0);
91 const closed = Promise.withResolvers<void>();
92 const socket = await Bun.connect({
93 hostname: "127.0.0.1",
94 port: old.port,
95 socket: { data() {}, close: () => closed.resolve(), error: () => closed.resolve() },
96 });
97 for (let i = 0; i < 200 && old.sessions() === 0; i++) {
98 await Bun.sleep(10); // real-timer: a real TCP accept lands on its own clock
99 }
100 expect(old.sessions()).toBe(1);
101 await start("release-login", old.port);
102 old.handOver();
103 await closed.promise;
104 expect(old.sessions()).toBe(0);
105 socket.end();
106 });
107 });
108
109 describe.skipIf(NO_OPENSSH)("a terminal on the old process at a handover", () => {
110 test("is told the forge is updating, and ends with that sentence", async () => {
111 const key = join(root, "id_ed25519");
112 const made = Bun.spawn([SSH_KEYGEN, "-t", "ed25519", "-N", "", "-q", "-f", key], { stdout: "pipe", stderr: "pipe" });
113 expect(await made.exited).toBe(0);
114 const opened = Promise.withResolvers<void>();
115 const old = await start("old-shell", 0, (session) => {
116 session.write("hello\r\n");
117 opened.resolve();
118 });
119 const client = Bun.spawn(
120 [
121 SSH, "-tt", "-p", String(old.port), "-i", key,
122 "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null",
123 "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
124 "[email protected]",
125 ],
126 { stdin: "pipe", stdout: "pipe", stderr: "pipe" },
127 );
128 await opened.promise;
129 await start("release-shell", old.port);
130 old.handOver();
131 const out = await new Response(client.stdout).text();
132 const err = await new Response(client.stderr).text();
133 await client.exited;
134 expect(out + err).toContain(HANDOVER_LINE);
135 expect(old.sessions()).toBe(0);
136 });
137 });
test/tui-capacity.test.ts −8
@@ -260,7 +260,6 @@ describe("tui capacity", () => {
260260 test("shows whether the forge can swap itself, and why not (#8383)", async () => {
261261 try {
262262 handover.migrate = true;
263 handover.ssh = false;
264263 let h = boot(carol);
265264 await h.press();
266265 // This app was built without selfSwap: FORGE_SELF_SWAP is off.
@@ -274,12 +273,6 @@ describe("tui capacity", () => {
274273 expect(h.text()).toContain("tcp_migrate_req 0");
275274 expect(h.text()).toContain("net.ipv4.tcp_migrate_req=1");
276275 h.stop();
277 handover.migrate = true;
278 handover.ssh = true;
279 h = boot(carol);
280 await h.press();
281 expect(h.text()).toContain("holds the SSH terminal");
282 h.stop();
283276 for (const who of [bob, bot]) {
284277 const m = boot(who);
285278 await m.press();
@@ -288,7 +281,6 @@ describe("tui capacity", () => {
288281 }
289282 } finally {
290283 delete handover.migrate;
291 delete handover.ssh;
292284 }
293285 });
294286 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs