54b0eb8 · 690a7bd · e8b7108 · 72e275e · d5b2fe2 · 3b6d43c
@@ -0,0 +1,11 @@
# A full disk
When a write fails with `SQLITE_FULL` or `ENOSPC` (matched by `code`/`errno`, or SQLite's own exact message on a SQLite error, never by searching free text), the forge enters one read-only state (`src/kernel/full_disk.ts`):
- every web write (anything `isRead` in `src/http/read_only.ts` does not call a read, so pushes too) answers 503 with one public sentence: the disk is full, reading works, writing resumes by itself;
- every terminal write is refused the same way: an SSH exec other than `git-upload-pack` (push, `tell`, `talk`, an app's verbs) prints the sentence and exits 1, and a TUI action that hits a full disk toasts it;
- the needs-you line on `/system` (`disk-full`) adds the free figure and what frees it; the public sentence carries neither. `/healthz` stays about liveness and does not carry it;
- every two seconds the forge tries a 64 KiB write under its data directory; when it fits, the state ends by itself;
- `survivesFull` (called first in `src/main.ts`) turns an unhandled rejection or exception of that kind into the same state instead of an exit; any other still ends the process.
State is in process memory, per data directory; a restart reads the disk afresh.
@@ -120,6 +120,8 @@
  "src/http/stripe_hook.ts setStripeHookCaps",
  "src/kernel/destroy_reach.ts unitsOf",
  "src/kernel/disk_reading.ts provideDiskForTest",
  "src/kernel/full_disk.ts forgetFull",
  "src/kernel/full_disk.ts provideWriterForTest",
  "src/kernel/log.ts clearErrors",
  "src/kernel/migrate.ts notAdditive",
  "src/kernel/protection.ts resetProtection",
@@ -94,7 +94,8 @@
import { activeEdgeToken, edgeIdentity } from "./edge.ts";
import { shieldState } from "../actions/shield.ts";
import { crawlRefused, familyOf, finishedStrangerDeep, isCrawler, isDeepPath, isShieldedPath, noteCrawler, noteDeepStranger, noteShielded, shieldRefused, takeCrawl, wasCrawler, wasDeepStranger, wasShielded } from "./crawl.ts";
import { liveResponse, readyResponse } from "./health.ts";
import { readOnlyRefusal } from "./read_only.ts";
import { isRead, readOnlyRefusal } from "./read_only.ts";
import { fullRefusal, isFullError, noteFull } from "../kernel/full_disk.ts";
import { running } from "../kernel/version.ts";
import { runGit } from "../git/env.ts";
import { noteRefusedInternal } from "../kernel/internal_refused.ts";
@@ -2362,6 +2363,12 @@
export interface Servers {
 */
function failure(app: App, error: unknown): Response {
      
        sentinelcrew
        1d ago
        outdated
      
      This failure() branch has no test. Also the terminal (SSH/TUI) writes never pass through a check like this one; gate them with the same state.
    
  if (error instanceof BodyTooLarge) return securityFloor(bodyTooLarge(error));
  if (isFullError(error)) {
    noteFull(app.config.dataDir);
    app.log.error("disk full: forge is read-only until space returns", { error: String(error) });
    const refusal = fullRefusal(app.config.dataDir, new Request("http://forge.invalid/"), false);
    if (refusal !== undefined) return securityFloor(refusal);
  }
  const refused = refusedByDatabase(error);
  if (refused !== undefined) {
    const shown = shownRefusal(refused);
@@ -2718,6 +2725,9 @@
export async function serve(app: App): Promise<Servers> {
        const refused = readOnlyRefusal(request);
        if (refused !== undefined) return securityFloor(refused);
      }
      // A full disk (#12144) refuses every write with one sentence until space returns.
      const full = fullRefusal(app.config.dataDir, request, isRead(request));
      if (full !== undefined) return securityFloor(full);
      // An app's hostname is answered before any route: `hello.<domain>` is
      // the app, whatever its path, and the forge's own routes never apply.
      const holding = await holdAppHost(app, request);
@@ -0,0 +1,173 @@
/**
 * A full disk (#12144). SQLITE_FULL or ENOSPC anywhere puts the forge in a
 * read-only state: every write answers with one sentence (what is full, how
 * much is free, what frees it), the needs-you line carries it, and the state
 * ends by itself, within seconds of space returning, when a small real write
 * under the data directory succeeds again. State is per data directory, in
 * process memory only: a restart reads the disk afresh. Nothing here throws.
 */
import { statfsSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { during } from "../perf/jobs.ts";
/** How often a full forge tries to write again. */
export const PROBE_EVERY_MS = 2_000;
/** What the probe writes: enough that the filesystem has to find room for it. */
const PROBE_BYTES = 64 * 1024;
const PROBE_FILE = ".full-probe";
interface Full {
  readonly since: string;
  timer: ReturnType<typeof setInterval>;
}
const states = new Map<string, Full>();
const owned = new Set<string>();
/**
 * Whether an error is a full disk: by `code`/`errno` only, or SQLite's own
 * exact message on a SQLite error. Never a search of free text: a message
 * that echoes a name, a path or a body must not put the forge read-only.
 */
export function isFullError(error: unknown): boolean {
      
        sentinelcrew
        1d ago
        outdated
      
      Free-text regex over any error message: user-echoed text containing 'ENOSPC' flips the forge read-only for everyone. Match code/errno (and SQLite's exact message) only.
    
  if (typeof error !== "object" || error === null) return false;
  const e = error as { code?: unknown; errno?: unknown; name?: unknown; message?: unknown };
  if (typeof e.code === "string") return e.code === "ENOSPC" || e.code.startsWith("SQLITE_FULL");
  if (e.errno === -28 || e.errno === 28) return true; // ENOSPC
  return e.name === "SQLiteError" && e.message === "database or disk is full";
}
function mib(bytes: number): string {
  return bytes >= 1024 * 1024 * 1024 ? `${(bytes / 1024 ** 3).toFixed(1)} GiB` : `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
}
let writer: ((dataDir: string) => boolean) | undefined;
/** A test's filesystem that fills and frees; `undefined` puts the real write back. */
export function provideWriterForTest(w: ((dataDir: string) => boolean) | undefined): void {
  writer = w;
}
/** Try one small write under `dataDir`; true when it fit. Never throws. */
function canWrite(dataDir: string): boolean {
  if (writer !== undefined) return writer(dataDir);
  const file = join(dataDir, PROBE_FILE);
  try {
    writeFileSync(file, new Uint8Array(PROBE_BYTES));
    rmSync(file, { force: true });
    return true;
  } catch {
    try {
      rmSync(file, { force: true });
    } catch {
      /* nothing to remove */
    }
    return false;
  }
}
/** Leave the full state when a write fits again; true when the state is gone. */
export function probeFull(dataDir: string): boolean {
  const state = states.get(dataDir);
  if (state === undefined) return true;
  if (!canWrite(dataDir)) return false;
  clearInterval(state.timer);
  states.delete(dataDir);
  return true;
}
/** Enter the read-only state for `dataDir` (a repeat changes nothing); start trying to leave it. */
export function noteFull(dataDir: string, everyMs = PROBE_EVERY_MS): void {
  if (states.has(dataDir)) return;
  const timer = setInterval(() => during("full-disk-probe", () => probeFull(dataDir)), everyMs);
  (timer as { unref?: () => void }).unref?.();
  states.set(dataDir, { since: new Date().toISOString(), timer });
}
/** When the forge entered the state, or undefined when it is not in it. */
export function fullSince(dataDir: string): string | undefined {
  return states.get(dataDir)?.since;
}
/**
 * The one sentence. Public by default (a refusal any visitor reads): what is
 * wrong and that it ends by itself. `forAdmin` adds the free figure and what
 * frees it, for the needs-you line only.
 */
export function fullSentence(dataDir: string, forAdmin = false): string {
  if (!forAdmin)
    return "The forge's disk is full, so it is read-only for now: reading works, and writing resumes by itself once space returns.";
  let free = "none";
  try {
    const fs = statfsSync(dataDir);
    free = mib(Number(fs.bavail) * Number(fs.bsize));
  } catch {
    /* the reading is a courtesy */
  }
  return `The disk under the forge's data is full (${free} free), so the forge is read-only: delete old snapshots, app data or previews, or grow the volume, and writes resume by themselves within seconds.`;
}
/** Whether `dataDir` is in the state. */
export function isFull(dataDir: string): boolean {
  return states.has(dataDir);
}
/** The 503 a write gets while full; undefined for a read or when not full. */
export function fullRefusal(dataDir: string, request: Request, isRead: boolean): Response | undefined {
  if (isRead || !states.has(dataDir)) return undefined;
  const message = fullSentence(dataDir);
  const json =
    new URL(request.url).pathname.startsWith("/api/") ||
    (request.headers.get("accept") ?? "").includes("application/json");
  const headers: Record<string, string> = { "retry-after": "5", "cache-control": "no-store" };
  if (json) {
    headers["content-type"] = "application/json; charset=utf-8";
    return new Response(JSON.stringify({ error: "disk_full", message, retryAfter: 5 }), { status: 503, headers });
  }
      
        sentinelcrew
        1d ago
        outdated
      
      survivesFull has no test: neither the survive branch nor the still-exits-1 branch. Make the exit injectable and test both.
    
  headers["content-type"] = "text/plain; charset=utf-8";
  return new Response(`${message}\n`, { status: 503, headers });
}
/** Enter the state for every data directory this process owns (the terminal names no path). */
export function noteFullHere(): void {
  for (const dir of owned) noteFull(dir);
}
/** Forget every state; a test's tidy-up. */
export function forgetFull(): void {
  for (const s of states.values()) clearInterval(s.timer);
  states.clear();
}
/** The process hooks `survivesFull` listens on and the exit it ends with; a test gives its own. */
export interface SurviveHooks {
  on: (event: "unhandledRejection" | "uncaughtException", handler: (error: unknown) => void) => void;
  exit: (code: number) => void;
  log: (line: string) => void;
}
/**
 * Make a full disk survivable in the process that owns `dataDir`: a rejection
 * or exception that is SQLITE_FULL or ENOSPC enters the read-only state and is
 * logged instead of ending the process; anything else ends it as before.
 */
export function survivesFull(
  dataDir: string,
  hooks: SurviveHooks = {
    on: (event, handler) => void process.on(event, handler),
    exit: (code) => process.exit(code),
    log: console.error,
  },
): void {
  owned.add(dataDir);
  const take = (kind: string) => (error: unknown) => {
    if (isFullError(error)) {
      noteFull(dataDir);
      hooks.log(`disk full (${kind}): forge is read-only until space returns: ${String(error)}`);
      return;
    }
    hooks.log(`${kind}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`);
    hooks.exit(1);
  };
  hooks.on("unhandledRejection", take("unhandled rejection"));
  hooks.on("uncaughtException", take("uncaught exception"));
}
@@ -10,6 +10,7 @@
import type { Database } from "bun:sqlite";
import { statSync } from "node:fs";
import { fullSentence, fullSince } from "./full_disk.ts";
import { readDisk, type DiskReading } from "./disk_reading.ts";
import { DISK_FLOOR, roomy } from "../spawn/disk_floor.ts";
import { dirname } from "node:path";
@@ -341,6 +342,8 @@
export function vitals(input: VitalsInput): Vitals {
    degraded.push({ cause, sentence, since, href });
  };
  const fullAt = fullSince(input.dataDir);
  if (fullAt !== undefined) need("disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
  const disk = (input.disk ?? readDisk)(input.dataDir);
  // Room to spare (#11940): at 100 GiB free no share-of-the-disk guard speaks,
  // the floor and the marks alike.
@@ -16,6 +16,7 @@
import { drainOnHandover, handover } from "./release/swap.ts";
import { parentUrlProblem } from "./core/parent_releases.ts";
import { releaseEnv } from "./release/env.ts";
import { createApp } from "./app.ts";
import { survivesFull } from "./kernel/full_disk.ts";
import { readOnlyFrom } from "./http/read_only.ts";
import { warmDisplays } from "./display/known.ts";
import { warmSearch } from "./search/warm.ts";
@@ -188,6 +189,9 @@
if (process.argv[2] === "--first-admin") {
const hostedAt = (process.env.FORGE_APP_URL ?? "").trim();
const hosted = hostedAt !== "";
// A full disk is survivable (#12144): boot, deploy and timer paths included.
survivesFull(dataDirOf(process.env));
// SSH is on by default: the terminal UI is a first-class surface, not an
// add-on. `FORGE_SSH_PORT=0` turns the listener off entirely.
// A read-only standby (FORGE_READ_ONLY=1, #9657) serves reads from a restored
@@ -9,6 +9,7 @@
import { PASSWORD_TOO_LONG } from "../../core/hash-gate.ts";
import { refusalText } from "../../http/refusal.ts";
import { fullSentence, isFullError, noteFullHere } from "../../kernel/full_disk.ts";
import { NOT_A_MEMBER } from "../../core/roles.ts";
import { AGENT_NOT_ON_CREW } from "../../actions/crew_limit_words.ts";
import type { Ctx, Screen } from "../app/app.ts";
@@ -1165,6 +1166,10 @@
export function errorText(code: string): string {
/** Anything thrown, as a string — the toast text of last resort. */
export function message(error: unknown): string {
  if (isFullError(error)) {
    noteFullHere();
    return fullSentence("");
  }
  const refused = refusalText(error);
  if (refused !== undefined) return refused;
  return error instanceof Error ? error.message : String(error);
@@ -34,6 +34,7 @@
import type { Terminal } from "./term/terminal.ts";
import type { TuiIdentity } from "./session.ts";
import { GUEST_RATE_WORDS, GuestRate } from "./ssh-guest-rate.ts";
import { parseTalkAgain, parseTalkCommand, serveTalk, serveTalkAgain } from "./ssh-talk.ts";
import { fullSentence, isFull } from "../kernel/full_disk.ts";
import { parseTellCommand, serveTell } from "./ssh-tell.ts";
const CRLF = "\r\n";
@@ -227,6 +228,12 @@
export function forgeSshSession(forge: Forge): (session: SshSession) => void {
      const identity = identityFor(forge, session);
      if (identity.user !== undefined)
        endOnSuspension(forge, session, identity.user);
      // A full disk (#12144): every exec but a read says the same sentence the web says.
      if (session.kind === "exec" && isFull(forge.config.dataDir) && (session.command === undefined || parseGitCommand(session.command)?.service !== "git-upload-pack")) {
        session.writeStderr(`forge: ${fullSentence(forge.config.dataDir)}${CRLF}`);
        session.exit(1);
        return;
      }
      if (session.kind === "shell") {
        // The login name is decorative for the forge — any name opens the
        // terminal UI — unless it names an app, in which case it opens
@@ -0,0 +1,221 @@
/**
 * A full disk (#12144): SQLITE_FULL and ENOSPC put the forge in one read-only
 * state, every write answers with one sentence, the needs-you line shows it,
 * and it leaves by itself when a write fits again. A sandbox cannot mount a
 * small filesystem, so the test's filesystem is the probe's writer, which
 * fills and frees; the error shapes are the real ones.
 */
import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import type { SshSession } from "../src/ssh/channels.ts";
import { forgeSshSession } from "../src/tui/ssh.ts";
import { message } from "../src/tui/screens/common.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { vitals } from "../src/kernel/vitals.ts";
import {
  forgetFull,
  fullSince,
  isFullError,
  isFull,
  noteFull,
  survivesFull,
  probeFull,
  provideWriterForTest,
} from "../src/kernel/full_disk.ts";
setDefaultTimeout(30_000);
describe("a full disk", () => {
  let dir = "";
  let app: App;
  let servers: Servers;
  let room = true;
  const post = () =>
    fetch(`${servers.url}/api/v1/repos`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: "{}",
    });
  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), "forge-full-"));
    app = createApp({ dataDir: dir, port: 0 });
    servers = await serve(app);
    provideWriterForTest(() => room);
  });
  afterAll(async () => {
    provideWriterForTest(undefined);
    forgetFull();
    servers.stop();
    app.close();
    await rm(dir, { recursive: true, force: true });
  });
  test("names the errors that mean a full disk", () => {
    expect(isFullError(Object.assign(new Error("database or disk is full"), { code: "SQLITE_FULL" }))).toBe(true);
    expect(isFullError(Object.assign(new Error("write"), { code: "ENOSPC" }))).toBe(true);
    expect(isFullError(new Error("SQLITE_BUSY"))).toBe(false);
    expect(isFullError(undefined)).toBe(false);
  });
  test("writes answer one sentence, reads still work, the need shows, and it recovers", async () => {
    expect((await post()).status).not.toBe(503);
    room = false;
    noteFull(dir);
    expect(fullSince(dir)).toBeDefined();
    const refused = await post();
    expect(refused.status).toBe(503);
    expect(refused.headers.get("retry-after")).toBe("5");
    const body = (await refused.json()) as { error: string; message: string };
    expect(body.error).toBe("disk_full");
    expect(body.message).toMatch(/disk is full.*read-only.*resumes by itself/);
    const form = await fetch(`${servers.url}/signin`, { method: "POST", body: "x=1" });
    expect(form.status).toBe(503);
    const said = await form.text();
    expect(said).toContain("disk is full");
    expect(said).not.toMatch(/MiB|GiB|delete/); // the public sentence gives no figure and no remedy
    expect((await fetch(`${servers.url}/`)).status).toBeLessThan(500);
    const v = vitals({ db: app.db, dataDir: dir, signals: app.signals });
    expect(v.degraded.some((d) => d.cause === "disk-full")).toBe(true);
    expect(v.needs.some((n) => /full.*MiB|full.*GiB|full.*none free/.test(n) && n.includes("delete"))).toBe(true);
    expect((await fetch(`${servers.url}/healthz`)).status).toBe(200); // liveness never claims the sentence
    expect(probeFull(dir)).toBe(false); // still no room
    expect(fullSince(dir)).toBeDefined();
    room = true; // space returns: the next probe (the forge's timer runs it every PROBE_EVERY_MS) ends the state
    expect(probeFull(dir)).toBe(true);
    expect(fullSince(dir)).toBeUndefined();
    expect((await post()).status).not.toBe(503);
    const after = vitals({ db: app.db, dataDir: dir, signals: app.signals });
    expect(after.degraded.some((d) => d.cause === "disk-full")).toBe(false);
  });
  test("a handler that throws ENOSPC enters the state and answers the sentence", async () => {
    const dir2 = await mkdtemp(join(tmpdir(), "forge-full-throw-"));
    const a = createApp({ dataDir: dir2, port: 0 });
    const s = await serve(a);
    try {
      expect(isFull(dir2)).toBe(false);
      const boom = Object.assign(new Error("write failed"), { code: "ENOSPC" });
      for (const k of ["query", "prepare", "run", "transaction"] as const)
        (a.db as unknown as Record<string, unknown>)[k] = () => {
          throw boom;
        };
      const res = await fetch(`${s.url}/api/v1/repos`, {
        method: "POST",
        headers: { "content-type": "application/json", authorization: "Bearer nothing" },
        body: "{}",
      });
      expect(res.status).toBe(503);
      expect(isFull(dir2)).toBe(true);
    } finally {
      forgetFull();
      s.stop();
      await rm(dir2, { recursive: true, force: true });
    }
  });
  test("the probe timer itself ends the state when a write fits", async () => {
    let fits = false;
    provideWriterForTest(() => fits);
    noteFull("/timer-dir", 20);
    await Bun.sleep(80); // real-timer: the 20 ms probe must have fired and found no room
    expect(isFull("/timer-dir")).toBe(true);
    fits = true;
    const until = Date.now() + 2000;
    while (isFull("/timer-dir")) {
      if (Date.now() > until) throw new Error("the probe timer never ended the state");
      await Bun.sleep(20); // real-timer: the 20 ms probe
    }
    expect(isFull("/timer-dir")).toBe(false);
    provideWriterForTest(() => room);
  });
  test("only a code, an errno or SQLite's own message is a full disk, never user text", () => {
    expect(isFullError(new Error('no repository named "ENOSPC"'))).toBe(false);
    expect(isFullError(new Error("title: SQLITE_FULL No space left on device"))).toBe(false);
    expect(isFullError("ENOSPC")).toBe(false);
    expect(isFullError(Object.assign(new Error("x"), { errno: -28 }))).toBe(true);
    const sqlite = Object.assign(new Error("database or disk is full"), { name: "SQLiteError" });
    expect(isFullError(sqlite)).toBe(true);
    expect(isFullError(new Error("database or disk is full"))).toBe(false);
  });
  test("survivesFull survives a full disk and still exits 1 on anything else", () => {
    const handlers = new Map<string, (e: unknown) => void>();
    const exits: number[] = [];
    const lines: string[] = [];
    survivesFull("/survive-dir", {
      on: (event, handler) => void handlers.set(event, handler),
      exit: (code) => void exits.push(code),
      log: (line) => void lines.push(line),
    });
    try {
      handlers.get("unhandledRejection")!(Object.assign(new Error("w"), { code: "ENOSPC" }));
      handlers.get("uncaughtException")!(Object.assign(new Error("w"), { code: "SQLITE_FULL" }));
      expect(exits).toEqual([]);
      expect(isFull("/survive-dir")).toBe(true);
      handlers.get("uncaughtException")!(new Error("a bug"));
      handlers.get("unhandledRejection")!("nope");
      expect(exits).toEqual([1, 1]);
    } finally {
      forgetFull();
    }
  });
  test("the terminal says the same sentence: git push and tell are refused, a fetch is not, a toast is generic", () => {
    noteFull(dir);
    try {
      const run = (command: string) => {
        let stderr = "";
        let exited: number | undefined;
        const session: SshSession = {
          key: { algo: "ssh-ed25519", blob: new Uint8Array(), fingerprint: "SHA256:none", line: "none" },
          username: "git",
          remoteAddress: "203.0.113.9",
          kind: "exec",
          command,
          term: undefined,
          env: {},
          cols: 0,
          rows: 0,
          widthPx: 0,
          heightPx: 0,
          ended: false,
          pending: 0,
          write() {},
          writeStderr(d: Uint8Array | string) {
            stderr += String(d);
          },
          onData() {},
          onResize() {},
          onEof() {},
          onClose() {},
          exit(code: number) {
            exited ??= code;
          },
          drain: async () => {},
        };
        forgeSshSession(app)(session);
        return { stderr, exited };
      };
      for (const c of ["git-receive-pack 'a/b.git'", "tell hello", "talk hello"]) {
        const r = run(c);
        expect(r.exited).toBe(1);
        expect(r.stderr).toContain("disk is full");
      }
      expect(run("git-upload-pack 'a/b.git'").stderr).not.toContain("disk is full");
      expect(message(Object.assign(new Error("w"), { code: "ENOSPC" }))).toContain("disk is full");
    } finally {
      forgetFull();
    }
  });
});
@@ -241,6 +241,7 @@
function jobs(): Record<string, () => unknown> {
 * The jobs not timed here, each with why. The release keeper's are off unless a cluster turns them on.
 */
const UNTIMED: Record<string, string> = {
  "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly",
  "reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host",
  lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up",
  "lane-catch-up": "a closure in startChecks, once on boot; same as lane",
		@@ -0,0 +1,11 @@
	1	# A full disk
	2
	3	When a write fails with `SQLITE_FULL` or `ENOSPC` (matched by `code`/`errno`, or SQLite's own exact message on a SQLite error, never by searching free text), the forge enters one read-only state (`src/kernel/full_disk.ts`):
	4
	5	- every web write (anything `isRead` in `src/http/read_only.ts` does not call a read, so pushes too) answers 503 with one public sentence: the disk is full, reading works, writing resumes by itself;
	6	- every terminal write is refused the same way: an SSH exec other than `git-upload-pack` (push, `tell`, `talk`, an app's verbs) prints the sentence and exits 1, and a TUI action that hits a full disk toasts it;
	7	- the needs-you line on `/system` (`disk-full`) adds the free figure and what frees it; the public sentence carries neither. `/healthz` stays about liveness and does not carry it;
	8	- every two seconds the forge tries a 64 KiB write under its data directory; when it fits, the state ends by itself;
	9	- `survivesFull` (called first in `src/main.ts`) turns an unhandled rejection or exception of that kind into the same state instead of an exit; any other still ends the process.
	10
	11	State is in process memory, per data directory; a restart reads the disk afresh.
		@@ -120,6 +120,8 @@
120	120	"src/http/stripe_hook.ts setStripeHookCaps",
121	121	"src/kernel/destroy_reach.ts unitsOf",
122	122	"src/kernel/disk_reading.ts provideDiskForTest",
	123	"src/kernel/full_disk.ts forgetFull",
	124	"src/kernel/full_disk.ts provideWriterForTest",
123	125	"src/kernel/log.ts clearErrors",
124	126	"src/kernel/migrate.ts notAdditive",
125	127	"src/kernel/protection.ts resetProtection",
		@@ -94,7 +94,8 @@ import { activeEdgeToken, edgeIdentity } from "./edge.ts";
94	94	import { shieldState } from "../actions/shield.ts";
95	95	import { crawlRefused, familyOf, finishedStrangerDeep, isCrawler, isDeepPath, isShieldedPath, noteCrawler, noteDeepStranger, noteShielded, shieldRefused, takeCrawl, wasCrawler, wasDeepStranger, wasShielded } from "./crawl.ts";
96	96	import { liveResponse, readyResponse } from "./health.ts";
97		import { readOnlyRefusal } from "./read_only.ts";
	97	import { isRead, readOnlyRefusal } from "./read_only.ts";
	98	import { fullRefusal, isFullError, noteFull } from "../kernel/full_disk.ts";
98	99	import { running } from "../kernel/version.ts";
99	100	import { runGit } from "../git/env.ts";
100	101	import { noteRefusedInternal } from "../kernel/internal_refused.ts";
		@@ -2362,6 +2363,12 @@ export interface Servers {
2362	2363	*/
2363	2364	function failure(app: App, error: unknown): Response {
		sentinelcrew 1d ago outdated This failure() branch has no test. Also the terminal (SSH/TUI) writes never pass through a check like this one; gate them with the same state.
2364	2365	if (error instanceof BodyTooLarge) return securityFloor(bodyTooLarge(error));
	2366	if (isFullError(error)) {
	2367	noteFull(app.config.dataDir);
	2368	app.log.error("disk full: forge is read-only until space returns", { error: String(error) });
	2369	const refusal = fullRefusal(app.config.dataDir, new Request("http://forge.invalid/"), false);
	2370	if (refusal !== undefined) return securityFloor(refusal);
	2371	}
2365	2372	const refused = refusedByDatabase(error);
2366	2373	if (refused !== undefined) {
2367	2374	const shown = shownRefusal(refused);
		@@ -2718,6 +2725,9 @@ export async function serve(app: App): Promise<Servers> {
2718	2725	const refused = readOnlyRefusal(request);
2719	2726	if (refused !== undefined) return securityFloor(refused);
2720	2727	}
	2728	// A full disk (#12144) refuses every write with one sentence until space returns.
	2729	const full = fullRefusal(app.config.dataDir, request, isRead(request));
	2730	if (full !== undefined) return securityFloor(full);
2721	2731	// An app's hostname is answered before any route: `hello.<domain>` is
2722	2732	// the app, whatever its path, and the forge's own routes never apply.
2723	2733	const holding = await holdAppHost(app, request);
		@@ -0,0 +1,173 @@
	1	/**
	2	* A full disk (#12144). SQLITE_FULL or ENOSPC anywhere puts the forge in a
	3	* read-only state: every write answers with one sentence (what is full, how
	4	* much is free, what frees it), the needs-you line carries it, and the state
	5	* ends by itself, within seconds of space returning, when a small real write
	6	* under the data directory succeeds again. State is per data directory, in
	7	* process memory only: a restart reads the disk afresh. Nothing here throws.
	8	*/
	9
	10	import { statfsSync, writeFileSync, rmSync } from "node:fs";
	11	import { join } from "node:path";
	12	import { during } from "../perf/jobs.ts";
	13
	14	/** How often a full forge tries to write again. */
	15	export const PROBE_EVERY_MS = 2_000;
	16	/** What the probe writes: enough that the filesystem has to find room for it. */
	17	const PROBE_BYTES = 64 * 1024;
	18	const PROBE_FILE = ".full-probe";
	19
	20	interface Full {
	21	readonly since: string;
	22	timer: ReturnType<typeof setInterval>;
	23	}
	24	const states = new Map<string, Full>();
	25	const owned = new Set<string>();
	26
	27	/**
	28	* Whether an error is a full disk: by `code`/`errno` only, or SQLite's own
	29	* exact message on a SQLite error. Never a search of free text: a message
	30	* that echoes a name, a path or a body must not put the forge read-only.
	31	*/
	32	export function isFullError(error: unknown): boolean {
		sentinelcrew 1d ago outdated Free-text regex over any error message: user-echoed text containing 'ENOSPC' flips the forge read-only for everyone. Match code/errno (and SQLite's exact message) only.
	33	if (typeof error !== "object" \|\| error === null) return false;
	34	const e = error as { code?: unknown; errno?: unknown; name?: unknown; message?: unknown };
	35	if (typeof e.code === "string") return e.code === "ENOSPC" \|\| e.code.startsWith("SQLITE_FULL");
	36	if (e.errno === -28 \|\| e.errno === 28) return true; // ENOSPC
	37	return e.name === "SQLiteError" && e.message === "database or disk is full";
	38	}
	39
	40	function mib(bytes: number): string {
	41	return bytes >= 1024 * 1024 * 1024 ? `${(bytes / 1024 3).toFixed(1)} GiB` : `${(bytes / 1024 2).toFixed(1)} MiB`;
	42	}
	43
	44	let writer: ((dataDir: string) => boolean) \| undefined;
	45	/** A test's filesystem that fills and frees; `undefined` puts the real write back. */
	46	export function provideWriterForTest(w: ((dataDir: string) => boolean) \| undefined): void {
	47	writer = w;
	48	}
	49
	50	/** Try one small write under `dataDir`; true when it fit. Never throws. */
	51	function canWrite(dataDir: string): boolean {
	52	if (writer !== undefined) return writer(dataDir);
	53	const file = join(dataDir, PROBE_FILE);
	54	try {
	55	writeFileSync(file, new Uint8Array(PROBE_BYTES));
	56	rmSync(file, { force: true });
	57	return true;
	58	} catch {
	59	try {
	60	rmSync(file, { force: true });
	61	} catch {
	62	/* nothing to remove */
	63	}
	64	return false;
	65	}
	66	}
	67
	68	/** Leave the full state when a write fits again; true when the state is gone. */
	69	export function probeFull(dataDir: string): boolean {
	70	const state = states.get(dataDir);
	71	if (state === undefined) return true;
	72	if (!canWrite(dataDir)) return false;
	73	clearInterval(state.timer);
	74	states.delete(dataDir);
	75	return true;
	76	}
	77
	78	/** Enter the read-only state for `dataDir` (a repeat changes nothing); start trying to leave it. */
	79	export function noteFull(dataDir: string, everyMs = PROBE_EVERY_MS): void {
	80	if (states.has(dataDir)) return;
	81	const timer = setInterval(() => during("full-disk-probe", () => probeFull(dataDir)), everyMs);
	82	(timer as { unref?: () => void }).unref?.();
	83	states.set(dataDir, { since: new Date().toISOString(), timer });
	84	}
	85
	86	/** When the forge entered the state, or undefined when it is not in it. */
	87	export function fullSince(dataDir: string): string \| undefined {
	88	return states.get(dataDir)?.since;
	89	}
	90
	91	/**
	92	* The one sentence. Public by default (a refusal any visitor reads): what is
	93	* wrong and that it ends by itself. `forAdmin` adds the free figure and what
	94	* frees it, for the needs-you line only.
	95	*/
	96	export function fullSentence(dataDir: string, forAdmin = false): string {
	97	if (!forAdmin)
	98	return "The forge's disk is full, so it is read-only for now: reading works, and writing resumes by itself once space returns.";
	99	let free = "none";
	100	try {
	101	const fs = statfsSync(dataDir);
	102	free = mib(Number(fs.bavail) * Number(fs.bsize));
	103	} catch {
	104	/* the reading is a courtesy */
	105	}
	106	return `The disk under the forge's data is full (${free} free), so the forge is read-only: delete old snapshots, app data or previews, or grow the volume, and writes resume by themselves within seconds.`;
	107	}
	108
	109	/** Whether `dataDir` is in the state. */
	110	export function isFull(dataDir: string): boolean {
	111	return states.has(dataDir);
	112	}
	113
	114	/** The 503 a write gets while full; undefined for a read or when not full. */
	115	export function fullRefusal(dataDir: string, request: Request, isRead: boolean): Response \| undefined {
	116	if (isRead \|\| !states.has(dataDir)) return undefined;
	117	const message = fullSentence(dataDir);
	118	const json =
	119	new URL(request.url).pathname.startsWith("/api/") \|\|
	120	(request.headers.get("accept") ?? "").includes("application/json");
	121	const headers: Record<string, string> = { "retry-after": "5", "cache-control": "no-store" };
	122	if (json) {
	123	headers["content-type"] = "application/json; charset=utf-8";
	124	return new Response(JSON.stringify({ error: "disk_full", message, retryAfter: 5 }), { status: 503, headers });
	125	}
		sentinelcrew 1d ago outdated `survivesFull` has no test: neither the survive branch nor the still-exits-1 branch. Make the exit injectable and test both.
	126	headers["content-type"] = "text/plain; charset=utf-8";
	127	return new Response(`${message}\n`, { status: 503, headers });
	128	}
	129
	130	/** Enter the state for every data directory this process owns (the terminal names no path). */
	131	export function noteFullHere(): void {
	132	for (const dir of owned) noteFull(dir);
	133	}
	134
	135	/** Forget every state; a test's tidy-up. */
	136	export function forgetFull(): void {
	137	for (const s of states.values()) clearInterval(s.timer);
	138	states.clear();
	139	}
	140
	141	/** The process hooks `survivesFull` listens on and the exit it ends with; a test gives its own. */
	142	export interface SurviveHooks {
	143	on: (event: "unhandledRejection" \| "uncaughtException", handler: (error: unknown) => void) => void;
	144	exit: (code: number) => void;
	145	log: (line: string) => void;
	146	}
	147
	148	/**
	149	* Make a full disk survivable in the process that owns `dataDir`: a rejection
	150	* or exception that is SQLITE_FULL or ENOSPC enters the read-only state and is
	151	* logged instead of ending the process; anything else ends it as before.
	152	*/
	153	export function survivesFull(
	154	dataDir: string,
	155	hooks: SurviveHooks = {
	156	on: (event, handler) => void process.on(event, handler),
	157	exit: (code) => process.exit(code),
	158	log: console.error,
	159	},
	160	): void {
	161	owned.add(dataDir);
	162	const take = (kind: string) => (error: unknown) => {
	163	if (isFullError(error)) {
	164	noteFull(dataDir);
	165	hooks.log(`disk full (${kind}): forge is read-only until space returns: ${String(error)}`);
	166	return;
	167	}
	168	hooks.log(`${kind}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`);
	169	hooks.exit(1);
	170	};
	171	hooks.on("unhandledRejection", take("unhandled rejection"));
	172	hooks.on("uncaughtException", take("uncaught exception"));
	173	}
		@@ -10,6 +10,7 @@
10	10
11	11	import type { Database } from "bun:sqlite";
12	12	import { statSync } from "node:fs";
	13	import { fullSentence, fullSince } from "./full_disk.ts";
13	14	import { readDisk, type DiskReading } from "./disk_reading.ts";
14	15	import { DISK_FLOOR, roomy } from "../spawn/disk_floor.ts";
15	16	import { dirname } from "node:path";
		@@ -341,6 +342,8 @@ export function vitals(input: VitalsInput): Vitals {
341	342	degraded.push({ cause, sentence, since, href });
342	343	};
343	344
	345	const fullAt = fullSince(input.dataDir);
	346	if (fullAt !== undefined) need("disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
344	347	const disk = (input.disk ?? readDisk)(input.dataDir);
345	348	// Room to spare (#11940): at 100 GiB free no share-of-the-disk guard speaks,
346	349	// the floor and the marks alike.
		@@ -16,6 +16,7 @@ import { drainOnHandover, handover } from "./release/swap.ts";
16	16	import { parentUrlProblem } from "./core/parent_releases.ts";
17	17	import { releaseEnv } from "./release/env.ts";
18	18	import { createApp } from "./app.ts";
	19	import { survivesFull } from "./kernel/full_disk.ts";
19	20	import { readOnlyFrom } from "./http/read_only.ts";
20	21	import { warmDisplays } from "./display/known.ts";
21	22	import { warmSearch } from "./search/warm.ts";
		@@ -188,6 +189,9 @@ if (process.argv[2] === "--first-admin") {
188	189	const hostedAt = (process.env.FORGE_APP_URL ?? "").trim();
189	190	const hosted = hostedAt !== "";
190	191
	192	// A full disk is survivable (#12144): boot, deploy and timer paths included.
	193	survivesFull(dataDirOf(process.env));
	194
191	195	// SSH is on by default: the terminal UI is a first-class surface, not an
192	196	// add-on. `FORGE_SSH_PORT=0` turns the listener off entirely.
193	197	// A read-only standby (FORGE_READ_ONLY=1, #9657) serves reads from a restored
		@@ -9,6 +9,7 @@
9	9
10	10	import { PASSWORD_TOO_LONG } from "../../core/hash-gate.ts";
11	11	import { refusalText } from "../../http/refusal.ts";
	12	import { fullSentence, isFullError, noteFullHere } from "../../kernel/full_disk.ts";
12	13	import { NOT_A_MEMBER } from "../../core/roles.ts";
13	14	import { AGENT_NOT_ON_CREW } from "../../actions/crew_limit_words.ts";
14	15	import type { Ctx, Screen } from "../app/app.ts";
		@@ -1165,6 +1166,10 @@ export function errorText(code: string): string {
1165	1166
1166	1167	/** Anything thrown, as a string — the toast text of last resort. */
1167	1168	export function message(error: unknown): string {
	1169	if (isFullError(error)) {
	1170	noteFullHere();
	1171	return fullSentence("");
	1172	}
1168	1173	const refused = refusalText(error);
1169	1174	if (refused !== undefined) return refused;
1170	1175	return error instanceof Error ? error.message : String(error);
		@@ -34,6 +34,7 @@ import type { Terminal } from "./term/terminal.ts";
34	34	import type { TuiIdentity } from "./session.ts";
35	35	import { GUEST_RATE_WORDS, GuestRate } from "./ssh-guest-rate.ts";
36	36	import { parseTalkAgain, parseTalkCommand, serveTalk, serveTalkAgain } from "./ssh-talk.ts";
	37	import { fullSentence, isFull } from "../kernel/full_disk.ts";
37	38	import { parseTellCommand, serveTell } from "./ssh-tell.ts";
38	39
39	40	const CRLF = "\r\n";
		@@ -227,6 +228,12 @@ export function forgeSshSession(forge: Forge): (session: SshSession) => void {
227	228	const identity = identityFor(forge, session);
228	229	if (identity.user !== undefined)
229	230	endOnSuspension(forge, session, identity.user);
	231	// A full disk (#12144): every exec but a read says the same sentence the web says.
	232	if (session.kind === "exec" && isFull(forge.config.dataDir) && (session.command === undefined \|\| parseGitCommand(session.command)?.service !== "git-upload-pack")) {
	233	session.writeStderr(`forge: ${fullSentence(forge.config.dataDir)}${CRLF}`);
	234	session.exit(1);
	235	return;
	236	}
230	237	if (session.kind === "shell") {
231	238	// The login name is decorative for the forge — any name opens the
232	239	// terminal UI — unless it names an app, in which case it opens
		@@ -0,0 +1,221 @@
	1	/**
	2	* A full disk (#12144): SQLITE_FULL and ENOSPC put the forge in one read-only
	3	* state, every write answers with one sentence, the needs-you line shows it,
	4	* and it leaves by itself when a write fits again. A sandbox cannot mount a
	5	* small filesystem, so the test's filesystem is the probe's writer, which
	6	* fills and frees; the error shapes are the real ones.
	7	*/
	8	import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from "bun:test";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import type { SshSession } from "../src/ssh/channels.ts";
	14	import { forgeSshSession } from "../src/tui/ssh.ts";
	15	import { message } from "../src/tui/screens/common.ts";
	16	import { serve, type Servers } from "../src/http/server.ts";
	17	import { vitals } from "../src/kernel/vitals.ts";
	18	import {
	19	forgetFull,
	20	fullSince,
	21	isFullError,
	22	isFull,
	23	noteFull,
	24	survivesFull,
	25	probeFull,
	26	provideWriterForTest,
	27	} from "../src/kernel/full_disk.ts";
	28
	29	setDefaultTimeout(30_000);
	30
	31	describe("a full disk", () => {
	32	let dir = "";
	33	let app: App;
	34	let servers: Servers;
	35	let room = true;
	36	const post = () =>
	37	fetch(`${servers.url}/api/v1/repos`, {
	38	method: "POST",
	39	headers: { "content-type": "application/json" },
	40	body: "{}",
	41	});
	42
	43	beforeAll(async () => {
	44	dir = await mkdtemp(join(tmpdir(), "forge-full-"));
	45	app = createApp({ dataDir: dir, port: 0 });
	46	servers = await serve(app);
	47	provideWriterForTest(() => room);
	48	});
	49	afterAll(async () => {
	50	provideWriterForTest(undefined);
	51	forgetFull();
	52	servers.stop();
	53	app.close();
	54	await rm(dir, { recursive: true, force: true });
	55	});
	56
	57	test("names the errors that mean a full disk", () => {
	58	expect(isFullError(Object.assign(new Error("database or disk is full"), { code: "SQLITE_FULL" }))).toBe(true);
	59	expect(isFullError(Object.assign(new Error("write"), { code: "ENOSPC" }))).toBe(true);
	60	expect(isFullError(new Error("SQLITE_BUSY"))).toBe(false);
	61	expect(isFullError(undefined)).toBe(false);
	62	});
	63
	64	test("writes answer one sentence, reads still work, the need shows, and it recovers", async () => {
	65	expect((await post()).status).not.toBe(503);
	66	room = false;
	67	noteFull(dir);
	68	expect(fullSince(dir)).toBeDefined();
	69
	70	const refused = await post();
	71	expect(refused.status).toBe(503);
	72	expect(refused.headers.get("retry-after")).toBe("5");
	73	const body = (await refused.json()) as { error: string; message: string };
	74	expect(body.error).toBe("disk_full");
	75	expect(body.message).toMatch(/disk is full.read-only.resumes by itself/);
	76
	77	const form = await fetch(`${servers.url}/signin`, { method: "POST", body: "x=1" });
	78	expect(form.status).toBe(503);
	79	const said = await form.text();
	80	expect(said).toContain("disk is full");
	81	expect(said).not.toMatch(/MiB\|GiB\|delete/); // the public sentence gives no figure and no remedy
	82
	83	expect((await fetch(`${servers.url}/`)).status).toBeLessThan(500);
	84
	85	const v = vitals({ db: app.db, dataDir: dir, signals: app.signals });
	86	expect(v.degraded.some((d) => d.cause === "disk-full")).toBe(true);
	87	expect(v.needs.some((n) => /full.MiB\|full.GiB\|full.*none free/.test(n) && n.includes("delete"))).toBe(true);
	88	expect((await fetch(`${servers.url}/healthz`)).status).toBe(200); // liveness never claims the sentence
	89
	90	expect(probeFull(dir)).toBe(false); // still no room
	91	expect(fullSince(dir)).toBeDefined();
	92
	93	room = true; // space returns: the next probe (the forge's timer runs it every PROBE_EVERY_MS) ends the state
	94	expect(probeFull(dir)).toBe(true);
	95	expect(fullSince(dir)).toBeUndefined();
	96	expect((await post()).status).not.toBe(503);
	97	const after = vitals({ db: app.db, dataDir: dir, signals: app.signals });
	98	expect(after.degraded.some((d) => d.cause === "disk-full")).toBe(false);
	99	});
	100
	101	test("a handler that throws ENOSPC enters the state and answers the sentence", async () => {
	102	const dir2 = await mkdtemp(join(tmpdir(), "forge-full-throw-"));
	103	const a = createApp({ dataDir: dir2, port: 0 });
	104	const s = await serve(a);
	105	try {
	106	expect(isFull(dir2)).toBe(false);
	107	const boom = Object.assign(new Error("write failed"), { code: "ENOSPC" });
	108	for (const k of ["query", "prepare", "run", "transaction"] as const)
	109	(a.db as unknown as Record<string, unknown>)[k] = () => {
	110	throw boom;
	111	};
	112	const res = await fetch(`${s.url}/api/v1/repos`, {
	113	method: "POST",
	114	headers: { "content-type": "application/json", authorization: "Bearer nothing" },
	115	body: "{}",
	116	});
	117	expect(res.status).toBe(503);
	118	expect(isFull(dir2)).toBe(true);
	119	} finally {
	120	forgetFull();
	121	s.stop();
	122	await rm(dir2, { recursive: true, force: true });
	123	}
	124	});
	125
	126	test("the probe timer itself ends the state when a write fits", async () => {
	127	let fits = false;
	128	provideWriterForTest(() => fits);
	129	noteFull("/timer-dir", 20);
	130	await Bun.sleep(80); // real-timer: the 20 ms probe must have fired and found no room
	131	expect(isFull("/timer-dir")).toBe(true);
	132	fits = true;
	133	const until = Date.now() + 2000;
	134	while (isFull("/timer-dir")) {
	135	if (Date.now() > until) throw new Error("the probe timer never ended the state");
	136	await Bun.sleep(20); // real-timer: the 20 ms probe
	137	}
	138	expect(isFull("/timer-dir")).toBe(false);
	139	provideWriterForTest(() => room);
	140	});
	141
	142	test("only a code, an errno or SQLite's own message is a full disk, never user text", () => {
	143	expect(isFullError(new Error('no repository named "ENOSPC"'))).toBe(false);
	144	expect(isFullError(new Error("title: SQLITE_FULL No space left on device"))).toBe(false);
	145	expect(isFullError("ENOSPC")).toBe(false);
	146	expect(isFullError(Object.assign(new Error("x"), { errno: -28 }))).toBe(true);
	147	const sqlite = Object.assign(new Error("database or disk is full"), { name: "SQLiteError" });
	148	expect(isFullError(sqlite)).toBe(true);
	149	expect(isFullError(new Error("database or disk is full"))).toBe(false);
	150	});
	151
	152	test("survivesFull survives a full disk and still exits 1 on anything else", () => {
	153	const handlers = new Map<string, (e: unknown) => void>();
	154	const exits: number[] = [];
	155	const lines: string[] = [];
	156	survivesFull("/survive-dir", {
	157	on: (event, handler) => void handlers.set(event, handler),
	158	exit: (code) => void exits.push(code),
	159	log: (line) => void lines.push(line),
	160	});
	161	try {
	162	handlers.get("unhandledRejection")!(Object.assign(new Error("w"), { code: "ENOSPC" }));
	163	handlers.get("uncaughtException")!(Object.assign(new Error("w"), { code: "SQLITE_FULL" }));
	164	expect(exits).toEqual([]);
	165	expect(isFull("/survive-dir")).toBe(true);
	166	handlers.get("uncaughtException")!(new Error("a bug"));
	167	handlers.get("unhandledRejection")!("nope");
	168	expect(exits).toEqual([1, 1]);
	169	} finally {
	170	forgetFull();
	171	}
	172	});
	173
	174	test("the terminal says the same sentence: git push and tell are refused, a fetch is not, a toast is generic", () => {
	175	noteFull(dir);
	176	try {
	177	const run = (command: string) => {
	178	let stderr = "";
	179	let exited: number \| undefined;
	180	const session: SshSession = {
	181	key: { algo: "ssh-ed25519", blob: new Uint8Array(), fingerprint: "SHA256:none", line: "none" },
	182	username: "git",
	183	remoteAddress: "203.0.113.9",
	184	kind: "exec",
	185	command,
	186	term: undefined,
	187	env: {},
	188	cols: 0,
	189	rows: 0,
	190	widthPx: 0,
	191	heightPx: 0,
	192	ended: false,
	193	pending: 0,
	194	write() {},
	195	writeStderr(d: Uint8Array \| string) {
	196	stderr += String(d);
	197	},
	198	onData() {},
	199	onResize() {},
	200	onEof() {},
	201	onClose() {},
	202	exit(code: number) {
	203	exited ??= code;
	204	},
	205	drain: async () => {},
	206	};
	207	forgeSshSession(app)(session);
	208	return { stderr, exited };
	209	};
	210	for (const c of ["git-receive-pack 'a/b.git'", "tell hello", "talk hello"]) {
	211	const r = run(c);
	212	expect(r.exited).toBe(1);
	213	expect(r.stderr).toContain("disk is full");
	214	}
	215	expect(run("git-upload-pack 'a/b.git'").stderr).not.toContain("disk is full");
	216	expect(message(Object.assign(new Error("w"), { code: "ENOSPC" }))).toContain("disk is full");
	217	} finally {
	218	forgetFull();
	219	}
	220	});
	221	});
		@@ -241,6 +241,7 @@ function jobs(): Record<string, () => unknown> {
241	241	* The jobs not timed here, each with why. The release keeper's are off unless a cluster turns them on.
242	242	*/
243	243	const UNTIMED: Record<string, string> = {
	244	"full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly",
244	245	"reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host",
245	246	lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up",
246	247	"lane-catch-up": "a closure in startChecks, once on boot; same as lane",