174c945
@@ -128,11 +128,29 @@
Steps 1–3 as files: `scripts/box-install.sh` (user, `/var/lib/forge`,
killed process or a reboot brings it back). Then the lines of
[RESTORE.md](RESTORE.md) with `<data-dir>` = `/var/lib/forge` as the `forge`
user, and `systemctl enable --now forge`. Nothing points at the box; compare
repository and app counts with production, kill the process and reboot to see
it return. Restoring reads the bucket by the card's lines only — no platform
service is involved. `test/forge-box-unit.test.ts` parses the unit.
repository and app counts with production, then run the drill below. Restoring
reads the bucket by the card's lines only — no platform service is involved.
`test/forge-box-unit.test.ts` parses the unit (`Restart=always`, `RestartSec`
between 1 and 10 seconds, `WantedBy=multi-user.target`).
Done on a machine: the drill itself (restore, counts, reboot) is the owner's to run.
### The drill: it comes back by itself
As root on the box, from the checkout (`/opt/forge`):
1. `scripts/box-drill.sh kill` sends SIGKILL to the forge's main process,
   waits for a new process to answer `/` with a 2xx or 3xx, and prints
   `back: pid N answers http://127.0.0.1:3000/ 3.1s after the kill`. It fails
   if `/` does not answer first, or not within 120 s (`DRILL_LIMIT=`).
2. `reboot`. Log back in and run `scripts/box-drill.sh boot`: it fails unless
   the unit is `enabled` and active and `/` answers, then prints when the
   forge came up, in seconds after the boot (`ActiveEnterTimestampMonotonic`).
   Nobody started it by hand, so that line is the proof.
Both take the URL as a second argument when the forge listens elsewhere
(default `http://127.0.0.1:${PORT:-3000}/`). Post both outputs on the issue
the drill closes. `test/box-drill.test.ts` drives the script against stubs.
## The vxrail is the box
The forge's production machine, the vxrail, becomes this box itself, so no
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Prove the forge box comes back by itself (docs/EXTRACT.md, "The box beside production").
#
#   scripts/box-drill.sh kill [url]   as root: SIGKILL the forge, wait for `/` to answer
#                                     from a new process, print how long it took.
#   scripts/box-drill.sh boot [url]   after `reboot`: the unit is enabled, active, came up
#                                     on its own this boot, and `/` answers; print when.
#
# url defaults to http://127.0.0.1:${PORT:-3000}/. It only reads systemd and the URL;
# the one thing it changes is the forge process it kills.
set -euo pipefail
unit=forge
mode="${1:-}"
url="${2:-http://127.0.0.1:${PORT:-3000}/}"
limit="${DRILL_LIMIT:-120}" # seconds to wait for `/` before failing
answers() {
  local code
  code="$(curl -s -o /dev/null -m 2 -w '%{http_code}' "$url" || true)"
  [[ "$code" =~ ^[23][0-9][0-9]$ ]]
}
main_pid() { systemctl show -p MainPID --value "$unit"; }
case "$mode" in
  kill)
    [ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; }
    answers || { echo "FAIL: $url does not answer before the kill" >&2; exit 1; }
    before="$(main_pid)"
    [ "$before" != 0 ] || { echo "FAIL: $unit has no main process" >&2; exit 1; }
    start="$(date +%s.%N)"
    kill -KILL "$before"
    echo "killed $unit pid $before at $(date -u +%FT%TZ)"
    while :; do
      now="$(main_pid)"
      if [ "$now" != 0 ] && [ "$now" != "$before" ] && answers; then break; fi
      if awk -v s="$start" -v n="$(date +%s.%N)" -v l="$limit" 'BEGIN{exit !(n-s>l)}'; then
        echo "FAIL: $url did not answer within ${limit}s of the kill" >&2
        exit 1
      fi
      sleep 0.2
    done
    took="$(awk -v s="$start" -v n="$(date +%s.%N)" 'BEGIN{printf "%.1f", n-s}')"
    echo "back: pid $now answers $url ${took}s after the kill"
    ;;
  boot)
    enabled="$(systemctl is-enabled "$unit" || true)"
    [ "$enabled" = enabled ] || { echo "FAIL: $unit is $enabled at boot, not enabled" >&2; exit 1; }
    systemctl is-active --quiet "$unit" || { echo "FAIL: $unit is not active" >&2; exit 1; }
    waited=0
    until answers; do
      [ "$waited" -lt "$limit" ] || { echo "FAIL: $url does not answer" >&2; exit 1; }
      sleep 1; waited=$((waited + 1))
    done
    # ActiveEnterTimestampMonotonic counts microseconds since this boot.
    up="$(systemctl show -p ActiveEnterTimestampMonotonic --value "$unit")"
    echo "booted $(uptime -s); $unit enabled, active $(awk -v u="$up" 'BEGIN{printf "%.1f", u/1e6}')s after boot, pid $(main_pid) answers $url"
    ;;
  *)
    echo "usage: $0 kill|boot [url]" >&2
    exit 2
    ;;
esac
@@ -0,0 +1,101 @@
/** scripts/box-drill.sh, driven against stub `systemctl`, `curl`, `id` and `uptime` on PATH. */
import { afterAll, expect, test } from "bun:test";
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
const script = join(import.meta.dir, "..", "scripts/box-drill.sh");
const dirs: string[] = [];
afterAll(() => {
  for (const d of dirs) rmSync(d, { recursive: true, force: true });
});
/** A bin dir whose systemctl reports the pid in $STATE/pid, and a new one once that process is gone. */
function stubs(opts: { enabled?: string; curlCode?: string } = {}) {
  const dir = mkdtempSync(join(tmpdir(), "box-drill-"));
  dirs.push(dir);
  const bin = join(dir, "bin");
  mkdirSync(bin);
  const put = (name: string, body: string) => {
    writeFileSync(join(bin, name), `#!/usr/bin/env bash\n${body}\n`);
    chmodSync(join(bin, name), 0o755);
  };
  put(
    "systemctl",
    `case "$1" in
  is-enabled) echo ${opts.enabled ?? "enabled"} ;;
  is-active) exit 0 ;;
  show)
    case "$3" in
      ActiveEnterTimestampMonotonic) echo 12345678 ;;
      MainPID)
        p="$(cat "${dir}/pid")"
        # gone, or a zombie nobody has reaped yet: either way, systemd would have restarted it
        if [ "$p" != 4242 ] && { ! kill -0 "$p" 2>/dev/null || grep -q ') Z ' "/proc/$p/stat" 2>/dev/null; }; then echo 4242 > "${dir}/pid"; p=4242; fi
        echo "$p" ;;
    esac ;;
esac`,
  );
  put("curl", `echo -n ${opts.curlCode ?? "200"}`);
  put("id", "echo 0");
  put("uptime", "echo 2026-11-12 10:00:00");
  return { dir, env: { ...process.env, PORT: undefined, PATH: `${bin}:${process.env.PATH}`, DRILL_LIMIT: "3" } };
}
function run(args: string[], env: Record<string, string | undefined>) {
  const r = Bun.spawnSync(["bash", script, ...args], { env, stdout: "pipe", stderr: "pipe" });
  return { code: r.exitCode, out: r.stdout.toString(), err: r.stderr.toString() };
}
test("the drill is executable and says how to use it", () => {
  expect(statSync(script).mode & 0o111).toBeGreaterThan(0);
  const r = run([], process.env);
  expect(r.code).toBe(2);
  expect(r.err).toContain("usage:");
});
test("kill: kills the forge, waits for a new process to answer /, and prints how long", () => {
  // An orphan, so whatever reaps orphans here clears it once it dies.
  const pid = Number(
    Bun.spawnSync(["sh", "-c", "sleep 60 >/dev/null 2>&1 & echo $!"]).stdout.toString().trim(),
  );
  const s = stubs();
  writeFileSync(join(s.dir, "pid"), String(pid));
  const r = run(["kill", "http://127.0.0.1:9/"], s.env);
  expect(r.err).toBe("");
  expect(r.code).toBe(0);
  expect(r.out).toContain(`killed forge pid ${pid}`);
  expect(r.out).toMatch(/back: pid 4242 answers http:\/\/127\.0\.0\.1:9\/ \d+\.\ds after the kill/);
  const stat = (() => {
    try {
      return readFileSync(`/proc/${pid}/stat`, "utf8");
    } catch {
      return "gone";
    }
  })();
  expect(stat === "gone" || stat.includes(") Z ")).toBe(true);
});
test("kill: fails when / never answers", () => {
  const s = stubs({ curlCode: "000" });
  writeFileSync(join(s.dir, "pid"), "4242");
  const r = run(["kill"], s.env);
  expect(r.code).toBe(1);
  expect(r.err).toContain("does not answer before the kill");
});
test("boot: the unit is enabled, active, and / answers; prints seconds after boot", () => {
  const s = stubs();
  writeFileSync(join(s.dir, "pid"), "4242");
  const r = run(["boot"], s.env);
  expect(r.code).toBe(0);
  expect(r.out).toContain("forge enabled, active 12.3s after boot, pid 4242 answers http://127.0.0.1:3000/");
});
test("boot: a unit not enabled at boot fails the drill", () => {
  const s = stubs({ enabled: "disabled" });
  writeFileSync(join(s.dir, "pid"), "4242");
  const r = run(["boot"], s.env);
  expect(r.code).toBe(1);
  expect(r.err).toContain("forge is disabled at boot");
});
@@ -37,6 +37,9 @@
test("the unit runs the forge from /opt/forge and always restarts", () => {
test("the unit starts at boot and never gives up restarting", () => {
  expect(unit.get("Install")!.get("WantedBy")).toEqual(["multi-user.target"]);
  const restartSec = Number(svc.get("RestartSec")?.[0]);
  expect(restartSec).toBeGreaterThanOrEqual(1);
  expect(restartSec).toBeLessThanOrEqual(10);
  expect(unit.get("Unit")!.get("StartLimitIntervalSec")).toEqual(["0"]);
});
		@@ -128,11 +128,29 @@ Steps 1–3 as files: `scripts/box-install.sh` (user, `/var/lib/forge`,
128	128	killed process or a reboot brings it back). Then the lines of
129	129	[RESTORE.md](RESTORE.md) with `<data-dir>` = `/var/lib/forge` as the `forge`
130	130	user, and `systemctl enable --now forge`. Nothing points at the box; compare
131		repository and app counts with production, kill the process and reboot to see
132		it return. Restoring reads the bucket by the card's lines only — no platform
133		service is involved. `test/forge-box-unit.test.ts` parses the unit.
	131	repository and app counts with production, then run the drill below. Restoring
	132	reads the bucket by the card's lines only — no platform service is involved.
	133	`test/forge-box-unit.test.ts` parses the unit (`Restart=always`, `RestartSec`
	134	between 1 and 10 seconds, `WantedBy=multi-user.target`).
134	135	Done on a machine: the drill itself (restore, counts, reboot) is the owner's to run.
135	136
	137	### The drill: it comes back by itself
	138
	139	As root on the box, from the checkout (`/opt/forge`):
	140
	141	1. `scripts/box-drill.sh kill` sends SIGKILL to the forge's main process,
	142	waits for a new process to answer `/` with a 2xx or 3xx, and prints
	143	`back: pid N answers http://127.0.0.1:3000/ 3.1s after the kill`. It fails
	144	if `/` does not answer first, or not within 120 s (`DRILL_LIMIT=`).
	145	2. `reboot`. Log back in and run `scripts/box-drill.sh boot`: it fails unless
	146	the unit is `enabled` and active and `/` answers, then prints when the
	147	forge came up, in seconds after the boot (`ActiveEnterTimestampMonotonic`).
	148	Nobody started it by hand, so that line is the proof.
	149
	150	Both take the URL as a second argument when the forge listens elsewhere
	151	(default `http://127.0.0.1:${PORT:-3000}/`). Post both outputs on the issue
	152	the drill closes. `test/box-drill.test.ts` drives the script against stubs.
	153
136	154	## The vxrail is the box
137	155
138	156	The forge's production machine, the vxrail, becomes this box itself, so no
		@@ -0,0 +1,64 @@
	1	#!/usr/bin/env bash
	2	# Prove the forge box comes back by itself (docs/EXTRACT.md, "The box beside production").
	3	#
	4	# scripts/box-drill.sh kill [url] as root: SIGKILL the forge, wait for `/` to answer
	5	# from a new process, print how long it took.
	6	# scripts/box-drill.sh boot [url] after `reboot`: the unit is enabled, active, came up
	7	# on its own this boot, and `/` answers; print when.
	8	#
	9	# url defaults to http://127.0.0.1:${PORT:-3000}/. It only reads systemd and the URL;
	10	# the one thing it changes is the forge process it kills.
	11	set -euo pipefail
	12
	13	unit=forge
	14	mode="${1:-}"
	15	url="${2:-http://127.0.0.1:${PORT:-3000}/}"
	16	limit="${DRILL_LIMIT:-120}" # seconds to wait for `/` before failing
	17
	18	answers() {
	19	local code
	20	code="$(curl -s -o /dev/null -m 2 -w '%{http_code}' "$url" \|\| true)"
	21	[[ "$code" =~ ^[23][0-9][0-9]$ ]]
	22	}
	23
	24	main_pid() { systemctl show -p MainPID --value "$unit"; }
	25
	26	case "$mode" in
	27	kill)
	28	[ "$(id -u)" = 0 ] \|\| { echo "run as root" >&2; exit 1; }
	29	answers \|\| { echo "FAIL: $url does not answer before the kill" >&2; exit 1; }
	30	before="$(main_pid)"
	31	[ "$before" != 0 ] \|\| { echo "FAIL: $unit has no main process" >&2; exit 1; }
	32	start="$(date +%s.%N)"
	33	kill -KILL "$before"
	34	echo "killed $unit pid $before at $(date -u +%FT%TZ)"
	35	while :; do
	36	now="$(main_pid)"
	37	if [ "$now" != 0 ] && [ "$now" != "$before" ] && answers; then break; fi
	38	if awk -v s="$start" -v n="$(date +%s.%N)" -v l="$limit" 'BEGIN{exit !(n-s>l)}'; then
	39	echo "FAIL: $url did not answer within ${limit}s of the kill" >&2
	40	exit 1
	41	fi
	42	sleep 0.2
	43	done
	44	took="$(awk -v s="$start" -v n="$(date +%s.%N)" 'BEGIN{printf "%.1f", n-s}')"
	45	echo "back: pid $now answers $url ${took}s after the kill"
	46	;;
	47	boot)
	48	enabled="$(systemctl is-enabled "$unit" \|\| true)"
	49	[ "$enabled" = enabled ] \|\| { echo "FAIL: $unit is $enabled at boot, not enabled" >&2; exit 1; }
	50	systemctl is-active --quiet "$unit" \|\| { echo "FAIL: $unit is not active" >&2; exit 1; }
	51	waited=0
	52	until answers; do
	53	[ "$waited" -lt "$limit" ] \|\| { echo "FAIL: $url does not answer" >&2; exit 1; }
	54	sleep 1; waited=$((waited + 1))
	55	done
	56	# ActiveEnterTimestampMonotonic counts microseconds since this boot.
	57	up="$(systemctl show -p ActiveEnterTimestampMonotonic --value "$unit")"
	58	echo "booted $(uptime -s); $unit enabled, active $(awk -v u="$up" 'BEGIN{printf "%.1f", u/1e6}')s after boot, pid $(main_pid) answers $url"
	59	;;
	60	*)
	61	echo "usage: $0 kill\|boot [url]" >&2
	62	exit 2
	63	;;
	64	esac
		@@ -0,0 +1,101 @@
	1	/** scripts/box-drill.sh, driven against stub `systemctl`, `curl`, `id` and `uptime` on PATH. */
	2	import { afterAll, expect, test } from "bun:test";
	3	import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
	4	import { tmpdir } from "node:os";
	5	import { join } from "node:path";
	6
	7	const script = join(import.meta.dir, "..", "scripts/box-drill.sh");
	8	const dirs: string[] = [];
	9	afterAll(() => {
	10	for (const d of dirs) rmSync(d, { recursive: true, force: true });
	11	});
	12
	13	/** A bin dir whose systemctl reports the pid in $STATE/pid, and a new one once that process is gone. */
	14	function stubs(opts: { enabled?: string; curlCode?: string } = {}) {
	15	const dir = mkdtempSync(join(tmpdir(), "box-drill-"));
	16	dirs.push(dir);
	17	const bin = join(dir, "bin");
	18	mkdirSync(bin);
	19	const put = (name: string, body: string) => {
	20	writeFileSync(join(bin, name), `#!/usr/bin/env bash\n${body}\n`);
	21	chmodSync(join(bin, name), 0o755);
	22	};
	23	put(
	24	"systemctl",
	25	`case "$1" in
	26	is-enabled) echo ${opts.enabled ?? "enabled"} ;;
	27	is-active) exit 0 ;;
	28	show)
	29	case "$3" in
	30	ActiveEnterTimestampMonotonic) echo 12345678 ;;
	31	MainPID)
	32	p="$(cat "${dir}/pid")"
	33	# gone, or a zombie nobody has reaped yet: either way, systemd would have restarted it
	34	if [ "$p" != 4242 ] && { ! kill -0 "$p" 2>/dev/null \|\| grep -q ') Z ' "/proc/$p/stat" 2>/dev/null; }; then echo 4242 > "${dir}/pid"; p=4242; fi
	35	echo "$p" ;;
	36	esac ;;
	37	esac`,
	38	);
	39	put("curl", `echo -n ${opts.curlCode ?? "200"}`);
	40	put("id", "echo 0");
	41	put("uptime", "echo 2026-11-12 10:00:00");
	42	return { dir, env: { ...process.env, PORT: undefined, PATH: `${bin}:${process.env.PATH}`, DRILL_LIMIT: "3" } };
	43	}
	44
	45	function run(args: string[], env: Record<string, string \| undefined>) {
	46	const r = Bun.spawnSync(["bash", script, ...args], { env, stdout: "pipe", stderr: "pipe" });
	47	return { code: r.exitCode, out: r.stdout.toString(), err: r.stderr.toString() };
	48	}
	49
	50	test("the drill is executable and says how to use it", () => {
	51	expect(statSync(script).mode & 0o111).toBeGreaterThan(0);
	52	const r = run([], process.env);
	53	expect(r.code).toBe(2);
	54	expect(r.err).toContain("usage:");
	55	});
	56
	57	test("kill: kills the forge, waits for a new process to answer /, and prints how long", () => {
	58	// An orphan, so whatever reaps orphans here clears it once it dies.
	59	const pid = Number(
	60	Bun.spawnSync(["sh", "-c", "sleep 60 >/dev/null 2>&1 & echo $!"]).stdout.toString().trim(),
	61	);
	62	const s = stubs();
	63	writeFileSync(join(s.dir, "pid"), String(pid));
	64	const r = run(["kill", "http://127.0.0.1:9/"], s.env);
	65	expect(r.err).toBe("");
	66	expect(r.code).toBe(0);
	67	expect(r.out).toContain(`killed forge pid ${pid}`);
	68	expect(r.out).toMatch(/back: pid 4242 answers http:\/\/127\.0\.0\.1:9\/ \d+\.\ds after the kill/);
	69	const stat = (() => {
	70	try {
	71	return readFileSync(`/proc/${pid}/stat`, "utf8");
	72	} catch {
	73	return "gone";
	74	}
	75	})();
	76	expect(stat === "gone" \|\| stat.includes(") Z ")).toBe(true);
	77	});
	78
	79	test("kill: fails when / never answers", () => {
	80	const s = stubs({ curlCode: "000" });
	81	writeFileSync(join(s.dir, "pid"), "4242");
	82	const r = run(["kill"], s.env);
	83	expect(r.code).toBe(1);
	84	expect(r.err).toContain("does not answer before the kill");
	85	});
	86
	87	test("boot: the unit is enabled, active, and / answers; prints seconds after boot", () => {
	88	const s = stubs();
	89	writeFileSync(join(s.dir, "pid"), "4242");
	90	const r = run(["boot"], s.env);
	91	expect(r.code).toBe(0);
	92	expect(r.out).toContain("forge enabled, active 12.3s after boot, pid 4242 answers http://127.0.0.1:3000/");
	93	});
	94
	95	test("boot: a unit not enabled at boot fails the drill", () => {
	96	const s = stubs({ enabled: "disabled" });
	97	writeFileSync(join(s.dir, "pid"), "4242");
	98	const r = run(["boot"], s.env);
	99	expect(r.code).toBe(1);
	100	expect(r.err).toContain("forge is disabled at boot");
	101	});
		@@ -37,6 +37,9 @@ test("the unit runs the forge from /opt/forge and always restarts", () => {
37	37
38	38	test("the unit starts at boot and never gives up restarting", () => {
39	39	expect(unit.get("Install")!.get("WantedBy")).toEqual(["multi-user.target"]);
	40	const restartSec = Number(svc.get("RestartSec")?.[0]);
	41	expect(restartSec).toBeGreaterThanOrEqual(1);
	42	expect(restartSec).toBeLessThanOrEqual(10);
40	43	expect(unit.get("Unit")!.get("StartLimitIntervalSec")).toEqual(["0"]);
41	44	});
42	45