trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

test/crew-sandbox-real.test.ts runs in the gate, or the gate says why it cannot #10715

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/10709-test-crew-sandbox-real-test-ts-runs-in-t → main 2 commits fork

  • Merged. scout/10709-test-crew-sandbox-real-test-ts-runs-in-t was merged into main as 8d9ac63.

Review

Sign in to review this change.
4 files changed +116 −65
scripts/gate.ts +6 −1
@@ -33,7 +33,12 @@ export type Exclusion = {
3333 };
3434
3535 export const EXCLUDED: readonly Exclusion[] = [
36 { file: "test/crew-sandbox-real.test.ts", reason: "landed unlisted and red locally; the builder admits it", issue: 1238 },
36 {
37 file: "test/crew-sandbox-real.test.ts",
38 reason:
39 "no ten green admit runs yet: the builder's sandbox refuses user namespaces, so the file only skips by name there and has passed nowhere",
40 issue: 10709,
41 },
3742 { file: "test/apps-api.test.ts", reason: "not yet admitted", issue: 261 },
3843 {
3944 file: "test/apps-open-host.test.ts",
test/crew-sandbox-escape.test.ts +2 −28
@@ -44,6 +44,7 @@ import { join } from "node:path";
4444 import { rgPath } from "@vscode/ripgrep";
4545 import { claudeSettings } from "../src/crew/claude.ts";
4646 import { sandboxAvailable, sandboxPolicy } from "../src/crew/sandbox.ts";
47 import { runtimeSettings, withoutParentProxy } from "./helpers/srt.ts";
4748
4849 const probe = await sandboxAvailable();
4950 const srt = join(import.meta.dir, "../node_modules/.bin/srt");
@@ -77,21 +78,6 @@ const WRITTEN = "escape-written.txt";
7778 /** Hosts off the allowlist that name the outside server. */
7879 const OFF_LIST = ["127.0.0.1", "localhost"] as const;
7980
80 /**
81 * The environment `srt` is launched with, minus any proxy setting. `srt`
82 * reads `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` of its own process as its
83 * *parent proxy* and chains every allowed request through it. On the check
84 * host that is the cell's egress proxy, which refuses `127.0.0.1:<port>`,
85 * so the allowed forge host:port got "Unable to connect" (#10602). A crew
86 * run's real sandbox is not launched behind one, so the test is not either.
87 */
88 function withoutParentProxy(env: NodeJS.ProcessEnv): Record<string, string> {
89 const kept: Record<string, string> = {};
90 for (const [key, value] of Object.entries(env))
91 if (value !== undefined && !/^(http|https|all|ftp|no)_proxy$/i.test(key)) kept[key] = value;
92 return kept;
93 }
94
9581 function routed(route: Route, path: string, pid: number): string {
9682 if (route === "by its path") return path;
9783 return `${route.replace("<pid>", String(pid))}${path}`;
@@ -270,19 +256,7 @@ console.log(JSON.stringify({
270256 });
271257 const real = claudeSettings(policy).sandbox as Record<string, any>;
272258 const settings = join(dataDir, "srt-settings.json");
273 writeFileSync(
274 settings,
275 JSON.stringify({
276 ripgrep: { command: rgPath },
277 network: { deniedDomains: [], ...real.network },
278 filesystem: {
279 denyRead: [],
280 denyWrite: [],
281 ...real.filesystem,
282 allowWrite: [work, ...(real.filesystem?.allowWrite ?? [])],
283 },
284 }),
285 );
259 writeFileSync(settings, JSON.stringify(runtimeSettings(real, work)));
286260 const proc = Bun.spawn([srt, "--settings", settings, "-c", `'${process.execPath}' probe.ts`], {
287261 cwd: work,
288262 env: withoutParentProxy(process.env),
test/crew-sandbox-real.test.ts +63 −36
@@ -8,59 +8,45 @@
88 * shape, so dropping `filesystem.denyRead` or flipping `enabled` fails here.
99 * A sandbox cannot be made inside a sandbox, so where one cannot be made (a
1010 * crew run, a container that refuses user namespaces) the test is skipped,
11 * with the reason on the console.
11 * naming itself and the reason on the console; the forge's check runs it.
1212 *
1313 * Outside localhost is checked twice: directly (the sandbox has a network of
1414 * its own, so it must fail) and through the sandbox's proxy, which is the
1515 * only way out. The policy allows loopback as the forge's `host:port` alone
16 * (`sandboxPolicy`), so the proxy refuses every other port.
16 * (`sandboxPolicy`), so the proxy refuses every other port. Through the
17 * proxy is a raw request to it, not `fetch`: srt puts 127.0.0.1 in NO_PROXY,
18 * which a client honours by going direct. `srt` is launched without proxy
19 * variables (`withoutParentProxy`), as the escape test is: it would chain
20 * through them, and the check host's egress proxy refuses loopback (#10709).
1721 *
1822 * Where a sandbox can be made, the runtime must be there too: a missing
19 * `rg` or `srt` fails the test rather than skipping it.
23 * `srt` or pinned `rg` (`@vscode/ripgrep`; the check host has none on its
24 * PATH) fails the test rather than skipping it.
2025 */
2126 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2227 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
2328 import { mkdtemp, rm } from "node:fs/promises";
2429 import { tmpdir } from "node:os";
2530 import { join } from "node:path";
31 import { rgPath } from "@vscode/ripgrep";
2632 import { claudeSettings } from "../src/crew/claude.ts";
2733 import { sandboxAvailable, sandboxPolicy } from "../src/crew/sandbox.ts";
34 import { runtimeSettings, withoutParentProxy } from "./helpers/srt.ts";
2835
2936 const probe = await sandboxAvailable();
3037 const srt = join(import.meta.dir, "../node_modules/.bin/srt");
3138 const nestedReason = !probe.ok
3239 ? `no sandbox can be made here (nested in a crew sandbox or user namespaces refused): ${probe.reason}`
3340 : undefined;
34 if (nestedReason !== undefined) console.warn(`skipped: ${nestedReason}`);
35 else if (Bun.which("rg") === null || !existsSync(srt))
41 if (nestedReason !== undefined)
42 console.warn(`skipped: test/crew-sandbox-real.test.ts: ${nestedReason}`);
43 else if (!existsSync(rgPath) || !existsSync(srt))
3644 throw new Error(
37 "a sandbox can be made here but its runtime is missing: install ripgrep (rg) and run bun install",
45 "a sandbox can be made here but its runtime is missing (srt or @vscode/ripgrep's rg): run bun install",
3846 );
3947
4048 const CANARY = "the-canary-must-not-be-read";
4149
42 /**
43 * Claude Code's sandbox settings as the runtime reads them. The gap from what
44 * Claude Code receives: the runtime wants `deniedDomains`, `denyRead` and
45 * `denyWrite` present (empty here), and `allowWrite` is the work directory
46 * alone, where Claude Code adds its own write grants. Network and denyRead
47 * are the generated ones, untouched.
48 */
49 function runtimeSettings(
50 sandbox: Record<string, any>,
51 workDir: string,
52 ): Record<string, unknown> {
53 return {
54 network: { deniedDomains: [], ...sandbox.network },
55 filesystem: {
56 denyRead: [],
57 denyWrite: [],
58 ...sandbox.filesystem,
59 allowWrite: [workDir, ...(sandbox.filesystem?.allowWrite ?? [])],
60 },
61 };
62 }
63
6450 describe.skipIf(nestedReason !== undefined)("a real sandboxed shell", () => {
6551 let dataDir: string;
6652 let work: string;
@@ -115,10 +101,39 @@ describe.skipIf(nestedReason !== undefined)("a real sandboxed shell", () => {
115101 `
116102 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
117103 const own = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("own") });
118 const get = async (port: number, proxy?: string) => {
119 try { return await (await fetch("http://127.0.0.1:" + port + "/", { proxy })).text(); }
104 const direct = async (port: number) => {
105 try { return await (await fetch("http://127.0.0.1:" + port + "/")).text(); }
120106 catch { return "refused"; }
121107 };
108 // A raw request in the absolute form a proxy takes, so NO_PROXY cannot send
109 // it direct: the body on a 2xx, "refused" on any other answer or none.
110 const viaProxy = (port: number) =>
111 new Promise<string>((resolve) => {
112 const proxy = process.env.HTTP_PROXY;
113 if (!proxy) return resolve("refused");
114 const p = new URL(proxy);
115 const auth = p.username
116 ? "proxy-authorization: Basic " +
117 btoa(decodeURIComponent(p.username) + ":" + decodeURIComponent(p.password)) + "\\r\\n"
118 : "";
119 let bytes = "";
120 Bun.connect({
121 hostname: p.hostname,
122 port: Number(p.port),
123 socket: {
124 open(socket) {
125 socket.write("GET http://127.0.0.1:" + port + "/ HTTP/1.1\\r\\nhost: 127.0.0.1:" + port + "\\r\\n" + auth + "connection: close\\r\\n\\r\\n");
126 },
127 data(_socket, chunk) { bytes += new TextDecoder().decode(chunk); },
128 close() {
129 const at = bytes.indexOf("\\r\\n\\r\\n");
130 resolve(at >= 0 && / 2\\d\\d/.test(bytes.split("\\r\\n")[0] ?? "") ? bytes.slice(at + 4) : "refused");
131 },
132 error() { resolve("refused"); },
133 connectError() { resolve("refused"); },
134 },
135 }).catch(() => resolve("refused"));
136 });
122137 const leaks: string[] = [];
123138 const walk = (path: string) => {
124139 let stat;
@@ -131,10 +146,10 @@ const walk = (path: string) => {
131146 };
132147 for (const path of ${JSON.stringify(denied)}) walk(path);
133148 console.log(JSON.stringify({
134 own: await get(own.port),
135 outside: await get(${outside.port}),
136 forge: await get(${forge.port}, process.env.HTTP_PROXY),
137 viaProxy: await get(${outside.port}, process.env.HTTP_PROXY),
149 own: await direct(own.port),
150 outside: await direct(${outside.port}),
151 forge: await viaProxy(${forge.port}),
152 viaProxy: await viaProxy(${outside.port}),
138153 leaks,
139154 }));
140155 own.stop(true);
@@ -147,7 +162,13 @@ own.stop(true);
147162 writeFileSync(settings, JSON.stringify(runtimeSettings(real, work)));
148163 const proc = Bun.spawn(
149164 [srt, "--settings", settings, "-c", `'${process.execPath}' probe.ts`],
150 { cwd: work, stdin: "ignore", stdout: "pipe", stderr: "pipe" },
165 {
166 cwd: work,
167 env: withoutParentProxy(process.env),
168 stdin: "ignore",
169 stdout: "pipe",
170 stderr: "pipe",
171 },
151172 );
152173 const [out, err, code] = await Promise.all([
153174 new Response(proc.stdout).text(),
@@ -227,7 +248,13 @@ console.log(JSON.stringify({ wrote, read, other }));
227248 writeFileSync(settings, JSON.stringify(runtimeSettings(real, work)));
228249 const proc = Bun.spawn(
229250 [srt, "--settings", settings, "-c", `'${process.execPath}' probe.ts`],
230 { cwd: work, stdin: "ignore", stdout: "pipe", stderr: "pipe" },
251 {
252 cwd: work,
253 env: withoutParentProxy(process.env),
254 stdin: "ignore",
255 stdout: "pipe",
256 stderr: "pipe",
257 },
231258 );
232259 const [out, err, code] = await Promise.all([
233260 new Response(proc.stdout).text(),
test/helpers/srt.ts added +45
@@ -0,0 +1,45 @@
1 /**
2 * Launching Claude Code's sandbox runtime (`srt`) from a test the way the
3 * check host allows: the shapes `crew-sandbox-escape.test.ts` and
4 * `crew-sandbox-real.test.ts` share (#10602, #10709).
5 */
6 import { rgPath } from "@vscode/ripgrep";
7
8 /**
9 * The environment `srt` is launched with, minus any proxy setting. `srt`
10 * reads `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` of its own process as its
11 * *parent proxy* and chains every allowed request through it. On the check
12 * host that is the cell's egress proxy, which refuses `127.0.0.1:<port>`,
13 * so the allowed forge host:port got "Unable to connect" (#10602). A crew
14 * run's real sandbox is not launched behind one, so the test is not either.
15 */
16 export function withoutParentProxy(env: NodeJS.ProcessEnv): Record<string, string> {
17 const kept: Record<string, string> = {};
18 for (const [key, value] of Object.entries(env))
19 if (value !== undefined && !/^(http|https|all|ftp|no)_proxy$/i.test(key)) kept[key] = value;
20 return kept;
21 }
22
23 /**
24 * Claude Code's sandbox settings as the runtime reads them. The gap from what
25 * Claude Code receives: the runtime wants `deniedDomains`, `denyRead` and
26 * `denyWrite` present (empty here), `allowWrite` is the work directory
27 * alone, where Claude Code adds its own write grants, and `ripgrep` is the
28 * pinned `@vscode/ripgrep` binary: the check host has no `rg` on its PATH.
29 * Network and denyRead are the generated ones, untouched.
30 */
31 export function runtimeSettings(
32 sandbox: Record<string, any>,
33 workDir: string,
34 ): Record<string, unknown> {
35 return {
36 ripgrep: { command: rgPath },
37 network: { deniedDomains: [], ...sandbox.network },
38 filesystem: {
39 denyRead: [],
40 denyWrite: [],
41 ...sandbox.filesystem,
42 allowWrite: [workDir, ...(sandbox.filesystem?.allowWrite ?? [])],
43 },
44 };
45 }

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs