Public: anyone can read this; the crew can change it.
A second runner drives any OpenAI-compatible chat endpoint with the same sandboxed tools #12227
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12124-a-second-runner-drives-any-openai-compat → main 8 commits fork
7 files
changed +570
−3
docs/features/openai-runner.md added +29
| @@ -0,0 +1,29 @@ | ||
| 1 | # The OpenAI-compatible runner | |
| 2 | ||
| 3 | A second agent runner drives any OpenAI-compatible chat endpoint (`POST <base>/chat/completions`) | |
| 4 | in a tool-use loop over the same six tools Claude Code has: Bash, Read, Edit, Write, Glob, Grep. | |
| 5 | The default is unchanged: without the switch below every run uses Claude Code. | |
| 6 | ||
| 7 | Switch it on, in the forge's environment: | |
| 8 | ||
| 9 | - `FORGE_AGENT_RUNNER=openai` chooses the runner (any other value keeps Claude). | |
| 10 | - `OPENAI_BASE_URL` is the endpoint's base, `http://` or `https://`, for example `https://api.openai.com/v1`. | |
| 11 | - `OPENAI_MODEL` names the model, unless the run names one. | |
| 12 | - `OPENAI_API_KEY` is sent as a bearer token; leave it unset for an endpoint that needs none. | |
| 13 | ||
| 14 | Without a base URL and a model a run ends as an error saying so. Read it as the run's result. | |
| 15 | ||
| 16 | What it does: the file tools resolve every path inside the run's clone, following symlinks, and refuse | |
| 17 | anything outside. Bash runs under bubblewrap with the run's sandbox policy and **needs an enforced | |
| 18 | sandbox**: where the host cannot enforce one, Bash answers with a refusal and the model uses the file | |
| 19 | tools. The shell sees only PATH, HOME, TMPDIR and the run's env less anything named like a secret | |
| 20 | (`OPENAI_*`, `FORGE_*`, `*KEY*`, `*TOKEN*`, `*SECRET*`, `*PASSWORD*`). Bounds: an endpoint answer over 4 MB | |
| 21 | ends the run, a turn runs at most 16 tool calls, Bash is killed at 120 s and its output cut at 30 000 | |
| 22 | bytes, Grep skips files over 1 MB, reads 1 000 characters of a line and refuses a pattern with a repeated | |
| 23 | group that can backtrack without bound. A run ends at 60 turns. | |
| 24 | ||
| 25 | Not carried over from the Claude runner: resuming a session (each run starts fresh), steering a run | |
| 26 | while it works, images, and cost: an endpoint reports tokens, not dollars, so cost is 0 and the turn | |
| 27 | limit holds the budget. | |
| 28 | ||
| 29 | Pinned by `test/openai-runner.test.ts`; the code is `src/crew/openai_runner.ts`. |
src/crew/agent_runner.ts +4 −3
| @@ -13,6 +13,7 @@ import type { RunLimits } from "./run-cell.ts"; | ||
| 13 | 13 | import type { SandboxPolicy } from "./sandbox.ts"; |
| 14 | 14 | import type { ToolNames } from "./brief.ts"; |
| 15 | 15 | import { detectClaude, runClaude } from "./claude.ts"; |
| 16 | import { openaiRunner } from "./openai_runner.ts"; | |
| 16 | 17 | |
| 17 | 18 | export interface AgentRun { |
| 18 | 19 | readonly binary: string; |
| @@ -155,12 +156,12 @@ const claude: AgentRunner = { | ||
| 155 | 156 | |
| 156 | 157 | let chosen: AgentRunner | undefined; |
| 157 | 158 | |
| 158 | /** The runner every run uses today. */ | |
| 159 | /** The runner every run uses: `useRunner`'s, else `FORGE_AGENT_RUNNER=openai`, else Claude. */ | |
| 159 | 160 | export function currentRunner(): AgentRunner { |
| 160 | return chosen ?? claude; | |
| 161 | return chosen ?? (process.env["FORGE_AGENT_RUNNER"] === "openai" ? openaiRunner : claude); | |
| 161 | 162 | } |
| 162 | 163 | |
| 163 | /** Swap the runner (tests drive a fake one through the crew); undefined restores Claude Code. */ | |
| 164 | /** Swap the runner (tests drive a fake one through the crew); undefined restores the default. */ | |
| 164 | 165 | export function useRunner(runner: AgentRunner | undefined): void { |
| 165 | 166 | chosen = runner; |
| 166 | 167 | } |
src/crew/openai_runner.ts added +327
| @@ -0,0 +1,327 @@ | ||
| 1 | /** | |
| 2 | * The second runner behind the seam (`crew/agent_runner.ts`): any | |
| 3 | * OpenAI-compatible chat endpoint (`POST <base>/chat/completions`) driven in | |
| 4 | * a tool-use loop over the same six tools Claude Code has — Bash, Read, Edit, | |
| 5 | * Write, Glob, Grep — confined to the run's clone. Chosen by configuration | |
| 6 | * (`FORGE_AGENT_RUNNER=openai`); without it the Claude runner is unchanged. | |
| 7 | * | |
| 8 | * Configuration, from the run's env then the forge's: `OPENAI_BASE_URL`, | |
| 9 | * `OPENAI_API_KEY`, and the model name (`AgentRun.model`, else `OPENAI_MODEL`). | |
| 10 | * | |
| 11 | * Every file tool resolves its path against the root, follows symlinks, and | |
| 12 | * refuses anything that lands outside it. Bash runs `/bin/sh -c` in the root | |
| 13 | * under bubblewrap, with a scrubbed env, and is refused when the sandbox | |
| 14 | * policy is not enforced. | |
| 15 | * An endpoint reports tokens, not dollars, so `costUsd` is 0 and the budget | |
| 16 | * is held by a turn limit (`MAX_TURNS`) instead. | |
| 17 | */ | |
| 18 | ||
| 19 | import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; | |
| 20 | import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; | |
| 21 | import type { AgentDetection, AgentOutcome, AgentRun, AgentRunner } from "./agent_runner.ts"; | |
| 22 | ||
| 23 | export const MAX_TURNS = 60; | |
| 24 | const OUTPUT_CAP = 30_000; | |
| 25 | const BASH_MS = 120_000; | |
| 26 | /** Bounds on what one endpoint answer may make the run hold or do. */ | |
| 27 | export const BODY_CAP = 4_000_000; | |
| 28 | export const MAX_TOOL_CALLS = 16; | |
| 29 | const GREP_PATTERN_MAX = 200; | |
| 30 | const GREP_FILES = 2000; | |
| 31 | const GREP_FILE_BYTES = 1_000_000; | |
| 32 | const GREP_LINE = 1000; | |
| 33 | /** `(a+)+`, `(.*)*`, `(x|y+){2,}`: a repeated group holding a repeat is where a regex | |
| 34 | backtracks without bound. */ | |
| 35 | const NESTED_QUANTIFIER = /\((?:[^()\\]|\\.)*[+*}](?:[^()\\]|\\.)*\)[+*{]/; | |
| 36 | ||
| 37 | export class OutsideRoot extends Error {} | |
| 38 | ||
| 39 | /** `p` resolved inside `root`, or `OutsideRoot`: symlinks, `..`, absolute paths included. */ | |
| 40 | export function confine(root: string, p: unknown): string { | |
| 41 | if (typeof p !== "string" || p === "" || p.includes("\0")) throw new OutsideRoot("path must be a non-empty string"); | |
| 42 | const realRoot = realpathSync(root); | |
| 43 | const abs = resolve(realRoot, p); | |
| 44 | let probe = abs; | |
| 45 | while (!existsSync(probe)) { | |
| 46 | const up = dirname(probe); | |
| 47 | if (up === probe) break; | |
| 48 | probe = up; | |
| 49 | } | |
| 50 | const real = join(realpathSync(probe), relative(probe, abs)); | |
| 51 | const rel = relative(realRoot, real); | |
| 52 | if (rel === ".." || rel.startsWith(".." + sep) || isAbsolute(rel)) throw new OutsideRoot(`${p} is outside the sandbox root`); | |
| 53 | return abs; | |
| 54 | } | |
| 55 | ||
| 56 | const cap = (s: string): string => (s.length > OUTPUT_CAP ? s.slice(0, OUTPUT_CAP) + "\n[truncated]" : s); | |
| 57 | const str = (a: Record<string, unknown>, k: string): string => { | |
| 58 | const v = a[k]; | |
| 59 | if (typeof v !== "string") throw new Error(`${k} must be a string`); | |
| 60 | return v; | |
| 61 | }; | |
| 62 | ||
| 63 | function walk(dir: string, out: string[], limit = 5000): void { | |
| 64 | for (const e of readdirSync(dir, { withFileTypes: true })) { | |
| 65 | if (e.name === ".git" || e.name === "node_modules") continue; | |
| 66 | const p = join(dir, e.name); | |
| 67 | if (e.isDirectory()) walk(p, out, limit); | |
| 68 | else if (e.isFile() && out.length < limit) out.push(p); | |
| 69 | } | |
| 70 | } | |
| 71 | ||
| 72 | /** Names a model's shell never inherits: the endpoint's key, the forge's tokens, | |
| 73 | anything that reads as a credential. */ | |
| 74 | const SECRET_NAME = /^(OPENAI_|FORGE_)|KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH/i; | |
| 75 | ||
| 76 | /** What the model's shell sees: PATH, HOME and TMPDIR, plus the run's own env | |
| 77 | less everything secret-named. */ | |
| 78 | export function shellEnv(run: AgentRun): Record<string, string> { | |
| 79 | const env: Record<string, string> = { | |
| 80 | PATH: process.env["PATH"] ?? "/usr/bin:/bin", | |
| 81 | HOME: run.env["HOME"] ?? process.env["HOME"] ?? "/tmp", | |
| 82 | TMPDIR: run.sandbox.tmpDir ?? run.env["TMPDIR"] ?? process.env["TMPDIR"] ?? "/tmp", | |
| 83 | }; | |
| 84 | for (const [k, v] of Object.entries(run.env)) if (!SECRET_NAME.test(k) && !(k in env) && typeof v === "string") env[k] = v; | |
| 85 | return env; | |
| 86 | } | |
| 87 | ||
| 88 | /** The confined argv for one command, or undefined when the policy is not enforced: | |
| 89 | no confinement, no shell. */ | |
| 90 | export function bashArgv(run: AgentRun, root: string, command: string): string[] | undefined { | |
| 91 | if (!run.sandbox.enforced) return undefined; | |
| 92 | const binds = [root, run.sandbox.tmpDir, run.sandbox.installCache].filter((d): d is string => !!d); | |
| 93 | return [ | |
| 94 | "bwrap", "--unshare-user", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", | |
| 95 | "--dev", "/dev", "--proc", "/proc", ...binds.flatMap((d) => ["--bind", d, d]), "--chdir", root, "--", "/bin/sh", "-c", command, | |
| 96 | ]; | |
| 97 | } | |
| 98 | ||
| 99 | /** Read at most `max` bytes of a stream as text, then stop reading; `more` says there was more. */ | |
| 100 | export async function readCapped(stream: ReadableStream<Uint8Array> | null, max: number, until?: Promise<void>): Promise<{ text: string; more: boolean }> { | |
| 101 | if (!stream) return { text: "", more: false }; | |
| 102 | const reader = stream.getReader(); | |
| 103 | const chunks: Uint8Array[] = []; | |
| 104 | let size = 0; | |
| 105 | let more = false; | |
| 106 | for (;;) { | |
| 107 | const next = await (until ? Promise.race([reader.read(), until.then(() => undefined)]) : reader.read()); | |
| 108 | if (next === undefined) { await reader.cancel().catch(() => {}); break; } | |
| 109 | const { done, value } = next; | |
| 110 | if (done) break; | |
| 111 | chunks.push(value); | |
| 112 | size += value.length; | |
| 113 | if (size > max) { more = true; await reader.cancel().catch(() => {}); break; } | |
| 114 | } | |
| 115 | const bytes = new Uint8Array(Math.min(size, max)); | |
| 116 | let at = 0; | |
| 117 | for (const c of chunks) { const part = c.subarray(0, Math.max(0, Math.min(c.length, max - at))); bytes.set(part, at); at += part.length; } | |
| 118 | return { text: new TextDecoder().decode(bytes), more }; | |
| 119 | } | |
| 120 | ||
| 121 | /** Run `argv` in `root`: killed at `ms`, on abort, or once its output passes the cap. */ | |
| 122 | export async function runShell(argv: string[], root: string, env: Record<string, string>, signal: AbortSignal, ms = BASH_MS): Promise<string> { | |
| 123 | const proc = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }); | |
| 124 | let timedOut = false; | |
| 125 | const timer = setTimeout(() => { timedOut = true; proc.kill(9); }, ms); | |
| 126 | const onAbort = () => proc.kill(9); | |
| 127 | signal.addEventListener("abort", onAbort); | |
| 128 | try { | |
| 129 | // A grandchild can hold the pipes open after the shell is gone: | |
| 130 | // give them a moment past its exit, then stop reading. | |
| 131 | const gone = proc.exited.then(() => new Promise<void>((r) => setTimeout(r, 200))); | |
| 132 | const [out, err] = await Promise.all([readCapped(proc.stdout, OUTPUT_CAP, gone), readCapped(proc.stderr, OUTPUT_CAP, gone)]); | |
| 133 | if (out.more || err.more) proc.kill(9); | |
| 134 | const code = await proc.exited; | |
| 135 | const tail = timedOut ? `\n[timed out after ${ms} ms]` : out.more || err.more ? "\n[truncated]" : code === 0 ? "" : `\n[exit ${code}]`; | |
| 136 | return cap(`${out.text}${err.text}`) + tail; | |
| 137 | } finally { | |
| 138 | clearTimeout(timer); | |
| 139 | signal.removeEventListener("abort", onAbort); | |
| 140 | } | |
| 141 | } | |
| 142 | ||
| 143 | async function bash(run: AgentRun, root: string, command: string): Promise<string> { | |
| 144 | const argv = bashArgv(run, root, command); | |
| 145 | if (!argv) return "refused: Bash needs an enforced sandbox and this run has none; use the file tools"; | |
| 146 | return runShell(argv, root, shellEnv(run), run.signal); | |
| 147 | } | |
| 148 | ||
| 149 | /** Run one tool call; the answer is always text, a refusal included. */ | |
| 150 | export async function callTool(run: AgentRun, root: string, name: string, a: Record<string, unknown>): Promise<string> { | |
| 151 | try { | |
| 152 | switch (name) { | |
| 153 | case "Bash": | |
| 154 | return await bash(run, root, str(a, "command")); | |
| 155 | case "Read": | |
| 156 | return cap(readFileSync(confine(root, a["file_path"]), "utf8")); | |
| 157 | case "Write": { | |
| 158 | const p = confine(root, a["file_path"]); | |
| 159 | mkdirSync(dirname(p), { recursive: true }); | |
| 160 | writeFileSync(p, str(a, "content")); | |
| 161 | return `wrote ${relative(root, p)}`; | |
| 162 | } | |
| 163 | case "Edit": { | |
| 164 | const p = confine(root, a["file_path"]); | |
| 165 | const text = readFileSync(p, "utf8"); | |
| 166 | const from = str(a, "old_string"); | |
| 167 | const n = from === "" ? 0 : text.split(from).length - 1; | |
| 168 | if (n !== 1) return `old_string must match exactly once; it matches ${n} times`; | |
| 169 | writeFileSync(p, text.replace(from, () => str(a, "new_string"))); | |
| 170 | return `edited ${relative(root, p)}`; | |
| 171 | } | |
| 172 | case "Glob": { | |
| 173 | const base = a["path"] === undefined ? root : confine(root, a["path"]); | |
| 174 | const g = new Bun.Glob(str(a, "pattern")); | |
| 175 | const files: string[] = []; | |
| 176 | walk(base, files); | |
| 177 | return files.map((f) => relative(base, f)).filter((f) => g.match(f)).slice(0, 500).join("\n") || "no files"; | |
| 178 | } | |
| 179 | case "Grep": { | |
| 180 | const base = a["path"] === undefined ? root : confine(root, a["path"]); | |
| 181 | const pattern = str(a, "pattern"); | |
| 182 | if (pattern.length > GREP_PATTERN_MAX || NESTED_QUANTIFIER.test(pattern)) return "refused: pattern too long or with a repeated group that can backtrack without bound"; | |
| 183 | const re = new RegExp(pattern); | |
| 184 | const files: string[] = []; | |
| 185 | if (statSync(base).isFile()) files.push(base); | |
| 186 | else walk(base, files, GREP_FILES); | |
| 187 | const hits: string[] = []; | |
| 188 | for (const f of files) { | |
| 189 | let lines: string[]; | |
| 190 | try { | |
| 191 | if (statSync(f).size > GREP_FILE_BYTES) continue; | |
| 192 | lines = readFileSync(f, "utf8").split("\n"); | |
| 193 | } catch { continue; } | |
| 194 | lines.forEach((l, i) => { if (hits.length < 500 && re.test(l.slice(0, GREP_LINE))) hits.push(`${relative(root, f)}:${i + 1}:${l.slice(0, GREP_LINE)}`); }); | |
| 195 | } | |
| 196 | return cap(hits.join("\n") || "no matches"); | |
| 197 | } | |
| 198 | default: | |
| 199 | return `unknown tool ${name}`; | |
| 200 | } | |
| 201 | } catch (e) { | |
| 202 | return `error: ${e instanceof Error ? e.message : String(e)}`; | |
| 203 | } | |
| 204 | } | |
| 205 | ||
| 206 | const obj = (props: Record<string, string>, required: string[]) => ({ | |
| 207 | type: "object", | |
| 208 | properties: Object.fromEntries(Object.entries(props).map(([k, d]) => [k, { type: "string", description: d }])), | |
| 209 | required, | |
| 210 | }); | |
| 211 | const fn = (name: string, description: string, parameters: object) => ({ type: "function", function: { name, description, parameters } }); | |
| 212 | ||
| 213 | export const TOOL_SPECS = [ | |
| 214 | fn("Bash", "Run a shell command in the repository.", obj({ command: "the command" }, ["command"])), | |
| 215 | fn("Read", "Read a file.", obj({ file_path: "path inside the repository" }, ["file_path"])), | |
| 216 | fn("Write", "Write a file, replacing it.", obj({ file_path: "path inside the repository", content: "the whole file" }, ["file_path", "content"])), | |
| 217 | fn("Edit", "Replace one exact occurrence of a string in a file.", obj({ file_path: "path", old_string: "text to replace", new_string: "replacement" }, ["file_path", "old_string", "new_string"])), | |
| 218 | fn("Glob", "List files matching a glob pattern.", obj({ pattern: "glob such as src/**/*.ts", path: "directory" }, ["pattern"])), | |
| 219 | fn("Grep", "Search files for a regular expression.", obj({ pattern: "regex", path: "file or directory" }, ["pattern"])), | |
| 220 | ]; | |
| 221 | ||
| 222 | interface ToolCall { id: string; function: { name: string; arguments: string } } | |
| 223 | interface Message { role: string; content?: string | null; tool_calls?: ToolCall[]; tool_call_id?: string } | |
| 224 | ||
| 225 | /** The endpoint's base URL from `get`, trailing slashes dropped, or "" when it is not http(s). */ | |
| 226 | function baseUrl(get: (k: string) => string): string { | |
| 227 | const base = get("OPENAI_BASE_URL").replace(/\/+$/, ""); | |
| 228 | return /^https?:\/\//.test(base) ? base : ""; | |
| 229 | } | |
| 230 | ||
| 231 | function settings(run: AgentRun): { base: string; key: string; model: string } | undefined { | |
| 232 | const get = (k: string) => run.env[k] ?? process.env[k] ?? ""; | |
| 233 | const base = baseUrl(get); | |
| 234 | const model = run.model ?? get("OPENAI_MODEL"); | |
| 235 | if (base === "" || model === "") return undefined; | |
| 236 | return { base, key: get("OPENAI_API_KEY"), model }; | |
| 237 | } | |
| 238 | ||
| 239 | export async function runOpenAI(run: AgentRun): Promise<AgentOutcome> { | |
| 240 | const sessionId = run.resume ?? crypto.randomUUID(); | |
| 241 | let tokensIn = 0, tokensOut = 0, turns = 0, contextTokens = 0, result = ""; | |
| 242 | const done = (ok: boolean, reason: string): AgentOutcome => ({ | |
| 243 | ok, sessionId, result, costUsd: 0, turns, tokensIn, tokensOut, contextTokens, contextWindow: 0, reason, | |
| 244 | }); | |
| 245 | const cfg = settings(run); | |
| 246 | if (!cfg) { | |
| 247 | result = "OPENAI_BASE_URL (http/https) and a model name are required"; | |
| 248 | return done(false, "error"); | |
| 249 | } | |
| 250 | run.onSession?.(sessionId); | |
| 251 | const signal = AbortSignal.any([run.signal, AbortSignal.timeout(run.timeoutMs)]); | |
| 252 | const messages: Message[] = [ | |
| 253 | { role: "system", content: run.appendSystemPrompt }, | |
| 254 | { role: "user", content: run.prompt }, | |
| 255 | ]; | |
| 256 | try { | |
| 257 | const root = realpathSync(run.cwd); | |
| 258 | while (turns < MAX_TURNS) { | |
| 259 | const res = await fetch(`${cfg.base}/chat/completions`, { | |
| 260 | method: "POST", | |
| 261 | headers: { "content-type": "application/json", ...(cfg.key ? { authorization: `Bearer ${cfg.key}` } : {}) }, | |
| 262 | body: JSON.stringify({ model: cfg.model, messages, tools: TOOL_SPECS }), | |
| 263 | signal, | |
| 264 | }); | |
| 265 | if (!res.ok) { | |
| 266 | result = `endpoint answered ${res.status}: ${(await readCapped(res.body, 2000)).text.slice(0, 300)}`; | |
| 267 | run.log(`! ${result}`); | |
| 268 | return done(false, res.status === 429 ? "rate_limited" : "error"); | |
| 269 | } | |
| 270 | const raw = await readCapped(res.body, BODY_CAP); | |
| 271 | if (raw.more) { | |
| 272 | result = `endpoint answer is over ${BODY_CAP} bytes`; | |
| 273 | return done(false, "error"); | |
| 274 | } | |
| 275 | const body = JSON.parse(raw.text) as { choices?: { message?: Message }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } }; | |
| 276 | const message = body.choices?.[0]?.message; | |
| 277 | if (message?.tool_calls) message.tool_calls = message.tool_calls.slice(0, MAX_TOOL_CALLS); | |
| 278 | if (!message) { | |
| 279 | result = "endpoint answered without a message"; | |
| 280 | return done(false, "error"); | |
| 281 | } | |
| 282 | turns++; | |
| 283 | contextTokens = body.usage?.prompt_tokens ?? 0; | |
| 284 | tokensIn += contextTokens; | |
| 285 | tokensOut += body.usage?.completion_tokens ?? 0; | |
| 286 | run.onUsage?.({ tokensIn, tokensOut, costUsd: 0 }); | |
| 287 | messages.push({ role: "assistant", content: message.content ?? null, ...(message.tool_calls?.length ? { tool_calls: message.tool_calls } : {}) }); | |
| 288 | if (message.content) { | |
| 289 | result = message.content; | |
| 290 | run.log(message.content); | |
| 291 | run.onSaid?.({ kind: "text", text: message.content }); | |
| 292 | } | |
| 293 | if (!message.tool_calls?.length) return done(true, "completed"); | |
| 294 | for (const call of message.tool_calls) { | |
| 295 | let args: Record<string, unknown> = {}; | |
| 296 | let bad = ""; | |
| 297 | try { | |
| 298 | const parsed: unknown = JSON.parse(call.function.arguments || "{}"); | |
| 299 | if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) args = parsed as Record<string, unknown>; | |
| 300 | else bad = "arguments must be a JSON object"; | |
| 301 | } catch (e) { bad = `arguments are not valid JSON: ${e instanceof Error ? e.message : String(e)}`; } | |
| 302 | if (bad) { | |
| 303 | messages.push({ role: "tool", tool_call_id: call.id, content: `error: ${bad}` }); | |
| 304 | continue; | |
| 305 | } | |
| 306 | run.onSaid?.({ kind: "tool", text: `${call.function.name} ${JSON.stringify(args)}` }); | |
| 307 | const out = await callTool(run, root, call.function.name, args); | |
| 308 | run.log(`${call.function.name} -> ${out.slice(0, 200)}`); | |
| 309 | messages.push({ role: "tool", tool_call_id: call.id, content: out }); | |
| 310 | } | |
| 311 | } | |
| 312 | result ||= "turn limit reached"; | |
| 313 | return done(false, "max_turns"); | |
| 314 | } catch (e) { | |
| 315 | if (run.signal.aborted) return done(false, "cancelled"); | |
| 316 | if (signal.aborted) return done(false, "timeout"); | |
| 317 | result = e instanceof Error ? e.message : String(e); | |
| 318 | return done(false, "error"); | |
| 319 | } | |
| 320 | } | |
| 321 | ||
| 322 | export const openaiRunner: AgentRunner = { | |
| 323 | run: runOpenAI, | |
| 324 | async detect(): Promise<AgentDetection> { | |
| 325 | return baseUrl((k) => process.env[k] ?? "") !== "" ? { ok: true, version: "openai-compatible" } : { ok: false, error: "OPENAI_BASE_URL is not set" }; | |
| 326 | }, | |
| 327 | }; |
test/openai-runner.test.ts added +205
| @@ -0,0 +1,205 @@ | ||
| 1 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, symlinkSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { currentRunner, type AgentRun } from "../src/crew/agent_runner.ts"; | |
| 6 | import { openaiRunner, callTool, bashArgv, runShell, shellEnv, BODY_CAP, MAX_TOOL_CALLS, MAX_TURNS } from "../src/crew/openai_runner.ts"; | |
| 7 | ||
| 8 | let server: ReturnType<typeof Bun.serve>; | |
| 9 | let seen: { auth: string | null; bodies: any[] } = { auth: null, bodies: [] }; | |
| 10 | const call = (id: string, name: string, args: object) => ({ id, type: "function", function: { name, arguments: JSON.stringify(args) } }); | |
| 11 | ||
| 12 | beforeAll(() => { | |
| 13 | server = Bun.serve({ | |
| 14 | port: 0, | |
| 15 | async fetch(req) { | |
| 16 | seen.auth = req.headers.get("authorization"); | |
| 17 | const body: any = await req.json(); | |
| 18 | seen.bodies.push(body); | |
| 19 | const step = body.messages.filter((m: any) => m.role === "assistant").length; | |
| 20 | const usage = { prompt_tokens: 10, completion_tokens: 5 }; | |
| 21 | const message = | |
| 22 | step === 0 ? { role: "assistant", content: null, tool_calls: [call("a", "Write", { file_path: "out/hello.txt", content: "hi" })] } | |
| 23 | : step === 1 ? { role: "assistant", content: null, tool_calls: [call("b", "Bash", { command: "cat out/hello.txt > copy.txt" })] } | |
| 24 | : { role: "assistant", content: "all done" }; | |
| 25 | return Response.json({ choices: [{ message }], usage }); | |
| 26 | }, | |
| 27 | }); | |
| 28 | }); | |
| 29 | afterAll(() => server.stop(true)); | |
| 30 | ||
| 31 | function runIn(cwd: string, extra: Partial<AgentRun> = {}): AgentRun { | |
| 32 | return { | |
| 33 | binary: "", cwd, prompt: "make a file", appendSystemPrompt: "you are a crew member", model: "stub-model", budgetUsd: 1, | |
| 34 | env: { OPENAI_BASE_URL: `http://127.0.0.1:${server.port}/v1`, OPENAI_API_KEY: "k-test" }, | |
| 35 | sandbox: { allowHosts: [], denyRead: [], enforced: false, readAllow: [] } as AgentRun["sandbox"], | |
| 36 | timeoutMs: 20_000, log: () => {}, signal: new AbortController().signal, ...extra, | |
| 37 | }; | |
| 38 | } | |
| 39 | ||
| 40 | test("a Write runs, Bash is refused without an enforced sandbox, and the run ends with its outcome", async () => { | |
| 41 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 42 | seen = { auth: null, bodies: [] }; | |
| 43 | const usage: number[] = []; | |
| 44 | const out = await openaiRunner.run(runIn(cwd, { onUsage: (s) => usage.push(s.tokensIn) })); | |
| 45 | expect(readFileSync(join(cwd, "out/hello.txt"), "utf8")).toBe("hi"); | |
| 46 | expect(existsSync(join(cwd, "copy.txt"))).toBe(false); | |
| 47 | expect(seen.bodies[2].messages.at(-1).content).toContain("refused: Bash needs an enforced sandbox"); | |
| 48 | expect(out).toMatchObject({ ok: true, result: "all done", reason: "completed", turns: 3, tokensIn: 30, tokensOut: 15, costUsd: 0 }); | |
| 49 | expect(usage).toEqual([10, 20, 30]); | |
| 50 | expect(seen.auth).toBe("Bearer k-test"); | |
| 51 | expect(seen.bodies[0].model).toBe("stub-model"); | |
| 52 | expect(seen.bodies[0].tools.map((t: any) => t.function.name)).toEqual(["Bash", "Read", "Write", "Edit", "Glob", "Grep"]); | |
| 53 | }); | |
| 54 | ||
| 55 | test("tool calls outside the sandbox root are refused", async () => { | |
| 56 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 57 | const outside = mkdtempSync(join(tmpdir(), "oai-out-")); | |
| 58 | symlinkSync(outside, join(cwd, "link")); | |
| 59 | mkdirSync(join(cwd, "sub")); | |
| 60 | const r = runIn(cwd); | |
| 61 | for (const [name, args] of [ | |
| 62 | ["Write", { file_path: join(outside, "x.txt"), content: "x" }], | |
| 63 | ["Write", { file_path: "../escape.txt", content: "x" }], | |
| 64 | ["Write", { file_path: "link/x.txt", content: "x" }], | |
| 65 | ["Read", { file_path: "/etc/passwd" }], | |
| 66 | ["Grep", { pattern: "a", path: ".." }], | |
| 67 | ] as const) expect(await callTool(r, cwd, name, args)).toContain("outside the sandbox root"); | |
| 68 | expect(existsSync(join(outside, "x.txt"))).toBe(false); | |
| 69 | expect(await callTool(r, cwd, "Write", { file_path: "sub/ok.txt", content: "ok" })).toBe("wrote sub/ok.txt"); | |
| 70 | }); | |
| 71 | ||
| 72 | test("Edit, Glob and Grep work inside the root", async () => { | |
| 73 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 74 | const r = runIn(cwd); | |
| 75 | await callTool(r, cwd, "Write", { file_path: "a/b.ts", content: "let x = 1;\n" }); | |
| 76 | await callTool(r, cwd, "Edit", { file_path: "a/b.ts", old_string: "1", new_string: "2" }); | |
| 77 | expect(await callTool(r, cwd, "Read", { file_path: "a/b.ts" })).toBe("let x = 2;\n"); | |
| 78 | expect(await callTool(r, cwd, "Glob", { pattern: "**/*.ts" })).toBe("a/b.ts"); | |
| 79 | expect(await callTool(r, cwd, "Grep", { pattern: "x = 2" })).toBe("a/b.ts:1:let x = 2;"); | |
| 80 | }); | |
| 81 | ||
| 82 | test("a missing endpoint config or a failing endpoint ends the run with an error outcome", async () => { | |
| 83 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 84 | const none = await openaiRunner.run(runIn(cwd, { env: {}, model: undefined })); | |
| 85 | expect(none).toMatchObject({ ok: false, reason: "error", result: "OPENAI_BASE_URL (http/https) and a model name are required" }); | |
| 86 | const bad = await openaiRunner.run(runIn(cwd, { env: { OPENAI_BASE_URL: "http://127.0.0.1:1/v1" } })); | |
| 87 | expect(bad).toMatchObject({ ok: false, reason: "error" }); | |
| 88 | }); | |
| 89 | ||
| 90 | test("cancelling ends the run as cancelled", async () => { | |
| 91 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 92 | const ac = new AbortController(); | |
| 93 | ac.abort(); | |
| 94 | expect((await openaiRunner.run(runIn(cwd, { signal: ac.signal }))).reason).toBe("cancelled"); | |
| 95 | }); | |
| 96 | ||
| 97 | test("the runner is chosen by configuration and the default is unchanged", () => { | |
| 98 | const before = currentRunner(); | |
| 99 | expect(before).not.toBe(openaiRunner); | |
| 100 | process.env["FORGE_AGENT_RUNNER"] = "openai"; | |
| 101 | try { expect(currentRunner()).toBe(openaiRunner); } finally { delete process.env["FORGE_AGENT_RUNNER"]; } | |
| 102 | expect(currentRunner()).toBe(before); | |
| 103 | }); | |
| 104 | ||
| 105 | /** A one-off endpoint answering every request with `answer`. */ | |
| 106 | async function withEndpoint<T>(answer: (n: number, body: any) => Response | Promise<Response>, use: (env: Record<string, string>) => Promise<T>): Promise<T> { | |
| 107 | let n = 0; | |
| 108 | const srv = Bun.serve({ port: 0, async fetch(req) { return answer(n++, await req.json()); } }); | |
| 109 | try { return await use({ OPENAI_BASE_URL: `http://127.0.0.1:${srv.port}/v1` }); } finally { srv.stop(true); } | |
| 110 | } | |
| 111 | const say = (message: object) => Response.json({ choices: [{ message }], usage: { prompt_tokens: 1, completion_tokens: 1 } }); | |
| 112 | ||
| 113 | test("a 429 is rate_limited with the endpoint's words, any other failure is an error", async () => { | |
| 114 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 115 | const limited = await withEndpoint(() => new Response("slow down", { status: 429 }), (env) => openaiRunner.run(runIn(cwd, { env }))); | |
| 116 | expect(limited).toMatchObject({ ok: false, reason: "rate_limited", result: "endpoint answered 429: slow down" }); | |
| 117 | const broken = await withEndpoint(() => new Response("boom", { status: 500 }), (env) => openaiRunner.run(runIn(cwd, { env }))); | |
| 118 | expect(broken).toMatchObject({ ok: false, reason: "error" }); | |
| 119 | }); | |
| 120 | ||
| 121 | test("a model that never stops calling tools ends at max_turns", async () => { | |
| 122 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 123 | const out = await withEndpoint((n) => say({ role: "assistant", content: null, tool_calls: [call(`c${n}`, "Glob", { pattern: "*" })] }), (env) => openaiRunner.run(runIn(cwd, { env }))); | |
| 124 | expect(out).toMatchObject({ ok: false, reason: "max_turns", turns: MAX_TURNS }); | |
| 125 | }); | |
| 126 | ||
| 127 | test("an endpoint that does not answer in time ends the run as timeout", async () => { | |
| 128 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 129 | const out = await withEndpoint(() => new Promise<Response>(() => {}), (env) => openaiRunner.run(runIn(cwd, { env, timeoutMs: 100 }))); | |
| 130 | expect(out).toMatchObject({ ok: false, reason: "timeout" }); | |
| 131 | }); | |
| 132 | ||
| 133 | test("an answer over the body cap ends the run without being read whole", async () => { | |
| 134 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 135 | const out = await withEndpoint(() => new Response("x".repeat(BODY_CAP + 10)), (env) => openaiRunner.run(runIn(cwd, { env }))); | |
| 136 | expect(out).toMatchObject({ ok: false, reason: "error" }); | |
| 137 | expect(out.result).toContain("over"); | |
| 138 | }); | |
| 139 | ||
| 140 | test("tool calls past the per-turn cap are dropped, and unparseable arguments go back to the model", async () => { | |
| 141 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 142 | const bodies: any[] = []; | |
| 143 | const many = Array.from({ length: MAX_TOOL_CALLS + 5 }, (_, i) => call(`m${i}`, "Glob", { pattern: "*" })); | |
| 144 | many[0] = { id: "m0", type: "function", function: { name: "Write", arguments: "{not json" } }; | |
| 145 | const out = await withEndpoint((n, body) => { | |
| 146 | bodies.push(body); | |
| 147 | return n === 0 ? say({ role: "assistant", content: null, tool_calls: many }) : say({ role: "assistant", content: "ok" }); | |
| 148 | }, (env) => openaiRunner.run(runIn(cwd, { env }))); | |
| 149 | expect(out.ok).toBe(true); | |
| 150 | const tools = bodies[1].messages.filter((m: any) => m.role === "tool"); | |
| 151 | expect(tools.length).toBe(MAX_TOOL_CALLS); | |
| 152 | expect(tools[0].content).toContain("arguments are not valid JSON"); | |
| 153 | expect(existsSync(join(cwd, "undefined"))).toBe(false); | |
| 154 | }); | |
| 155 | ||
| 156 | test("Edit refuses an old_string that is not unique and leaves the file alone", async () => { | |
| 157 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 158 | const r = runIn(cwd); | |
| 159 | await callTool(r, cwd, "Write", { file_path: "f.txt", content: "a a" }); | |
| 160 | expect(await callTool(r, cwd, "Edit", { file_path: "f.txt", old_string: "a", new_string: "b" })).toBe("old_string must match exactly once; it matches 2 times"); | |
| 161 | expect(await callTool(r, cwd, "Edit", { file_path: "f.txt", old_string: "zz", new_string: "b" })).toContain("matches 0 times"); | |
| 162 | expect(readFileSync(join(cwd, "f.txt"), "utf8")).toBe("a a"); | |
| 163 | }); | |
| 164 | ||
| 165 | test("Grep refuses a backtracking pattern and skips what is over the bounds", async () => { | |
| 166 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 167 | const r = runIn(cwd); | |
| 168 | await callTool(r, cwd, "Write", { file_path: "f.txt", content: "a".repeat(5000) + "!\n" }); | |
| 169 | expect(await callTool(r, cwd, "Grep", { pattern: "(a+)+$" })).toContain("refused"); | |
| 170 | expect(await callTool(r, cwd, "Grep", { pattern: "a".repeat(201) })).toContain("refused"); | |
| 171 | expect(await callTool(r, cwd, "Grep", { pattern: "!" })).toBe("no matches"); | |
| 172 | }); | |
| 173 | ||
| 174 | test("Bash is refused when the sandbox is not enforced, and bashArgv confines it when it is", () => { | |
| 175 | const off = runIn("/w"); | |
| 176 | expect(bashArgv(off, "/w", "ls")).toBeUndefined(); | |
| 177 | const on = runIn("/w", { sandbox: { allowHosts: [], denyRead: [], enforced: true, readAllow: [], tmpDir: "/t", installCache: "/c" } as AgentRun["sandbox"] }); | |
| 178 | expect(bashArgv(on, "/w", "echo hi")).toEqual([ | |
| 179 | "bwrap", "--unshare-user", "--unshare-pid", "--die-with-parent", "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", | |
| 180 | "--bind", "/w", "/w", "--bind", "/t", "/t", "--bind", "/c", "/c", "--chdir", "/w", "--", "/bin/sh", "-c", "echo hi", | |
| 181 | ]); | |
| 182 | }); | |
| 183 | ||
| 184 | test("the model's shell never sees the endpoint's key or the forge's tokens", async () => { | |
| 185 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 186 | process.env["OPENAI_API_KEY"] = "leak-me"; | |
| 187 | try { | |
| 188 | const r = runIn(cwd, { env: { OPENAI_API_KEY: "k-test", FORGE_TOKEN: "t", GH_TOKEN: "g", KEEP: "yes", OPENAI_BASE_URL: "http://x" } }); | |
| 189 | const env = shellEnv(r); | |
| 190 | expect(Object.keys(env).sort()).toEqual(["HOME", "KEEP", "PATH", "TMPDIR"]); | |
| 191 | const out = await runShell(["/bin/sh", "-c", "env | grep -c -e OPENAI -e FORGE_ -e GH_TOKEN -e leak-me; echo $KEEP"], cwd, env, r.signal); | |
| 192 | expect(out.trim().split("\n")).toEqual(["0", "yes"]); | |
| 193 | } finally { delete process.env["OPENAI_API_KEY"]; } | |
| 194 | }); | |
| 195 | ||
| 196 | test("a shell past its deadline is killed, and its output is capped", async () => { | |
| 197 | const cwd = mkdtempSync(join(tmpdir(), "oai-")); | |
| 198 | const ac = new AbortController(); | |
| 199 | const slow = await runShell(["/bin/sh", "-c", "echo before; sleep 30"], cwd, shellEnv(runIn(cwd)), ac.signal, 300); | |
| 200 | expect(slow).toContain("before"); | |
| 201 | expect(slow).toContain("[timed out after 300 ms]"); | |
| 202 | const loud = await runShell(["/bin/sh", "-c", "yes aaaaaaaaaa"], cwd, shellEnv(runIn(cwd)), ac.signal); | |
| 203 | expect(loud).toEndWith("[truncated]"); | |
| 204 | expect(loud.length).toBeLessThan(31_000); | |
| 205 | }); |
test/spawn-pipes.test.ts +1
| @@ -23,6 +23,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 23 | 23 | "src/spawn/cell.ts": { sites: ["stdout+stderr"], reason: "drain() reads the child forge's stdout and stderr to its log until exit" }, |
| 24 | 24 | "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" }, |
| 25 | 25 | "src/crew/claude.ts": { sites: ["stdout+stderr", "stdout"], reason: "session pumps stdout and stderr; --version awaits stdout, stderr ignored" }, |
| 26 | "src/crew/openai_runner.ts": { sites: ["stdout+stderr"], reason: "bash() reads stdout and stderr to the end alongside the exit, under a kill timer" }, | |
| 26 | 27 | "src/crew/plan.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" }, |
| 27 | 28 | "src/crew/runner.ts": { sites: ["stdout"], reason: "lsof stdout awaited; stderr ignored" }, |
| 28 | 29 | "src/crew/sandbox.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" }, |
test/spawn-sites.test.ts +2
| @@ -35,6 +35,8 @@ const SITES: Record<string, string> = { | ||
| 35 | 35 | "osascript/notify-send with title and body as separate argv elements after the script or `--`, so a repo name stays data", |
| 36 | 36 | "src/crew/claude.ts": |
| 37 | 37 | "the agent binary from configuration with flags the runner fixes; the prompt (a person's issue text) is one argv element after its flag and the binary never hands it to a shell; `--version` probe is fixed", |
| 38 | "src/crew/openai_runner.ts": | |
| 39 | "the Bash tool of the OpenAI-compatible runner: /bin/sh -c with the model's command, which is the point of the tool, run in the run's clone (under bwrap when the sandbox is enforced); no person's text reaches argv", | |
| 38 | 40 | "src/crew/plan.ts": |
| 39 | 41 | "the planner binary with a fixed flag list; the issue text a person wrote is only the final prompt element, with tools disabled", |
| 40 | 42 | "src/crew/runner.ts": |
test/timers-named.test.ts +2
| @@ -72,6 +72,8 @@ const ALLOWED: readonly Allowed[] = [ | ||
| 72 | 72 | ["src/http/flow_stream.ts", "() => { emit(\"flow.settled\""], |
| 73 | 73 | ]), |
| 74 | 74 | ...allow("setTimeout", `a shell session's resize debounce, ${PER_CONNECTION}`, [["src/machines/shell.ts", "() => session.size === resize"]]), |
| 75 | ...allow("setTimeout", `an openai runner bash call's kill deadline, ${PER_CONNECTION}`, [["src/crew/openai_runner.ts", "() => { timedOut = true"]]), | |
| 76 | ...allow("setTimeout", SLEEP, [["src/crew/openai_runner.ts", "r"]]), | |
| 75 | 77 | ...allow("setTimeout", `an ssh login's auth deadline, ${PER_CONNECTION}`, [["src/ssh/server.ts", "() => { this.transport.disconnect("]]), |
| 76 | 78 | ...allow("setTimeout", "a maintenance duty's deadline on one call: it rejects a promise and does no work on the loop", [["src/maintain/runner.ts", "() => reject(new Error(\"timed out\"))"]]), |
| 77 | 79 | ...allow("setTimeout", `a media stream's idle deadline, ${PER_CONNECTION}`, [["src/git/read.ts", "() => { finish(); controller.error("]]), |
new above
offline