Public: anyone can read this; the crew can change it.

The review and build briefs say a preview's /__forge/ and pill come from the host forge, so changes to them are judged by test #6255

Closed opened this
Closed without merging

scout/forge:scout/6100-the-review-and-build-briefs-say-a-previe → main 6 commits fork 3 threads · 3 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
4 files changed +107 −0
docs/CREW.md +4
@@ -36,6 +36,10 @@ Nesting a sandbox is not possible there, so `test/crew-sandbox.test.ts` checks
3636 the settings that enforce it (`allowLocalBinding` on, `allowAllUnixSockets` off,
3737 the `denyRead` list) and not the boundary itself.
3838
39 ## Previews
40
41 On an app host `/__forge/*` and the injected forge pill are answered by the host forge, not by the pull's branch, so when a diff (or the issue, for the builder) touches `src/apps/door.ts`, `toolbar.ts`, `feedback.ts`, `door_pulls.ts` or the `/__forge` handling in `proxy.ts`, the reviewer's and the builder's briefs say to judge them by the request-level tests, not by the preview (`touchesHostDoor` in `src/crew/brief.ts`).
42
3943 ## Where a run's time went
4044
4145 Every run that started ends its log with one `timing: run ended …s after the run started — clone … · brief … · harness …` line (plus `wrap-up` when the reserve was used), whatever way it ended; `src/apps/timing.ts` draws it.
src/crew/brief.ts +37
@@ -43,6 +43,8 @@ export interface BriefInput {
4343 readonly forgeUrl: string;
4444 readonly isApp: boolean;
4545 readonly previewUrl?: string | undefined;
46 /** The task is about `/__forge/*` or the pill, which a preview cannot show. */
47 readonly hostDoor?: boolean | undefined;
4648 /** Where this repository's app answers; the shell may reach these and no other app. */
4749 readonly ownApp?: OwnApp | undefined;
4850 /** Whether a session is being continued (the task was already given). */
@@ -163,6 +165,7 @@ export function systemPrompt(input: BriefInput): string {
163165 : [
164166 `- The preview of your pull is at ${input.previewUrl} once it is up.`,
165167 ]),
168 ...(input.hostDoor === true ? [HOST_DOOR_LINE] : []),
166169 ...ownAppLines(input.ownApp),
167170 );
168171 if (input.policy !== undefined)
@@ -242,6 +245,39 @@ export interface TaskInput extends BriefInput {
242245 readonly reserve?: Reserve | undefined;
243246 }
244247
248 const HOST_DOOR_FILES = /src\/apps\/(?:door|toolbar|feedback|door_pulls)\.ts\b/;
249 const DIFF_HEADER = /^diff --git a\/(\S+) b\/\S+$/gm;
250
251 /** One rule for both briefs: a host-door file, or proxy.ts when `__forge` is in the same text. */
252 function hostDoorRule(files: string, text: string): boolean {
253 return HOST_DOOR_FILES.test(files) || (/src\/apps\/proxy\.ts\b/.test(files) && text.includes("__forge"));
254 }
255
256 /**
257 * Whether a diff changes the host forge's doors: judged by its `diff --git`
258 * file headers, so prose that merely mentions a file does not count. A preview
259 * cannot show these.
260 */
261 export function touchesHostDoor(diff: string | undefined): boolean {
262 if (diff === undefined) return false;
263 for (const m of diff.matchAll(DIFF_HEADER)) if (hostDoorRule(m[1] ?? "", diff)) return true;
264 return false;
265 }
266
outdated

Wording says "in the diff", but the builder sees this line before any diff exists. Make it "the request-level tests" without the diff reference, or have a builder-specific line.

267 /** Whether the builder's task (the issue's title and body) is about the host forge's doors. */
268 export function hostDoorFor(issue: { readonly title: string; readonly body: string }): boolean {
269 return hostDoorRule(`${issue.title}\n${issue.body}`, `${issue.title}\n${issue.body}`);
270 }
271
272 /** The builder's decision, as the runner makes it: never for a review, which reads the diff instead. */
273 export function builderHostDoor(reviewing: boolean, issue: { readonly title: string; readonly body: string }): boolean {
274 return !reviewing && hostDoorFor(issue);
275 }
276
277 /** Said beside the preview line when the change is to what a preview cannot show. */
278 export const HOST_DOOR_LINE =
279 "- `/__forge/*` and the forge pill on this preview are served by the host forge, not this branch: judge them by the request-level tests that come with the change, not by the preview.";
280
245281 /** The reviewer's latest verdict, and the final answer of the run that did not push for it. */
246282 export interface LastTry {
247283 readonly reviewer: string;
@@ -387,6 +423,7 @@ function taskBody(input: TaskInput): string {
387423 : [
388424 `- This pull's preview, at this head, is at ${input.previewUrl}. Use it as a person would: curl its pages, try the doors it offers. Judge this preview only — another pull's preview is not this change.`,
389425 ]),
426 ...(touchesHostDoor(input.reviewDiff) ? [HOST_DOOR_LINE] : []),
390427 ...(input.looks !== undefined && input.looks.length > 0
391428 ? [
392429 `- The forge took pictures of the preview at this head; read them (they are PNG files, phone then desktop): ${input.looks.join(", ")}`,
src/crew/runner.ts +2
@@ -98,6 +98,7 @@ import {
9898 type LastTry,
9999 type RunReason,
100100 type TaskInput,
101 builderHostDoor,
101102 } from "./brief.ts";
102103 import { policyBrief } from "../contract/audit.ts";
103104 import {
@@ -1009,6 +1010,7 @@ async function work(
10091010 forgeUrl: options.forgeUrl,
10101011 isApp: record !== undefined,
10111012 previewUrl,
1013 hostDoor: builderHostDoor(reviewing, issue),
10121014 ownApp: ownApp(app, repo),

This wiring is untested. Removing the line fails no test. Route it through a function the test calls.

10131015 policy,
10141016 // A review reads one head afresh: a resumed session would carry an
test/crew-brief-host-door.test.ts added +64
@@ -0,0 +1,64 @@
1 /** A preview cannot show `/__forge/*` or the pill: the briefs say so only when the change is to them. */
2 import { describe, expect, test } from "bun:test";
3 import { HOST_DOOR_LINE, builderHostDoor, hostDoorFor, systemPrompt, taskPrompt, type BriefInput, type TaskInput } from "../src/crew/brief.ts";
4
5 const base = {
6 agent: { user: { username: "bot", displayName: "" }, brief: "" },
7 repo: { ownerName: "alice", name: "square", defaultBranch: "main", autonomyDirection: "" },
8 issue: { number: 1, title: "t", body: "b", authorName: "alice" },
9 pull: { number: 1 },
10 branch: "bot/1",
11 forgeUrl: "http://forge.test",
12 isApp: true,
13 previewUrl: "http://p.test",
14 comments: [],
15 reviews: [],
16 resuming: false,
17 };
18
19 const review = (reviewDiff: string): string =>
20 taskPrompt({ ...base, reason: "review", reviewDiff } as unknown as TaskInput);
21 const build = (title: string, body: string): string =>
22 systemPrompt({
23 ...base,
24 issue: { ...base.issue, title, body },
25 hostDoor: hostDoorFor({ title, body }),
outdated

The test re-implements the detection with a private regex instead of calling touchesHostDoor, so it proves nothing about the real builder path. Call the production function.

26 } as unknown as BriefInput);
27
28 const FILES = ["door", "toolbar", "feedback", "door_pulls"].map((f) => `src/apps/${f}.ts`);
29
30 describe("the host-door line", () => {
31 test("the reviewer's brief has it for a diff touching src/apps/door.ts, not for src/web/", () => {
32 expect(review("diff --git a/src/apps/door.ts b/src/apps/door.ts\n+x")).toContain(HOST_DOOR_LINE);
33 expect(review("diff --git a/src/web/home.ts b/src/web/home.ts\n+x")).not.toContain(HOST_DOOR_LINE);
34 });
35 test("proxy.ts counts only for its /__forge handling", () => {
36 expect(review("diff --git a/src/apps/proxy.ts b/src/apps/proxy.ts\n+ if (path.startsWith(\"/__forge\"))")).toContain(HOST_DOOR_LINE);
37 expect(review("diff --git a/src/apps/proxy.ts b/src/apps/proxy.ts\n+ x")).not.toContain(HOST_DOOR_LINE);
38 });
39 test("each of the five host-door files triggers both briefs", () => {
40 for (const file of FILES) {
41 expect(review(`diff --git a/${file} b/${file}\n+x`)).toContain(HOST_DOOR_LINE);
42 expect(hostDoorFor({ title: "t", body: `Change ${file}` })).toBe(true);
43 expect(build("t", `Change ${file}`)).toContain(HOST_DOOR_LINE);
44 }
45 expect(hostDoorFor({ title: "t", body: "src/apps/proxy.ts handles /__forge" })).toBe(true);
46 expect(hostDoorFor({ title: "Fix src/apps/toolbar.ts", body: "" })).toBe(true);
47 });
48 test("hostDoorFor is false for other issues, and the builder's brief is silent", () => {
49 for (const body of ["Change src/web/home.ts", "src/apps/proxy.ts only", "src/apps/doors.ts", ""]) {
50 expect(hostDoorFor({ title: "t", body })).toBe(false);
51 expect(build("t", body)).not.toContain(HOST_DOOR_LINE);
52 }
53 });
54 test("a diff that only mentions a host-door file in prose does not trigger the line", () => {
55 const docs = "diff --git a/docs/notes.md b/docs/notes.md\n+ see src/apps/door.ts and src/apps/proxy.ts __forge";
56 expect(review(docs)).not.toContain(HOST_DOOR_LINE);
57 });
58 test("builderHostDoor is the runner's decision: on for a build, never for a review", () => {
59 const issue = { title: "t", body: "Change src/apps/door.ts" };
60 expect(builderHostDoor(false, issue)).toBe(true);
61 expect(builderHostDoor(true, issue)).toBe(false);
62 expect(builderHostDoor(false, { title: "t", body: "x" })).toBe(false);
63 });
64 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs