dd0f009 · 3fc5117 · 63d74ee · f226ce5 · ac47c38 · b5ca382
@@ -1711,6 +1711,7 @@
forge icon.
```sh
bun test              # protocol-level + adversarial suite
bun scripts/design-lint.ts   # no colour or off-scale font-size beyond docs/design/baseline.json, no class docs/DESIGN.md §5 does not name
bun scripts/cold-read.ts <url> [--repo owner/name] [--write [--allow-text-only]] [--file-issues N [--file-into owner/name]]   # cold-read audit (#5882): a no-context model reads the top pages (signed in with FORGE_TOKEN, sent only to FORGE_URL's forge; the signed-in pages go to the model endpoint, so use a throwaway account) at 390 and 1440 px; needs ANTHROPIC_API_KEY
bun x tsc --noEmit    # zero type errors is the standard
bun run check         # exactly what the forge runs against every push
bun run check:record  # the same, and rewrites scripts/test-times.json
@@ -0,0 +1,13 @@
You have never seen this product and know nothing about it. You are shown one
page of it as it looks on a screen {{width}} px wide ({{device}}). Judge only what
is in front of you.
Answer in under 100 words in total, as JSON and nothing else:
{"confusing": ["what is confusing or unexplained, one short sentence each"],
 "clickFirst": "the one thing you would click or tap first, and why",
 "clarity": 1-5}
`clarity` is how sure you are, in five seconds, of what this page is for and
what to do next: 1 = no idea, 5 = obvious. Name the part of the page you blame
(a button, a heading, a list) in each confusion, so it can be judged.
@@ -0,0 +1,156 @@
/**
 * `bun scripts/cold-read.ts <forge-url> [--repo owner/name] [--write [--allow-text-only]]
 *   [--file-issues <parent> [--file-into owner/name] [--max N]]`
 *
 * A no-context agent reads each top page at 390 px and 1440 px and says, in
 * under 100 words, what is confusing and what it would click first. The prompt
 * is `docs/design/cold-read-prompt.md`. The reader is signed in with
 * `FORGE_TOKEN` (a bearer the forge honours on pages), so /migrate and
 * /settings show their real first run. A page that is not a 200 at its own
 * URL (a redirect, a 404) is refused, never scored. `--repo` adds that
 * repository's page, pulls and issues after checking it exists. The fetched
 * server-rendered HTML is shot with headless Chromium at the width (its text
 * when there is no browser; `--write` then refuses unless `--allow-text-only`, which marks the score `text-only`) and sent, with the prompt, to the model named by `COLD_READ_MODEL`
 * (default claude-sonnet-5-5) through `ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL`.
 *
      
        sentinelcrew
        19d ago
        outdated
      
      readFileSync is imported twice (readFileSync as read). Use the one name.
    
 * `FORGE_TOKEN` goes only to the forge `FORGE_URL` names; any other <forge-url> is refused before a request.
 * Privacy: the signed-in pages (/settings can show an email, token names, linked accounts) are sent in full to the
 * model endpoint, `ANTHROPIC_BASE_URL` included. Use a throwaway account's token unless that is acceptable. The shot
 * is loaded from file://, so it carries no CSP header, and Chromium runs `--no-sandbox`: read only your own forge.
 *
 * It prints a score per page. `--write` stores them under `coldRead` in
 * `docs/design/baseline.json`; `--file-issues N` files each confusion as an
 * issue in `--file-into` (default `--repo`) on `FORGE_URL` (which must be the forge read), linking back to #N: at most `--max` (default 20), and
 * none whose title is already open.
 */
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { findChromium } from "../src/looks/capture.ts";
import { PAGES, REPO, filingPlan, repoPages, tokenRefusal, withColdRead, writeRefusal, type ColdRead } from "../src/design/coldread.ts";
import { coldRead, type Shot } from "../src/design/coldrun.ts";
const BASELINE = "docs/design/baseline.json";
const args = process.argv.slice(2);
const base = args.find((a) => /^https?:\/\//.test(a))?.replace(/\/$/, "");
const write = args.includes("--write");
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined);
const parent = opt("--file-issues") === undefined ? undefined : Number(opt("--file-issues"));
const repo = opt("--repo");
const into = opt("--file-into") ?? repo;
const cap = Number(opt("--max") ?? 20);
if (!base || (parent !== undefined && !Number.isInteger(parent)) || (repo !== undefined && !REPO.test(repo)) || (into !== undefined && !REPO.test(into)) || !Number.isInteger(cap) || cap < 0) {
  console.error("usage: bun scripts/cold-read.ts <url> [--repo owner/name] [--write [--allow-text-only]] [--file-issues <issue number> [--file-into owner/name] [--max N]]");
  process.exit(2);
}
const token = process.env["FORGE_TOKEN"];
      
        sentinelcrew
        19d ago
        outdated
      
      High: this bearer is sent to base (the --repo probe on the next line, and fetchPage) before anything checks that base is the forge in FORGE_URL. Run the same-origin check here at startup, before the first fetch, not only in filingPlan. Then a mistyped or foreign URL never receives the token.
    
if (!token) {
  console.error("FORGE_TOKEN is not set: the reader must be signed in, or /migrate and /settings are the sign-in page");
  process.exit(2);
}
const unsafe = tokenRefusal(base, process.env["FORGE_URL"]);
if (unsafe) {
  console.error(unsafe);
  process.exit(2);
}
const auth = { authorization: `Bearer ${token}` };
if (repo !== undefined) {
      
        sentinelcrew
        19d ago
        outdated
      
      This follows redirects silently and doesn't check res.status. The Chromium path has the same problem. Reject a reading whose final URL or status isn't the page asked for, so a sign-in or 404 page is never scored under another page's name.
    
  const r = await fetch(`${base}/api/v1/repos/${repo}`, { headers: auth, redirect: "manual" });
  if (r.status !== 200) {
    console.error(`--repo ${repo}: ${base} answered ${r.status}`);
    process.exit(2);
  }
}
const key = process.env["ANTHROPIC_API_KEY"];
if (!key) {
  console.error("ANTHROPIC_API_KEY is not set: the audit needs a model to read the pages");
  process.exit(2);
}
const api = (process.env["ANTHROPIC_BASE_URL"] ?? "https://api.anthropic.com").replace(/\/$/, "");
const model = process.env["COLD_READ_MODEL"] ?? "claude-sonnet-5-5";
const prompt = readFileSync(new URL("../docs/design/cold-read-prompt.md", import.meta.url), "utf8");
      
        sentinelcrew
        19d ago
        outdated
      
      Low: there is no timeout on the Chromium process, so a page that never finishes loading hangs the whole run. Pass --timeout= / --virtual-time-budget=, or kill it after a few seconds. Also note: loaded from file://, the page has lost the forge's CSP response header, and runs with --no-sandbox.
    
async function shoot(html: string, path: string, w: { width: number; height: number }): Promise<Shot> {
  const binary = findChromium();
  if (binary) {
    const dir = mkdtempSync(join(process.env["TMPDIR"] ?? tmpdir(), "coldread-"));
    try {
      // The first paint is server-rendered; <base> lets its stylesheets load from the forge.
      await Bun.write(join(dir, "p.html"), html.replace(/<head[^>]*>/i, (h) => `${h}<base href="${base + path}">`));
      const out = join(dir, "p.png");
      const p = Bun.spawn([binary, "--headless", "--no-sandbox", "--hide-scrollbars", "--virtual-time-budget=5000", `--window-size=${w.width},${w.height}`, `--screenshot=${out}`, `file://${join(dir, "p.html")}`], { stdout: "ignore", stderr: "ignore" });
      const timer = setTimeout(() => p.kill(), 15_000); // a page that never settles must not hang the run
      await p.exited;
      clearTimeout(timer);
      const data = Buffer.from(readFileSync(out)).toString("base64");
      return { kind: "image", block: { type: "image", source: { type: "base64", media_type: "image/png", data } } };
    } catch {
      /* fall through to text */
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  }
  const text = html.replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").slice(0, 8000);
  return { kind: "text", block: { type: "text", text: `The page's text:\n${text}` } };
}
async function ask(shot: unknown, text: string): Promise<string> {
  const content = [shot, { type: "text", text }];
  const res = await fetch(`${api}/v1/messages`, {
      
        sentinelcrew
        19d ago
        outdated
      
      Medium: this is the signed-in HTML of /settings and /migrate (it can include email, token names, linked accounts), and it is sent in full to ANTHROPIC_BASE_URL, which is also overridable. Say this in the header and README, or read /settings with a fresh throwaway account. An operator's private settings should not reach a third-party model by default.
    
    method: "POST",
    headers: { "x-api-key": key!, "anthropic-version": "2023-06-01", "content-type": "application/json" },
    body: JSON.stringify({ model, max_tokens: 400, messages: [{ role: "user", content }] }),
  });
  if (!res.ok) throw new Error(`model answered ${res.status}`);
  const j = (await res.json()) as { content?: { type: string; text?: string }[] };
  return j.content?.find((c) => c.type === "text")?.text ?? "";
      
        sentinelcrew
        19d ago
        outdated
      
      This has no cap and no de-duplication, so one run can file up to 112 issues and a rerun files them all again. Cap it, and skip a title that is already open.
    
}
const run = await coldRead([...PAGES, ...(repo ? repoPages(repo) : [])], prompt, {
  fetchPage: async (path) => {
    const res = await fetch(base + path, { headers: auth, redirect: "manual" });
    return { status: res.status, location: res.headers.get("location"), html: res.status === 200 ? await res.text() : "" };
  },
  shoot,
  ask,
  log: (l) => console.error(l),
});
const { pages, findings } = run;
for (const [page, why] of Object.entries(run.failed)) console.error(`NOT SCORED ${page}: ${why.join("; ")}`);
if (run.mode === "text-only") console.error("text-only: no Chromium, so 390 and 1440 were sent the same words; phone and desktop are not comparable");
const refused = write ? writeRefusal(run, args.includes("--allow-text-only")) : null;
if (refused) {
  console.error(refused);
  process.exit(1);
}
if (write) {
  const record: ColdRead = { at: new Date().toISOString(), ...(parent !== undefined ? { issue: parent } : {}), mode: run.mode, pages };
  writeFileSync(BASELINE, JSON.stringify(withColdRead(JSON.parse(readFileSync(BASELINE, "utf8")), record), null, 2) + "\n");
  console.log(`wrote coldRead to ${BASELINE}`);
}
if (parent !== undefined) {
  const forge = process.env["FORGE_URL"];
  // The target is checked before anything is fetched; the open list then decides what is filed.
  const early = filingPlan({ forge, base, into, open: [], findings: [], parent, cap });
  if ("error" in early) {
    console.error(early.error);
    process.exit(2);
  }
  const list = await fetch(`${early.url}?state=open&limit=500`, { headers: auth });
  if (!list.ok) {
    console.error(`could not list open issues (${list.status}); filing nothing`);
    process.exit(1);
  }
  const open = (((await list.json()) as { items?: { title?: string }[] }).items ?? []).flatMap((i) => (i.title ? [i.title] : []));
  const plan = filingPlan({ forge, base, into, open, findings, parent, cap }) as Exclude<ReturnType<typeof filingPlan>, { error: string }>;
  console.log(`${findings.length} findings, ${plan.filing.length} to file (cap ${cap}, ${plan.skipped} skipped as open or over the cap)`);
  for (const f of plan.filing) {
    const r = await fetch(plan.url, {
      method: "POST",
      headers: { ...auth, "content-type": "application/json" },
      body: JSON.stringify({ title: f.title, body: f.body }),
    });
    console.log(`${r.status} ${f.page} ${f.device}`);
  }
}
@@ -302,7 +302,7 @@
if (import.meta.main) {
  const root = process.cwd();
  const now = scan(root);
  if (process.argv.includes("--update")) {
    const baseline: Baseline = { files: now.files };
    const baseline = { ...JSON.parse(readFileSync(join(root, BASELINE), "utf8")), files: now.files }; // keeps `coldRead` (#5882)
    writeFileSync(join(root, BASELINE), JSON.stringify(baseline, null, 2) + "\n");
    console.log(`wrote ${BASELINE}`);
  } else {
@@ -0,0 +1,178 @@
/**
 * The cold-read audit's plain data: which pages, at which widths, how an
 * answer is read, scored and merged into `docs/design/baseline.json`. No
 * network and no model here; `scripts/cold-read.ts` does those.
 */
/** Order of work: /migrate, /, a new forge's first run, then the rest by traffic. */
export const PAGES = ["/migrate", "/", "/forges/new", "/settings"] as const;
      
        sentinelcrew
        19d ago
        outdated
      
      /trevato/forge, /pulls and /issues are 404 on the preview and on production, and /migrate and /settings redirect to /signin. Take the repo as an input and sign the reader in, or this list scores 404 and sign-in pages.
    
/** The repository pages, for `--repo owner/name`: repo, pulls, issues. */
export const repoPages = (repo: string): string[] => [`/${repo}`, `/${repo}/pulls`, `/${repo}/issues`];
export const REPO = /^[A-Za-z0-9_.-]{1,64}\/[A-Za-z0-9_.-]{1,64}$/;
/**
 * A reading is of the page asked for or it is nothing: a redirect (a sign-in),
 * a 404 or any non-200 is refused with the reason, never scored under the
 * asked-for page's name. `location` is the redirect target when there is one.
 */
export function refusal(path: string, status: number, location?: string | null): string | null {
  if (status >= 300 && status < 400) return `${path} redirects to ${location ?? "elsewhere"} (${status}), not the page itself`;
  if (status !== 200) return `${path} answered ${status}`;
  return null;
}
/** Titles of findings already open, normalised, so a second run skips them. */
export const titleKey = (t: string): string => t.trim().toLowerCase();
/** What to file: not above `cap`, none already open, none twice. */
export function toFile<T extends { title: string }>(found: readonly T[], open: ReadonlySet<string>, cap: number): T[] {
  const seen = new Set(open);
  const out: T[] = [];
  for (const f of found) {
    const k = titleKey(f.title);
    if (seen.has(k) || out.length >= cap) continue;
    seen.add(k);
    out.push(f);
  }
  return out;
}
export const WIDTHS = [
  { width: 390, height: 844, device: "phone" },
  { width: 1440, height: 900, device: "desktop" },
] as const;
export const MAX_WORDS = 100;
export interface Answer {
  readonly confusing: readonly string[];
  readonly clickFirst: string;
  readonly clarity: number;
}
const words = (s: string): number => s.split(/\s+/).filter(Boolean).length;
/** The model's reply as an Answer, or why it is not one. Untrusted text in, bounded data out. */
export function parseAnswer(raw: string): Answer | string {
  const start = raw.indexOf("{");
  const end = raw.lastIndexOf("}");
  if (start < 0 || end < start) return "no JSON object in the reply";
  let o: unknown;
  try {
    o = JSON.parse(raw.slice(start, end + 1));
  } catch {
    return "the reply is not valid JSON";
  }
  if (typeof o !== "object" || o === null) return "the reply is not an object";
  const r = o as Record<string, unknown>;
  const { confusing, clickFirst, clarity } = r;
  if (!Array.isArray(confusing) || !confusing.every((c) => typeof c === "string")) return "confusing must be a list of strings";
  if (typeof clickFirst !== "string") return "clickFirst must be a string";
  if (typeof clarity !== "number" || !Number.isInteger(clarity) || clarity < 1 || clarity > 5) return "clarity must be an integer 1-5";
  const list = confusing.map((c) => c.trim()).filter(Boolean).slice(0, 8);
  return { confusing: list, clickFirst: clickFirst.trim(), clarity };
}
export const answerWords = (a: Answer): number => a.confusing.reduce((n, c) => n + words(c), words(a.clickFirst));
export interface PageScore {
  /** Mean clarity (1-5) over the widths that answered. */
  readonly clarity: number;
  readonly phone: number | null;
  readonly desktop: number | null;
  readonly confusions: number;
  readonly words: number;
}
export interface Reading {
  readonly device: string;
  readonly answer: Answer;
}
export function scorePage(readings: readonly Reading[]): PageScore | null {
  if (readings.length === 0) return null;
  const of = (d: string) => readings.find((r) => r.device === d)?.answer.clarity ?? null;
  const mean = readings.reduce((n, r) => n + r.answer.clarity, 0) / readings.length;
  return {
    clarity: Math.round(mean * 10) / 10,
    phone: of("phone"),
    desktop: of("desktop"),
    confusions: readings.reduce((n, r) => n + r.answer.confusing.length, 0),
    words: readings.reduce((n, r) => n + answerWords(r.answer), 0),
  };
}
export interface ColdRead {
  readonly at: string;
  /** The parent issue, when `--file-issues` named one. */
  readonly issue?: number;
  /** `text-only` when there was no Chromium: the two widths read the same words. */
  readonly mode: "screenshot" | "text-only";
  readonly pages: Record<string, PageScore>;
}
/** The baseline with `coldRead` set, every other key kept as it was. */
export function withColdRead(baseline: Record<string, unknown>, coldRead: ColdRead): Record<string, unknown> {
  return { ...baseline, coldRead };
}
/** One finding as an issue: a title that names the page and the blamed text, a body that links back. */
export function findingIssue(page: string, device: string, text: string, parent: number): { title: string; body: string } {
  const title = `Cold read ${page} (${device}): ${text}`.slice(0, 120);
  return {
    title,
    body: `A no-context agent reading ${page} at ${device} width said:\n\n> ${text.replace(/\n/g, " ")}\n\nFound by the cold-read audit (\`bun scripts/cold-read.ts\`), part of #${parent}.`,
  };
}
/** Why `--write` must not store this run, or null. A baseline is never written from a partial or text-only read. */
export function writeRefusal(run: { failed: Record<string, string[]>; mode: string }, allowTextOnly: boolean): string | null {
  if (run.mode === "text-only" && !allowTextOnly) return "refusing to --write a text-only score (no Chromium): install it, or pass --allow-text-only to record it marked text-only";
  if (Object.keys(run.failed).length > 0) return "refusing to --write: some pages were not read (is FORGE_TOKEN honoured on pages?)";
  return null;
}
/**
 * Where findings are filed and which of them: `error` when the target is not
 * the forge that was read, or not named; otherwise the issues URL and the
 * findings that are neither open already, repeated, nor over `cap`.
 */
export function filingPlan(o: {
  forge: string | undefined;
  base: string;
  into: string | undefined;
  open: readonly string[];
  findings: readonly { page: string; device: string; text: string }[];
  parent: number;
  cap: number;
}): { error: string } | { url: string; filing: { title: string; body: string; page: string; device: string }[]; skipped: number } {
  if (!o.forge) return { error: "--file-issues needs FORGE_URL" };
  if (!o.into) return { error: "--file-issues needs --repo or --file-into: the repository to file into" };
  let same = false;
  try {
    same = new URL(o.forge).origin === new URL(o.base).origin;
  } catch {
    /* not a URL: not the same */
  }
  if (!same) return { error: `FORGE_URL (${o.forge}) is not ${o.base}: findings are filed only into the forge that was read` };
  const mine = o.findings.map((f) => ({ ...f, ...findingIssue(f.page, f.device, f.text, o.parent) }));
  const filing = toFile(mine, new Set(o.open.map(titleKey)), o.cap);
  return { url: `${o.forge.replace(/\/$/, "")}/api/v1/repos/${o.into}/issues`, filing, skipped: mine.length - filing.length };
}
/**
 * Whether `FORGE_TOKEN` may be sent to `base`: only to the forge `FORGE_URL`
 * names. Checked before the first request, so a mistyped or foreign URL never
 * receives the token. Null when it may; otherwise why not.
 */
export function tokenRefusal(base: string, forge: string | undefined): string | null {
  if (!forge) return "FORGE_URL is not set: the token is sent only to the forge it names";
  try {
    if (new URL(base).origin === new URL(forge).origin) return null;
  } catch {
    /* fall through */
  }
  return `${base} is not the forge in FORGE_URL (${forge}): FORGE_TOKEN is not sent there`;
}
@@ -0,0 +1,60 @@
/**
 * The cold-read loop with its three outside things injected: fetching a page,
 * shooting it, asking the model. `scripts/cold-read.ts` passes the real ones;
 * the test passes stubs.
 */
import { MAX_WORDS, WIDTHS, answerWords, parseAnswer, refusal, scorePage, type PageScore, type Reading } from "./coldread.ts";
export interface Shot {
  /** "image" when Chromium shot it, "text" when only the page's text is sent. */
  readonly kind: "image" | "text";
  readonly block: unknown;
}
export interface Deps {
  fetchPage(path: string): Promise<{ status: number; location: string | null; html: string }>;
  shoot(html: string, path: string, w: { width: number; height: number }): Promise<Shot>;
  ask(shot: unknown, prompt: string): Promise<string>;
  log(line: string): void;
}
export interface Run {
  readonly pages: Record<string, PageScore>;
  readonly findings: { page: string; device: string; text: string }[];
  /** Pages that could not be read, with why. */
  readonly failed: Record<string, string[]>;
  /** "text-only" when any reading had no screenshot: phone and desktop then saw the same words. */
  readonly mode: "screenshot" | "text-only";
}
export async function coldRead(pagePaths: readonly string[], promptTemplate: string, deps: Deps): Promise<Run> {
  const pages: Run["pages"] = {};
  const findings: Run["findings"] = [];
  const failed: Record<string, string[]> = {};
  let textOnly = false;
  for (const page of pagePaths) {
    const readings: Reading[] = [];
    for (const w of WIDTHS) {
      try {
        const res = await deps.fetchPage(page);
        const why = refusal(page, res.status, res.location);
        if (why) throw new Error(why);
        const shot = await deps.shoot(res.html, page, w);
        if (shot.kind === "text") textOnly = true;
        const prompt = promptTemplate.replace("{{width}}", String(w.width)).replace("{{device}}", w.device);
        const answer = parseAnswer(await deps.ask(shot.block, prompt));
        if (typeof answer === "string") throw new Error(answer);
        if (answerWords(answer) > MAX_WORDS) deps.log(`${page} ${w.device}: over ${MAX_WORDS} words (${answerWords(answer)}), kept`);
        readings.push({ device: w.device, answer });
        for (const c of answer.confusing) findings.push({ page, device: w.device, text: c });
        deps.log(`${page} ${w.device}: clarity ${answer.clarity}; first click: ${answer.clickFirst}`);
      } catch (e) {
        (failed[page] ??= []).push(`${w.device}: ${(e as Error).message}`);
        deps.log(`${page} ${w.device}: ${(e as Error).message}`);
      }
    }
    const s = scorePage(readings);
    if (s) pages[page] = s;
  }
  return { pages, findings, failed, mode: textOnly ? "text-only" : "screenshot" };
}
@@ -0,0 +1,107 @@
import { describe, expect, test } from "bun:test";
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { coldRead } from "../src/design/coldrun.ts";
import { tokenRefusal, filingPlan, writeRefusal, PAGES, REPO, findingIssue, refusal, repoPages, toFile, parseAnswer, scorePage, withColdRead } from "../src/design/coldread.ts";
const ok = (clarity: number, n = 1) => ({ confusing: Array(n).fill("the banner is unclear"), clickFirst: "the big button", clarity });
describe("cold-read audit (#5882)", () => {
  test("reads the model's JSON and refuses what is not", () => {
    expect(parseAnswer('sure: {"confusing":["a"],"clickFirst":"b","clarity":4}')).toEqual({ confusing: ["a"], clickFirst: "b", clarity: 4 });
    expect(typeof parseAnswer("no")).toBe("string");
    expect(typeof parseAnswer('{"confusing":"a","clickFirst":"b","clarity":4}')).toBe("string");
    expect(typeof parseAnswer('{"confusing":[],"clickFirst":"b","clarity":9}')).toBe("string");
  });
  test("scores a page by the mean of its widths", () => {
    const s = scorePage([{ device: "phone", answer: ok(2, 2) }, { device: "desktop", answer: ok(4) }]);
    expect(s).toMatchObject({ clarity: 3, phone: 2, desktop: 4, confusions: 3 });
    expect(scorePage([])).toBeNull();
  });
  test("writes coldRead and keeps the rest of the baseline", () => {
    const out = withColdRead({ files: { a: 1 } }, { at: "x", issue: 5882, mode: "screenshot", pages: {} });
    expect(out["files"]).toEqual({ a: 1 });
    expect(out["coldRead"]).toBeDefined();
  });
  test("a finding links back to its parent", () => {
    expect(findingIssue("/migrate", "phone", "the button", 5882).body).toContain("#5882");
  });
  test("the order starts /migrate, /, new forge; the prompt is in the repository", () => {
    expect(PAGES.slice(0, 3)).toEqual(["/migrate", "/", "/forges/new"]);
    expect(readFileSync("docs/design/cold-read-prompt.md", "utf8")).toContain("under 100 words");
  });
  test("a redirect or a non-200 is refused, a 200 is read", () => {
    expect(refusal("/migrate", 303, "/signin")).toContain("/signin");
    expect(refusal("/a/b", 404)).toContain("404");
    expect(refusal("/", 200)).toBeNull();
  });
  test("repo pages come from the repo, which must be owner/name", () => {
    expect(repoPages("a/b")).toEqual(["/a/b", "/a/b/pulls", "/a/b/issues"]);
    expect(REPO.test("a/b")).toBe(true);
    expect(REPO.test("a/b/../c")).toBe(false);
  });
  test("filing is capped and skips titles already open or repeated", () => {
    const f = [{ title: "A" }, { title: "b" }, { title: "B" }, { title: "c" }];
    expect(toFile(f, new Set(["a"]), 10).map((x) => x.title)).toEqual(["b", "c"]);
    expect(toFile(f, new Set(), 2).map((x) => x.title)).toEqual(["A", "b"]);
  });
  test("design-lint --update keeps coldRead", async () => {
    const dir = mkdtempSync(join(process.env["TMPDIR"] ?? "/tmp", "dl-"));
    Bun.spawnSync(["mkdir", "-p", join(dir, "docs/design/areas"), join(dir, "src/web")]);
    writeFileSync(join(dir, "docs/DESIGN.md"), "");
    writeFileSync(join(dir, "docs/design/baseline.json"), JSON.stringify({ files: {}, coldRead: { at: "x", issue: 1, pages: {} } }));
    const p = Bun.spawnSync(["bun", join(process.cwd(), "scripts/design-lint.ts"), "--update"], { cwd: dir });
    expect(p.stderr.toString()).toBe("");
    expect(JSON.parse(readFileSync(join(dir, "docs/design/baseline.json"), "utf8")).coldRead.issue).toBe(1);
  });
  test("the loop reads, refuses, scores and flags text-only, with a stubbed model", async () => {
    const asked: string[] = [];
    const run = await coldRead(["/", "/migrate"], "w={{width}} d={{device}}", {
      fetchPage: async (p) => (p === "/" ? { status: 200, location: null, html: "<p>hi</p>" } : { status: 303, location: "/signin", html: "" }),
      shoot: async (html) => ({ kind: "text", block: html }),
      ask: async (_s, prompt) => (asked.push(prompt), '{"confusing":["what is this"],"clickFirst":"sign in","clarity":3}'),
      log: () => {},
    });
    expect(asked).toEqual(["w=390 d=phone", "w=1440 d=desktop"]);
    expect(run.pages["/"]).toMatchObject({ clarity: 3, phone: 3, desktop: 3 });
    expect(run.pages["/migrate"]).toBeUndefined();
    expect(run.failed["/migrate"]?.[0]).toContain("/signin");
    expect(run.findings).toHaveLength(2);
    expect(run.mode).toBe("text-only");
  });
  test("a reply that is not an answer fails the page, not the run", async () => {
    const run = await coldRead(["/"], "", { fetchPage: async () => ({ status: 200, location: null, html: "" }), shoot: async () => ({ kind: "image", block: 1 }), ask: async () => "nope", log: () => {} });
    expect(run.pages).toEqual({});
    expect(run.failed["/"]).toHaveLength(2);
    expect(run.mode).toBe("screenshot");
  });
  test("--write is refused for a text-only or partial read, allowed otherwise", () => {
    expect(writeRefusal({ failed: {}, mode: "screenshot" }, false)).toBeNull();
    expect(writeRefusal({ failed: {}, mode: "text-only" }, false)).toContain("text-only");
    expect(writeRefusal({ failed: {}, mode: "text-only" }, true)).toBeNull();
    expect(writeRefusal({ failed: { "/migrate": ["phone: 303"] }, mode: "screenshot" }, true)).toContain("not read");
  });
  test("filing names its target, stays on the forge it read, skips open titles, honours the cap", () => {
    const o = { forge: "https://f.example/", base: "https://f.example", into: "a/b", open: [], findings: [{ page: "/", device: "phone", text: "x" }, { page: "/", device: "desktop", text: "y" }], parent: 1, cap: 10 };
    const p = filingPlan(o);
    expect("url" in p && p.url).toBe("https://f.example/api/v1/repos/a/b/issues");
    expect("filing" in p && p.filing).toHaveLength(2);
    expect(filingPlan({ ...o, forge: undefined })).toHaveProperty("error");
    expect(filingPlan({ ...o, into: undefined })).toHaveProperty("error");
    expect(filingPlan({ ...o, forge: "https://other.example" })).toHaveProperty("error");
    expect(filingPlan({ ...o, forge: "nonsense" })).toHaveProperty("error");
    const open = filingPlan(o) as { filing: { title: string }[] };
    const again = filingPlan({ ...o, open: [open.filing[0]!.title.toUpperCase()] });
    expect("filing" in again && again.filing).toHaveLength(1);
    const capped = filingPlan({ ...o, cap: 1 });
    expect("skipped" in capped && capped.skipped).toBe(1);
  });
  test("the token goes only to the forge FORGE_URL names", () => {
    expect(tokenRefusal("https://f.example", "https://f.example/")).toBeNull();
    expect(tokenRefusal("https://evil.example", "https://f.example")).toContain("not sent");
    expect(tokenRefusal("http://f.example", "https://f.example")).toContain("not sent");
    expect(tokenRefusal("https://f.example", undefined)).toContain("FORGE_URL");
    expect(tokenRefusal("https://f.example", "nonsense")).toContain("not sent");
  });
});
		@@ -1711,6 +1711,7 @@ forge icon.
1711	1711	```sh
1712	1712	bun test # protocol-level + adversarial suite
1713	1713	bun scripts/design-lint.ts # no colour or off-scale font-size beyond docs/design/baseline.json, no class docs/DESIGN.md §5 does not name
	1714	bun scripts/cold-read.ts <url> [--repo owner/name] [--write [--allow-text-only]] [--file-issues N [--file-into owner/name]] # cold-read audit (#5882): a no-context model reads the top pages (signed in with FORGE_TOKEN, sent only to FORGE_URL's forge; the signed-in pages go to the model endpoint, so use a throwaway account) at 390 and 1440 px; needs ANTHROPIC_API_KEY
1714	1715	bun x tsc --noEmit # zero type errors is the standard
1715	1716	bun run check # exactly what the forge runs against every push
1716	1717	bun run check:record # the same, and rewrites scripts/test-times.json
		@@ -0,0 +1,13 @@
	1	You have never seen this product and know nothing about it. You are shown one
	2	page of it as it looks on a screen {{width}} px wide ({{device}}). Judge only what
	3	is in front of you.
	4
	5	Answer in under 100 words in total, as JSON and nothing else:
	6
	7	{"confusing": ["what is confusing or unexplained, one short sentence each"],
	8	"clickFirst": "the one thing you would click or tap first, and why",
	9	"clarity": 1-5}
	10
	11	`clarity` is how sure you are, in five seconds, of what this page is for and
	12	what to do next: 1 = no idea, 5 = obvious. Name the part of the page you blame
	13	(a button, a heading, a list) in each confusion, so it can be judged.
		@@ -0,0 +1,156 @@
	1	/**
	2	* `bun scripts/cold-read.ts <forge-url> [--repo owner/name] [--write [--allow-text-only]]
	3	* [--file-issues <parent> [--file-into owner/name] [--max N]]`
	4	*
	5	* A no-context agent reads each top page at 390 px and 1440 px and says, in
	6	* under 100 words, what is confusing and what it would click first. The prompt
	7	* is `docs/design/cold-read-prompt.md`. The reader is signed in with
	8	* `FORGE_TOKEN` (a bearer the forge honours on pages), so /migrate and
	9	* /settings show their real first run. A page that is not a 200 at its own
	10	* URL (a redirect, a 404) is refused, never scored. `--repo` adds that
	11	* repository's page, pulls and issues after checking it exists. The fetched
	12	* server-rendered HTML is shot with headless Chromium at the width (its text
	13	* when there is no browser; `--write` then refuses unless `--allow-text-only`, which marks the score `text-only`) and sent, with the prompt, to the model named by `COLD_READ_MODEL`
	14	* (default claude-sonnet-5-5) through `ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL`.
	15	*
		sentinelcrew 19d ago outdated `readFileSync` is imported twice (`readFileSync as read`). Use the one name.
	16	* `FORGE_TOKEN` goes only to the forge `FORGE_URL` names; any other <forge-url> is refused before a request.
	17	* Privacy: the signed-in pages (/settings can show an email, token names, linked accounts) are sent in full to the
	18	* model endpoint, `ANTHROPIC_BASE_URL` included. Use a throwaway account's token unless that is acceptable. The shot
	19	* is loaded from file://, so it carries no CSP header, and Chromium runs `--no-sandbox`: read only your own forge.
	20	*
	21	* It prints a score per page. `--write` stores them under `coldRead` in
	22	* `docs/design/baseline.json`; `--file-issues N` files each confusion as an
	23	* issue in `--file-into` (default `--repo`) on `FORGE_URL` (which must be the forge read), linking back to #N: at most `--max` (default 20), and
	24	* none whose title is already open.
	25	*/
	26	import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
	27	import { join } from "node:path";
	28	import { tmpdir } from "node:os";
	29	import { findChromium } from "../src/looks/capture.ts";
	30	import { PAGES, REPO, filingPlan, repoPages, tokenRefusal, withColdRead, writeRefusal, type ColdRead } from "../src/design/coldread.ts";
	31	import { coldRead, type Shot } from "../src/design/coldrun.ts";
	32
	33	const BASELINE = "docs/design/baseline.json";
	34	const args = process.argv.slice(2);
	35	const base = args.find((a) => /^https?:\/\//.test(a))?.replace(/\/$/, "");
	36	const write = args.includes("--write");
	37	const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined);
	38	const parent = opt("--file-issues") === undefined ? undefined : Number(opt("--file-issues"));
	39	const repo = opt("--repo");
	40	const into = opt("--file-into") ?? repo;
	41	const cap = Number(opt("--max") ?? 20);
	42	if (!base \|\| (parent !== undefined && !Number.isInteger(parent)) \|\| (repo !== undefined && !REPO.test(repo)) \|\| (into !== undefined && !REPO.test(into)) \|\| !Number.isInteger(cap) \|\| cap < 0) {
	43	console.error("usage: bun scripts/cold-read.ts <url> [--repo owner/name] [--write [--allow-text-only]] [--file-issues <issue number> [--file-into owner/name] [--max N]]");
	44	process.exit(2);
	45	}
	46	const token = process.env["FORGE_TOKEN"];
		sentinelcrew 19d ago outdated High: this bearer is sent to `base` (the `--repo` probe on the next line, and `fetchPage`) before anything checks that `base` is the forge in FORGE_URL. Run the same-origin check here at startup, before the first fetch, not only in `filingPlan`. Then a mistyped or foreign URL never receives the token.
	47	if (!token) {
	48	console.error("FORGE_TOKEN is not set: the reader must be signed in, or /migrate and /settings are the sign-in page");
	49	process.exit(2);
	50	}
	51	const unsafe = tokenRefusal(base, process.env["FORGE_URL"]);
	52	if (unsafe) {
	53	console.error(unsafe);
	54	process.exit(2);
	55	}
	56	const auth = { authorization: `Bearer ${token}` };
	57	if (repo !== undefined) {
		sentinelcrew 19d ago outdated This follows redirects silently and doesn't check `res.status`. The Chromium path has the same problem. Reject a reading whose final URL or status isn't the page asked for, so a sign-in or 404 page is never scored under another page's name.
	58	const r = await fetch(`${base}/api/v1/repos/${repo}`, { headers: auth, redirect: "manual" });
	59	if (r.status !== 200) {
	60	console.error(`--repo ${repo}: ${base} answered ${r.status}`);
	61	process.exit(2);
	62	}
	63	}
	64	const key = process.env["ANTHROPIC_API_KEY"];
	65	if (!key) {
	66	console.error("ANTHROPIC_API_KEY is not set: the audit needs a model to read the pages");
	67	process.exit(2);
	68	}
	69	const api = (process.env["ANTHROPIC_BASE_URL"] ?? "https://api.anthropic.com").replace(/\/$/, "");
	70	const model = process.env["COLD_READ_MODEL"] ?? "claude-sonnet-5-5";
	71	const prompt = readFileSync(new URL("../docs/design/cold-read-prompt.md", import.meta.url), "utf8");
		sentinelcrew 19d ago outdated Low: there is no timeout on the Chromium process, so a page that never finishes loading hangs the whole run. Pass `--timeout=` / `--virtual-time-budget=`, or kill it after a few seconds. Also note: loaded from file://, the page has lost the forge's CSP response header, and runs with `--no-sandbox`.
	72
	73	async function shoot(html: string, path: string, w: { width: number; height: number }): Promise<Shot> {
	74	const binary = findChromium();
	75	if (binary) {
	76	const dir = mkdtempSync(join(process.env["TMPDIR"] ?? tmpdir(), "coldread-"));
	77	try {
	78	// The first paint is server-rendered; <base> lets its stylesheets load from the forge.
	79	await Bun.write(join(dir, "p.html"), html.replace(/<head[^>]*>/i, (h) => `${h}<base href="${base + path}">`));
	80	const out = join(dir, "p.png");
	81	const p = Bun.spawn([binary, "--headless", "--no-sandbox", "--hide-scrollbars", "--virtual-time-budget=5000", `--window-size=${w.width},${w.height}`, `--screenshot=${out}`, `file://${join(dir, "p.html")}`], { stdout: "ignore", stderr: "ignore" });
	82	const timer = setTimeout(() => p.kill(), 15_000); // a page that never settles must not hang the run
	83	await p.exited;
	84	clearTimeout(timer);
	85	const data = Buffer.from(readFileSync(out)).toString("base64");
	86	return { kind: "image", block: { type: "image", source: { type: "base64", media_type: "image/png", data } } };
	87	} catch {
	88	/* fall through to text */
	89	} finally {
	90	rmSync(dir, { recursive: true, force: true });
	91	}
	92	}
	93	const text = html.replace(/<(script\|style)[\s\S]*?<\/\1>/gi, " ").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").slice(0, 8000);
	94	return { kind: "text", block: { type: "text", text: `The page's text:\n${text}` } };
	95	}
	96
	97	async function ask(shot: unknown, text: string): Promise<string> {
	98	const content = [shot, { type: "text", text }];
	99	const res = await fetch(`${api}/v1/messages`, {
		sentinelcrew 19d ago outdated Medium: this is the signed-in HTML of /settings and /migrate (it can include email, token names, linked accounts), and it is sent in full to `ANTHROPIC_BASE_URL`, which is also overridable. Say this in the header and README, or read /settings with a fresh throwaway account. An operator's private settings should not reach a third-party model by default.
	100	method: "POST",
	101	headers: { "x-api-key": key!, "anthropic-version": "2023-06-01", "content-type": "application/json" },
	102	body: JSON.stringify({ model, max_tokens: 400, messages: [{ role: "user", content }] }),
	103	});
	104	if (!res.ok) throw new Error(`model answered ${res.status}`);
	105	const j = (await res.json()) as { content?: { type: string; text?: string }[] };
	106	return j.content?.find((c) => c.type === "text")?.text ?? "";
		sentinelcrew 19d ago outdated This has no cap and no de-duplication, so one run can file up to 112 issues and a rerun files them all again. Cap it, and skip a title that is already open.
	107	}
	108
	109	const run = await coldRead([...PAGES, ...(repo ? repoPages(repo) : [])], prompt, {
	110	fetchPage: async (path) => {
	111	const res = await fetch(base + path, { headers: auth, redirect: "manual" });
	112	return { status: res.status, location: res.headers.get("location"), html: res.status === 200 ? await res.text() : "" };
	113	},
	114	shoot,
	115	ask,
	116	log: (l) => console.error(l),
	117	});
	118	const { pages, findings } = run;
	119	for (const [page, why] of Object.entries(run.failed)) console.error(`NOT SCORED ${page}: ${why.join("; ")}`);
	120	if (run.mode === "text-only") console.error("text-only: no Chromium, so 390 and 1440 were sent the same words; phone and desktop are not comparable");
	121	const refused = write ? writeRefusal(run, args.includes("--allow-text-only")) : null;
	122	if (refused) {
	123	console.error(refused);
	124	process.exit(1);
	125	}
	126
	127	if (write) {
	128	const record: ColdRead = { at: new Date().toISOString(), ...(parent !== undefined ? { issue: parent } : {}), mode: run.mode, pages };
	129	writeFileSync(BASELINE, JSON.stringify(withColdRead(JSON.parse(readFileSync(BASELINE, "utf8")), record), null, 2) + "\n");
	130	console.log(`wrote coldRead to ${BASELINE}`);
	131	}
	132	if (parent !== undefined) {
	133	const forge = process.env["FORGE_URL"];
	134	// The target is checked before anything is fetched; the open list then decides what is filed.
	135	const early = filingPlan({ forge, base, into, open: [], findings: [], parent, cap });
	136	if ("error" in early) {
	137	console.error(early.error);
	138	process.exit(2);
	139	}
	140	const list = await fetch(`${early.url}?state=open&limit=500`, { headers: auth });
	141	if (!list.ok) {
	142	console.error(`could not list open issues (${list.status}); filing nothing`);
	143	process.exit(1);
	144	}
	145	const open = (((await list.json()) as { items?: { title?: string }[] }).items ?? []).flatMap((i) => (i.title ? [i.title] : []));
	146	const plan = filingPlan({ forge, base, into, open, findings, parent, cap }) as Exclude<ReturnType<typeof filingPlan>, { error: string }>;
	147	console.log(`${findings.length} findings, ${plan.filing.length} to file (cap ${cap}, ${plan.skipped} skipped as open or over the cap)`);
	148	for (const f of plan.filing) {
	149	const r = await fetch(plan.url, {
	150	method: "POST",
	151	headers: { ...auth, "content-type": "application/json" },
	152	body: JSON.stringify({ title: f.title, body: f.body }),
	153	});
	154	console.log(`${r.status} ${f.page} ${f.device}`);
	155	}
	156	}
		@@ -302,7 +302,7 @@ if (import.meta.main) {
302	302	const root = process.cwd();
303	303	const now = scan(root);
304	304	if (process.argv.includes("--update")) {
305		const baseline: Baseline = { files: now.files };
	305	const baseline = { ...JSON.parse(readFileSync(join(root, BASELINE), "utf8")), files: now.files }; // keeps `coldRead` (#5882)
306	306	writeFileSync(join(root, BASELINE), JSON.stringify(baseline, null, 2) + "\n");
307	307	console.log(`wrote ${BASELINE}`);
308	308	} else {
		@@ -0,0 +1,178 @@
	1	/**
	2	* The cold-read audit's plain data: which pages, at which widths, how an
	3	* answer is read, scored and merged into `docs/design/baseline.json`. No
	4	* network and no model here; `scripts/cold-read.ts` does those.
	5	*/
	6
	7	/** Order of work: /migrate, /, a new forge's first run, then the rest by traffic. */
	8	export const PAGES = ["/migrate", "/", "/forges/new", "/settings"] as const;
		sentinelcrew 19d ago outdated `/trevato/forge`, `/pulls` and `/issues` are 404 on the preview and on production, and `/migrate` and `/settings` redirect to `/signin`. Take the repo as an input and sign the reader in, or this list scores 404 and sign-in pages.
	9
	10	/** The repository pages, for `--repo owner/name`: repo, pulls, issues. */
	11	export const repoPages = (repo: string): string[] => [`/${repo}`, `/${repo}/pulls`, `/${repo}/issues`];
	12
	13	export const REPO = /^[A-Za-z0-9_.-]{1,64}\/[A-Za-z0-9_.-]{1,64}$/;
	14
	15	/**
	16	* A reading is of the page asked for or it is nothing: a redirect (a sign-in),
	17	* a 404 or any non-200 is refused with the reason, never scored under the
	18	* asked-for page's name. `location` is the redirect target when there is one.
	19	*/
	20	export function refusal(path: string, status: number, location?: string \| null): string \| null {
	21	if (status >= 300 && status < 400) return `${path} redirects to ${location ?? "elsewhere"} (${status}), not the page itself`;
	22	if (status !== 200) return `${path} answered ${status}`;
	23	return null;
	24	}
	25
	26	/** Titles of findings already open, normalised, so a second run skips them. */
	27	export const titleKey = (t: string): string => t.trim().toLowerCase();
	28
	29	/** What to file: not above `cap`, none already open, none twice. */
	30	export function toFile<T extends { title: string }>(found: readonly T[], open: ReadonlySet<string>, cap: number): T[] {
	31	const seen = new Set(open);
	32	const out: T[] = [];
	33	for (const f of found) {
	34	const k = titleKey(f.title);
	35	if (seen.has(k) \|\| out.length >= cap) continue;
	36	seen.add(k);
	37	out.push(f);
	38	}
	39	return out;
	40	}
	41
	42	export const WIDTHS = [
	43	{ width: 390, height: 844, device: "phone" },
	44	{ width: 1440, height: 900, device: "desktop" },
	45	] as const;
	46
	47	export const MAX_WORDS = 100;
	48
	49	export interface Answer {
	50	readonly confusing: readonly string[];
	51	readonly clickFirst: string;
	52	readonly clarity: number;
	53	}
	54
	55	const words = (s: string): number => s.split(/\s+/).filter(Boolean).length;
	56
	57	/** The model's reply as an Answer, or why it is not one. Untrusted text in, bounded data out. */
	58	export function parseAnswer(raw: string): Answer \| string {
	59	const start = raw.indexOf("{");
	60	const end = raw.lastIndexOf("}");
	61	if (start < 0 \|\| end < start) return "no JSON object in the reply";
	62	let o: unknown;
	63	try {
	64	o = JSON.parse(raw.slice(start, end + 1));
	65	} catch {
	66	return "the reply is not valid JSON";
	67	}
	68	if (typeof o !== "object" \|\| o === null) return "the reply is not an object";
	69	const r = o as Record<string, unknown>;
	70	const { confusing, clickFirst, clarity } = r;
	71	if (!Array.isArray(confusing) \|\| !confusing.every((c) => typeof c === "string")) return "confusing must be a list of strings";
	72	if (typeof clickFirst !== "string") return "clickFirst must be a string";
	73	if (typeof clarity !== "number" \|\| !Number.isInteger(clarity) \|\| clarity < 1 \|\| clarity > 5) return "clarity must be an integer 1-5";
	74	const list = confusing.map((c) => c.trim()).filter(Boolean).slice(0, 8);
	75	return { confusing: list, clickFirst: clickFirst.trim(), clarity };
	76	}
	77
	78	export const answerWords = (a: Answer): number => a.confusing.reduce((n, c) => n + words(c), words(a.clickFirst));
	79
	80	export interface PageScore {
	81	/** Mean clarity (1-5) over the widths that answered. */
	82	readonly clarity: number;
	83	readonly phone: number \| null;
	84	readonly desktop: number \| null;
	85	readonly confusions: number;
	86	readonly words: number;
	87	}
	88
	89	export interface Reading {
	90	readonly device: string;
	91	readonly answer: Answer;
	92	}
	93
	94	export function scorePage(readings: readonly Reading[]): PageScore \| null {
	95	if (readings.length === 0) return null;
	96	const of = (d: string) => readings.find((r) => r.device === d)?.answer.clarity ?? null;
	97	const mean = readings.reduce((n, r) => n + r.answer.clarity, 0) / readings.length;
	98	return {
	99	clarity: Math.round(mean * 10) / 10,
	100	phone: of("phone"),
	101	desktop: of("desktop"),
	102	confusions: readings.reduce((n, r) => n + r.answer.confusing.length, 0),
	103	words: readings.reduce((n, r) => n + answerWords(r.answer), 0),
	104	};
	105	}
	106
	107	export interface ColdRead {
	108	readonly at: string;
	109	/** The parent issue, when `--file-issues` named one. */
	110	readonly issue?: number;
	111	/** `text-only` when there was no Chromium: the two widths read the same words. */
	112	readonly mode: "screenshot" \| "text-only";
	113	readonly pages: Record<string, PageScore>;
	114	}
	115
	116	/** The baseline with `coldRead` set, every other key kept as it was. */
	117	export function withColdRead(baseline: Record<string, unknown>, coldRead: ColdRead): Record<string, unknown> {
	118	return { ...baseline, coldRead };
	119	}
	120
	121	/** One finding as an issue: a title that names the page and the blamed text, a body that links back. */
	122	export function findingIssue(page: string, device: string, text: string, parent: number): { title: string; body: string } {
	123	const title = `Cold read ${page} (${device}): ${text}`.slice(0, 120);
	124	return {
	125	title,
	126	body: `A no-context agent reading ${page} at ${device} width said:\n\n> ${text.replace(/\n/g, " ")}\n\nFound by the cold-read audit (\`bun scripts/cold-read.ts\`), part of #${parent}.`,
	127	};
	128	}
	129
	130	/** Why `--write` must not store this run, or null. A baseline is never written from a partial or text-only read. */
	131	export function writeRefusal(run: { failed: Record<string, string[]>; mode: string }, allowTextOnly: boolean): string \| null {
	132	if (run.mode === "text-only" && !allowTextOnly) return "refusing to --write a text-only score (no Chromium): install it, or pass --allow-text-only to record it marked text-only";
	133	if (Object.keys(run.failed).length > 0) return "refusing to --write: some pages were not read (is FORGE_TOKEN honoured on pages?)";
	134	return null;
	135	}
	136
	137	/**
	138	* Where findings are filed and which of them: `error` when the target is not
	139	* the forge that was read, or not named; otherwise the issues URL and the
	140	* findings that are neither open already, repeated, nor over `cap`.
	141	*/
	142	export function filingPlan(o: {
	143	forge: string \| undefined;
	144	base: string;
	145	into: string \| undefined;
	146	open: readonly string[];
	147	findings: readonly { page: string; device: string; text: string }[];
	148	parent: number;
	149	cap: number;
	150	}): { error: string } \| { url: string; filing: { title: string; body: string; page: string; device: string }[]; skipped: number } {
	151	if (!o.forge) return { error: "--file-issues needs FORGE_URL" };
	152	if (!o.into) return { error: "--file-issues needs --repo or --file-into: the repository to file into" };
	153	let same = false;
	154	try {
	155	same = new URL(o.forge).origin === new URL(o.base).origin;
	156	} catch {
	157	/* not a URL: not the same */
	158	}
	159	if (!same) return { error: `FORGE_URL (${o.forge}) is not ${o.base}: findings are filed only into the forge that was read` };
	160	const mine = o.findings.map((f) => ({ ...f, ...findingIssue(f.page, f.device, f.text, o.parent) }));
	161	const filing = toFile(mine, new Set(o.open.map(titleKey)), o.cap);
	162	return { url: `${o.forge.replace(/\/$/, "")}/api/v1/repos/${o.into}/issues`, filing, skipped: mine.length - filing.length };
	163	}
	164
	165	/**
	166	* Whether `FORGE_TOKEN` may be sent to `base`: only to the forge `FORGE_URL`
	167	* names. Checked before the first request, so a mistyped or foreign URL never
	168	* receives the token. Null when it may; otherwise why not.
	169	*/
	170	export function tokenRefusal(base: string, forge: string \| undefined): string \| null {
	171	if (!forge) return "FORGE_URL is not set: the token is sent only to the forge it names";
	172	try {
	173	if (new URL(base).origin === new URL(forge).origin) return null;
	174	} catch {
	175	/* fall through */
	176	}
	177	return `${base} is not the forge in FORGE_URL (${forge}): FORGE_TOKEN is not sent there`;
	178	}
		@@ -0,0 +1,60 @@
	1	/**
	2	* The cold-read loop with its three outside things injected: fetching a page,
	3	* shooting it, asking the model. `scripts/cold-read.ts` passes the real ones;
	4	* the test passes stubs.
	5	*/
	6	import { MAX_WORDS, WIDTHS, answerWords, parseAnswer, refusal, scorePage, type PageScore, type Reading } from "./coldread.ts";
	7
	8	export interface Shot {
	9	/** "image" when Chromium shot it, "text" when only the page's text is sent. */
	10	readonly kind: "image" \| "text";
	11	readonly block: unknown;
	12	}
	13
	14	export interface Deps {
	15	fetchPage(path: string): Promise<{ status: number; location: string \| null; html: string }>;
	16	shoot(html: string, path: string, w: { width: number; height: number }): Promise<Shot>;
	17	ask(shot: unknown, prompt: string): Promise<string>;
	18	log(line: string): void;
	19	}
	20
	21	export interface Run {
	22	readonly pages: Record<string, PageScore>;
	23	readonly findings: { page: string; device: string; text: string }[];
	24	/** Pages that could not be read, with why. */
	25	readonly failed: Record<string, string[]>;
	26	/** "text-only" when any reading had no screenshot: phone and desktop then saw the same words. */
	27	readonly mode: "screenshot" \| "text-only";
	28	}
	29
	30	export async function coldRead(pagePaths: readonly string[], promptTemplate: string, deps: Deps): Promise<Run> {
	31	const pages: Run["pages"] = {};
	32	const findings: Run["findings"] = [];
	33	const failed: Record<string, string[]> = {};
	34	let textOnly = false;
	35	for (const page of pagePaths) {
	36	const readings: Reading[] = [];
	37	for (const w of WIDTHS) {
	38	try {
	39	const res = await deps.fetchPage(page);
	40	const why = refusal(page, res.status, res.location);
	41	if (why) throw new Error(why);
	42	const shot = await deps.shoot(res.html, page, w);
	43	if (shot.kind === "text") textOnly = true;
	44	const prompt = promptTemplate.replace("{{width}}", String(w.width)).replace("{{device}}", w.device);
	45	const answer = parseAnswer(await deps.ask(shot.block, prompt));
	46	if (typeof answer === "string") throw new Error(answer);
	47	if (answerWords(answer) > MAX_WORDS) deps.log(`${page} ${w.device}: over ${MAX_WORDS} words (${answerWords(answer)}), kept`);
	48	readings.push({ device: w.device, answer });
	49	for (const c of answer.confusing) findings.push({ page, device: w.device, text: c });
	50	deps.log(`${page} ${w.device}: clarity ${answer.clarity}; first click: ${answer.clickFirst}`);
	51	} catch (e) {
	52	(failed[page] ??= []).push(`${w.device}: ${(e as Error).message}`);
	53	deps.log(`${page} ${w.device}: ${(e as Error).message}`);
	54	}
	55	}
	56	const s = scorePage(readings);
	57	if (s) pages[page] = s;
	58	}
	59	return { pages, findings, failed, mode: textOnly ? "text-only" : "screenshot" };
	60	}
		@@ -0,0 +1,107 @@
	1	import { describe, expect, test } from "bun:test";
	2	import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
	3	import { join } from "node:path";
	4	import { coldRead } from "../src/design/coldrun.ts";
	5	import { tokenRefusal, filingPlan, writeRefusal, PAGES, REPO, findingIssue, refusal, repoPages, toFile, parseAnswer, scorePage, withColdRead } from "../src/design/coldread.ts";
	6
	7	const ok = (clarity: number, n = 1) => ({ confusing: Array(n).fill("the banner is unclear"), clickFirst: "the big button", clarity });
	8
	9	describe("cold-read audit (#5882)", () => {
	10	test("reads the model's JSON and refuses what is not", () => {
	11	expect(parseAnswer('sure: {"confusing":["a"],"clickFirst":"b","clarity":4}')).toEqual({ confusing: ["a"], clickFirst: "b", clarity: 4 });
	12	expect(typeof parseAnswer("no")).toBe("string");
	13	expect(typeof parseAnswer('{"confusing":"a","clickFirst":"b","clarity":4}')).toBe("string");
	14	expect(typeof parseAnswer('{"confusing":[],"clickFirst":"b","clarity":9}')).toBe("string");
	15	});
	16	test("scores a page by the mean of its widths", () => {
	17	const s = scorePage([{ device: "phone", answer: ok(2, 2) }, { device: "desktop", answer: ok(4) }]);
	18	expect(s).toMatchObject({ clarity: 3, phone: 2, desktop: 4, confusions: 3 });
	19	expect(scorePage([])).toBeNull();
	20	});
	21	test("writes coldRead and keeps the rest of the baseline", () => {
	22	const out = withColdRead({ files: { a: 1 } }, { at: "x", issue: 5882, mode: "screenshot", pages: {} });
	23	expect(out["files"]).toEqual({ a: 1 });
	24	expect(out["coldRead"]).toBeDefined();
	25	});
	26	test("a finding links back to its parent", () => {
	27	expect(findingIssue("/migrate", "phone", "the button", 5882).body).toContain("#5882");
	28	});
	29	test("the order starts /migrate, /, new forge; the prompt is in the repository", () => {
	30	expect(PAGES.slice(0, 3)).toEqual(["/migrate", "/", "/forges/new"]);
	31	expect(readFileSync("docs/design/cold-read-prompt.md", "utf8")).toContain("under 100 words");
	32	});
	33
	34	test("a redirect or a non-200 is refused, a 200 is read", () => {
	35	expect(refusal("/migrate", 303, "/signin")).toContain("/signin");
	36	expect(refusal("/a/b", 404)).toContain("404");
	37	expect(refusal("/", 200)).toBeNull();
	38	});
	39	test("repo pages come from the repo, which must be owner/name", () => {
	40	expect(repoPages("a/b")).toEqual(["/a/b", "/a/b/pulls", "/a/b/issues"]);
	41	expect(REPO.test("a/b")).toBe(true);
	42	expect(REPO.test("a/b/../c")).toBe(false);
	43	});
	44	test("filing is capped and skips titles already open or repeated", () => {
	45	const f = [{ title: "A" }, { title: "b" }, { title: "B" }, { title: "c" }];
	46	expect(toFile(f, new Set(["a"]), 10).map((x) => x.title)).toEqual(["b", "c"]);
	47	expect(toFile(f, new Set(), 2).map((x) => x.title)).toEqual(["A", "b"]);
	48	});
	49	test("design-lint --update keeps coldRead", async () => {
	50	const dir = mkdtempSync(join(process.env["TMPDIR"] ?? "/tmp", "dl-"));
	51	Bun.spawnSync(["mkdir", "-p", join(dir, "docs/design/areas"), join(dir, "src/web")]);
	52	writeFileSync(join(dir, "docs/DESIGN.md"), "");
	53	writeFileSync(join(dir, "docs/design/baseline.json"), JSON.stringify({ files: {}, coldRead: { at: "x", issue: 1, pages: {} } }));
	54	const p = Bun.spawnSync(["bun", join(process.cwd(), "scripts/design-lint.ts"), "--update"], { cwd: dir });
	55	expect(p.stderr.toString()).toBe("");
	56	expect(JSON.parse(readFileSync(join(dir, "docs/design/baseline.json"), "utf8")).coldRead.issue).toBe(1);
	57	});
	58	test("the loop reads, refuses, scores and flags text-only, with a stubbed model", async () => {
	59	const asked: string[] = [];
	60	const run = await coldRead(["/", "/migrate"], "w={{width}} d={{device}}", {
	61	fetchPage: async (p) => (p === "/" ? { status: 200, location: null, html: "<p>hi</p>" } : { status: 303, location: "/signin", html: "" }),
	62	shoot: async (html) => ({ kind: "text", block: html }),
	63	ask: async (_s, prompt) => (asked.push(prompt), '{"confusing":["what is this"],"clickFirst":"sign in","clarity":3}'),
	64	log: () => {},
	65	});
	66	expect(asked).toEqual(["w=390 d=phone", "w=1440 d=desktop"]);
	67	expect(run.pages["/"]).toMatchObject({ clarity: 3, phone: 3, desktop: 3 });
	68	expect(run.pages["/migrate"]).toBeUndefined();
	69	expect(run.failed["/migrate"]?.[0]).toContain("/signin");
	70	expect(run.findings).toHaveLength(2);
	71	expect(run.mode).toBe("text-only");
	72	});
	73	test("a reply that is not an answer fails the page, not the run", async () => {
	74	const run = await coldRead(["/"], "", { fetchPage: async () => ({ status: 200, location: null, html: "" }), shoot: async () => ({ kind: "image", block: 1 }), ask: async () => "nope", log: () => {} });
	75	expect(run.pages).toEqual({});
	76	expect(run.failed["/"]).toHaveLength(2);
	77	expect(run.mode).toBe("screenshot");
	78	});
	79	test("--write is refused for a text-only or partial read, allowed otherwise", () => {
	80	expect(writeRefusal({ failed: {}, mode: "screenshot" }, false)).toBeNull();
	81	expect(writeRefusal({ failed: {}, mode: "text-only" }, false)).toContain("text-only");
	82	expect(writeRefusal({ failed: {}, mode: "text-only" }, true)).toBeNull();
	83	expect(writeRefusal({ failed: { "/migrate": ["phone: 303"] }, mode: "screenshot" }, true)).toContain("not read");
	84	});
	85	test("filing names its target, stays on the forge it read, skips open titles, honours the cap", () => {
	86	const o = { forge: "https://f.example/", base: "https://f.example", into: "a/b", open: [], findings: [{ page: "/", device: "phone", text: "x" }, { page: "/", device: "desktop", text: "y" }], parent: 1, cap: 10 };
	87	const p = filingPlan(o);
	88	expect("url" in p && p.url).toBe("https://f.example/api/v1/repos/a/b/issues");
	89	expect("filing" in p && p.filing).toHaveLength(2);
	90	expect(filingPlan({ ...o, forge: undefined })).toHaveProperty("error");
	91	expect(filingPlan({ ...o, into: undefined })).toHaveProperty("error");
	92	expect(filingPlan({ ...o, forge: "https://other.example" })).toHaveProperty("error");
	93	expect(filingPlan({ ...o, forge: "nonsense" })).toHaveProperty("error");
	94	const open = filingPlan(o) as { filing: { title: string }[] };
	95	const again = filingPlan({ ...o, open: [open.filing[0]!.title.toUpperCase()] });
	96	expect("filing" in again && again.filing).toHaveLength(1);
	97	const capped = filingPlan({ ...o, cap: 1 });
	98	expect("skipped" in capped && capped.skipped).toBe(1);
	99	});
	100	test("the token goes only to the forge FORGE_URL names", () => {
	101	expect(tokenRefusal("https://f.example", "https://f.example/")).toBeNull();
	102	expect(tokenRefusal("https://evil.example", "https://f.example")).toContain("not sent");
	103	expect(tokenRefusal("http://f.example", "https://f.example")).toContain("not sent");
	104	expect(tokenRefusal("https://f.example", undefined)).toContain("FORGE_URL");
	105	expect(tokenRefusal("https://f.example", "nonsense")).toContain("not sent");
	106	});
	107	});