Public: anyone can read this; the crew can change it.
`bun scripts/home-read.ts <url>` prints a page as the eye reads it, with its characters, words and fetch time, so the #5735 cold read starts from the same text every time #5781
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/5765-bun-scripts-home-read-ts-url-prints-a-pa → main 3 commits fork 3 threads · 3 unresolved
4 files
changed +251
−0
README.md +2
| @@ -346,6 +346,8 @@ or the last change landed in the past 7 days ("Last change: …, 3 days ago"); | ||
| 346 | 346 | a quiet app says nothing more. The terminal's first screen opens with the same |
| 347 | 347 | lines, from the same queries (`src/core/client-apps.ts`). |
| 348 | 348 | |
| 349 | **Reading the home as the eye does (#5735, step 1).** `bun scripts/home-read.ts <url>` fetches a page and prints its visible text, then one line: `chars N · words N · fetched in N ms`. Set `FORGE_COOKIE` (for example `forge_session=…; forge_mode=studio`) to read it as the owner; the cookie is sent and never printed. It exits non-zero on any status but 200. The conversion is `pageText` in `src/web/page_text.ts`: scripts, styles, templates, the head and anything `hidden` or `aria-hidden="true"` are dropped, link text stays, blocks sit on their own lines, entities are decoded. The method then continues: (2) a fresh agent with no context reads only that text and answers in at most 150 words, (3) the steward scores it against ground truth on coverage of the 5 things that mattered, errors, and seconds, (4) repeat after every feed change. The steward runs steps 2–4 and logs them on #5735. | |
| 350 | ||
| 349 | 351 | Home shows nine figures first, from `src/flow/kpis.ts`: shipped per hour (with |
| 350 | 352 | a 24 h sparkline), the live version and since when, the median check time, the crew builders |
| 351 | 353 | running of their slots, and the crew's spend today of the forge's daily |
scripts/home-read.ts added +26
| @@ -0,0 +1,26 @@ | ||
| 1 | /** | |
| 2 | * `bun scripts/home-read.ts <url>` prints a page as the eye reads it, then | |
| 3 | * `chars N · words N · fetched in N ms` (#5765): step 1 of the home cold read | |
| 4 | * (#5735), so every read starts from the same text. `FORGE_COOKIE`, when set, | |
| 5 | * is sent as the cookie and never printed. Exits 1 on a non-200 status. | |
| 6 | */ | |
| 7 | import { pageText } from "../src/web/page_text.ts"; | |
| 8 | ||
| 9 | const url = process.argv[2]; | |
| 10 | if (!url || !/^https?:\/\//i.test(url)) { | |
| 11 | console.error("usage: bun scripts/home-read.ts <http(s)://url>"); | |
| 12 | process.exit(2); | |
| 13 | } | |
| 14 | const cookie = process.env.FORGE_COOKIE; | |
| 15 | const started = performance.now(); | |
| 16 | const res = await fetch(url, { headers: cookie ? { cookie } : {}, redirect: "manual", signal: AbortSignal.timeout(15_000) }); | |
| 17 | const html = await res.text(); | |
| 18 | const ms = Math.round(performance.now() - started); | |
| 19 | if (res.status !== 200) { | |
| 20 | console.error(`${url} answered ${res.status}`); | |
| 21 | process.exit(1); | |
| 22 | } | |
| 23 | const text = pageText(html); | |
| 24 | const words = text.split(/\s+/).filter(Boolean).length; | |
| 25 | console.log(text); | |
| 26 | console.log(`chars ${text.length} · words ${words} · fetched in ${ms} ms`); |
src/web/page_text.ts added +106
| @@ -0,0 +1,106 @@ | ||
| 1 | /** | |
| 2 | * A page as the eye reads it (#5765): HTML in, visible text out. One pass over | |
| 3 | * the tags with a stack of the elements being skipped, so nested tags cannot | |
| 4 | * confuse it. Scripts, styles, templates, the head and anything `hidden` or | |
| 5 | * `aria-hidden="true"` are dropped; link text stays; block elements sit on | |
| 6 | * their own lines; entities are decoded; whitespace is collapsed. | |
| 7 | */ | |
| 8 | ||
| 9 | const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]); | |
| 10 | const SKIPPED = new Set(["script", "style", "template", "head", "noscript"]); | |
| 11 | const BLOCK = new Set([ | |
| 12 | "address", "article", "aside", "blockquote", "br", "dd", "details", "dialog", "div", "dl", "dt", "fieldset", "figcaption", | |
| 13 | "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", | |
| 14 | "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul", | |
| 15 | ]); | |
| 16 | /** Inline elements that run into their neighbours; every other tag is a word break. */ | |
| 17 | const RUN = new Set(["abbr", "b", "bdi", "bdo", "cite", "code", "del", "dfn", "em", "i", "ins", "kbd", "mark", "q", "s", "samp", "small", "strong", "sub", "sup", "time", "u", "var", "wbr"]); | |
| 18 | const NAMED: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", middot: "·", hellip: "…", mdash: "—", ndash: "–", rarr: "→", larr: "←" }; | |
| 19 | ||
| 20 | function decode(s: string): string { | |
| 21 | return s.replace(/&(?:#(\d{1,7})|#[xX]([0-9a-fA-F]{1,6})|([a-zA-Z]+));/g, (m, dec, hex, name) => { | |
| 22 | if (name) return NAMED[name] ?? m; | |
| 23 | const cp = dec ? Number(dec) : parseInt(hex, 16); | |
| 24 | return cp > 0 && cp <= 0x10ffff && !(cp >= 0xd800 && cp <= 0xdfff) ? String.fromCodePoint(cp) : m; | |
| 25 | }); | |
| 26 | } | |
| 27 | ||
| 28 | /** True when the tag has a real `hidden` attribute (not `hidden="false"`) or `aria-hidden="true"`; quoted values never count. */ | |
outdated
Bug: this regex also matches |
||
| 29 | function hides(attrs: string): boolean { | |
| 30 | const re = /([^\s"'=\/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g; | |
| 31 | for (const m of attrs.matchAll(re)) { | |
| 32 | const name = m[1]!.toLowerCase(); | |
| 33 | const value = (m[2] ?? m[3] ?? m[4] ?? "").trim().toLowerCase(); | |
| 34 | if (name === "hidden" && value !== "false") return true; | |
| 35 | if (name === "aria-hidden" && value === "true") return true; | |
| 36 | } | |
| 37 | return false; | |
| 38 | } | |
| 39 | ||
| 40 | export function pageText(html: string): string { | |
| 41 | const lower = html.toLowerCase(); | |
| 42 | let out = ""; | |
| 43 | const skip: string[] = []; // open elements that are not shown, innermost last | |
| 44 | let i = 0; | |
| 45 | while (i < html.length) { | |
| 46 | const lt = html.indexOf("<", i); | |
| 47 | if (lt < 0) { | |
| 48 | if (!skip.length) out += html.slice(i); | |
| 49 | break; | |
| 50 | } | |
| 51 | if (!skip.length) out += html.slice(i, lt); | |
| 52 | if (html.startsWith("<!--", lt)) { | |
| 53 | const end = html.indexOf("-->", lt + 4); | |
| 54 | i = end < 0 ? html.length : end + 3; | |
| 55 | continue; | |
| 56 | } | |
| 57 | const m = /^<(\/?)([a-zA-Z][a-zA-Z0-9-]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>/.exec(html.slice(lt, lt + 8192)); | |
| 58 | if (!m) { | |
| 59 | // Not a tag (`<!doctype`, a lone `<`): drop a declaration, keep a stray `<`. | |
| 60 | const decl = /^<[!?][^>]*>/.exec(html.slice(lt, lt + 8192)); | |
| 61 | if (decl) i = lt + decl[0].length; | |
| 62 | else { | |
| 63 | if (!skip.length) out += "<"; | |
| 64 | i = lt + 1; | |
| 65 | } | |
| 66 | continue; | |
| 67 | } | |
| 68 | i = lt + m[0].length; | |
| 69 | const closing = m[1] === "/"; | |
| 70 | const name = m[2]!.toLowerCase(); | |
| 71 | const attrs = m[3]!; | |
| 72 | if (closing) { | |
| 73 | const at = skip.lastIndexOf(name); | |
| 74 | if (at >= 0) skip.length = at; | |
| 75 | else if (!skip.length && BLOCK.has(name)) out += "\n"; | |
| 76 | continue; | |
| 77 | } | |
| 78 | const raw = name === "script" || name === "style"; | |
| 79 | if (raw && (skip.length || SKIPPED.has(name))) { | |
| 80 | // Raw-text elements hold no tags: jump to their close. | |
| 81 | const close = lower.indexOf(`</${name}`, i); | |
| 82 | const gt = close < 0 ? -1 : html.indexOf(">", close); | |
| 83 | i = gt < 0 ? html.length : gt + 1; | |
| 84 | continue; | |
| 85 | } | |
| 86 | if (skip.length) { | |
| 87 | if (!VOID.has(name) && !attrs.endsWith("/")) skip.push(name); | |
| 88 | continue; | |
| 89 | } | |
| 90 | if (SKIPPED.has(name) || hides(attrs)) { | |
| 91 | if (!VOID.has(name) && !attrs.endsWith("/")) skip.push(name); | |
| 92 | continue; | |
| 93 | } | |
| 94 | if (BLOCK.has(name)) out += "\n"; | |
| 95 | else if (name === "img") { | |
| 96 | const alt = /\balt\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(attrs); | |
| 97 | const text = alt?.[1] ?? alt?.[2]; | |
| 98 | if (text) out += ` ${text} `; | |
| 99 | } else if (!RUN.has(name)) out += " "; | |
| 100 | } | |
| 101 | return decode(out) | |
| 102 | .split("\n") | |
| 103 | .map((l) => l.replace(/\s+/g, " ").trim()) | |
| 104 | .filter(Boolean) | |
| 105 | .join("\n"); | |
| 106 | } |
test/page-text.test.ts added +117
| @@ -0,0 +1,117 @@ | ||
| 1 | /** The home as the eye reads it (#5765): the cold read of #5735 starts from this text. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createSession } from "../src/core/sessions.ts"; | |
| 8 | import { createRepo as createRepoRow } from "../src/core/repos.ts"; | |
| 9 | import { id } from "../src/kernel/id.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import { pageText } from "../src/web/page_text.ts"; | |
| 13 | import { promote } from "./helpers/members.ts"; | |
| 14 | ||
| 15 | let dataDir: string; | |
| 16 | let app: App; | |
| 17 | let servers: Servers; | |
| 18 | let cookie: string; | |
| 19 | ||
| 20 | beforeAll(async () => { | |
| 21 | dataDir = await mkdtemp(join(tmpdir(), "forge-page-text-")); | |
| 22 | app = createApp({ dataDir, port: 0 }); | |
| 23 | servers = await serve(app); | |
| 24 | const r = await createUser(app.db, { | |
| 25 | origin: "seed", | |
| 26 | trusted: true, | |
| 27 | username: "keeper", | |
| 28 | email: "[email protected]", | |
| 29 | password: "a-good-passphrase", | |
| 30 | adminIfFirst: true, | |
| 31 | }); | |
| 32 | if (!r.ok) throw new Error(r.error); | |
| 33 | promote(app.db, r.value); | |
| 34 | cookie = `forge_session=${createSession(app.db, r.value.id).token}; forge_mode=studio`; | |
| 35 | }); | |
| 36 | ||
| 37 | afterAll(async () => { | |
| 38 | servers.stop(); | |
| 39 | app.close(); | |
| 40 | await rm(dataDir, { recursive: true, force: true }); | |
| 41 | }); | |
| 42 | ||
| 43 | const get = async (path: string) => (await fetch(`${servers.url}${path}`, { headers: { cookie } })).text(); | |
| 44 | ||
| 45 | test("the real home reads as text: heading and waiting count, no CSS, script or tag", async () => { | |
| 46 | const owner = app.db.query<{ id: string }, []>("SELECT id FROM user LIMIT 1").get()!.id; | |
| 47 | const repo = await createRepoRow(app.db, app.reposRoot, { ownerId: owner, name: "wait", visibility: "public", description: "wait" }); | |
| 48 | if (!repo.ok) throw new Error(repo.error); | |
| 49 | for (const [i, h] of [30, 40, 50].entries()) { | |
| 50 | const issueId = id(); | |
| 51 | const number = app.db.query<{ issue_seq: number }, [string]>("UPDATE repo SET issue_seq = issue_seq + 1 WHERE id = ?1 RETURNING issue_seq").get(repo.value.id)!.issue_seq; | |
| 52 | app.db | |
| 53 | .query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', ?4, '', ?5, 'open', ?6)`) | |
| 54 | .run(issueId, repo.value.id, number, `Stale ${i}`, owner, new Date(Date.now() - h * 3_600_000).toISOString()); | |
| 55 | app.db | |
| 56 | .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id) VALUES (?1, 'main', ?2, ?3, ?3, ?4)`) | |
| 57 | .run(issueId, `b/${i}`, "0".repeat(40), repo.value.id); | |
| 58 | } | |
| 59 | const raw = await get("/"); | |
| 60 | expect(raw).toContain("<script"); | |
| 61 | expect(raw).toContain("<style"); | |
| 62 | const text = pageText(raw); | |
| 63 | expect(text).toContain("What you asked for"); | |
| 64 | expect(text).toMatch(/3 waiting on you, oldest 5\d h/); | |
| 65 | expect(text).not.toMatch(/<\/?[a-z]/i); | |
| 66 | expect(text).not.toMatch(/[{};]\s*$|:root|@media|function\s*\(|=>|var\(--/m); | |
| 67 | expect(text.length).toBeLessThan(raw.length / 2); | |
| 68 | }); | |
Add a case where |
||
| 69 | ||
| 70 | test("entities decode; hidden, template, script, style and head blocks are dropped; link text stays", () => { | |
| 71 | const html = `<!doctype html><html><head><title>T</title><style>p{color:red}</style></head><body> | |
| 72 | <script>var x = "</div> shown?";</script> | |
| 73 | <h1>Tom & Jerry <3 AB ·</h1> | |
| 74 | <p>See <a href="/x">the <b>bold</b> link</a>, ok.</p> | |
| 75 | <div hidden>secret <div>nested</div> still secret</div> | |
| 76 | <div aria-hidden="true"><span>icon</span><script>"</div>"</script> gone</div> | |
| 77 | <div aria-hidden="false">kept</div> | |
| 78 | <template><p>tpl</p></template> | |
| 79 | <ul><li>one</li><li>two words</li></ul><!-- comment --></body></html>`; | |
| 80 | const text = pageText(html); | |
| 81 | expect(text).toBe("Tom & Jerry <3 AB ·\nSee the bold link, ok.\nkept\none\ntwo words"); | |
| 82 | }); | |
| 83 | ||
| 84 | const run = async (args: string[], env: Record<string, string> = {}) => { | |
| 85 | const proc = Bun.spawn(["bun", "scripts/home-read.ts", ...args], { | |
| 86 | env: { ...process.env, FORGE_COOKIE: "", ...env }, | |
| 87 | stdout: "pipe", | |
| 88 | stderr: "pipe", | |
| 89 | }); | |
| 90 | const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); | |
| 91 | return { out, err, code }; | |
| 92 | }; | |
| 93 | ||
| 94 | test("scripts/home-read.ts prints the text and the stats line, sends FORGE_COOKIE and never prints it", async () => { | |
| 95 | const r = await run([`${servers.url}/`], { FORGE_COOKIE: cookie }); | |
| 96 | expect(r.code).toBe(0); | |
| 97 | expect(r.out.trimEnd().split("\n").pop()).toMatch(/^chars \d+ · words \d+ · fetched in \d+ ms$/); | |
| 98 | expect(r.out).toContain("What you asked for"); | |
| 99 | const token = cookie.match(/forge_session=([^;]+)/)![1]!; | |
| 100 | expect(r.out).not.toContain(token); | |
| 101 | expect(r.err).not.toContain(token); | |
| 102 | }); | |
| 103 | ||
| 104 | test("scripts/home-read.ts exits 1 on a non-200 and 2 on a non-http argument", async () => { | |
| 105 | const denied = await run([`${servers.url}/admin`]); | |
| 106 | expect(denied.code).toBe(1); | |
| 107 | expect(denied.err).toContain("answered"); | |
| 108 | expect((await run(["file:///etc/passwd"])).code).toBe(2); | |
| 109 | expect((await run([])).code).toBe(2); | |
| 110 | }); | |
| 111 | ||
| 112 | test("`hidden` inside another attribute's value does not hide; only a real attribute does", () => { | |
| 113 | const html = `<p class="x hidden y">class kept</p><a title="a hidden gem" href="/g">title kept</a> | |
| 114 | <p data-x='aria-hidden="true"'>data kept</p><p hidden="false">false kept</p> | |
| 115 | <p class="a" hidden>gone</p><p title="t" aria-hidden='true'>gone too</p><p HIDDEN="">gone three</p>`; | |
| 116 | expect(pageText(html)).toBe("class kept\ntitle kept\ndata kept\nfalse kept"); | |
| 117 | }); |
new above
offline
Non-blocking: a
<dialog>withoutopenis not rendered, but this reads it as visible. On the preview home, the output ends withCommand palette ↑↓ navigate ↵ open esc close Keyboard shortcuts Close, which the eye never sees. That text goes into every cold read and into its word count. One fix is to treat adialogtag with noopenattribute likehiddeninhides(). A closed<details>body is the same case, apart from its<summary>.