Public: anyone can read this; the crew can change it.

Each deepsec sweep records the findings a person marked false positive, so blocking the gate has a measured rate #15267

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15257-each-deepsec-sweep-records-the-findings → main 4 commits fork 3 threads · 3 unresolved

  • Merged. scout/15257-each-deepsec-sweep-records-the-findings was merged into main as c7babf2.

Review

Sign in to review this change.
5 files changed +304 −2
docs/features/deepsec-findings.md +2 −2
@@ -34,8 +34,8 @@ A finding is matched by its title and its file. The file must be the whole path,
3434
3535 ## In the terminal
3636
37 Filing is not a terminal screen: a script or a check runs it. The reports it opens are ordinary sealed issues. In the terminal they show in the repository's issues and under the P0 Trust pillar, like every other report.
37 Filing is not a terminal screen: a script or a check runs it. The reports it opens are ordinary sealed issues. In the terminal they show in the repository's issues and under the P0 Trust pillar, like every other report. The terminal's System screen shows a forge admin the same false-positive line as /system, under "deepsec findings".
3838
3939 ## Not yet
4040
41 A finding does not block the gate yet. It will block it once deepsec's false-positive rate has been measured over two weeks of runs. The weekly sweep (#15026, `src/checks/deepsec_sweep.ts`) files through the same path. You can still run the script yourself on a run of your own.
41 A finding does not block the gate yet. It will block it once deepsec's false-positive rate has been measured over two weeks of runs. The rate is measured now: a report closed without a change (`dropped`) is a false positive, counted against the sweep whose commit its `Found by` line names (`src/checks/deepsec_rate.ts`). /system shows a forge admin one line for the forge's own repository, for example `deepsec false positives, last 14 days: 4 of 11 findings (36%) closed without a change, 4 not yet judged, over 2 sweeps.` Under 10 findings it says too few to measure. The line holds counts only, never a report's text. The weekly sweep (#15026, `src/checks/deepsec_sweep.ts`) files through the same path. You can still run the script yourself on a run of your own.
src/checks/deepsec_rate.ts added +100
@@ -0,0 +1,100 @@
1 /**
2 * deepsec's false-positive rate (#15257): of the sealed reports deepsec filed in the last
3 * {@link RATE_DAYS} days, how many a person closed without a change. A finding blocks the gate
4 * only once this has been measured over two weeks (docs/features/deepsec-findings.md); this
5 * measures it and changes nothing about the gate.
6 *
7 * A report counts when this door opened it (`filedFor`'s shape in `deepsec.ts`: sealed, in the
8 * owner's name, `### ` first and a `Found by` line), and is counted against its sweep: the commit
9 * that `Found by … on <commit>` names. Closed `dropped` (closed without a change) is a false
10 * positive; closed `done` is a real one; open is not yet judged.
11 *
12 * A sealed report's text never leaves this module: the body is read only for the commit it
13 * names, and what comes out is counts.
14 */
15
16 import type { Database } from "bun:sqlite";
17 import { getRepo } from "../core/repos.ts";
18 import { ownRepoPath } from "../kernel/own_repo.ts";
19
20 /** The window the rate is measured over. */
21 export const RATE_DAYS = 14;
22 /** Fewer findings than this in the window and the rate is "too few", not a number. */
23 export const RATE_MIN = 10;
24
25 export interface SweepCount {
26 /**
27 * The first nine hex characters of the commit the sweep read; null when the report names none.
28 */
29 readonly commit: string | null;
30 readonly findings: number;
31 readonly falsePositives: number;
32 readonly open: number;
33 }
34
35 export interface Rate {
36 /** ISO start of the window. */
37 readonly since: string;
38 readonly findings: number;
39 readonly falsePositives: number;
40 /** Reports nobody has judged yet. */
41 readonly open: number;
42 /** The same counts, one per sweep, most findings first. */
43 readonly sweeps: readonly SweepCount[];
44 }
45
46 /** The commit a report's `Found by … on <commit>, <date>.` line names. */
47 const FOUND_ON = /\nFound by [^\n]*? on ([0-9a-f]{9}), \d{4}-\d{2}-\d{2}\./;
48
49 /**
50 * Counts deepsec's reports in `repoId` over the last {@link RATE_DAYS} days, against each sweep.
51 */
52 export function falsePositiveRate(db: Database, repo: { readonly id: string; readonly ownerId: string }, now: Date): Rate {
53 const since = new Date(now.getTime() - RATE_DAYS * 86_400_000).toISOString();
54 const rows = db
55 .query<{ state: string; close_reason: string | null; body: string }, [string, string, string]>(
56 `SELECT state, close_reason, body FROM issue
57 WHERE repo_id = ?1 AND kind = 'issue' AND sealed = 1 AND author_id = ?2 AND created_at >= ?3
58 AND substr(body, 1, 4) = '### ' AND instr(body, char(10) || 'Found by ') > 0`,
59 )
60 .all(repo.id, repo.ownerId, since);
61 const by = new Map<string | null, { findings: number; falsePositives: number; open: number }>();
62 let falsePositives = 0;
63 let open = 0;
64 for (const r of rows) {
65 const commit = FOUND_ON.exec(r.body)?.[1] ?? null;
66 const c = by.get(commit) ?? { findings: 0, falsePositives: 0, open: 0 };
67 c.findings += 1;
68 if (r.state === "open") {
69 c.open += 1;
70 open += 1;
71 } else if (r.close_reason === "dropped") {
72 c.falsePositives += 1;
73 falsePositives += 1;
74 }
75 by.set(commit, c);
76 }
77 const sweeps = [...by].map(([commit, c]) => ({ commit, ...c })).sort((a, b) => b.findings - a.findings);
78 return { since, findings: rows.length, falsePositives, open, sweeps };
79 }
80
81 /** The rate as one line: counts only, never a report's text. */
82 export function rateLine(r: Rate): string {
83 const head = `deepsec false positives, last ${RATE_DAYS} days`;
84 if (r.findings < RATE_MIN)
85 return `${head}: too few to measure (${r.findings} finding${r.findings === 1 ? "" : "s"}, ${RATE_MIN} needed).`;
86 const pct = Math.round((100 * r.falsePositives) / r.findings);
87 const sweeps = r.sweeps.length === 1 ? "1 sweep" : `${r.sweeps.length} sweeps`;
outdated

"closed as not exploitable" is not what dropped means (it also covers duplicates). Say "closed without a change". Principle 1: drop the trailing "Findings do not block the gate." here, which the doc already says.

88 const open = r.open > 0 ? `, ${r.open} not yet judged` : "";
89 return `${head}: ${r.falsePositives} of ${r.findings} findings (${pct}%) closed without a change${open}, over ${sweeps}.`;
90 }
91
92 /**
93 * The rate for the forge's own repository (`ownRepoPath()`), as /system and the terminal's System
94 * screen show it; null when that repository is not on this forge.
95 */
96 export function ownRepoRate(db: Database, now: Date): Rate | null {
97 const [owner = "", name = ""] = ownRepoPath().split("/");
98 const repo = getRepo(db, owner, name);
99 return repo === undefined ? null : falsePositiveRate(db, repo, now);
100 }
src/tui/screens/system.ts +11
@@ -70,6 +70,7 @@ import { childDirOf, listChildrenNow } from "../../actions/spawn.ts";
7070 import { adoptionsOf } from "../../spawn/cell.ts";
7171 import type { AdoptionRow } from "../../spawn/adopted.ts";
7272 import { withheldFindings, type WithheldFinding } from "../../crew/sentinel.ts";
73 import { ownRepoRate, rateLine, type Rate } from "../../checks/deepsec_rate.ts";
7374 import { childrenUse, realSelf, selfUse, sizeWords, useWords, type ChildUseRow, type SelfUse } from "../../spawn/usage.ts";
7475
7576 const WIDTH = 72;
@@ -550,6 +551,14 @@ export function withheldLines(rows: readonly WithheldFinding[], inner = WIDTH -
550551 return ["Names the sentinel left out", ...rows.map((r) => truncate(`#${pad(String(r.issue), 7)} ${pad(r.slug, 24)} ${r.repoId}`, inner))];
551552 }
552553
554 /**
555 * deepsec's false-positive rate on the forge's own repository, last 14 days, as /system's
556 * "deepsec findings" says it (#15257): counts only. Nothing when the repository is not here.
557 */
558 export function deepsecRateLines(rate: Rate | null, inner = WIDTH - 6): string[] {
559 return rate === null ? [] : ["deepsec findings", ...wrap(rateLine(rate), inner)];
560 }
561
553562 /** The egress fault, one line, as /system says it; nothing when egress is enforced. */
554563 export function egressLines(session: Session): string[] {
555564 const said = egressReportedLine(session.db);
@@ -574,6 +583,7 @@ export const entry: PaletteEntry = {
574583 const kids = childLines(childrenUse(listChildrenNow(session.app), (bodyId) => childDirOf(session.app, bodyId)));
575584 const left = adoptedLines(adoptionsOf(session.app));
576585 const hid = withheldLines(withheldFindings(session.db));
586 const rate = deepsecRateLines(ownRepoRate(session.db, new Date()));
577587 const who = askingLines(session);
578588 ctx.push(systemScreen(ctx, [
579589 ...forgeLines(readHealth(session.app, session.user)),
@@ -585,6 +595,7 @@ export const entry: PaletteEntry = {
585595 ...(kids.length === 0 ? [] : ["", ...kids]),
586596 ...(left.length === 0 ? [] : ["", ...left]),
587597 ...(hid.length === 0 ? [] : ["", ...hid]),
598 ...(rate.length === 0 ? [] : ["", ...rate]),
588599 "",
589600 ...systemLines(liveMachine().read()),
590601 ...samplerLines(droppedRows()),
src/web/pages/system.ts +12
@@ -23,6 +23,7 @@
2323 */
2424 import { askingOf, type Asker } from "../../perf/asking.ts";
2525 import { sentinelRepoPath, withheldFindings, type WithheldFinding } from "../../crew/sentinel.ts";
26 import { ownRepoRate, rateLine, RATE_MIN, type Rate } from "../../checks/deepsec_rate.ts";
2627 import { turnStatsOf } from "../../http/app_turn.ts";
2728 import { mayReadRefusals } from "../../core/first_ask_refusals.ts";
2829 import { outsideDependencies } from "../../keeper/at_boot.ts";
@@ -239,6 +240,16 @@ export function adoptedSection(rows: readonly AdoptionRow[]): Html {
239240 )}</ul></section>`;
240241 }
241242
243 /**
244 * deepsec's false-positive rate on the forge's own repository, last 14 days (#15257): one line of
245 * counts, never a sealed report's text. Nothing when the repository is not here.
246 */
247 export function deepsecRateSection(rate: Rate | null): Html {
248 if (rate === null) return html``;
249 return html`<section class="settings-block" id="sys-deepsec-rate" data-findings="${String(rate.findings)}" data-measured="${String(rate.findings >= RATE_MIN)}"><h2>deepsec findings</h2>
250 <p>${rateLine(rate)}</p></section>`;
251 }
252
242253 /**
243254 * The sentinel's open issues that left a private repository's name out (#14909), each with the
244255 * name: the issue is filed where others read it, this page is a forge admin's. Nothing when none.
@@ -746,6 +757,7 @@ export async function systemPage(ctx: Ctx, patterns: () => readonly string[]): P
746757 ${childrenSection(children)}
747758 ${adoptedSection(owner ? adoptionsOf(ctx.app) : [])}
748759 ${owner ? withheldSection(timePhase(ctx.request, "withheld", () => withheldFindings(ctx.app.db)), sentinelRepoPath()) : html``}
760 ${owner ? deepsecRateSection(timePhase(ctx.request, "deepsec-rate", () => ownRepoRate(ctx.app.db, new Date()))) : html``}
749761 ${inProcessSection(inProcess)}
750762 <section class="settings-block"><h2>Disks and network</h2>
outdated

Principle 1 and clean code: the repo lookup is inline in the template and duplicates the TUI's ownRate. Move it into src/checks/deepsec_rate.ts as one function and call it here and in the TUI. Also add a test that a non-admin gets no section.

751763 <ul class="list" id="sys-disks">${at("sys-disks")}</ul></section>
test/deepsec-rate.test.ts added +179
@@ -0,0 +1,179 @@
1 /**
2 * deepsec's false-positive rate (#15257): reports filed by the deepsec door, closed as dropped,
3 * counted against their sweep over 14 days; "too few" under 10; counts only on /system.
4 */
5
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { openIssue } from "../src/actions/issues.ts";
12 import { createRepo } from "../src/actions/repos.ts";
13 import { reportFor } from "../src/checks/deepsec.ts";
14 import type { DeepsecFinding } from "../src/checks/deepsec_findings.ts";
15 import { falsePositiveRate, rateLine, RATE_MIN } from "../src/checks/deepsec_rate.ts";
16 import type { Repo } from "../src/core/repos.ts";
17 import { createUser, getUserByName, type User } from "../src/core/users.ts";
18 import { deepsecRateSection } from "../src/web/pages/system.ts";
19 import { deepsecRateLines, entry as systemEntry } from "../src/tui/screens/system.ts";
20 import { createToken } from "../src/core/tokens.ts";
21 import { serve } from "../src/http/server.ts";
22 import { createForgeApp, forgeCommands } from "../src/tui/forge.ts";
23 import { headlessTerminal } from "../src/tui/term/terminal.ts";
24 import { settleApp } from "./helpers/tui.ts";
25
26 const SECRET = "Token compared with == in the vault";
27 const finding = (n: number): DeepsecFinding => ({
28 file: `src/f${n}.ts`,
29 title: `${SECRET} ${n}`,
30 severity: "HIGH",
31 lines: [n],
32 description: "secret detail",
33 fix: "",
34 confirmed: true,
35 reasoning: "",
36 });
37
38 let dataDir: string;
39 let app: App;
40 let alice: User;
41 let bob: User;
42 let repo: Repo;
43 let servers: Awaited<ReturnType<typeof serve>>;
44 let savedOwn: string | undefined;
45 const now = new Date();
46
47 function file(n: number, commit: string, closed?: "done" | "dropped", daysAgo = 0): void {
48 const made = openIssue(app, alice, repo, { ...reportFor(finding(n), { commit, now }), sealed: true });
49 if (!made.ok) throw new Error(String(made.error));
50 const at = new Date(now.getTime() - daysAgo * 86_400_000).toISOString();
51 app.db
52 .query("UPDATE issue SET created_at = ?1, state = ?2, close_reason = ?3, closed_at = ?4 WHERE id = ?5")
53 .run(at, closed === undefined ? "open" : "closed", closed ?? null, closed === undefined ? null : at, made.value.id);
54 }
55
56 describe("deepsec false-positive rate", () => {
57 beforeAll(async () => {
58 dataDir = await mkdtemp(join(tmpdir(), "forge-deepsec-rate-"));
59 app = createApp({ dataDir, port: 0 });
60 const made = await createUser(app.db, {
61 origin: "seed",
62 trusted: true,
63 username: "alice",
64 email: "[email protected]",
65 password: "a-good-passphrase",
66 adminIfFirst: true,
67 });
68 if (!made.ok) throw new Error(made.error);
69 alice = getUserByName(app.db, "alice")!;
70 const created = await createRepo(app, alice, { name: "engine", visibility: "public" });
71 if (!created.ok) throw new Error(String(created.error));
72 repo = created.value;
73 const other = await createUser(app.db, {
74 origin: "seed",
75 trusted: true,
76 username: "bob",
77 email: "[email protected]",
78 password: "a-good-passphrase",
79 });
80 if (!other.ok) throw new Error(other.error);
81 bob = other.value;
82 // alice/engine stands for the forge's own repository here.
83 savedOwn = process.env.FORGE_SENTINEL_REPO;
84 process.env.FORGE_SENTINEL_REPO = "alice/engine";
85 servers = await serve(app);
86 });
87 afterAll(async () => {
88 if (savedOwn === undefined) delete process.env.FORGE_SENTINEL_REPO;
89 else process.env.FORGE_SENTINEL_REPO = savedOwn;
90 await servers.stop();
91 app.db.close();
92 await rm(dataDir, { recursive: true, force: true });
93 });
94
95 test("under 10 findings the line says too few", () => {
96 const a = "a".repeat(40);
97 file(1, a, "dropped");
98 file(2, a);
99 const rate = falsePositiveRate(app.db, repo, now);
100 expect(rate.findings).toBe(2);
101 expect(rateLine(rate)).toBe(`deepsec false positives, last 14 days: too few to measure (2 findings, ${RATE_MIN} needed).`);
102 });
103
104 test("a report closed dropped counts against its sweep; old reports and other issues do not count; the line carries counts only", () => {
105 const a = "a".repeat(40);
106 const b = "b".repeat(40);
107 for (let n = 3; n <= 7; n++) file(n, a, n === 3 ? "dropped" : n === 4 ? "done" : undefined);
108 for (let n = 8; n <= 11; n++) file(n, b, n <= 9 ? "dropped" : "done");
109 file(12, b, "dropped", 15); // outside the window
110 // An ordinary issue, sealed but not shaped as deepsec's report, is not a finding.
111 const other = openIssue(app, alice, repo, { title: "a sealed note", body: "nothing", sealed: true });
112 if (!other.ok) throw new Error(String(other.error));
113
114 const rate = falsePositiveRate(app.db, repo, now);
115 expect(rate.findings).toBe(11);
116 expect(rate.falsePositives).toBe(4);
117 expect(rate.open).toBe(4);
118 expect(rate.sweeps).toEqual([
119 { commit: "aaaaaaaaa", findings: 7, falsePositives: 2, open: 4 },
120 { commit: "bbbbbbbbb", findings: 4, falsePositives: 2, open: 0 },
121 ]);
122 const line = rateLine(rate);
123 expect(line).toBe("deepsec false positives, last 14 days: 4 of 11 findings (36%) closed without a change, 4 not yet judged, over 2 sweeps.");
124
125 const html = String(deepsecRateSection(rate));
126 expect(html).toContain('data-measured="true"');
127 expect(html).toContain("4 of 11 findings (36%)");
128 expect(html).not.toContain(SECRET);
129 expect(html).not.toContain("secret detail");
130 expect(String(deepsecRateSection(null))).toBe("");
131
132 // The terminal's System screen says the same line.
133 const said = deepsecRateLines(rate, 200);
134 expect(said).toEqual(["deepsec findings", line]);
135 expect(said.join("\n")).not.toContain(SECRET);
136 expect(deepsecRateLines(null)).toEqual([]);
137 });
138
139 test("/system shows the section to the owner; a member and a guest get no deepsec text", async () => {
140 const page = async (token?: string) => {
141 const res = await fetch(servers.url + "/system", {
142 headers: token === undefined ? {} : { authorization: `Bearer ${token}` },
143 redirect: "manual",
144 });
145 return { status: res.status, body: await res.text() };
146 };
147 const owner = await page(createToken(app.db, alice.id, "t", "admin").token);
148 expect(owner.status).toBe(200);
149 expect(owner.body).toContain('id="sys-deepsec-rate"');
150 expect(owner.body).toContain("4 of 11 findings (36%) closed without a change");
151 expect(owner.body).not.toContain(SECRET);
152
153 for (const seen of [await page(createToken(app.db, bob.id, "t", "admin").token), await page()]) {
154 expect(seen.status).not.toBe(200);
155 expect(seen.body).not.toContain("sys-deepsec-rate");
156 expect(seen.body).not.toContain("deepsec");
157 }
158 });
159
160 test("the terminal's System screen says the line to an admin, and is not offered to a member or a guest", async () => {
161 const identity = (user?: User) => ({ sshUsername: user?.username ?? "git", remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user });
162 const open = (user?: User) => createForgeApp(app, identity(user), { terminal: headlessTerminal(200, 120), exitHooks: false });
163 for (const who of [bob, undefined]) {
164 const { app: tui, session } = open(who);
165 expect(forgeCommands(session)(tui.ctx).map((c) => c.id)).not.toContain("system");
166 tui.stop?.();
167 }
168 const { app: tui, session } = open(alice);
169 expect(forgeCommands(session)(tui.ctx).map((c) => c.id)).toContain("system");
170 tui.start();
171 systemEntry.run(session, tui.ctx);
172 await settleApp(tui);
173 const text = tui.screen.toText();
174 tui.stop?.();
175 expect(text).toContain("deepsec findings");
176 expect(text).toContain("4 of 11 findings (36%)");
177 expect(text).not.toContain(SECRET);
178 });
179 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs