Public: anyone can read this; the crew can change it.

No token-shaped string is tracked: a test scans the repository's files for keys, PEM blocks and bearer tokens #14399

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14395-no-token-shaped-string-is-tracked-a-test → main 2 commits fork

  • Merged. mason/14395-no-token-shaped-string-is-tracked-a-test was merged into main as bfcee5c.

Review

Sign in to review this change.
3 files changed +144 −6
scripts/tree.ts +5 −6
@@ -83,12 +83,11 @@ export function trackedFiles(root: string): string[] {
8383 stdout: "pipe",
8484 stderr: "ignore",
8585 });
86 if (listed.exitCode === 0)
87 return listed.stdout
88 .toString()
89 .split("\0")
90 .filter((file) => file !== "");
91 return walk(root);
86 const files =
87 listed.exitCode === 0 ? listed.stdout.toString().split("\0").filter((file) => file !== "") : [];
88 // No list, or an empty one (the check directory may sit inside another repository that
89 // tracks none of it): every file the checkout wrote stands for the tracked ones.
90 return files.length > 0 ? files : walk(root);
9291 }
9392
9493 /** The tree's problems, read from disk; a tracked file deleted in the work tree is skipped. */
test/no-secrets-tracked.test.ts added +131
@@ -0,0 +1,131 @@
1 /**
2 * No token-shaped string is tracked (#14395): every tracked file is read (`trackedFiles`: `git
3 * ls-files`, or the walked checkout where there is no git), and a
4 * PEM private key header, an `sk-` / `ghp_` / `xox` / AWS key-id shaped string, or a bearer
5 * token literal fails the test with its file, line and rule: never its value.
6 *
7 * The release valve is `test/secrets-allow.json`, one entry per file and rule with the reason
8 * the string is a fixture. A false positive is listed there; a rule below is never loosened to
9 * let one through. An entry that no longer matches fails too, so the list only shrinks.
10 */
11
12 import { expect, test } from "bun:test";
13 import { readFileSync, statSync } from "node:fs";
14 import { join } from "node:path";
15 import { trackedFiles } from "../scripts/tree.ts";
16 import allowed from "./secrets-allow.json";
17
18 type Rule = { name: string; pattern: RegExp; tokenShaped?: (match: string) => boolean };
19 type Finding = { file: string; line: number; rule: string };
20 type Allowance = { file: string; rule: string; reason: string };
21
22 /**
23 * A run of 20 or more letters and digits holding both: what a random key looks like, and what
24 * a fixture spelled from words joined by hyphens (`sk-ant-oat01-the-test-key`) does not.
25 */
26 const randomRun = (value: string): boolean =>
27 (value.match(/[A-Za-z0-9]{20,}/g) ?? []).some((run) => /[A-Za-z]/.test(run) && /[0-9]/.test(run));
28
29 /** Exact vendor shapes match as they are; the generic prefixes must also look random. */
30 const RULES: Rule[] = [
31 { name: "pem-private-key", pattern: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----/g },
32 { name: "sk-key", pattern: /\bsk-[A-Za-z0-9_-]{20,}/g, tokenShaped: randomRun },
33 { name: "github-token", pattern: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/g },
34 { name: "slack-token", pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}/g },
35 { name: "aws-key-id", pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g },
36 { name: "bearer-token", pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{20,}/g, tokenShaped: randomRun },
37 ];
38
39 /** Scan one file's text; the result names where and which rule, and never carries the value. */
40 function scanText(file: string, text: string): Finding[] {
41 const found: Finding[] = [];
42 const lines = text.split("\n");
43 for (let i = 0; i < lines.length; i++) {
44 for (const rule of RULES) {
45 for (const m of lines[i]!.matchAll(rule.pattern)) {
46 if (!rule.tokenShaped || rule.tokenShaped(m[0])) {
47 found.push({ file, line: i + 1, rule: rule.name });
48 break;
49 }
50 }
51 }
52 }
53 return found;
54 }
55
56 const root = join(import.meta.dir, "..");
57 const MAX_BYTES = 4 * 1024 * 1024;
58
59 function trackedFindings(): Finding[] {
60 const findings: Finding[] = [];
61 for (const file of trackedFiles(root)) {
62 const path = join(root, file);
63 let size: number;
64 try {
65 const st = statSync(path);
66 if (!st.isFile()) continue;
67 size = st.size;
68 } catch {
69 continue; // listed but deleted in this checkout
70 }
71 if (size > MAX_BYTES) continue;
72 const bytes = readFileSync(path);
73 if (bytes.includes(0)) continue; // binary
74 findings.push(...scanText(file, bytes.toString("utf8")));
75 }
76 return findings;
77 }
78
79 const isAllowed = (f: Finding, list: Allowance[]): boolean => list.some((a) => a.file === f.file && a.rule === f.rule);
80
81 const findings = trackedFindings();
82 const list = allowed as Allowance[];
83
84 test("no tracked file holds a token-shaped string outside the allow-list", () => {
85 const fresh = findings.filter((f) => !isAllowed(f, list)).map((f) => `${f.file}:${f.line} ${f.rule}`);
86 expect(
87 fresh,
88 `token-shaped strings are tracked (value withheld). Remove them; if one is a fixture, list its file and rule with a reason in test/secrets-allow.json: ${fresh.join(", ")}`,
89 ).toEqual([]);
90 });
91
92 test("every allow-list entry has a reason and still matches: the list only shrinks", () => {
93 for (const a of list) {
94 expect(a.reason.trim().length, `${a.file} ${a.rule} needs a reason`).toBeGreaterThan(10);
95 expect(RULES.map((r) => r.name)).toContain(a.rule);
96 }
97 const stale = list.filter((a) => !findings.some((f) => f.file === a.file && f.rule === a.rule));
98 expect(stale.map((a) => `${a.file} ${a.rule}`), "these entries match nothing now; remove them").toEqual([]);
99 });
100
101 test("the scanner catches a planted sample of every rule, and the finding withholds the value", () => {
102 // Built in memory from pieces, so this file never holds a token-shaped literal itself.
103 const random = (n: number, alphabet: string) =>
104 Array.from({ length: n }, (_, i) => alphabet[(i * 7 + 3) % alphabet.length]).join("");
105 const alnum = "aB3dE5gH7jK9mN1pQ2sT4vW6yZ8";
106 const samples: Record<string, string> = {
107 "pem-private-key": ["-----BEGIN", "RSA PRIVATE", "KEY-----"].join(" "),
108 "sk-key": ["sk", "ant", "api03", random(60, alnum + "_-")].join("-"),
109 "github-token": ["gh", "p_", random(36, alnum)].join(""),
110 "slack-token": ["xo", "xb-", "1234567890", "-", random(24, alnum)].join(""),
111 "aws-key-id": ["AK", "IA", random(16, "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567")].join(""),
112 "bearer-token": ["Authorization:", "Bear" + "er", random(40, alnum)].join(" "),
113 };
114 expect(Object.keys(samples).sort()).toEqual(RULES.map((r) => r.name).sort());
115 for (const [rule, value] of Object.entries(samples)) {
116 const text = `first line\nconst planted = "${value}";\n`;
117 const got = scanText("planted.ts", text);
118 expect(got, rule).toEqual([{ file: "planted.ts", line: 2, rule }]);
119 expect(JSON.stringify(got)).not.toContain(value);
120 }
121 });
122
123 test("word-built fixtures and interpolated headers are not token-shaped", () => {
124 const quiet = [
125 ["sk", "ant", "oat01", "the", "owners", "work", "account", "never", "for", "the", "crew"].join("-"),
126 "headers: { authorization: `Bear" + "er ${token}` }",
127 "Bear" + "er forge_not-a-real-token",
128 "-----BEGIN PUBLIC KEY-----",
129 ];
130 for (const text of quiet) expect(scanText("quiet.ts", text), text).toEqual([]);
131 });
test/secrets-allow.json added +8
@@ -0,0 +1,8 @@
1 [
2 { "file": "test/apps-proxy-unavailable.test.ts", "rule": "github-token", "reason": "alphabet-and-digits stand-in proving a failed deploy's error is redacted on the app's unavailable page" },
3 { "file": "test/desk.test.ts", "rule": "aws-key-id", "reason": "alphabet stand-in proving the desk refuses a secret-shaped note" },
4 { "file": "test/mcp-preview.test.ts", "rule": "github-token", "reason": "alphabet-and-digits stand-in proving the MCP preview status redacts a failed deploy's error" },
5 { "file": "test/notify-rollback.test.ts", "rule": "bearer-token", "reason": "alphabet-and-digits stand-in proving a rollback's last words are masked before they reach a card" },
6 { "file": "test/origins-oauth.test.ts", "rule": "github-token", "reason": "alphabet-and-digits stand-ins proving redactSecrets scrubs GitHub tokens from log lines" },
7 { "file": "test/sentinel.test.ts", "rule": "github-token", "reason": "alternating-case alphabet stand-in proving the sentinel never quotes a token in what it files" }
8 ]

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs