Public: anyone can read this; the crew can change it.

The secret scan reads cat-file replies exactly and fails closed on anything it cannot parse #14980

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14959-the-secret-scan-reads-cat-file-replies-e → main 1 commit fork

  • Merged. mason/14959-the-secret-scan-reads-cat-file-replies-e was merged into main as ac4bef5.

Review

Sign in to review this change.
2 files changed +89 −12
src/checks/scan.ts +44 −11
@@ -37,8 +37,9 @@ const askable = (path: string): boolean => !path.includes("\n") && !path.include
3737
3838 /**
3939 * `oid:path → text` for each named blob, skipping the large and the binary:
40 * one `cat-file` for them all. Null when git could not be read at all, so a
41 * caller that must not pass on silence (the hold) can tell.
40 * one `cat-file` for them all. Null when git could not be read at all or its
41 * reply could not be parsed (`batchReplies`), so a caller that must not pass
42 * on silence (the hold) can tell.
4243 */
4344 async function readFiles(dir: string, wanted: readonly { readonly oid: string; readonly path: string }[]): Promise<Read | null> {
4445 const out: Read = { texts: new Map(), skipped: 0 };
@@ -49,24 +50,56 @@ async function readFiles(dir: string, wanted: readonly { readonly oid: string; r
4950 asked.length = MAX_FILES;
5051 }
5152 if (asked.length === 0) return out;
52 const raw = await runGitRaw(["cat-file", "--batch"], { cwd: dir, stdin: asked.map((w) => `${w.oid}:${w.path}\n`).join(""), timeoutMs: 60_000 });
53 const requests = asked.map((w) => `${w.oid}:${w.path}`);
54 const raw = await runGitRaw(["cat-file", "--batch"], { cwd: dir, stdin: requests.map((r) => `${r}\n`).join(""), timeoutMs: 60_000 });
5355 if (!raw.ok) return null;
54 const bytes = raw.value;
56 const replies = batchReplies(raw.value, requests);
57 if (replies === null) return null;
58 out.skipped += replies.skipped;
59 for (const [request, text] of replies.texts) out.texts.set(request, text);
60 return out;
61 }
62
63 /** A found object's header: `<oid> <type> <size>`, nothing more. */
64 const FOUND = /^[0-9a-f]{40}(?:[0-9a-f]{24})? (blob|tree|commit|tag) (\d+)$/;
65 /**
66 * A request git could not answer: echoed, then a word. Read from the end: a name may hold spaces.
67 */
68 const UNANSWERED = / (?:missing|ambiguous)$/;
69
70 /**
71 * Each `cat-file --batch` reply read exactly, one per request in order: a
72 * found object's body is stepped over whatever its type (a submodule's
73 * commit, a tree), so the next reply starts where it should, and only a
74 * blob's is kept. Null on anything else (an unknown header, a body that runs
75 * past the end or lacks its newline, a reply too few or too many): a reply
76 * misread would put one file's text under another's name, so the scan fails
77 * rather than guess.
78 */
79 export function batchReplies(bytes: Uint8Array, requests: readonly string[]): Read | null {
80 const out: Read = { texts: new Map(), skipped: 0 };
5581 const decoder = new TextDecoder();
5682 let at = 0;
57 for (const w of asked) {
83 for (const request of requests) {
5884 const nl = bytes.indexOf(10, at);
59 if (nl < 0) break;
60 const header = decoder.decode(bytes.subarray(at, nl)).split(" ");
85 if (nl < 0) return null;
86 const header = decoder.decode(bytes.subarray(at, nl));
6187 at = nl + 1;
62 if (header[1] !== "blob") continue; // "missing", or a submodule's commit
63 const size = Number(header[2]);
88 const found = FOUND.exec(header);
89 if (found === null) {
90 // Echoed from the request's own `<oid>:`, so a reply is tied to what was asked.
91 if (UNANSWERED.test(header) && header.startsWith(request.slice(0, request.indexOf(":") + 1))) continue;
92 return null;
93 }
94 const size = Number(found[2]);
95 if (!Number.isSafeInteger(size) || at + size >= bytes.length || bytes[at + size] !== 10) return null;
6496 const body = bytes.subarray(at, at + size);
6597 at += size + 1;
66 if (size <= MAX_BYTES) out.texts.set(`${w.oid}:${w.path}`, decoder.decode(body));
98 if (found[1] !== "blob") continue;
99 if (size <= MAX_BYTES) out.texts.set(request, decoder.decode(body));
67100 else out.skipped++;
68101 }
69 return out;
102 return at === bytes.length ? out : null;
70103 }
71104
72105 async function listTree(dir: string, oid: string): Promise<string[]> {
test/code-scanning.test.ts +45 −1
@@ -13,7 +13,7 @@ import { acceptFinding } from "../src/actions/scan_accept.ts";
1313 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
1414 import { createAgent } from "../src/actions/crew.ts";
1515 import { createApp, type App } from "../src/app.ts";
16 import { scanNightly, scanPush, scansSettled } from "../src/checks/scan.ts";
16 import { batchReplies, presentAt, scanNightly, scanPush, scansSettled } from "../src/checks/scan.ts";
1717 import { advisoryFindings, secretFindings, staticFindings, versionBelow } from "../src/checks/scan_rules.ts";
1818 import { lastScan, listFindings, nightlyDue, openHigh } from "../src/checks/scan_store.ts";
1919 import type { Repo } from "../src/core/repos.ts";
@@ -243,3 +243,47 @@ describe("a push", () => {
243243 expect(page).toContain("partial: 1 file not read");
244244 });
245245 });
246
247 describe("cat-file replies (#14959)", () => {
248 test("a gitlink's commit body is stepped over: the file after it is still read", async () => {
249 const clone = join(work, "square");
250 expect((await git(["checkout", "-q", "-b", "gl", "origin/main"], clone)).code).toBe(0);
251 expect((await git(["pull", "-q", "origin", "main"], clone)).code).toBe(0);
252 const base = (await git(["rev-parse", "HEAD"], clone)).out.trim();
253 // A submodule pointing at a commit the store holds: `cat-file` answers it with that commit's
254 // body.
255 expect((await git(["update-index", "--add", "--cacheinfo", `160000,${base},sub`], clone)).code).toBe(0);
256 await writeFile(join(clone, "t.ts"), `export const key = "${AWS}";\n`);
257 expect((await git(["add", "t.ts"], clone)).code).toBe(0);
258 expect((await git(["-c", "user.name=a", "-c", "user.email=a@x", "commit", "-qm", "gitlink"], clone)).code).toBe(0);
259 const head = (await git(["rev-parse", "HEAD"], clone)).out.trim();
260 expect((await git(["push", "-q", "origin", "gl"], clone)).code).toBe(0);
261 await scansSettled(app, repo.id);
262 await scanPush(app, repo, [{ ref: "refs/heads/gl", oldOid: base, newOid: head }]);
263 const secret = listFindings(app.db, repo.id).find((f) => f.oid === head && f.path === "t.ts");
264 expect(secret?.line).toBe(1);
265 // The hold reads the same way: the gitlink asked first does not hide the secret after it.
266 const sub = { ...secret!, id: -2, path: "sub", fingerprint: "none" };
267 expect((await presentAt(app, repo, head, [sub, secret!]))?.map((f) => f.path)).toEqual(["t.ts"]);
268 }, 30_000);
269
270 test("a missing name shaped like a blob header is read from the end and shifts no later reply", async () => {
271 const tip = await tipOf();
272 const deploy = listFindings(app.db, repo.id).find((f) => f.kind === "secret" && f.path === "deploy.ts")!;
273 const ghost = { ...deploy, id: -1, path: "gone blob 100 x", fingerprint: "none" };
274 expect((await presentAt(app, repo, tip, [ghost, deploy]))?.map((f) => f.path)).toEqual(["deploy.ts"]);
275 });
276
277 test("a reply it cannot parse fails the read, never skips it", () => {
278 const enc = (s: string) => new TextEncoder().encode(s);
279 const oid = "a".repeat(40);
280 const asked = [`${oid}:x`, `${oid}:y`];
281 expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid}:y missing\n`), asked)?.texts).toEqual(new Map([[`${oid}:x`, "hi"]]));
282 expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid} wat 2\nhi\n`), asked)).toBeNull(); // unknown header
283 expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid} blob 9\nhi\n`), asked)).toBeNull(); // body past the end
284 expect(batchReplies(enc(`${oid} blob 2\nhi!${oid}:y missing\n`), asked)).toBeNull(); // body without its newline
285 expect(batchReplies(enc(`${oid} blob 2\nhi\n`), asked)).toBeNull(); // too few
286 expect(batchReplies(enc(`${oid}:x missing\n${oid}:y missing\n${oid}:z missing\n`), asked)).toBeNull(); // too many
287 expect(batchReplies(enc(`${"b".repeat(40)}:x missing\n${oid}:y missing\n`), asked)).toBeNull(); // not this request's
288 });
289 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs