Public: anyone can read this; the crew can change it.
The secret scan reads cat-file replies exactly and fails closed on anything it cannot parse #14980
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14959-the-secret-scan-reads-cat-file-replies-e → main 1 commit fork
2 files
changed +89
−12
src/checks/scan.ts +44 −11
| @@ -37,8 +37,9 @@ const askable = (path: string): boolean => !path.includes("\n") && !path.include | ||
| 37 | 37 | |
| 38 | 38 | /** |
| 39 | 39 | * `oid:path → text` for each named blob, skipping the large and the binary: |
| 40 | * one `cat-file` for them all. Null when git could not be read at all, so a | |
| 41 | * caller that must not pass on silence (the hold) can tell. | |
| 40 | * one `cat-file` for them all. Null when git could not be read at all or its | |
| 41 | * reply could not be parsed (`batchReplies`), so a caller that must not pass | |
| 42 | * on silence (the hold) can tell. | |
| 42 | 43 | */ |
| 43 | 44 | async function readFiles(dir: string, wanted: readonly { readonly oid: string; readonly path: string }[]): Promise<Read | null> { |
| 44 | 45 | const out: Read = { texts: new Map(), skipped: 0 }; |
| @@ -49,24 +50,56 @@ async function readFiles(dir: string, wanted: readonly { readonly oid: string; r | ||
| 49 | 50 | asked.length = MAX_FILES; |
| 50 | 51 | } |
| 51 | 52 | if (asked.length === 0) return out; |
| 52 | const raw = await runGitRaw(["cat-file", "--batch"], { cwd: dir, stdin: asked.map((w) => `${w.oid}:${w.path}\n`).join(""), timeoutMs: 60_000 }); | |
| 53 | const requests = asked.map((w) => `${w.oid}:${w.path}`); | |
| 54 | const raw = await runGitRaw(["cat-file", "--batch"], { cwd: dir, stdin: requests.map((r) => `${r}\n`).join(""), timeoutMs: 60_000 }); | |
| 53 | 55 | if (!raw.ok) return null; |
| 54 | const bytes = raw.value; | |
| 56 | const replies = batchReplies(raw.value, requests); | |
| 57 | if (replies === null) return null; | |
| 58 | out.skipped += replies.skipped; | |
| 59 | for (const [request, text] of replies.texts) out.texts.set(request, text); | |
| 60 | return out; | |
| 61 | } | |
| 62 | ||
| 63 | /** A found object's header: `<oid> <type> <size>`, nothing more. */ | |
| 64 | const FOUND = /^[0-9a-f]{40}(?:[0-9a-f]{24})? (blob|tree|commit|tag) (\d+)$/; | |
| 65 | /** | |
| 66 | * A request git could not answer: echoed, then a word. Read from the end: a name may hold spaces. | |
| 67 | */ | |
| 68 | const UNANSWERED = / (?:missing|ambiguous)$/; | |
| 69 | ||
| 70 | /** | |
| 71 | * Each `cat-file --batch` reply read exactly, one per request in order: a | |
| 72 | * found object's body is stepped over whatever its type (a submodule's | |
| 73 | * commit, a tree), so the next reply starts where it should, and only a | |
| 74 | * blob's is kept. Null on anything else (an unknown header, a body that runs | |
| 75 | * past the end or lacks its newline, a reply too few or too many): a reply | |
| 76 | * misread would put one file's text under another's name, so the scan fails | |
| 77 | * rather than guess. | |
| 78 | */ | |
| 79 | export function batchReplies(bytes: Uint8Array, requests: readonly string[]): Read | null { | |
| 80 | const out: Read = { texts: new Map(), skipped: 0 }; | |
| 55 | 81 | const decoder = new TextDecoder(); |
| 56 | 82 | let at = 0; |
| 57 | for (const w of asked) { | |
| 83 | for (const request of requests) { | |
| 58 | 84 | const nl = bytes.indexOf(10, at); |
| 59 | if (nl < 0) break; | |
| 60 | const header = decoder.decode(bytes.subarray(at, nl)).split(" "); | |
| 85 | if (nl < 0) return null; | |
| 86 | const header = decoder.decode(bytes.subarray(at, nl)); | |
| 61 | 87 | at = nl + 1; |
| 62 | if (header[1] !== "blob") continue; // "missing", or a submodule's commit | |
| 63 | const size = Number(header[2]); | |
| 88 | const found = FOUND.exec(header); | |
| 89 | if (found === null) { | |
| 90 | // Echoed from the request's own `<oid>:`, so a reply is tied to what was asked. | |
| 91 | if (UNANSWERED.test(header) && header.startsWith(request.slice(0, request.indexOf(":") + 1))) continue; | |
| 92 | return null; | |
| 93 | } | |
| 94 | const size = Number(found[2]); | |
| 95 | if (!Number.isSafeInteger(size) || at + size >= bytes.length || bytes[at + size] !== 10) return null; | |
| 64 | 96 | const body = bytes.subarray(at, at + size); |
| 65 | 97 | at += size + 1; |
| 66 | if (size <= MAX_BYTES) out.texts.set(`${w.oid}:${w.path}`, decoder.decode(body)); | |
| 98 | if (found[1] !== "blob") continue; | |
| 99 | if (size <= MAX_BYTES) out.texts.set(request, decoder.decode(body)); | |
| 67 | 100 | else out.skipped++; |
| 68 | 101 | } |
| 69 | return out; | |
| 102 | return at === bytes.length ? out : null; | |
| 70 | 103 | } |
| 71 | 104 | |
| 72 | 105 | async function listTree(dir: string, oid: string): Promise<string[]> { |
test/code-scanning.test.ts +45 −1
| @@ -13,7 +13,7 @@ import { acceptFinding } from "../src/actions/scan_accept.ts"; | ||
| 13 | 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 14 | 14 | import { createAgent } from "../src/actions/crew.ts"; |
| 15 | 15 | import { createApp, type App } from "../src/app.ts"; |
| 16 | import { scanNightly, scanPush, scansSettled } from "../src/checks/scan.ts"; | |
| 16 | import { batchReplies, presentAt, scanNightly, scanPush, scansSettled } from "../src/checks/scan.ts"; | |
| 17 | 17 | import { advisoryFindings, secretFindings, staticFindings, versionBelow } from "../src/checks/scan_rules.ts"; |
| 18 | 18 | import { lastScan, listFindings, nightlyDue, openHigh } from "../src/checks/scan_store.ts"; |
| 19 | 19 | import type { Repo } from "../src/core/repos.ts"; |
| @@ -243,3 +243,47 @@ describe("a push", () => { | ||
| 243 | 243 | expect(page).toContain("partial: 1 file not read"); |
| 244 | 244 | }); |
| 245 | 245 | }); |
| 246 | ||
| 247 | describe("cat-file replies (#14959)", () => { | |
| 248 | test("a gitlink's commit body is stepped over: the file after it is still read", async () => { | |
| 249 | const clone = join(work, "square"); | |
| 250 | expect((await git(["checkout", "-q", "-b", "gl", "origin/main"], clone)).code).toBe(0); | |
| 251 | expect((await git(["pull", "-q", "origin", "main"], clone)).code).toBe(0); | |
| 252 | const base = (await git(["rev-parse", "HEAD"], clone)).out.trim(); | |
| 253 | // A submodule pointing at a commit the store holds: `cat-file` answers it with that commit's | |
| 254 | // body. | |
| 255 | expect((await git(["update-index", "--add", "--cacheinfo", `160000,${base},sub`], clone)).code).toBe(0); | |
| 256 | await writeFile(join(clone, "t.ts"), `export const key = "${AWS}";\n`); | |
| 257 | expect((await git(["add", "t.ts"], clone)).code).toBe(0); | |
| 258 | expect((await git(["-c", "user.name=a", "-c", "user.email=a@x", "commit", "-qm", "gitlink"], clone)).code).toBe(0); | |
| 259 | const head = (await git(["rev-parse", "HEAD"], clone)).out.trim(); | |
| 260 | expect((await git(["push", "-q", "origin", "gl"], clone)).code).toBe(0); | |
| 261 | await scansSettled(app, repo.id); | |
| 262 | await scanPush(app, repo, [{ ref: "refs/heads/gl", oldOid: base, newOid: head }]); | |
| 263 | const secret = listFindings(app.db, repo.id).find((f) => f.oid === head && f.path === "t.ts"); | |
| 264 | expect(secret?.line).toBe(1); | |
| 265 | // The hold reads the same way: the gitlink asked first does not hide the secret after it. | |
| 266 | const sub = { ...secret!, id: -2, path: "sub", fingerprint: "none" }; | |
| 267 | expect((await presentAt(app, repo, head, [sub, secret!]))?.map((f) => f.path)).toEqual(["t.ts"]); | |
| 268 | }, 30_000); | |
| 269 | ||
| 270 | test("a missing name shaped like a blob header is read from the end and shifts no later reply", async () => { | |
| 271 | const tip = await tipOf(); | |
| 272 | const deploy = listFindings(app.db, repo.id).find((f) => f.kind === "secret" && f.path === "deploy.ts")!; | |
| 273 | const ghost = { ...deploy, id: -1, path: "gone blob 100 x", fingerprint: "none" }; | |
| 274 | expect((await presentAt(app, repo, tip, [ghost, deploy]))?.map((f) => f.path)).toEqual(["deploy.ts"]); | |
| 275 | }); | |
| 276 | ||
| 277 | test("a reply it cannot parse fails the read, never skips it", () => { | |
| 278 | const enc = (s: string) => new TextEncoder().encode(s); | |
| 279 | const oid = "a".repeat(40); | |
| 280 | const asked = [`${oid}:x`, `${oid}:y`]; | |
| 281 | expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid}:y missing\n`), asked)?.texts).toEqual(new Map([[`${oid}:x`, "hi"]])); | |
| 282 | expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid} wat 2\nhi\n`), asked)).toBeNull(); // unknown header | |
| 283 | expect(batchReplies(enc(`${oid} blob 2\nhi\n${oid} blob 9\nhi\n`), asked)).toBeNull(); // body past the end | |
| 284 | expect(batchReplies(enc(`${oid} blob 2\nhi!${oid}:y missing\n`), asked)).toBeNull(); // body without its newline | |
| 285 | expect(batchReplies(enc(`${oid} blob 2\nhi\n`), asked)).toBeNull(); // too few | |
| 286 | expect(batchReplies(enc(`${oid}:x missing\n${oid}:y missing\n${oid}:z missing\n`), asked)).toBeNull(); // too many | |
| 287 | expect(batchReplies(enc(`${"b".repeat(40)}:x missing\n${oid}:y missing\n`), asked)).toBeNull(); // not this request's | |
| 288 | }); | |
| 289 | }); |
new above
offline