Public: anyone can read this; the crew can change it.
Every gate keeps its log, so a red gate can be read without re-running the pull #6389
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6351-every-gate-keeps-its-log-so-a-red-gate-c → main 6 commits fork 3 threads · 3 unresolved
8 files
changed +245
−3
src/checks/gate_log.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * Every gate keeps its log: per repository, the last 20 gates, each as one | |
| 3 | * small file under `checks/gates/<owner>/<name>/` holding its per-file | |
| 4 | * results and the last 200 lines of its output, so a red gate is read | |
| 5 | * without running the pull again. | |
| 6 | */ | |
| 7 | ||
| 8 | import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { fileResults, type FileResult } from "./gate_last.ts"; | |
| 11 | ||
| 12 | export const KEEP_GATES = 20; | |
| 13 | export const TAIL_LINES = 200; | |
| 14 | ||
| 15 | export type KeptGate = { | |
| 16 | id: string; | |
| 17 | at: number; | |
| 18 | commit: string; | |
| 19 | ok: boolean; | |
| 20 | files: Record<string, FileResult>; | |
| 21 | tail: string[]; | |
| 22 | }; | |
| 23 | ||
| 24 | const ID = /^\d{13}-[0-9a-f]{4,40}$/; | |
| 25 | const SEGMENT = /^[A-Za-z0-9._-]+$/; | |
| 26 | ||
| 27 | export const gateId = (now: number, commit: string): string => | |
| 28 | `${String(Math.max(0, Math.trunc(now))).padStart(13, "0")}-${commit.toLowerCase().slice(0, 12)}`; | |
| 29 | ||
| 30 | const dirOf = (dataDir: string, owner: string, name: string): string | undefined => | |
| 31 | SEGMENT.test(owner) && SEGMENT.test(name) && owner !== ".." && name !== ".." | |
| 32 | ? join(dataDir, "checks", "gates", owner, name) | |
| 33 | : undefined; | |
| 34 | ||
| 35 | /** Keep one gate's record and drop all but the newest 20. Returns its id. */ | |
| 36 | export function keepGateLog( | |
| 37 | dataDir: string, | |
| 38 | repo: string, | |
| 39 | commit: string, | |
| 40 | ok: boolean, | |
| 41 | lines: readonly string[], | |
| 42 | now: number, | |
| 43 | ): string | undefined { | |
| 44 | const [owner = "", name = ""] = repo.split("/"); | |
| 45 | const dir = dirOf(dataDir, owner, name); | |
| 46 | const id = gateId(now, commit); | |
| 47 | if (dir === undefined || !ID.test(id)) return undefined; | |
| 48 | const kept: KeptGate = { | |
| 49 | id, | |
| 50 | at: now, | |
| 51 | commit: id.slice(14), | |
| 52 | ok, | |
| 53 | files: fileResults(lines), | |
| 54 | tail: lines.slice(-TAIL_LINES), | |
| 55 | }; | |
| 56 | mkdirSync(dir, { recursive: true }); | |
| 57 | writeFileSync(join(dir, `${id}.json`), JSON.stringify(kept)); | |
| 58 | const ids = readdirSync(dir) | |
| 59 | .filter((f) => f.endsWith(".json") && ID.test(f.slice(0, -5))) | |
| 60 | .sort(); | |
| 61 | for (const old of ids.slice(0, Math.max(0, ids.length - KEEP_GATES))) | |
| 62 | rmSync(join(dir, old), { force: true }); | |
| 63 | return id; | |
| 64 | } | |
| 65 | ||
| 66 | export function readGateLog( | |
| 67 | dataDir: string, | |
| 68 | owner: string, | |
| 69 | name: string, | |
| 70 | id: string, | |
| 71 | ): KeptGate | undefined { | |
| 72 | const dir = dirOf(dataDir, owner, name); | |
| 73 | if (dir === undefined || !ID.test(id)) return undefined; | |
| 74 | try { | |
| 75 | const parsed = JSON.parse(readFileSync(join(dir, `${id}.json`), "utf8")) as KeptGate; | |
| 76 | return Array.isArray(parsed.tail) ? parsed : undefined; | |
| 77 | } catch { | |
| 78 | return undefined; | |
| 79 | } | |
| 80 | } |
src/checks/landing.ts +5 −1
| @@ -26,6 +26,7 @@ import type { Pull } from "../core/pulls.ts"; | ||
| 26 | 26 | import type { Repo } from "../core/repos.ts"; |
| 27 | 27 | import { failuresOf } from "./failures.ts"; |
| 28 | 28 | import { keepForgeGate } from "./gate_last.ts"; |
| 29 | import { keepGateLog } from "./gate_log.ts"; | |
| 29 | 30 | import { checksFor } from "./service.ts"; |
| 30 | 31 | import { landVerdict, type LandVerdict } from "./tiers.ts"; |
| 31 | 32 | import { ownProbeHeaders } from "../kernel/probe.ts"; |
| @@ -58,8 +59,11 @@ export async function gateCandidate( | ||
| 58 | 59 | // Cut off, not judged: no verdict, and not the gate log a red reads (#3155). |
| 59 | 60 | if (done.interrupted === true) return { lands: false, interrupted: true }; |
| 60 | 61 | keepForgeGate(app.config.dataDir, app.config.forgeRepo, `${repo.ownerName}/${repo.name}`, done.lines); |
| 62 | const slug = `${repo.ownerName}/${repo.name}`; | |
| 63 | const kept = keepGateLog(app.config.dataDir, slug, commitOid, done.ok, done.lines, Date.now()); | |
| 61 | 64 | const failed = failuresOf(done.lines); |
| 62 | return landVerdict(done.ok, failed.tests, failed.files); | |
| 65 | const link = kept === undefined ? undefined : `/api/v1/repos/${slug}/gates/${kept}/log`; | |
| 66 | return landVerdict(done.ok, failed.tests, failed.files, link); | |
| 63 | 67 | } |
| 64 | 68 | |
| 65 | 69 | /** |
src/checks/tiers.ts +4 −2
| @@ -270,16 +270,18 @@ export function landVerdict( | ||
| 270 | 270 | green: boolean, |
| 271 | 271 | failedTests: readonly string[], |
| 272 | 272 | failedFiles: readonly string[] = [], |
| 273 | logLink?: string, | |
| 273 | 274 | ): Judged { |
| 274 | 275 | if (green) return { lands: true }; |
| 275 | 276 | const names = failedTests.map((name) => `- ${name}`).join("\n"); |
| 277 | const kept = logLink === undefined ? "" : `\n\nThe gate's log: ${logLink}`; | |
| 276 | 278 | return { |
| 277 | 279 | lands: false, |
| 278 | 280 | reopen: true, |
| 279 | 281 | comment: |
| 280 | 282 | failedTests.length === 0 |
| 281 | ? "The full gate on land was red and named no test; the pull is open again." | |
| 282 | : `The full gate on land was red; the pull is open again. Red tests:\n${names}`, | |
| 283 | ? `The full gate on land was red and named no test; the pull is open again.${kept}` | |
| 284 | : `The full gate on land was red; the pull is open again. Red tests:\n${names}${kept}`, | |
| 283 | 285 | learn: [...new Set(failedFiles)], |
| 284 | 286 | }; |
| 285 | 287 | } |
src/contract/openapi/routes/getApiV1ReposOwnerNameGatesIdLog.ts added +3
| @@ -0,0 +1,3 @@ | ||
| 1 | import { REPO, type Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { method: "get", path: `${REPO}/gates/{id}/log`, summary: "Read a kept full gate's log: the last 200 lines as text/plain, under a one-line header; the last 20 gates of a repository are kept; 404 when none is kept under that id", auth: false }; |
src/contract/openapi/routes/index.ts +1
| @@ -81,6 +81,7 @@ export { route as getApiV1ReposOwnerNameFlow } from "./getApiV1ReposOwnerNameFlo | ||
| 81 | 81 | export { route as getApiV1ReposOwnerNameFollowing } from "./getApiV1ReposOwnerNameFollowing.ts"; |
| 82 | 82 | export { route as getApiV1ReposOwnerNameFollowingProposals } from "./getApiV1ReposOwnerNameFollowingProposals.ts"; |
| 83 | 83 | export { route as getApiV1ReposOwnerNameForks } from "./getApiV1ReposOwnerNameForks.ts"; |
| 84 | export { route as getApiV1ReposOwnerNameGatesIdLog } from "./getApiV1ReposOwnerNameGatesIdLog.ts"; | |
| 84 | 85 | export { route as getApiV1ReposOwnerNameHooks } from "./getApiV1ReposOwnerNameHooks.ts"; |
| 85 | 86 | export { route as getApiV1ReposOwnerNameIssues } from "./getApiV1ReposOwnerNameIssues.ts"; |
| 86 | 87 | export { route as getApiV1ReposOwnerNameIssuesNumber } from "./getApiV1ReposOwnerNameIssuesNumber.ts"; |
src/http/api.ts +13
| @@ -387,6 +387,7 @@ import { readUsage, usageDays, USAGE_DAYS_MAX } from "../actions/usage.ts"; | ||
| 387 | 387 | import { readReturns, reportSpend, RETURNS_DAYS } from "../actions/returns.ts"; |
| 388 | 388 | import { fleetSelection, readFleetGraph } from "../actions/fleet-graph.ts"; |
| 389 | 389 | import { carriedBy } from "../core/carried.ts"; |
| 390 | import { readGateLog } from "../checks/gate_log.ts"; | |
| 390 | 391 | import { readRunLog, subscribeRunLog } from "../runs/log.ts"; |
| 391 | 392 | import { blobForRender, commitLog, listRefs, listTree } from "../git/read.ts"; |
| 392 | 393 | import { repoDir } from "../git/repos.ts"; |
| @@ -6284,6 +6285,18 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 6284 | 6285 | return timePhase(ctx.request, "render", () => json(found)); |
| 6285 | 6286 | }); |
| 6286 | 6287 | |
| 6288 | // A kept gate's log (`checks/gate_log.ts`): the last 200 lines, as text. | |
| 6289 | router.add("GET", "/api/v1/repos/:owner/:name/gates/:id/log", (ctx) => { | |
This route is the door the landing note links to, and no test requests it. Add a route test: a guest and a non-member get 404 on a private repository's kept gate. The owner gets 200 with text/plain, |
||
| 6290 | const repo = loadRepo(ctx); | |
| 6291 | if (!repo) return notFound(); | |
| 6292 | const kept = readGateLog(ctx.app.config.dataDir, repo.ownerName, repo.name, ctx.params.id ?? ""); | |
| 6293 | if (!kept) return notFound(); | |
| 6294 | const head = `# gate ${kept.ok ? "green" : "red"} at ${new Date(kept.at).toISOString()}, commit ${kept.commit}\n`; | |
| 6295 | return new Response(`${head}${kept.tail.join("\n")}\n`, { | |
| 6296 | headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" }, | |
| 6297 | }); | |
| 6298 | }); | |
| 6299 | ||
| 6287 | 6300 | // The log: text, or a live tail as server-sent events (`?follow=1`, or |
| 6288 | 6301 | // an `Accept: text/event-stream`). Lines already written come first, |
| 6289 | 6302 | // then each new one as it lands; `done` carries the finished run. |
test/gate-log.test.ts added +137
| @@ -0,0 +1,137 @@ | ||
| 1 | /** Every gate keeps its log, per repository, the last 20 (#6351). */ | |
| 2 | import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { KEEP_GATES, keepGateLog, readGateLog } from "../src/checks/gate_log.ts"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 9 | import { createToken } from "../src/core/tokens.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { forgeHttp, type ForgeHttp } from "../src/http/server.ts"; | |
| 12 | import { quietLogs } from "./helpers/globals.ts"; | |
| 13 | import { landVerdict } from "../src/checks/tiers.ts"; | |
| 14 | ||
| 15 | quietLogs(); | |
| 16 | ||
| 17 | const dirs: string[] = []; | |
| 18 | const data = (): string => { | |
| 19 | const d = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "gate-log-")); | |
| 20 | dirs.push(d); | |
| 21 | return d; | |
| 22 | }; | |
| 23 | afterEach(() => dirs.splice(0).forEach((d) => rmSync(d, { recursive: true, force: true }))); | |
| 24 | ||
| 25 | const red = ["test/a.test.ts:", "(pass) one", "(fail) two"]; | |
| 26 | ||
| 27 | test("a red gate outside forgeRepo leaves a readable log", () => { | |
| 28 | const d = data(); | |
| 29 | const lines = [...Array.from({ length: 300 }, (_, i) => `line ${i}`), ...red]; | |
| 30 | const id = keepGateLog(d, "ann/app", "abcdef1234567890", false, lines, 1_000)!; | |
| 31 | const kept = readGateLog(d, "ann", "app", id)!; | |
| 32 | expect(kept.ok).toBe(false); | |
| 33 | expect(kept.files).toEqual({ "test/a.test.ts": "fail" }); | |
| 34 | expect(kept.tail).toHaveLength(200); | |
| 35 | expect(kept.tail.at(-1)).toBe("(fail) two"); | |
| 36 | }); | |
| 37 | ||
| 38 | test("the 21st gate drops the oldest", () => { | |
| 39 | const d = data(); | |
| 40 | const ids = Array.from({ length: KEEP_GATES + 1 }, (_, i) => | |
| 41 | keepGateLog(d, "ann/app", "abcdef12", true, red, 1_000 + i)!, | |
| 42 | ); | |
| 43 | expect(readGateLog(d, "ann", "app", ids[0]!)).toBeUndefined(); | |
| 44 | for (const id of ids.slice(1)) expect(readGateLog(d, "ann", "app", id)).toBeDefined(); | |
| 45 | expect(readGateLog(d, "bob", "app", ids[1]!)).toBeUndefined(); | |
| 46 | }); | |
| 47 | ||
| 48 | test("odd names and ids are refused", () => { | |
| 49 | const d = data(); | |
| 50 | expect(keepGateLog(d, "../x", "abcdef12", true, red, 1)).toBeUndefined(); | |
| 51 | expect(readGateLog(d, "ann", "app", "../../x")).toBeUndefined(); | |
| 52 | }); | |
| 53 | ||
| 54 | test("the landing note names the red tests and links the kept log", () => { | |
| 55 | const note = (tests: string[]): string => | |
| 56 | (landVerdict(false, tests, [], "/api/v1/repos/ann/app/gates/x/log") as { comment: string }) | |
| 57 | .comment; | |
| 58 | expect(note(["two"])).toContain("- two"); | |
| 59 | expect(note(["two"])).toContain("/gates/x/log"); | |
| 60 | expect(note([])).toContain("/gates/x/log"); | |
| 61 | }); | |
| 62 | ||
| 63 | describe("GET /api/v1/repos/:owner/:name/gates/:id/log", () => { | |
| 64 | let dataDir: string; | |
| 65 | let app: App; | |
| 66 | let http: ForgeHttp; | |
| 67 | let id: string; | |
| 68 | const token: Record<string, string> = {}; | |
| 69 | ||
| 70 | const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => { | |
| 71 | if (!r.ok) throw new Error(JSON.stringify(r.error)); | |
| 72 | return r.value; | |
| 73 | }; | |
| 74 | const get = async (path: string, who?: string) => { | |
| 75 | const res = await http.handle( | |
| 76 | new Request(`http://forge.test${path}`, { | |
| 77 | headers: who === undefined ? {} : { authorization: `Bearer ${token[who]}` }, | |
| 78 | }), | |
| 79 | ); | |
| 80 | return { res, text: await res.text() }; | |
| 81 | }; | |
| 82 | ||
| 83 | beforeAll(async () => { | |
| 84 | dataDir = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "gate-route-")); | |
| 85 | app = createApp({ dataDir, port: 0 }); | |
| 86 | http = forgeHttp(app); | |
| 87 | const mk = async (username: string, adminIfFirst = false) => | |
| 88 | ok( | |
| 89 | await createUser(app.db, { | |
| 90 | origin: "seed", | |
| 91 | trusted: true, | |
| 92 | username, | |
| 93 | email: `${username}@forge.local`, | |
| 94 | password: "a-good-passphrase", | |
| 95 | adminIfFirst, | |
| 96 | }), | |
| 97 | ); | |
| 98 | const alice = await mk("alice", true); | |
| 99 | const mallory = await mk("mallory"); | |
| 100 | const repo = ok(await createRepo(app, alice, { name: "vault", visibility: "private" })); | |
| 101 | ok( | |
| 102 | await commitToBranch(app, alice, repo, { | |
| 103 | message: "Start", | |
| 104 | files: [{ path: "README.md", content: "# vault\n" }], | |
| 105 | }), | |
| 106 | ); | |
| 107 | token.alice = createToken(app.db, alice.id, "t", "rw").token; | |
| 108 | token.mallory = createToken(app.db, mallory.id, "t", "rw").token; | |
| 109 | id = keepGateLog(dataDir, "alice/vault", "abcdef1234567890", false, red, 5_000)!; | |
| 110 | }, 60_000); | |
| 111 | ||
| 112 | afterAll(() => { | |
| 113 | app?.close(); | |
| 114 | rmSync(dataDir, { recursive: true, force: true }); | |
| 115 | }); | |
| 116 | ||
| 117 | const path = (which = id): string => `/api/v1/repos/alice/vault/gates/${which}/log`; | |
| 118 | ||
| 119 | test("a guest and a non-member get 404 on a private repository's gate", async () => { | |
| 120 | expect((await get(path())).res.status).toBe(404); | |
| 121 | expect((await get(path(), "mallory")).res.status).toBe(404); | |
| 122 | }); | |
| 123 | ||
| 124 | test("the owner reads it as no-store text under a header line", async () => { | |
| 125 | const { res, text } = await get(path(), "alice"); | |
| 126 | expect(res.status).toBe(200); | |
| 127 | expect(res.headers.get("content-type")).toBe("text/plain; charset=utf-8"); | |
| 128 | expect(res.headers.get("cache-control")).toBe("no-store"); | |
| 129 | expect(text.startsWith("# gate red at")).toBe(true); | |
| 130 | expect(text.trimEnd().split("\n").at(-1)).toBe("(fail) two"); | |
| 131 | }); | |
| 132 | ||
| 133 | test("an unknown id is 404 for the owner", async () => { | |
| 134 | expect((await get(path("0000000005000-123456abcdef"), "alice")).res.status).toBe(404); | |
| 135 | expect((await get(path("nope"), "alice")).res.status).toBe(404); | |
| 136 | }); | |
| 137 | }); |
test/security-floor-routes.test.ts +2
| @@ -41,6 +41,7 @@ const FLEET = "needs a fleet operation, started by an operator action"; | ||
| 41 | 41 | const SECRET = "needs a secret scope an operator has set"; |
| 42 | 42 | const GO = "needs a short code minted by a share; the seed shares nothing"; |
| 43 | 43 | const ARCH = "needs an archived-home operation id; the seed archives nothing"; |
| 44 | const GATE = "needs a kept gate log, written only when a full gate runs on land"; | |
| 44 | 45 | |
| 45 | 46 | /** GET patterns the seed cannot fill → why. Requested with a placeholder all the same. */ |
| 46 | 47 | const UNSEEDED: Record<string, string> = { |
| @@ -56,6 +57,7 @@ const UNSEEDED: Record<string, string> = { | ||
| 56 | 57 | "/api/v1/orgs/:org": ORG, |
| 57 | 58 | "/api/v1/orgs/:org/teams": ORG, |
| 58 | 59 | "/api/v1/orgs/:org/teams/:team/members": ORG, |
| 60 | "/api/v1/repos/:owner/:name/gates/:id/log": GATE, | |
| 59 | 61 | "/api/v1/repos/:owner/:name/waves/:version": WAVE, |
| 60 | 62 | "/api/v1/secrets/:scope": SECRET, |
| 61 | 63 | "/api/v1/user/origins/:id/browse/:owner/:name": ORIGIN, |
new above
offline
Not blocking: this now writes to disk for every repository's gate, and a throw here (for example ENOSPC) fails the landing rather than just losing the link. Wrapping it in try/catch and returning
undefinedon error would keep a disk problem from blocking flow (#6263).