Public: anyone can read this; the crew can change it.

Every gate keeps its log, so a red gate can be read without re-running the pull #6389

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6351-every-gate-keeps-its-log-so-a-red-gate-c → main 6 commits fork 3 threads · 3 unresolved

  • Merged. scout/6351-every-gate-keeps-its-log-so-a-red-gate-c was merged into main as b2e952f.

Review

Sign in to review this change.
8 files changed +245 −3
src/checks/gate_log.ts added +80
@@ -0,0 +1,80 @@
1 /**
2 * Every gate keeps its log: per repository, the last 20 gates, each as one
3 * small file under `checks/gates/<owner>/<name>/` holding its per-file
4 * results and the last 200 lines of its output, so a red gate is read
5 * without running the pull again.
6 */
7
8 import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
9 import { join } from "node:path";
10 import { fileResults, type FileResult } from "./gate_last.ts";
11
12 export const KEEP_GATES = 20;
13 export const TAIL_LINES = 200;
14
15 export type KeptGate = {
16 id: string;
17 at: number;
18 commit: string;
19 ok: boolean;
20 files: Record<string, FileResult>;
21 tail: string[];
22 };
23
24 const ID = /^\d{13}-[0-9a-f]{4,40}$/;
25 const SEGMENT = /^[A-Za-z0-9._-]+$/;
26
27 export const gateId = (now: number, commit: string): string =>
28 `${String(Math.max(0, Math.trunc(now))).padStart(13, "0")}-${commit.toLowerCase().slice(0, 12)}`;
29
30 const dirOf = (dataDir: string, owner: string, name: string): string | undefined =>
31 SEGMENT.test(owner) && SEGMENT.test(name) && owner !== ".." && name !== ".."
32 ? join(dataDir, "checks", "gates", owner, name)
33 : undefined;
34
35 /** Keep one gate's record and drop all but the newest 20. Returns its id. */
36 export function keepGateLog(
37 dataDir: string,
38 repo: string,
39 commit: string,
40 ok: boolean,
41 lines: readonly string[],
42 now: number,
43 ): string | undefined {
44 const [owner = "", name = ""] = repo.split("/");
45 const dir = dirOf(dataDir, owner, name);
46 const id = gateId(now, commit);
47 if (dir === undefined || !ID.test(id)) return undefined;
48 const kept: KeptGate = {
49 id,
50 at: now,
51 commit: id.slice(14),
52 ok,
53 files: fileResults(lines),
54 tail: lines.slice(-TAIL_LINES),
55 };
56 mkdirSync(dir, { recursive: true });
57 writeFileSync(join(dir, `${id}.json`), JSON.stringify(kept));
58 const ids = readdirSync(dir)
59 .filter((f) => f.endsWith(".json") && ID.test(f.slice(0, -5)))
60 .sort();
61 for (const old of ids.slice(0, Math.max(0, ids.length - KEEP_GATES)))
62 rmSync(join(dir, old), { force: true });
63 return id;
64 }
65
66 export function readGateLog(
67 dataDir: string,
68 owner: string,
69 name: string,
70 id: string,
71 ): KeptGate | undefined {
72 const dir = dirOf(dataDir, owner, name);
73 if (dir === undefined || !ID.test(id)) return undefined;
74 try {
75 const parsed = JSON.parse(readFileSync(join(dir, `${id}.json`), "utf8")) as KeptGate;
76 return Array.isArray(parsed.tail) ? parsed : undefined;
77 } catch {
78 return undefined;
79 }
80 }
src/checks/landing.ts +5 −1
@@ -26,6 +26,7 @@ import type { Pull } from "../core/pulls.ts";
2626 import type { Repo } from "../core/repos.ts";
2727 import { failuresOf } from "./failures.ts";
2828 import { keepForgeGate } from "./gate_last.ts";
29 import { keepGateLog } from "./gate_log.ts";
2930 import { checksFor } from "./service.ts";
3031 import { landVerdict, type LandVerdict } from "./tiers.ts";
3132 import { ownProbeHeaders } from "../kernel/probe.ts";
@@ -58,8 +59,11 @@ export async function gateCandidate(
5859 // Cut off, not judged: no verdict, and not the gate log a red reads (#3155).
5960 if (done.interrupted === true) return { lands: false, interrupted: true };
6061 keepForgeGate(app.config.dataDir, app.config.forgeRepo, `${repo.ownerName}/${repo.name}`, done.lines);
62 const slug = `${repo.ownerName}/${repo.name}`;
63 const kept = keepGateLog(app.config.dataDir, slug, commitOid, done.ok, done.lines, Date.now());

Not blocking: this now writes to disk for every repository's gate, and a throw here (for example ENOSPC) fails the landing rather than just losing the link. Wrapping it in try/catch and returning undefined on error would keep a disk problem from blocking flow (#6263).

6164 const failed = failuresOf(done.lines);
62 return landVerdict(done.ok, failed.tests, failed.files);
65 const link = kept === undefined ? undefined : `/api/v1/repos/${slug}/gates/${kept}/log`;
66 return landVerdict(done.ok, failed.tests, failed.files, link);
6367 }
6468
6569 /**
src/checks/tiers.ts +4 −2
@@ -270,16 +270,18 @@ export function landVerdict(
270270 green: boolean,
271271 failedTests: readonly string[],
272272 failedFiles: readonly string[] = [],
273 logLink?: string,
273274 ): Judged {
274275 if (green) return { lands: true };
275276 const names = failedTests.map((name) => `- ${name}`).join("\n");
277 const kept = logLink === undefined ? "" : `\n\nThe gate's log: ${logLink}`;
276278 return {
277279 lands: false,
278280 reopen: true,
279281 comment:
280282 failedTests.length === 0
281 ? "The full gate on land was red and named no test; the pull is open again."
282 : `The full gate on land was red; the pull is open again. Red tests:\n${names}`,
283 ? `The full gate on land was red and named no test; the pull is open again.${kept}`
284 : `The full gate on land was red; the pull is open again. Red tests:\n${names}${kept}`,
283285 learn: [...new Set(failedFiles)],
284286 };
285287 }
src/contract/openapi/routes/getApiV1ReposOwnerNameGatesIdLog.ts added +3
@@ -0,0 +1,3 @@
1 import { REPO, type Route } from "../shared.ts";
2
3 export const route: Route = { method: "get", path: `${REPO}/gates/{id}/log`, summary: "Read a kept full gate's log: the last 200 lines as text/plain, under a one-line header; the last 20 gates of a repository are kept; 404 when none is kept under that id", auth: false };
src/contract/openapi/routes/index.ts +1
@@ -81,6 +81,7 @@ export { route as getApiV1ReposOwnerNameFlow } from "./getApiV1ReposOwnerNameFlo
8181 export { route as getApiV1ReposOwnerNameFollowing } from "./getApiV1ReposOwnerNameFollowing.ts";
8282 export { route as getApiV1ReposOwnerNameFollowingProposals } from "./getApiV1ReposOwnerNameFollowingProposals.ts";
8383 export { route as getApiV1ReposOwnerNameForks } from "./getApiV1ReposOwnerNameForks.ts";
84 export { route as getApiV1ReposOwnerNameGatesIdLog } from "./getApiV1ReposOwnerNameGatesIdLog.ts";
8485 export { route as getApiV1ReposOwnerNameHooks } from "./getApiV1ReposOwnerNameHooks.ts";
8586 export { route as getApiV1ReposOwnerNameIssues } from "./getApiV1ReposOwnerNameIssues.ts";
8687 export { route as getApiV1ReposOwnerNameIssuesNumber } from "./getApiV1ReposOwnerNameIssuesNumber.ts";
src/http/api.ts +13
@@ -387,6 +387,7 @@ import { readUsage, usageDays, USAGE_DAYS_MAX } from "../actions/usage.ts";
387387 import { readReturns, reportSpend, RETURNS_DAYS } from "../actions/returns.ts";
388388 import { fleetSelection, readFleetGraph } from "../actions/fleet-graph.ts";
389389 import { carriedBy } from "../core/carried.ts";
390 import { readGateLog } from "../checks/gate_log.ts";
390391 import { readRunLog, subscribeRunLog } from "../runs/log.ts";
391392 import { blobForRender, commitLog, listRefs, listTree } from "../git/read.ts";
392393 import { repoDir } from "../git/repos.ts";
@@ -6284,6 +6285,18 @@ export function apiRoutes(router: Router<Handler>): void {
62846285 return timePhase(ctx.request, "render", () => json(found));
62856286 });
62866287
6288 // A kept gate's log (`checks/gate_log.ts`): the last 200 lines, as text.
6289 router.add("GET", "/api/v1/repos/:owner/:name/gates/:id/log", (ctx) => {

This route is the door the landing note links to, and no test requests it. Add a route test: a guest and a non-member get 404 on a private repository's kept gate. The owner gets 200 with text/plain, no-store and the # gate red at header line. An unknown id gets 404.

6290 const repo = loadRepo(ctx);
6291 if (!repo) return notFound();
6292 const kept = readGateLog(ctx.app.config.dataDir, repo.ownerName, repo.name, ctx.params.id ?? "");
6293 if (!kept) return notFound();
6294 const head = `# gate ${kept.ok ? "green" : "red"} at ${new Date(kept.at).toISOString()}, commit ${kept.commit}\n`;
6295 return new Response(`${head}${kept.tail.join("\n")}\n`, {
6296 headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" },
6297 });
6298 });
6299
62876300 // The log: text, or a live tail as server-sent events (`?follow=1`, or
62886301 // an `Accept: text/event-stream`). Lines already written come first,
62896302 // then each new one as it lands; `done` carries the finished run.
test/gate-log.test.ts added +137
@@ -0,0 +1,137 @@
1 /** Every gate keeps its log, per repository, the last 20 (#6351). */
2 import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
3 import { mkdtempSync, rmSync } from "node:fs";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { KEEP_GATES, keepGateLog, readGateLog } from "../src/checks/gate_log.ts";
7 import { createApp, type App } from "../src/app.ts";
8 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
9 import { createToken } from "../src/core/tokens.ts";
10 import { createUser } from "../src/core/users.ts";
11 import { forgeHttp, type ForgeHttp } from "../src/http/server.ts";
12 import { quietLogs } from "./helpers/globals.ts";
13 import { landVerdict } from "../src/checks/tiers.ts";
14
15 quietLogs();
16
17 const dirs: string[] = [];
18 const data = (): string => {
19 const d = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "gate-log-"));
20 dirs.push(d);
21 return d;
22 };
23 afterEach(() => dirs.splice(0).forEach((d) => rmSync(d, { recursive: true, force: true })));
24
25 const red = ["test/a.test.ts:", "(pass) one", "(fail) two"];
26
27 test("a red gate outside forgeRepo leaves a readable log", () => {
28 const d = data();
29 const lines = [...Array.from({ length: 300 }, (_, i) => `line ${i}`), ...red];
30 const id = keepGateLog(d, "ann/app", "abcdef1234567890", false, lines, 1_000)!;
31 const kept = readGateLog(d, "ann", "app", id)!;
32 expect(kept.ok).toBe(false);
33 expect(kept.files).toEqual({ "test/a.test.ts": "fail" });
34 expect(kept.tail).toHaveLength(200);
35 expect(kept.tail.at(-1)).toBe("(fail) two");
36 });
37
38 test("the 21st gate drops the oldest", () => {
39 const d = data();
40 const ids = Array.from({ length: KEEP_GATES + 1 }, (_, i) =>
41 keepGateLog(d, "ann/app", "abcdef12", true, red, 1_000 + i)!,
42 );
43 expect(readGateLog(d, "ann", "app", ids[0]!)).toBeUndefined();
44 for (const id of ids.slice(1)) expect(readGateLog(d, "ann", "app", id)).toBeDefined();
45 expect(readGateLog(d, "bob", "app", ids[1]!)).toBeUndefined();
46 });
47
48 test("odd names and ids are refused", () => {
49 const d = data();
50 expect(keepGateLog(d, "../x", "abcdef12", true, red, 1)).toBeUndefined();
51 expect(readGateLog(d, "ann", "app", "../../x")).toBeUndefined();
52 });
53
54 test("the landing note names the red tests and links the kept log", () => {
55 const note = (tests: string[]): string =>
56 (landVerdict(false, tests, [], "/api/v1/repos/ann/app/gates/x/log") as { comment: string })
57 .comment;
58 expect(note(["two"])).toContain("- two");
59 expect(note(["two"])).toContain("/gates/x/log");
60 expect(note([])).toContain("/gates/x/log");
61 });
62
63 describe("GET /api/v1/repos/:owner/:name/gates/:id/log", () => {
64 let dataDir: string;
65 let app: App;
66 let http: ForgeHttp;
67 let id: string;
68 const token: Record<string, string> = {};
69
70 const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => {
71 if (!r.ok) throw new Error(JSON.stringify(r.error));
72 return r.value;
73 };
74 const get = async (path: string, who?: string) => {
75 const res = await http.handle(
76 new Request(`http://forge.test${path}`, {
77 headers: who === undefined ? {} : { authorization: `Bearer ${token[who]}` },
78 }),
79 );
80 return { res, text: await res.text() };
81 };
82
83 beforeAll(async () => {
84 dataDir = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "gate-route-"));
85 app = createApp({ dataDir, port: 0 });
86 http = forgeHttp(app);
87 const mk = async (username: string, adminIfFirst = false) =>
88 ok(
89 await createUser(app.db, {
90 origin: "seed",
91 trusted: true,
92 username,
93 email: `${username}@forge.local`,
94 password: "a-good-passphrase",
95 adminIfFirst,
96 }),
97 );
98 const alice = await mk("alice", true);
99 const mallory = await mk("mallory");
100 const repo = ok(await createRepo(app, alice, { name: "vault", visibility: "private" }));
101 ok(
102 await commitToBranch(app, alice, repo, {
103 message: "Start",
104 files: [{ path: "README.md", content: "# vault\n" }],
105 }),
106 );
107 token.alice = createToken(app.db, alice.id, "t", "rw").token;
108 token.mallory = createToken(app.db, mallory.id, "t", "rw").token;
109 id = keepGateLog(dataDir, "alice/vault", "abcdef1234567890", false, red, 5_000)!;
110 }, 60_000);
111
112 afterAll(() => {
113 app?.close();
114 rmSync(dataDir, { recursive: true, force: true });
115 });
116
117 const path = (which = id): string => `/api/v1/repos/alice/vault/gates/${which}/log`;
118
119 test("a guest and a non-member get 404 on a private repository's gate", async () => {
120 expect((await get(path())).res.status).toBe(404);
121 expect((await get(path(), "mallory")).res.status).toBe(404);
122 });
123
124 test("the owner reads it as no-store text under a header line", async () => {
125 const { res, text } = await get(path(), "alice");
126 expect(res.status).toBe(200);
127 expect(res.headers.get("content-type")).toBe("text/plain; charset=utf-8");
128 expect(res.headers.get("cache-control")).toBe("no-store");
129 expect(text.startsWith("# gate red at")).toBe(true);
130 expect(text.trimEnd().split("\n").at(-1)).toBe("(fail) two");
131 });
132
133 test("an unknown id is 404 for the owner", async () => {
134 expect((await get(path("0000000005000-123456abcdef"), "alice")).res.status).toBe(404);
135 expect((await get(path("nope"), "alice")).res.status).toBe(404);
136 });
137 });
test/security-floor-routes.test.ts +2
@@ -41,6 +41,7 @@ const FLEET = "needs a fleet operation, started by an operator action";
4141 const SECRET = "needs a secret scope an operator has set";
4242 const GO = "needs a short code minted by a share; the seed shares nothing";
4343 const ARCH = "needs an archived-home operation id; the seed archives nothing";
44 const GATE = "needs a kept gate log, written only when a full gate runs on land";
4445
4546 /** GET patterns the seed cannot fill → why. Requested with a placeholder all the same. */
4647 const UNSEEDED: Record<string, string> = {
@@ -56,6 +57,7 @@ const UNSEEDED: Record<string, string> = {
5657 "/api/v1/orgs/:org": ORG,
5758 "/api/v1/orgs/:org/teams": ORG,
5859 "/api/v1/orgs/:org/teams/:team/members": ORG,
60 "/api/v1/repos/:owner/:name/gates/:id/log": GATE,
5961 "/api/v1/repos/:owner/:name/waves/:version": WAVE,
6062 "/api/v1/secrets/:scope": SECRET,
6163 "/api/v1/user/origins/:id/browse/:owner/:name": ORIGIN,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs