Public: anyone can read this; the crew can change it.

Every gate keeps its log, so a red gate can be read without re-running the pull #6366

Parked opened this
Closed without merging

scout/forge:scout/6351-every-gate-keeps-its-log-so-a-red-gate-c → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
8 files changed +177 −3
src/checks/gate_log.ts added +94
@@ -0,0 +1,94 @@
1 /**
2 * Every gate keeps its log: per repository, the last 20 gates, each as one
3 * small file under `checks/gates/<owner>/<name>/` holding its per-file
4 * results and the last 200 lines of its output, so a red gate is read
5 * without running the pull again.
6 */
7
8 import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
9 import { join } from "node:path";
10 import { fileResults, type FileResult } from "./gate_last.ts";
11
12 export const KEEP_GATES = 20;
13 export const TAIL_LINES = 200;
14
15 export type KeptGate = {
16 id: string;
17 at: number;
18 commit: string;
19 ok: boolean;
20 files: Record<string, FileResult>;
21 tail: string[];
22 };
23
24 const ID = /^\d{13}-[0-9a-f]{4,40}$/;
25 const SEGMENT = /^[A-Za-z0-9._-]+$/;
26
27 export const gateId = (now: number, commit: string): string =>
28 `${String(Math.max(0, Math.trunc(now))).padStart(13, "0")}-${commit.toLowerCase().slice(0, 12)}`;
29
30 const dirOf = (dataDir: string, owner: string, name: string): string | undefined =>
31 SEGMENT.test(owner) && SEGMENT.test(name) && owner !== ".." && name !== ".."
32 ? join(dataDir, "checks", "gates", owner, name)
33 : undefined;
34
35 /** Keep one gate's record and drop all but the newest 20. Returns its id. */
36 export function keepGateLog(
37 dataDir: string,
38 repo: string,
39 commit: string,
40 ok: boolean,
41 lines: readonly string[],
42 now: number,
43 ): string | undefined {
44 const [owner = "", name = ""] = repo.split("/");
45 const dir = dirOf(dataDir, owner, name);
46 const id = gateId(now, commit);
47 if (dir === undefined || !ID.test(id)) return undefined;
48 const kept: KeptGate = {
49 id,
50 at: now,
51 commit: id.slice(14),
52 ok,
53 files: fileResults(lines),
54 tail: lines.slice(-TAIL_LINES),
55 };
56 mkdirSync(dir, { recursive: true });
57 writeFileSync(join(dir, `${id}.json`), JSON.stringify(kept));
58 const ids = readdirSync(dir)
59 .filter((f) => f.endsWith(".json") && ID.test(f.slice(0, -5)))
60 .sort();
61 for (const old of ids.slice(0, Math.max(0, ids.length - KEEP_GATES)))
62 rmSync(join(dir, old), { force: true });
63 return id;
64 }
65
66 export function readGateLog(
67 dataDir: string,
68 owner: string,
69 name: string,
70 id: string,
71 ): KeptGate | undefined {
72 const dir = dirOf(dataDir, owner, name);
73 if (dir === undefined || !ID.test(id)) return undefined;
74 try {
75 const parsed = JSON.parse(readFileSync(join(dir, `${id}.json`), "utf8")) as KeptGate;
76 return Array.isArray(parsed.tail) ? parsed : undefined;
77 } catch {
78 return undefined;
79 }
80 }
81
82 export function listGateLogs(dataDir: string, owner: string, name: string): string[] {
83 const dir = dirOf(dataDir, owner, name);
84 if (dir === undefined) return [];
85 try {
86 return readdirSync(dir)
87 .filter((f) => f.endsWith(".json") && ID.test(f.slice(0, -5)))
88 .map((f) => f.slice(0, -5))
89 .sort()
90 .reverse();
91 } catch {
92 return [];
93 }
94 }
src/checks/landing.ts +5 −1
@@ -26,6 +26,7 @@ import type { Pull } from "../core/pulls.ts";
2626 import type { Repo } from "../core/repos.ts";
2727 import { failuresOf } from "./failures.ts";
2828 import { keepForgeGate } from "./gate_last.ts";
29 import { keepGateLog } from "./gate_log.ts";
2930 import { checksFor } from "./service.ts";
3031 import { landVerdict, type LandVerdict } from "./tiers.ts";
3132 import { ownProbeHeaders } from "../kernel/probe.ts";
@@ -58,8 +59,11 @@ export async function gateCandidate(
5859 // Cut off, not judged: no verdict, and not the gate log a red reads (#3155).
5960 if (done.interrupted === true) return { lands: false, interrupted: true };
6061 keepForgeGate(app.config.dataDir, app.config.forgeRepo, `${repo.ownerName}/${repo.name}`, done.lines);
62 const slug = `${repo.ownerName}/${repo.name}`;
63 const kept = keepGateLog(app.config.dataDir, slug, commitOid, done.ok, done.lines, Date.now());
6164 const failed = failuresOf(done.lines);
62 return landVerdict(done.ok, failed.tests, failed.files);
65 const link = kept === undefined ? undefined : `/api/v1/repos/${slug}/gates/${kept}/log`;
66 return landVerdict(done.ok, failed.tests, failed.files, link);
6367 }
6468
6569 /**
src/checks/tiers.ts +4 −2
@@ -270,16 +270,18 @@ export function landVerdict(
270270 green: boolean,
271271 failedTests: readonly string[],
272272 failedFiles: readonly string[] = [],
273 logLink?: string,
273274 ): Judged {
274275 if (green) return { lands: true };
275276 const names = failedTests.map((name) => `- ${name}`).join("\n");
277 const kept = logLink === undefined ? "" : `\n\nThe gate's log: ${logLink}`;
276278 return {
277279 lands: false,
278280 reopen: true,
279281 comment:
280282 failedTests.length === 0
281 ? "The full gate on land was red and named no test; the pull is open again."
282 : `The full gate on land was red; the pull is open again. Red tests:\n${names}`,
283 ? `The full gate on land was red and named no test; the pull is open again.${kept}`
284 : `The full gate on land was red; the pull is open again. Red tests:\n${names}${kept}`,
283285 learn: [...new Set(failedFiles)],
284286 };
285287 }
src/contract/openapi/routes/getApiV1ReposOwnerNameGatesIdLog.ts added +3
@@ -0,0 +1,3 @@
1 import { REPO, type Route } from "../shared.ts";
2
3 export const route: Route = { method: "get", path: `${REPO}/gates/{id}/log`, summary: "Read a kept full gate's log: the last 200 lines as text/plain, under a one-line header; the last 20 gates of a repository are kept; 404 when none is kept under that id", auth: false };
src/contract/openapi/routes/index.ts +1
@@ -100,6 +100,7 @@ export { route as getApiV1ReposOwnerNamePullsNumberThreads } from "./getApiV1Rep
100100 export { route as getApiV1ReposOwnerNameReports } from "./getApiV1ReposOwnerNameReports.ts";
101101 export { route as getApiV1ReposOwnerNameRuns } from "./getApiV1ReposOwnerNameRuns.ts";
102102 export { route as getApiV1ReposOwnerNameRunsId } from "./getApiV1ReposOwnerNameRunsId.ts";
103 export { route as getApiV1ReposOwnerNameGatesIdLog } from "./getApiV1ReposOwnerNameGatesIdLog.ts";
103104 export { route as getApiV1ReposOwnerNameRunsIdLog } from "./getApiV1ReposOwnerNameRunsIdLog.ts";
104105 export { route as getApiV1ReposOwnerNameTags } from "./getApiV1ReposOwnerNameTags.ts";
105106 export { route as getApiV1ReposOwnerNameUpstream } from "./getApiV1ReposOwnerNameUpstream.ts";
src/http/api.ts +13
@@ -387,6 +387,7 @@ import { readUsage, usageDays, USAGE_DAYS_MAX } from "../actions/usage.ts";
387387 import { readReturns, reportSpend, RETURNS_DAYS } from "../actions/returns.ts";
388388 import { fleetSelection, readFleetGraph } from "../actions/fleet-graph.ts";
389389 import { carriedBy } from "../core/carried.ts";
390 import { readGateLog } from "../checks/gate_log.ts";
390391 import { readRunLog, subscribeRunLog } from "../runs/log.ts";
391392 import { blobForRender, commitLog, listRefs, listTree } from "../git/read.ts";
392393 import { repoDir } from "../git/repos.ts";
@@ -6270,6 +6271,18 @@ export function apiRoutes(router: Router<Handler>): void {
62706271 return timePhase(ctx.request, "render", () => json(found));
62716272 });
62726273
6274 // A kept gate's log (`checks/gate_log.ts`): the last 200 lines, as text.
6275 router.add("GET", "/api/v1/repos/:owner/:name/gates/:id/log", (ctx) => {
6276 const repo = loadRepo(ctx);
6277 if (!repo) return notFound();
6278 const kept = readGateLog(ctx.app.config.dataDir, repo.ownerName, repo.name, ctx.params.id ?? "");
6279 if (!kept) return notFound();
6280 const head = `# gate ${kept.ok ? "green" : "red"} at ${new Date(kept.at).toISOString()}, commit ${kept.commit}\n`;
6281 return new Response(`${head}${kept.tail.join("\n")}\n`, {
6282 headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" },
6283 });
6284 });
6285
62736286 // The log: text, or a live tail as server-sent events (`?follow=1`, or
62746287 // an `Accept: text/event-stream`). Lines already written come first,
62756288 // then each new one as it lands; `done` carries the finished run.
test/gate-log.test.ts added +55
@@ -0,0 +1,55 @@
1 /** Every gate keeps its log, per repository, the last 20 (#6351). */
2 import { afterEach, expect, test } from "bun:test";
3 import { mkdtempSync, rmSync } from "node:fs";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { KEEP_GATES, keepGateLog, listGateLogs, readGateLog } from "../src/checks/gate_log.ts";
7 import { landVerdict } from "../src/checks/tiers.ts";
8
9 const dirs: string[] = [];
10 const data = (): string => {
11 const d = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "gate-log-"));
12 dirs.push(d);
13 return d;
14 };
15 afterEach(() => dirs.splice(0).forEach((d) => rmSync(d, { recursive: true, force: true })));
16
17 const red = ["test/a.test.ts:", "(pass) one", "(fail) two"];
18
19 test("a red gate outside forgeRepo leaves a readable log", () => {
20 const d = data();
21 const lines = [...Array.from({ length: 300 }, (_, i) => `line ${i}`), ...red];
22 const id = keepGateLog(d, "ann/app", "abcdef1234567890", false, lines, 1_000)!;
23 const kept = readGateLog(d, "ann", "app", id)!;
24 expect(kept.ok).toBe(false);
25 expect(kept.files).toEqual({ "test/a.test.ts": "fail" });
26 expect(kept.tail).toHaveLength(200);
27 expect(kept.tail.at(-1)).toBe("(fail) two");
28 });
29
30 test("the 21st gate drops the oldest", () => {
31 const d = data();
32 const ids = Array.from({ length: KEEP_GATES + 1 }, (_, i) =>
33 keepGateLog(d, "ann/app", "abcdef12", true, red, 1_000 + i)!,
34 );
35 const left = listGateLogs(d, "ann", "app");
36 expect(left).toHaveLength(KEEP_GATES);
37 expect(left).not.toContain(ids[0]!);
38 expect(readGateLog(d, "ann", "app", ids[0]!)).toBeUndefined();
39 expect(listGateLogs(d, "bob", "app")).toEqual([]);
40 });
41
42 test("odd names and ids are refused", () => {
43 const d = data();
44 expect(keepGateLog(d, "../x", "abcdef12", true, red, 1)).toBeUndefined();
45 expect(readGateLog(d, "ann", "app", "../../x")).toBeUndefined();
46 });
47
48 test("the landing note names the red tests and links the kept log", () => {
49 const note = (tests: string[]): string =>
50 (landVerdict(false, tests, [], "/api/v1/repos/ann/app/gates/x/log") as { comment: string })
51 .comment;
52 expect(note(["two"])).toContain("- two");
53 expect(note(["two"])).toContain("/gates/x/log");
54 expect(note([])).toContain("/gates/x/log");
55 });
test/security-floor-routes.test.ts +2
@@ -41,6 +41,7 @@ const FLEET = "needs a fleet operation, started by an operator action";
4141 const SECRET = "needs a secret scope an operator has set";
4242 const GO = "needs a short code minted by a share; the seed shares nothing";
4343 const ARCH = "needs an archived-home operation id; the seed archives nothing";
44 const GATE = "needs a kept gate log, written only when a full gate runs on land";
4445
4546 /** GET patterns the seed cannot fill → why. Requested with a placeholder all the same. */
4647 const UNSEEDED: Record<string, string> = {
@@ -56,6 +57,7 @@ const UNSEEDED: Record<string, string> = {
5657 "/api/v1/orgs/:org": ORG,
5758 "/api/v1/orgs/:org/teams": ORG,
5859 "/api/v1/orgs/:org/teams/:team/members": ORG,
60 "/api/v1/repos/:owner/:name/gates/:id/log": GATE,
5961 "/api/v1/repos/:owner/:name/waves/:version": WAVE,
6062 "/api/v1/secrets/:scope": SECRET,
6163 "/api/v1/user/origins/:id/browse/:owner/:name": ORIGIN,

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs