Public: anyone can read this; the crew can change it.
A test that only reads the tree never makes a lane's red: OpenAPI coverage, MCP docs, src imports and the README gates join TREE_GATES #8189
Merged
scoutcrew
opened this
Check · Review · Land · Live done — Landed and live in production.
Merged
scout/forge:scout/8184-a-test-that-only-reads-the-tree-never-ma → main · 2 commits ·fork
-
Merged. scout/8184-a-test-that-only-reads-the-tree-never-ma was merged into main as 2dce28a.
Review
Sign in to review this change.
2 files
changed +39
−7
src/checks/shared_red.ts +15 −2
| @@ -49,17 +49,30 @@ export const NO_TEST_NAME = "(unnamed)"; | ||
| 49 | 49 | * reserving it ("keeper", "feed", "favicon.ico") failed it, main green. |
| 50 | 50 | * OpenAPI coverage (#8140): each pull adding an /api/v1 route without its |
| 51 | 51 | * row, or a route file without regenerating the index, failed it, main green. |
| 52 | * The rest are top-level tests named whole (#8184): each reads only the tree. | |
| 52 | 53 | */ |
| 53 | 54 | export const TREE_GATES: readonly string[] = [ |
| 54 | 55 | "design check: nothing added outside the system (#2678) > ", |
| 55 | 56 | "no polling in src/web > ", |
| 56 | 57 | "reserved names > every top-level page of the forge is a reserved name", |
| 58 | // A route or tool a pull added without its row in docs/openapi or docs/MCP.md. | |
| 57 | 59 | "every registered /api/v1 route has an OpenAPI row or is listed as not yet described", |
| 60 | "NOT_YET_DESCRIBED has no entry that is now described", | |
| 61 | "NOT_YET_DESCRIBED is sorted and names registered routes", | |
| 62 | "every tool in tools/list has a row in docs/MCP.md", | |
| 63 | "every row in docs/MCP.md names a tool that exists", | |
| 64 | // An import a pull added from scripts/ or test/ into src/. | |
| 65 | "nothing under src/ imports from outside src/ (scripts/, test/, …)", | |
| 66 | // A README a pull grew past its limit, or a docs/ link it broke. | |
| 67 | "README.md stays under 80 KB", | |
| 68 | "every docs/ link in README.md resolves", | |
| 69 | // A route file a pull added without regenerating the index. | |
| 58 | 70 | "the gathered routes > the index is current: run `bun scripts/openapi-routes.ts`", |
| 59 | 71 | ]; |
| 60 | 72 | |
| 61 | /** Whether a failing test's name is one of {@link TREE_GATES}'. */ | |
| 62 | export const isTreeGate = (test: string): boolean => TREE_GATES.some((g) => test.startsWith(g)); | |
| 73 | /** Whether a failing test's name is one of {@link TREE_GATES}': under a suite prefix (`… > `), or a test's full name exactly. */ | |
| 74 | export const isTreeGate = (test: string): boolean => | |
| 75 | TREE_GATES.some((g) => (g.endsWith(" > ") ? test.startsWith(g) : test === g)); | |
| 63 | 76 | |
| 64 | 77 | export interface SharedTest { |
| 65 | 78 | readonly repoId: string; |
test/shared-red.test.ts +24 −5
| @@ -87,6 +87,9 @@ describe("a red many pulls share", () => { | ||
| 87 | 87 | const gate = "reserved names > every top-level page of the forge is a reserved name"; |
| 88 | 88 | for (const i of ["i1", "i2", "i3"]) red(i, [gate]); |
| 89 | 89 | expect(shared(red("i4", [gate]))).toBe(false); |
| 90 | const index = "the gathered routes > the index is current: run `bun scripts/openapi-routes.ts`"; | |
| 91 | for (const i of ["i1", "i2", "i3"]) red(i, [index]); | |
| 92 | expect(shared(red("i4", [index]))).toBe(false); | |
| 90 | 93 | expect(sharedTests(db)).toEqual([]); |
| 91 | 94 | // The rest of the suite is no gate: three pulls on it still share it. |
| 92 | 95 | const other = "reserved names > the forge's own nouns are reserved too"; |
| @@ -94,7 +97,7 @@ describe("a red many pulls share", () => { | ||
| 94 | 97 | expect(shared(red("i3", [other]))).toBe(true); |
| 95 | 98 | }); |
| 96 | 99 | |
| 97 | test("an /api/v1 route a pull left undescribed is that pull's own red, not the lane's (#8140)", () => { | |
| 100 | test("three pulls each adding a route with no OpenAPI row are three pulls' own reds (#8184)", () => { | |
| 98 | 101 | const gate = "every registered /api/v1 route has an OpenAPI row or is listed as not yet described"; |
| 99 | 102 | for (const i of ["i1", "i2", "i3"]) red(i, [gate]); |
| 100 | 103 | expect(shared(red("i4", [gate]))).toBe(false); |
| @@ -102,6 +105,12 @@ describe("a red many pulls share", () => { | ||
| 102 | 105 | for (const i of ["i1", "i2", "i3"]) red(i, [index]); |
| 103 | 106 | expect(shared(red("i4", [index]))).toBe(false); |
| 104 | 107 | expect(sharedTests(db)).toEqual([]); |
| 108 | const v = vitals({ db, dataDir: tmpdir(), signals: { errors: new ErrorLedger(), refused: new RefusedLedger() } }); | |
| 109 | expect(v.degraded.filter((d) => d.cause.startsWith("shared:"))).toEqual([]); | |
| 110 | // Named whole, not by prefix: a top-level test it begins does not hide. | |
| 111 | const longer = `${gate} twice`; | |
| 112 | for (const i of ["i1", "i2"]) red(i, [longer]); | |
| 113 | expect(shared(red("i3", [longer]))).toBe(true); | |
| 105 | 114 | }); |
| 106 | 115 | |
| 107 | 116 | test("every tree gate names a suite, or a test in it, that exists", () => { |
| @@ -119,6 +128,15 @@ describe("a red many pulls share", () => { | ||
| 119 | 128 | expect(suiteNames('test("top", () => {});\n').has("top")).toBe(true); |
| 120 | 129 | }); |
| 121 | 130 | |
| 131 | test("a top-level test is named bare; one inside a describe is not", () => { | |
| 132 | const names = suiteNames('test("t", () => {});\ndescribe("A", () => {\n test("u", () => {});\n});\ntest("v", () => {});\n'); | |
| 133 | expect(names.has("t")).toBe(true); | |
| 134 | expect(names.has("v")).toBe(true); | |
| 135 | expect(names.has("A > u")).toBe(true); | |
| 136 | expect(names.has("u")).toBe(false); | |
| 137 | expect(names.has("A > v")).toBe(false); | |
| 138 | }); | |
| 139 | ||
| 122 | 140 | test("one pull red three times is its own red, not the lane's", () => { |
| 123 | 141 | red("i1", ["X"]); |
| 124 | 142 | red("i1", ["X"]); |
| @@ -280,13 +298,14 @@ describe("a failure outside a test names what it threw (#5727)", () => { | ||
| 280 | 298 | }); |
| 281 | 299 | }); |
| 282 | 300 | |
| 283 | /** Each `describe > test` a test file declares, every test paired only with the describe it sits under, plus `describe > ` for each suite. */ | |
| 301 | /** Each `describe > test` a test file declares, every test paired only with the describe it sits under, plus `describe > ` for each suite and each top-level test's bare name. */ | |
| 284 | 302 | function suiteNames(text: string): Set<string> { |
| 285 | 303 | const names = new Set<string>(); |
| 286 | 304 | let suite: string | undefined; |
| 287 | for (const m of text.matchAll(/\b(describe|test)\("((?:[^"\\]|\\.)*)"/g)) { | |
| 288 | if (m[1] === "describe") names.add(`${(suite = m[2])} > `); | |
| 289 | else names.add(suite === undefined ? m[2]! : `${suite} > ${m[2]}`); | |
| 305 | for (const m of text.matchAll(/^([ \t]*)(describe|test)\("((?:[^"\\]|\\.)*)"/gm)) { | |
| 306 | if (m[2] === "describe") names.add(`${(suite = m[3])} > `); | |
| 307 | else if (m[1] === "") names.add(m[3]!); | |
| 308 | else if (suite !== undefined) names.add(`${suite} > ${m[3]}`); | |
| 290 | 309 | } |
| 291 | 310 | return names; |
| 292 | 311 | } |
new above
offline