fc50062 · d820484 · edaa486
@@ -25,6 +25,22 @@
issue, and only when its evidence changed; one whose issue was closed and
returned is filed again. It files in `FORGE_SENTINEL_REPO` (default
`trevato/forge`) as the `sentinel` agent, else the first admin.
**How it is labelled** (#8856). Each finding declares a kind, and is filed
with its labels, so the bug clock (`src/flow/bugs.ts`) runs from the filing
and nobody has to sort it:
| Kind | Labels | Clock |
| --- | --- | --- |
| `security` (a hand-installed trigger) | `bug`, `high` | 4 h |
| `defect` (every other finding, and the performance watcher's slow routes) | `bug` | 48 h |
A label the repository lacks is made (`src/hunt/label.ts`, shared by the
sentinel and `src/perf/duty.ts`); labels already on the issue stay, and an
issue adopted after its record was lost is labelled again. A budget breach
also gets `P0 Trust`. A kind joins the table with the first detector that
files it: the route hunters (`src/hunt/door.ts`, `links.ts`, `contract.ts`)
file nothing yet, so there is no `broken` or `polish` kind.
**What it shows.** To a site admin only: the board's forge strip and the
terminal palette's `Sentinel` command (as a toast) show the count of open findings and the
last hunt's time. A hunt is synchronous, so there is no "hunting" state to show.
@@ -23,7 +23,6 @@
import { getAgentByName } from "../core/agents.ts";
import {
  getIssueById,
  getLabel,
  setIssueLabels as writeLabels,
  type Issue,
} from "../core/issues.ts";
import { canWrite, getRepo, type Repo } from "../core/repos.ts";
@@ -48,6 +47,8 @@
import {
} from "../checks/shared_red.ts";
import { readRunLogTail } from "../runs/log.ts";
import { clip } from "../text/clip.ts";
import type { FindingKind } from "../hunt/finding.ts";
import { addLabels, kindLabels } from "../hunt/label.ts";
const log = createLog("sentinel");
@@ -71,6 +72,8 @@
export interface Finding {
  readonly evidence: readonly string[];
  readonly pillar: string;
  readonly lookFor: string;
  /** How it is filed: its labels, and so its clock (#8856). */
  readonly kind: FindingKind;
  /** A budget breach: filed at once, labelled, left unassigned for a builder (#3002). */
  readonly urgent?: true;
  /** What a person or a run would do, when it is mechanical and reversible. */
@@ -131,6 +134,8 @@
export function strayTriggers(db: Database): Finding[] {
        `sql: ${clip(t.sql ?? "", QUOTE_MAX)}`,
      ],
      pillar: "Trust: the database does only what the schema says",
      // Someone's hand on the database: a trigger can refuse or rewrite any write.
      kind: "security",
      lookFor: `\`SELECT name FROM sqlite_master WHERE type = 'trigger'\` lists only triggers the schema creates.`,
      fix: `Drop it at boot like \`dropLegacyNewAccountGuards\` does (\`DROP TRIGGER "${t.name}"\`), with a test that plants it and boots.`,
    }));
@@ -172,6 +177,7 @@
export function heldSlots(db: Database, now: number): Finding[] {
          : []),
      ],
      pillar: "Reliability: the gate always has a slot",
      kind: "defect",
      lookFor: `No ${w.kind} run queued for ${QUEUE_WAIT_MAX_MS / 60_000} minutes while the slots are free; a cancelled run's process is gone with it.`,
      fix: "End the process of a cancelled run on the cancel path, and free its slot.",
    });
@@ -213,6 +219,7 @@
export function ledgerStreaks(db: Database, now: number): Finding[] {
        `a check passed after the first stop: run ${green.id} at ${green.finished_at}`,
      ],
      pillar: "Throughput: red checks end in a fix, not a stop",
      kind: "defect",
      lookFor: "A retry decision that is not stop, or a green check after a stop.",
    });
  }
@@ -250,6 +257,7 @@
export function ledgerStreaks(db: Database, now: number): Finding[] {
        ...(skipped > 0 ? [`${skipped} skips of a run with nothing new are not counted`] : []),
      ],
      pillar: "Throughput: the crew spends on work, not on being refused",
      kind: "defect",
      lookFor: "Fewer than 20 guard decisions a day for the repository.",
    });
  }
@@ -321,6 +329,7 @@
export function runStreaks(db: Database): Finding[] {
          `runs: ${hits.slice(0, 5).map((r) => r.id).join(", ")}`,
        ],
        pillar: "Trust: a green check means something",
        kind: "defect",
        lookFor: "A green check run for the repository's newest head.",
      });
    } else {
@@ -344,6 +353,7 @@
export function runStreaks(db: Database): Finding[] {
          `runs: ${pushes.slice(0, 5).map((r) => r.id).join(", ")}`,
        ],
        pillar: "Throughput: a crew run can finish what it starts",
        kind: "defect",
        lookFor: "An agent run that pushes its branch from inside its sandbox.",
      });
    }
@@ -371,6 +381,7 @@
export function repeatedErrors(now: number): Finding[] {
        `line: ${clip(lines[0]?.message ?? "", QUOTE_MAX)}`,
      ],
      pillar: "Reliability: a quiet log",
      kind: "defect",
      lookFor: "No ERROR line of this shape in an hour.",
    });
  }
@@ -414,6 +425,7 @@
export function degradedCauses(db: Database, now: number, forgeRepoId?: string,
        ...sharedLines(db, cause, now, forgeRepoId, dataDir),
      ],
      pillar: "P0 Trust: the forge says when it is not well, and is well again",
      kind: "defect",
      lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
      incident: new Date(d.opened).toISOString(),
      cause,
@@ -467,6 +479,7 @@
export function budgetBreaches(db: Database, now: number, forgeRepoId?: string,
        ...sharedLines(db, s.cause, now, forgeRepoId, dataDir),
      ],
      pillar: "P0 Trust: the forge says when it is not well, and is well again",
      kind: "defect",
      lookFor: `\`${s.cause}\` is gone from the degraded list and the cause is fixed, not waited out.`,
      fix: s.cause.startsWith("red:")
        ? "Fix the failing test on main in one pull. This is the only issue for it: pulls red on the same test wait for main, and no other builder is handed it while this is open."
@@ -613,27 +626,23 @@
const bodyOf = (f: Finding, now: string): string =>
  ].join("\n");
/**
 * A breach is crew work at once: the P0 Trust label, and no assignee, so a
 * builder may start it on the next tick (#3002). The sentinel never builds,
 * so it never assigns itself: a finding parked on it is held from every
 * builder. The label may fail alone (no such label): the issue is filed
 * either way, and the failure is said.
 * Every finding is filed sorted (#8856): its kind's labels (`bug`, with
 * `high` for a security hole), through `hunt/label.ts`, so the bug clock runs
 * from the moment it is filed. A breach is crew work at once: the P0 Trust label as
 * well, and no assignee, so a builder may start it on the next tick (#3002).
 * The sentinel never builds, so it never assigns itself: a finding parked on
 * it is held from every builder. Labels already on the issue stay. The
 * labelling may fail alone: the issue is filed either way, and the failure is said.
 */
function labelBreach(app: App, repo: Repo, actor: User, issue: Issue): void {
function labelFinding(app: App, repo: Repo, actor: User, issue: Issue, f: Finding): void {
  try {
    // The forge acts on its own finding: no access check stands between it
    // and its label, as none stands between it and the issue it just filed.
    const label = getLabel(app.db, repo.id, "P0 Trust");
    if (label === undefined) log.warn("sentinel has no P0 Trust label to apply", { issue: issue.number });
    else {
      writeLabels(app.db, issue.id, [label.id]);
      app.bus.append({
        kind: "issue.labeled",
        repoId: repo.id,
        actorId: actor.id,
        payload: { number: issue.number, labels: [label.name] },
      });
    const wanted = kindLabels(app, repo.id, f.kind);
    if (f.urgent) {
      const label = getLabel(app.db, repo.id, "P0 Trust");
      if (label === undefined) log.warn("sentinel has no P0 Trust label to apply", { issue: issue.number });
      else wanted.push(label);
    }
    addLabels(app, repo.id, actor.id, issue, wanted);
  } catch (error) {
    log.error("sentinel could not label", { issue: issue.number, error: String(error) });
  }
@@ -986,19 +995,21 @@
function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string
 * Read, file, record. One open issue per finding: a tracked finding whose
 * issue is open is commented on only when its evidence changed; one whose
 * issue is closed and returns is a new issue. A setting away from its default
 * is no finding: the capacity page says so beside it (#3269).
 * is no finding: the capacity page says so beside it (#3269). `find` is the
 * detectors, `findFaults` unless a test files findings of its own.
 */
export function hunt(
  app: App,
  repoPath: string,
  now = Date.now(),
  find: typeof findFaults = findFaults,
): HuntResult {
  const at = new Date(now).toISOString();
  let filed = 0;
  let commented = 0;
  const [owner = "", name = ""] = repoPath.split("/");
  const repo = getRepo(app.db, owner, name);
  const findings = findFaults(app.db, now, repo?.id, app.config.dataDir);
  const findings = find(app.db, now, repo?.id, app.config.dataDir);
  const actor = sentinelActor(app.db);
  if (repo !== undefined && actor !== undefined) {
    settleShared(app, repo, actor, now);
@@ -1059,7 +1070,11 @@
export function hunt(
          }
          issueId = made.value.id;
          filed++;
          if (f.urgent) labelBreach(app, repo, actor, made.value);
          labelFinding(app, repo, actor, made.value, f);
        } else {
          // Adopted: its labels may have been lost with its record.
          const issue = getIssueById(app.db, issueId);
          if (issue !== undefined) labelFinding(app, repo, actor, issue, f);
        }
        app.db
          .query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen, incident) VALUES (?1, ?2, ?3, ?4, ?4, ?5)")
@@ -18,3 +18,21 @@
export const found = (detector: string, subject: string, title: string, evidence
export const seenLine = (path: string, status: number, location?: string | null): string =>
  `GET ${path} → ${status}${location ? ` → ${location}` : ""}`;
/**
 * What a finding is, so it is filed already sorted (#8856): a security hole is
 * a `high` bug (the 4 h clock in `flow/bugs.ts`), any other defect a bug (48 h).
 * A kind joins this list with the first detector that files it.
 */
export type FindingKind = "security" | "defect";
export const KIND_LABELS: Readonly<Record<FindingKind, readonly string[]>> = {
  security: ["bug", "high"],
  defect: ["bug"],
};
/** Each label a kind is filed with, and the colour it is made in when the repository has none by that name. */
export const LABEL_COLORS: Readonly<Record<string, string>> = {
  bug: "d73a4a",
  high: "b60205",
};
@@ -0,0 +1,35 @@
/**
 * Filing a finding sorted (#8856): the labels of its kind, made in the
 * repository when it has none by that name, added to what the issue already
 * carries. Shared by every detector that files: the sentinel and the
 * performance watcher. The forge acts on its own finding, so no access check
 * stands between it and the labels, as none stands between it and the issue.
 */
import type { App } from "../app.ts";
import { createLabel, getLabel, labelsFor, setIssueLabels, type Issue, type Label } from "../core/issues.ts";
import { KIND_LABELS, LABEL_COLORS, type FindingKind } from "./finding.ts";
export function getOrMakeLabel(app: App, repoId: string, name: string): Label | undefined {
  const had = getLabel(app.db, repoId, name);
  if (had !== undefined) return had;
  const made = createLabel(app.db, repoId, name, LABEL_COLORS[name] ?? "ededed");
  return made.ok ? made.value : undefined;
}
export const kindLabels = (app: App, repoId: string, kind: FindingKind): Label[] =>
  KIND_LABELS[kind].flatMap((name) => getOrMakeLabel(app, repoId, name) ?? []);
/** Adds `wanted` to the issue's labels and says so on the bus; labels already on it stay. */
export function addLabels(app: App, repoId: string, actorId: string, issue: Issue, wanted: readonly Label[]): void {
  const had = labelsFor(app.db, issue.id);
  const added = wanted.filter((w) => !had.some((h) => h.id === w.id));
  if (added.length === 0) return;
  setIssueLabels(app.db, issue.id, [...had, ...added].map((l) => l.id));
  app.bus.append({
    kind: "issue.labeled",
    repoId,
    actorId,
    payload: { number: issue.number, labels: added.map((l) => l.name) },
  });
}
@@ -2,7 +2,8 @@
 * The `performance-watcher` duty (#7559): once a minute, judge the record and
 * file each new finding, of a route or of a git subcommand (#8158), as one
 * issue in the forge's own repository, opened by its owner. A title already
 * open is not filed twice, and neither is a key this process has filed.
 * open is not filed twice, and neither is a key this process has filed. A
 * slow route or subcommand is a defect: filed as a `bug`, on its clock (#8856).
 */
import type { App } from "../app.ts";
@@ -10,11 +11,14 @@
import { openIssue } from "../actions/issues.ts";
import { getUserById } from "../core/users.ts";
import { getRepo } from "../core/repos.ts";
import { listIssues } from "../core/issues.ts";
import type { FindingKind } from "../hunt/finding.ts";
import { addLabels, kindLabels } from "../hunt/label.ts";
import { createGitRecord } from "./git.ts";
import type { PerfRecord } from "./record.ts";
import { evaluate, evaluateGit, fresh } from "./watch.ts";
export const TICK_MS = 60_000;
const KIND: FindingKind = "defect";
export function startPerfWatcher(
  app: App,
@@ -36,7 +40,8 @@
export function startPerfWatcher(
      const findings = [...evaluate(record, now()), ...evaluateGit(gitRecord, now())];
      for (const f of fresh(findings, filed)) {
        if (open.has(f.title)) continue;
        openIssue(app, actor, repo, { title: f.title, body: f.evidence });
        const made = openIssue(app, actor, repo, { title: f.title, body: f.evidence });
        if (made.ok) addLabels(app, repo.id, actor.id, made.value, kindLabels(app, repo.id, KIND));
      }
    } catch (error) {
      app.log.warn("performance watcher failed", { error: String(error) });
@@ -4,7 +4,7 @@
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { listIssues } from "../src/core/issues.ts";
import { labelsFor, listIssues } from "../src/core/issues.ts";
import { getRepo } from "../src/core/repos.ts";
import { createUser } from "../src/core/users.ts";
import { startPerfWatcher } from "../src/perf/duty.ts";
@@ -84,3 +84,9 @@
test("the watcher files a slow git subcommand once (#8158)", async () => {
  stop();
  expect(titles().filter((t) => t.startsWith("git "))).toEqual(["git rev-parse is over its budget"]);
});
test("a slow route is filed as a bug, so its clock runs from the filing (#8856)", () => {
  const repo = getRepo(app.db, "alice", "forge")!;
  const issue = listIssues(app.db, repo.id, { state: "open" }).find((i) => i.title === "/ is over its budget: kpis")!;
  expect(labelsFor(app.db, issue.id).map((l) => l.name)).toEqual(["bug"]);
});
@@ -21,6 +21,7 @@
import { createRun } from "../src/core/runs.ts";
import { createUser, type User } from "../src/core/users.ts";
import { recordCap } from "../src/crew/guard.ts";
import { sharedCause } from "../src/checks/shared_red.ts";
import { bugClockAlarm, HIGH_BUG_MS } from "../src/flow/bugs.ts";
import {
  budgetBreaches,
  degradedCauses,
@@ -691,7 +692,7 @@
describe("a budget breach is crew work at once", () => {
    const issue = open[0]!;
    // The sentinel never builds, so it never assigns itself (#3002).
    expect(issue.assigneeId).toBeNull();
    expect(labelsFor(bapp.db, issue.id).map((l) => l.name)).toEqual(["P0 Trust"]);
    expect(labelsFor(bapp.db, issue.id).map((l) => l.name).sort()).toEqual(["P0 Trust", "bug"]);
    expect(listEvents(bapp.db, { kind: "issue.assigned" }).length).toBe(0);
  });
@@ -979,3 +980,83 @@
describe("a degraded finding whose cause has stayed closed", () => {
    expect(hunt(happ, "alice/forge", t).filed).toBe(1);
  });
});
describe("a finding is filed sorted (#8856)", () => {
  let kapp: App;
  let krepo: Repo;
  let kdir: string;
  const names = (issueId: string): string[] => labelsFor(kapp.db, issueId).map((l) => l.name).sort();
  const openTitled = (part: string) =>
    listIssues(kapp.db, krepo.id, { state: "open", limit: 200 }).find((i) => i.title.includes(part));
  beforeAll(async () => {
    kdir = await mkdtemp(join(tmpdir(), "forge-sentinel-kind-"));
    kapp = createApp({ dataDir: kdir, port: 0 });
    const a = await createUser(kapp.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!a.ok) throw new Error(a.error);
    const made = await createRepo(kapp, a.value, { name: "forge", visibility: "private" });
    if (!made.ok) throw new Error(String(made.error));
    krepo = made.value;
    kapp.db.exec(
      `CREATE TRIGGER planted_guard BEFORE INSERT ON repo
       BEGIN SELECT RAISE(ABORT, 'forge: paused'); END`,
    );
  });
  afterAll(async () => {
    kapp.db.close();
    await rm(kdir, { recursive: true, force: true });
  });
  test("a security finding lands as a high bug, and is past its clock after 4 h", () => {
    const found = findFaults(kapp.db, NOW).find((f) => f.key === "trigger:planted_guard");
    expect(found?.kind).toBe("security");
    expect(hunt(kapp, "alice/forge", NOW).filed).toBe(1);
    const issue = openTitled("planted_guard")!;
    // The repository had no `bug` or `high`: the sentinel made them.
    expect(names(issue.id)).toEqual(["bug", "high"]);
    const filedAt = Date.parse(issue.createdAt);
    expect(bugClockAlarm(kapp.db, new Date(filedAt + HIGH_BUG_MS - 60_000)).fired).toBe(false);
    const alarm = bugClockAlarm(kapp.db, new Date(filedAt + HIGH_BUG_MS + 60_000));
    expect(alarm.fired).toBe(true);
    expect(alarm.href).toBe(`/alice/forge/issues/${issue.number}`);
    expect(alarm.says).toContain("4 h");
  });
  test("still present, it only comments; re-filed after close, it keeps its labels", () => {
    expect(hunt(kapp, "alice/forge", NOW + 60_000).filed).toBe(0);
    const issue = openTitled("planted_guard")!;
    kapp.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(issue.id);
    expect(hunt(kapp, "alice/forge", NOW + 120_000).filed).toBe(1);
    const again = openTitled("planted_guard")!;
    expect(again.id).not.toBe(issue.id);
    expect(names(again.id)).toEqual(["bug", "high"]);
  });
  test("an issue adopted after its record was lost gets its labels", () => {
    const lost = () => [
      { key: "log:lost", title: "The log says lost 3 times", evidence: ["lost"], pillar: "Reliability", lookFor: "No lost.", kind: "defect" as const },
    ];
    expect(hunt(kapp, "alice/forge", NOW, lost).filed).toBe(1);
    const issue = openTitled("log says lost")!;
    // The record is lost and so are the labels: the next hunt adopts the issue, files nothing, and labels it again.
    kapp.db.query("DELETE FROM sentinel_finding WHERE key = 'log:lost'").run();
    kapp.db.query("DELETE FROM issue_label WHERE issue_id = ?1").run(issue.id);
    expect(hunt(kapp, "alice/forge", NOW + 60_000, lost).filed).toBe(0);
    expect(openTitled("log says lost")!.id).toBe(issue.id);
    expect(names(issue.id)).toEqual(["bug"]);
  });
  test("another defect is a bug on the 48 h clock", () => {
    const defect = () => [
      { key: "log:x", title: "The log says x 3 times", evidence: ["x"], pillar: "Reliability", lookFor: "No x.", kind: "defect" as const },
    ];
    expect(hunt(kapp, "alice/forge", NOW, defect).filed).toBe(1);
    expect(names(openTitled("log says x")!.id)).toEqual(["bug"]);
  });
});
		@@ -25,6 +25,22 @@ issue, and only when its evidence changed; one whose issue was closed and
25	25	returned is filed again. It files in `FORGE_SENTINEL_REPO` (default
26	26	`trevato/forge`) as the `sentinel` agent, else the first admin.
27	27
	28	How it is labelled (#8856). Each finding declares a kind, and is filed
	29	with its labels, so the bug clock (`src/flow/bugs.ts`) runs from the filing
	30	and nobody has to sort it:
	31
	32	\| Kind \| Labels \| Clock \|
	33	\| --- \| --- \| --- \|
	34	\| `security` (a hand-installed trigger) \| `bug`, `high` \| 4 h \|
	35	\| `defect` (every other finding, and the performance watcher's slow routes) \| `bug` \| 48 h \|
	36
	37	A label the repository lacks is made (`src/hunt/label.ts`, shared by the
	38	sentinel and `src/perf/duty.ts`); labels already on the issue stay, and an
	39	issue adopted after its record was lost is labelled again. A budget breach
	40	also gets `P0 Trust`. A kind joins the table with the first detector that
	41	files it: the route hunters (`src/hunt/door.ts`, `links.ts`, `contract.ts`)
	42	file nothing yet, so there is no `broken` or `polish` kind.
	43
28	44	What it shows. To a site admin only: the board's forge strip and the
29	45	terminal palette's `Sentinel` command (as a toast) show the count of open findings and the
30	46	last hunt's time. A hunt is synchronous, so there is no "hunting" state to show.
		@@ -23,7 +23,6 @@ import { getAgentByName } from "../core/agents.ts";
23	23	import {
24	24	getIssueById,
25	25	getLabel,
26		setIssueLabels as writeLabels,
27	26	type Issue,
28	27	} from "../core/issues.ts";
29	28	import { canWrite, getRepo, type Repo } from "../core/repos.ts";
		@@ -48,6 +47,8 @@ import {
48	47	} from "../checks/shared_red.ts";
49	48	import { readRunLogTail } from "../runs/log.ts";
50	49	import { clip } from "../text/clip.ts";
	50	import type { FindingKind } from "../hunt/finding.ts";
	51	import { addLabels, kindLabels } from "../hunt/label.ts";
51	52
52	53	const log = createLog("sentinel");
53	54
		@@ -71,6 +72,8 @@ export interface Finding {
71	72	readonly evidence: readonly string[];
72	73	readonly pillar: string;
73	74	readonly lookFor: string;
	75	/** How it is filed: its labels, and so its clock (#8856). */
	76	readonly kind: FindingKind;
74	77	/** A budget breach: filed at once, labelled, left unassigned for a builder (#3002). */
75	78	readonly urgent?: true;
76	79	/** What a person or a run would do, when it is mechanical and reversible. */
		@@ -131,6 +134,8 @@ export function strayTriggers(db: Database): Finding[] {
131	134	`sql: ${clip(t.sql ?? "", QUOTE_MAX)}`,
132	135	],
133	136	pillar: "Trust: the database does only what the schema says",
	137	// Someone's hand on the database: a trigger can refuse or rewrite any write.
	138	kind: "security",
134	139	lookFor: `\`SELECT name FROM sqlite_master WHERE type = 'trigger'\` lists only triggers the schema creates.`,
135	140	fix: `Drop it at boot like \`dropLegacyNewAccountGuards\` does (\`DROP TRIGGER "${t.name}"\`), with a test that plants it and boots.`,
136	141	}));
		@@ -172,6 +177,7 @@ export function heldSlots(db: Database, now: number): Finding[] {
172	177	: []),
173	178	],
174	179	pillar: "Reliability: the gate always has a slot",
	180	kind: "defect",
175	181	lookFor: `No ${w.kind} run queued for ${QUEUE_WAIT_MAX_MS / 60_000} minutes while the slots are free; a cancelled run's process is gone with it.`,
176	182	fix: "End the process of a cancelled run on the cancel path, and free its slot.",
177	183	});
		@@ -213,6 +219,7 @@ export function ledgerStreaks(db: Database, now: number): Finding[] {
213	219	`a check passed after the first stop: run ${green.id} at ${green.finished_at}`,
214	220	],
215	221	pillar: "Throughput: red checks end in a fix, not a stop",
	222	kind: "defect",
216	223	lookFor: "A retry decision that is not stop, or a green check after a stop.",
217	224	});
218	225	}
		@@ -250,6 +257,7 @@ export function ledgerStreaks(db: Database, now: number): Finding[] {
250	257	...(skipped > 0 ? [`${skipped} skips of a run with nothing new are not counted`] : []),
251	258	],
252	259	pillar: "Throughput: the crew spends on work, not on being refused",
	260	kind: "defect",
253	261	lookFor: "Fewer than 20 guard decisions a day for the repository.",
254	262	});
255	263	}
		@@ -321,6 +329,7 @@ export function runStreaks(db: Database): Finding[] {
321	329	`runs: ${hits.slice(0, 5).map((r) => r.id).join(", ")}`,
322	330	],
323	331	pillar: "Trust: a green check means something",
	332	kind: "defect",
324	333	lookFor: "A green check run for the repository's newest head.",
325	334	});
326	335	} else {
		@@ -344,6 +353,7 @@ export function runStreaks(db: Database): Finding[] {
344	353	`runs: ${pushes.slice(0, 5).map((r) => r.id).join(", ")}`,
345	354	],
346	355	pillar: "Throughput: a crew run can finish what it starts",
	356	kind: "defect",
347	357	lookFor: "An agent run that pushes its branch from inside its sandbox.",
348	358	});
349	359	}
		@@ -371,6 +381,7 @@ export function repeatedErrors(now: number): Finding[] {
371	381	`line: ${clip(lines[0]?.message ?? "", QUOTE_MAX)}`,
372	382	],
373	383	pillar: "Reliability: a quiet log",
	384	kind: "defect",
374	385	lookFor: "No ERROR line of this shape in an hour.",
375	386	});
376	387	}
		@@ -414,6 +425,7 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string,
414	425	...sharedLines(db, cause, now, forgeRepoId, dataDir),
415	426	],
416	427	pillar: "P0 Trust: the forge says when it is not well, and is well again",
	428	kind: "defect",
417	429	lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
418	430	incident: new Date(d.opened).toISOString(),
419	431	cause,
		@@ -467,6 +479,7 @@ export function budgetBreaches(db: Database, now: number, forgeRepoId?: string,
467	479	...sharedLines(db, s.cause, now, forgeRepoId, dataDir),
468	480	],
469	481	pillar: "P0 Trust: the forge says when it is not well, and is well again",
	482	kind: "defect",
470	483	lookFor: `\`${s.cause}\` is gone from the degraded list and the cause is fixed, not waited out.`,
471	484	fix: s.cause.startsWith("red:")
472	485	? "Fix the failing test on main in one pull. This is the only issue for it: pulls red on the same test wait for main, and no other builder is handed it while this is open."
		@@ -613,27 +626,23 @@ const bodyOf = (f: Finding, now: string): string =>
613	626	].join("\n");
614	627
615	628	/**
616		* A breach is crew work at once: the P0 Trust label, and no assignee, so a
617		* builder may start it on the next tick (#3002). The sentinel never builds,
618		* so it never assigns itself: a finding parked on it is held from every
619		* builder. The label may fail alone (no such label): the issue is filed
620		* either way, and the failure is said.
	629	* Every finding is filed sorted (#8856): its kind's labels (`bug`, with
	630	* `high` for a security hole), through `hunt/label.ts`, so the bug clock runs
	631	* from the moment it is filed. A breach is crew work at once: the P0 Trust label as
	632	* well, and no assignee, so a builder may start it on the next tick (#3002).
	633	* The sentinel never builds, so it never assigns itself: a finding parked on
	634	* it is held from every builder. Labels already on the issue stay. The
	635	* labelling may fail alone: the issue is filed either way, and the failure is said.
621	636	*/
622		function labelBreach(app: App, repo: Repo, actor: User, issue: Issue): void {
	637	function labelFinding(app: App, repo: Repo, actor: User, issue: Issue, f: Finding): void {
623	638	try {
624		// The forge acts on its own finding: no access check stands between it
625		// and its label, as none stands between it and the issue it just filed.
626		const label = getLabel(app.db, repo.id, "P0 Trust");
627		if (label === undefined) log.warn("sentinel has no P0 Trust label to apply", { issue: issue.number });
628		else {
629		writeLabels(app.db, issue.id, [label.id]);
630		app.bus.append({
631		kind: "issue.labeled",
632		repoId: repo.id,
633		actorId: actor.id,
634		payload: { number: issue.number, labels: [label.name] },
635		});
	639	const wanted = kindLabels(app, repo.id, f.kind);
	640	if (f.urgent) {
	641	const label = getLabel(app.db, repo.id, "P0 Trust");
	642	if (label === undefined) log.warn("sentinel has no P0 Trust label to apply", { issue: issue.number });
	643	else wanted.push(label);
636	644	}
	645	addLabels(app, repo.id, actor.id, issue, wanted);
637	646	} catch (error) {
638	647	log.error("sentinel could not label", { issue: issue.number, error: String(error) });
639	648	}
		@@ -986,19 +995,21 @@ function closeHeld(app: App, repo: Repo, actor: User, active: ReadonlySet<string
986	995	* Read, file, record. One open issue per finding: a tracked finding whose
987	996	* issue is open is commented on only when its evidence changed; one whose
988	997	* issue is closed and returns is a new issue. A setting away from its default
989		* is no finding: the capacity page says so beside it (#3269).
	998	* is no finding: the capacity page says so beside it (#3269). `find` is the
	999	* detectors, `findFaults` unless a test files findings of its own.
990	1000	*/
991	1001	export function hunt(
992	1002	app: App,
993	1003	repoPath: string,
994	1004	now = Date.now(),
	1005	find: typeof findFaults = findFaults,
995	1006	): HuntResult {
996	1007	const at = new Date(now).toISOString();
997	1008	let filed = 0;
998	1009	let commented = 0;
999	1010	const [owner = "", name = ""] = repoPath.split("/");
1000	1011	const repo = getRepo(app.db, owner, name);
1001		const findings = findFaults(app.db, now, repo?.id, app.config.dataDir);
	1012	const findings = find(app.db, now, repo?.id, app.config.dataDir);
1002	1013	const actor = sentinelActor(app.db);
1003	1014	if (repo !== undefined && actor !== undefined) {
1004	1015	settleShared(app, repo, actor, now);
		@@ -1059,7 +1070,11 @@ export function hunt(
1059	1070	}
1060	1071	issueId = made.value.id;
1061	1072	filed++;
1062		if (f.urgent) labelBreach(app, repo, actor, made.value);
	1073	labelFinding(app, repo, actor, made.value, f);
	1074	} else {
	1075	// Adopted: its labels may have been lost with its record.
	1076	const issue = getIssueById(app.db, issueId);
	1077	if (issue !== undefined) labelFinding(app, repo, actor, issue, f);
1063	1078	}
1064	1079	app.db
1065	1080	.query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen, incident) VALUES (?1, ?2, ?3, ?4, ?4, ?5)")
		@@ -18,3 +18,21 @@ export const found = (detector: string, subject: string, title: string, evidence
18	18
19	19	export const seenLine = (path: string, status: number, location?: string \| null): string =>
20	20	`GET ${path} → ${status}${location ? ` → ${location}` : ""}`;
	21
	22	/**
	23	* What a finding is, so it is filed already sorted (#8856): a security hole is
	24	* a `high` bug (the 4 h clock in `flow/bugs.ts`), any other defect a bug (48 h).
	25	* A kind joins this list with the first detector that files it.
	26	*/
	27	export type FindingKind = "security" \| "defect";
	28
	29	export const KIND_LABELS: Readonly<Record<FindingKind, readonly string[]>> = {
	30	security: ["bug", "high"],
	31	defect: ["bug"],
	32	};
	33
	34	/** Each label a kind is filed with, and the colour it is made in when the repository has none by that name. */
	35	export const LABEL_COLORS: Readonly<Record<string, string>> = {
	36	bug: "d73a4a",
	37	high: "b60205",
	38	};
		@@ -0,0 +1,35 @@
	1	/**
	2	* Filing a finding sorted (#8856): the labels of its kind, made in the
	3	* repository when it has none by that name, added to what the issue already
	4	* carries. Shared by every detector that files: the sentinel and the
	5	* performance watcher. The forge acts on its own finding, so no access check
	6	* stands between it and the labels, as none stands between it and the issue.
	7	*/
	8
	9	import type { App } from "../app.ts";
	10	import { createLabel, getLabel, labelsFor, setIssueLabels, type Issue, type Label } from "../core/issues.ts";
	11	import { KIND_LABELS, LABEL_COLORS, type FindingKind } from "./finding.ts";
	12
	13	export function getOrMakeLabel(app: App, repoId: string, name: string): Label \| undefined {
	14	const had = getLabel(app.db, repoId, name);
	15	if (had !== undefined) return had;
	16	const made = createLabel(app.db, repoId, name, LABEL_COLORS[name] ?? "ededed");
	17	return made.ok ? made.value : undefined;
	18	}
	19
	20	export const kindLabels = (app: App, repoId: string, kind: FindingKind): Label[] =>
	21	KIND_LABELS[kind].flatMap((name) => getOrMakeLabel(app, repoId, name) ?? []);
	22
	23	/** Adds `wanted` to the issue's labels and says so on the bus; labels already on it stay. */
	24	export function addLabels(app: App, repoId: string, actorId: string, issue: Issue, wanted: readonly Label[]): void {
	25	const had = labelsFor(app.db, issue.id);
	26	const added = wanted.filter((w) => !had.some((h) => h.id === w.id));
	27	if (added.length === 0) return;
	28	setIssueLabels(app.db, issue.id, [...had, ...added].map((l) => l.id));
	29	app.bus.append({
	30	kind: "issue.labeled",
	31	repoId,
	32	actorId,
	33	payload: { number: issue.number, labels: added.map((l) => l.name) },
	34	});
	35	}
		@@ -2,7 +2,8 @@
2	2	* The `performance-watcher` duty (#7559): once a minute, judge the record and
3	3	* file each new finding, of a route or of a git subcommand (#8158), as one
4	4	* issue in the forge's own repository, opened by its owner. A title already
5		* open is not filed twice, and neither is a key this process has filed.
	5	* open is not filed twice, and neither is a key this process has filed. A
	6	* slow route or subcommand is a defect: filed as a `bug`, on its clock (#8856).
6	7	*/
7	8
8	9	import type { App } from "../app.ts";
		@@ -10,11 +11,14 @@ import { openIssue } from "../actions/issues.ts";
10	11	import { getUserById } from "../core/users.ts";
11	12	import { getRepo } from "../core/repos.ts";
12	13	import { listIssues } from "../core/issues.ts";
	14	import type { FindingKind } from "../hunt/finding.ts";
	15	import { addLabels, kindLabels } from "../hunt/label.ts";
13	16	import { createGitRecord } from "./git.ts";
14	17	import type { PerfRecord } from "./record.ts";
15	18	import { evaluate, evaluateGit, fresh } from "./watch.ts";
16	19
17	20	export const TICK_MS = 60_000;
	21	const KIND: FindingKind = "defect";
18	22
19	23	export function startPerfWatcher(
20	24	app: App,
		@@ -36,7 +40,8 @@ export function startPerfWatcher(
36	40	const findings = [...evaluate(record, now()), ...evaluateGit(gitRecord, now())];
37	41	for (const f of fresh(findings, filed)) {
38	42	if (open.has(f.title)) continue;
39		openIssue(app, actor, repo, { title: f.title, body: f.evidence });
	43	const made = openIssue(app, actor, repo, { title: f.title, body: f.evidence });
	44	if (made.ok) addLabels(app, repo.id, actor.id, made.value, kindLabels(app, repo.id, KIND));
40	45	}
41	46	} catch (error) {
42	47	app.log.warn("performance watcher failed", { error: String(error) });
		@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
4	4	import { join } from "node:path";
5	5	import { createRepo } from "../src/actions/repos.ts";
6	6	import { createApp, type App } from "../src/app.ts";
7		import { listIssues } from "../src/core/issues.ts";
	7	import { labelsFor, listIssues } from "../src/core/issues.ts";
8	8	import { getRepo } from "../src/core/repos.ts";
9	9	import { createUser } from "../src/core/users.ts";
10	10	import { startPerfWatcher } from "../src/perf/duty.ts";
		@@ -84,3 +84,9 @@ test("the watcher files a slow git subcommand once (#8158)", async () => {
84	84	stop();
85	85	expect(titles().filter((t) => t.startsWith("git "))).toEqual(["git rev-parse is over its budget"]);
86	86	});
	87
	88	test("a slow route is filed as a bug, so its clock runs from the filing (#8856)", () => {
	89	const repo = getRepo(app.db, "alice", "forge")!;
	90	const issue = listIssues(app.db, repo.id, { state: "open" }).find((i) => i.title === "/ is over its budget: kpis")!;
	91	expect(labelsFor(app.db, issue.id).map((l) => l.name)).toEqual(["bug"]);
	92	});
		@@ -21,6 +21,7 @@ import { createRun } from "../src/core/runs.ts";
21	21	import { createUser, type User } from "../src/core/users.ts";
22	22	import { recordCap } from "../src/crew/guard.ts";
23	23	import { sharedCause } from "../src/checks/shared_red.ts";
	24	import { bugClockAlarm, HIGH_BUG_MS } from "../src/flow/bugs.ts";
24	25	import {
25	26	budgetBreaches,
26	27	degradedCauses,
		@@ -691,7 +692,7 @@ describe("a budget breach is crew work at once", () => {
691	692	const issue = open[0]!;
692	693	// The sentinel never builds, so it never assigns itself (#3002).
693	694	expect(issue.assigneeId).toBeNull();
694		expect(labelsFor(bapp.db, issue.id).map((l) => l.name)).toEqual(["P0 Trust"]);
	695	expect(labelsFor(bapp.db, issue.id).map((l) => l.name).sort()).toEqual(["P0 Trust", "bug"]);
695	696	expect(listEvents(bapp.db, { kind: "issue.assigned" }).length).toBe(0);
696	697	});
697	698
		@@ -979,3 +980,83 @@ describe("a degraded finding whose cause has stayed closed", () => {
979	980	expect(hunt(happ, "alice/forge", t).filed).toBe(1);
980	981	});
981	982	});
	983
	984	describe("a finding is filed sorted (#8856)", () => {
	985	let kapp: App;
	986	let krepo: Repo;
	987	let kdir: string;
	988	const names = (issueId: string): string[] => labelsFor(kapp.db, issueId).map((l) => l.name).sort();
	989	const openTitled = (part: string) =>
	990	listIssues(kapp.db, krepo.id, { state: "open", limit: 200 }).find((i) => i.title.includes(part));
	991	beforeAll(async () => {
	992	kdir = await mkdtemp(join(tmpdir(), "forge-sentinel-kind-"));
	993	kapp = createApp({ dataDir: kdir, port: 0 });
	994	const a = await createUser(kapp.db, {
	995	origin: "seed",
	996	trusted: true,
	997	username: "alice",
	998	email: "[email protected]",
	999	password: "a-good-passphrase",
	1000	adminIfFirst: true,
	1001	});
	1002	if (!a.ok) throw new Error(a.error);
	1003	const made = await createRepo(kapp, a.value, { name: "forge", visibility: "private" });
	1004	if (!made.ok) throw new Error(String(made.error));
	1005	krepo = made.value;
	1006	kapp.db.exec(
	1007	`CREATE TRIGGER planted_guard BEFORE INSERT ON repo
	1008	BEGIN SELECT RAISE(ABORT, 'forge: paused'); END`,
	1009	);
	1010	});
	1011	afterAll(async () => {
	1012	kapp.db.close();
	1013	await rm(kdir, { recursive: true, force: true });
	1014	});
	1015
	1016	test("a security finding lands as a high bug, and is past its clock after 4 h", () => {
	1017	const found = findFaults(kapp.db, NOW).find((f) => f.key === "trigger:planted_guard");
	1018	expect(found?.kind).toBe("security");
	1019	expect(hunt(kapp, "alice/forge", NOW).filed).toBe(1);
	1020	const issue = openTitled("planted_guard")!;
	1021	// The repository had no `bug` or `high`: the sentinel made them.
	1022	expect(names(issue.id)).toEqual(["bug", "high"]);
	1023	const filedAt = Date.parse(issue.createdAt);
	1024	expect(bugClockAlarm(kapp.db, new Date(filedAt + HIGH_BUG_MS - 60_000)).fired).toBe(false);
	1025	const alarm = bugClockAlarm(kapp.db, new Date(filedAt + HIGH_BUG_MS + 60_000));
	1026	expect(alarm.fired).toBe(true);
	1027	expect(alarm.href).toBe(`/alice/forge/issues/${issue.number}`);
	1028	expect(alarm.says).toContain("4 h");
	1029	});
	1030
	1031	test("still present, it only comments; re-filed after close, it keeps its labels", () => {
	1032	expect(hunt(kapp, "alice/forge", NOW + 60_000).filed).toBe(0);
	1033	const issue = openTitled("planted_guard")!;
	1034	kapp.db.query("UPDATE issue SET state = 'closed' WHERE id = ?1").run(issue.id);
	1035	expect(hunt(kapp, "alice/forge", NOW + 120_000).filed).toBe(1);
	1036	const again = openTitled("planted_guard")!;
	1037	expect(again.id).not.toBe(issue.id);
	1038	expect(names(again.id)).toEqual(["bug", "high"]);
	1039	});
	1040
	1041	test("an issue adopted after its record was lost gets its labels", () => {
	1042	const lost = () => [
	1043	{ key: "log:lost", title: "The log says lost 3 times", evidence: ["lost"], pillar: "Reliability", lookFor: "No lost.", kind: "defect" as const },
	1044	];
	1045	expect(hunt(kapp, "alice/forge", NOW, lost).filed).toBe(1);
	1046	const issue = openTitled("log says lost")!;
	1047	// The record is lost and so are the labels: the next hunt adopts the issue, files nothing, and labels it again.
	1048	kapp.db.query("DELETE FROM sentinel_finding WHERE key = 'log:lost'").run();
	1049	kapp.db.query("DELETE FROM issue_label WHERE issue_id = ?1").run(issue.id);
	1050	expect(hunt(kapp, "alice/forge", NOW + 60_000, lost).filed).toBe(0);
	1051	expect(openTitled("log says lost")!.id).toBe(issue.id);
	1052	expect(names(issue.id)).toEqual(["bug"]);
	1053	});
	1054
	1055	test("another defect is a bug on the 48 h clock", () => {
	1056	const defect = () => [
	1057	{ key: "log:x", title: "The log says x 3 times", evidence: ["x"], pillar: "Reliability", lookFor: "No x.", kind: "defect" as const },
	1058	];
	1059	expect(hunt(kapp, "alice/forge", NOW, defect).filed).toBe(1);
	1060	expect(names(openTitled("log says x")!.id)).toEqual(["bug"]);
	1061	});
	1062	});