Public: anyone can read this; the crew can change it.

/api/v1/health and /system report the cgroup's memory and CPU limit, not the host's #12539

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12518-api-v1-health-and-system-report-the-cgro → main 11 commits fork 2 threads · 2 unresolved

  • Merged. scout/12518-api-v1-health-and-system-report-the-cgro was merged into main as 18eaf38.

Review

Sign in to review this change.
10 files changed +231 −24
docs/features/system.md added +40
@@ -0,0 +1,40 @@
1 # The machine's size on /system and in health (#12518)
2
3 A forge in a pod may use only what its cgroup allows, and that can be much
4 less than the host it runs on. `/system`, the terminal's System screen and
5 `GET /api/v1/health` report what this forge may use. They do not report the
6 host's whole size.
7
8 ## Where the limits come from
9
10 Each value is read when it is asked for:
11
12 - **CPUs**: cgroup v2 `/sys/fs/cgroup/cpu.max`, `quota / period`, rounded
13 down. The result is at least 1 and at most the host's count. A `max` quota,
14 a missing file or a malformed one means the host's count
15 (`availableParallelism()`).
16 - **Memory**: cgroup v2 `/sys/fs/cgroup/memory.max`. If that file is missing,
17 cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes` is read instead. A
18 value counts only when it is a whole number below the host's memory
19 (`totalmem()`). Otherwise the host's memory is used: `max`, v1's huge
20 "unset" number, or a missing or malformed file.
21
22 **cgroup v1 caveat:** on cgroup v1, only the memory limit is read. The CPU
23 limit (`cpu.cfs_quota_us` / `cpu.cfs_period_us`) is not, so a v1 pod reports
24 the host's CPU count even when its quota is lower.
25
26 ## What each door shows
27
28 - **`GET /api/v1/health`**, read by an admin: the full answer carries
29 `machine: { cpus, memoryBytes, limitedBy }`. `memory.totalBytes` is the
30 same memory limit. A signed-in reader who is not an admin, and a stranger,
31 get the short answer, which has no `machine` key.
32 - **`machine.limitedBy`** is `"cgroup"` when the cgroup holds the CPUs or the
33 memory below the host's, and `"host"` when both are the host's own.
34 - **`/system`**: the summary has a `limited by cgroup` pill next to the core
35 count. The footprint's Machine fact ends with `, limited by cgroup`.
36 - **The terminal's System screen**: the first line reads
37 `CPU … over N cores · limited by cgroup · load …`.
38
39 When no limit is set below the host, none of these say "cgroup", and
40 `limitedBy` is `"host"`.
src/core/capacity.ts +70 −10
@@ -14,7 +14,7 @@
1414 import type { Database } from "bun:sqlite";
1515 import { readFileSync } from "node:fs";
1616 import { isIP } from "node:net";
17 import { availableParallelism } from "node:os";
17 import { availableParallelism, totalmem } from "node:os";
1818 import { PAYMENT_HOSTS } from "./egress_grants.ts";
1919 import { memoryLimitBytes } from "../system/memory_limit.ts";
2020
@@ -253,15 +253,76 @@ export function provideCpusForTest(cpus: number | undefined): void {
253253 */
254254 export function cpuLimit(file = "/sys/fs/cgroup/cpu.max"): number {
255255 if (cpusForTest !== undefined) return cpusForTest;
outdated

Sentinel host memoryBytes: 0 just to reuse the parser. Read the cpu limit without a fake host memory.

256 const machine = availableParallelism();
257 try {
258 const [quota, period] = readFileSync(file, "utf8").trim().split(/\s+/).map(Number);
259 if (quota !== undefined && period !== undefined && quota > 0 && period > 0)
260 return Math.max(1, Math.min(machine, Math.floor(quota / period)));
261 } catch {
262 // No cgroup v2 CPU controller here: the machine's count.
256 return cpusOf(readOrNull(file), availableParallelism());
257 }
258
259 /** The CPUs a cgroup v2 `cpu.max` allows, rounded down, never below one nor above the host's. */
260 export function cpusOf(cpuMax: string | null | undefined, hostCpus: number): number {
261 const [quota, period] = (cpuMax ?? "").trim().split(/\s+/).map(Number);
262 return quota !== undefined && period !== undefined && quota > 0 && period > 0
263 ? Math.max(1, Math.min(hostCpus, Math.floor(quota / period)))
264 : hostCpus;
265 }
266
267 /** The cgroup files `cgroupLimits` reads, as their contents; null where a file is not there. */
268 export interface CgroupFiles {
269 /** cgroup v2 `cpu.max`: "quota period" in µs, or "max period". */
270 readonly cpuMax?: string | null;
271 /** cgroup v2 `memory.max`: bytes, or "max". */
272 readonly memoryMax?: string | null;
273 /** cgroup v1 `memory.limit_in_bytes`: bytes; a huge number when unset. */
274 readonly memoryV1?: string | null;
275 }
276
277 /** What this forge may use, and whether a cgroup or the host's own size says so. */
278 export interface MachineLimits {
279 readonly cpus: number;
280 readonly memoryBytes: number;
281 readonly cpuLimited: boolean;
282 readonly memoryLimited: boolean;
283 /** "cgroup" when either limit is the cgroup's, else "host". */
284 readonly limitedBy: "cgroup" | "host";
285 }
286
287 /**
288 * The limits a cgroup sets, from its files' contents (#12518): `cpu.max` rounded down,
289 * never below one; `memory.max`, then v1's `memory.limit_in_bytes`. Unset ("max"), missing,
290 * malformed or not below the host: the host's. Pure, so a test hands in fixture contents.
291 */
292 export function cgroupLimits(files: CgroupFiles, host: { readonly cpus: number; readonly memoryBytes: number }): MachineLimits {
293 const cpus = cpusOf(files.cpuMax, host.cpus);
294 let memoryBytes = host.memoryBytes;
295 for (const text of [files.memoryMax, files.memoryV1]) {
296 const t = (text ?? "").trim();
297 const n = /^\d+$/.test(t) ? Number(t) : NaN;
298 if (Number.isFinite(n) && n > 0 && n < host.memoryBytes) {
299 memoryBytes = n;
300 break;
301 }
263302 }
264 return machine;
303 const cpuLimited = cpus < host.cpus;
304 const memoryLimited = memoryBytes !== host.memoryBytes;
305 return { cpus, memoryBytes, cpuLimited, memoryLimited, limitedBy: cpuLimited || memoryLimited ? "cgroup" : "host" };
306 }
307
308 const readOrNull = (file: string): string | null => {
309 try {
310 return readFileSync(file, "utf8");
311 } catch {
312 return null;
313 }
314 };
315
316 /**
317 * This pod's limits, read from its cgroup (v2, with v1's memory as the
318 * fallback), else the host's.
319 */
320 export function machineLimits(dir = "/sys/fs/cgroup"): MachineLimits {
321 const limits = cgroupLimits(
322 { cpuMax: readOrNull(`${dir}/cpu.max`), memoryMax: readOrNull(`${dir}/memory.max`), memoryV1: readOrNull(`${dir}/memory/memory.limit_in_bytes`) },
323 { cpus: availableParallelism(), memoryBytes: totalmem() },
324 );
325 return cpusForTest === undefined ? limits : { ...limits, cpus: cpusForTest };
265326 }
266327
267328 /**
@@ -320,7 +381,6 @@ export function floorMiBForMemory(bytes: number): number {
320381 return Number.isFinite(n) && n > 0 ? n : 0;
321382 }
322383
323
324384 /**
325385 * What this pod's cgroup holds in memory now: `memory.current` less the inactive file cache
326386 * (the kernel gives that back under pressure, as `MemAvailable` counts it). Null when
src/kernel/vitals.ts +11 −2
@@ -41,7 +41,7 @@ import { running as runningVersion } from "./version.ts";
4141 import { rollbackNeed } from "../release/launch.ts";
4242 import { behindStanding } from "../release/behind.ts";
4343 import { newestDrills } from "../core/domain_drills.ts";
44 import { capacityValue, podMemoryUsedBytes } from "../core/capacity.ts";
44 import { capacityValue, machineLimits, podMemoryUsedBytes, type MachineLimits } from "../core/capacity.ts";
4545 import { memoryLimitBytes } from "../system/memory_limit.ts";
4646 import { limitValue } from "../settings/limits.ts";
4747 import { unwrittenFor, SILENT_AFTER_S } from "../http/access_log.ts";
@@ -143,6 +143,11 @@ export interface Vitals {
143143 /** The pod's memory limit where one is set (cgroup), else the host's memory. */
144144 totalBytes: number;
145145 };
146 /**
147 * The CPUs and memory this forge may use, and whether its cgroup or the
148 * host says so (#12518).
149 */
150 readonly machine: { cpus: number; memoryBytes: number; limitedBy: "cgroup" | "host" };
146151 readonly queue: Record<string, number>;
147152 readonly lane: Lane;
148153 readonly stuck: {
@@ -177,6 +182,8 @@ export interface VitalsInput {
177182 readonly diskFreeMin?: number;
178183 /** The pod's memory limit and usage; read from the cgroup unless a test gives them. */
179184 readonly podMemory?: PodMemory;
185 /** The pod's CPU and memory limits; `machineLimits()` unless a test gives them. */
186 readonly limits?: MachineLimits;
180187 /**
181188 * The last walk of the database file, asked for only while a db-size need
182189 * fires; `dbSize.last` unless a test gives one.
@@ -398,6 +405,7 @@ export function memoryReading(
398405
399406 export function vitals(input: VitalsInput): Vitals {
400407 const now = input.now ?? Date.now();
408 const limits = input.limits ?? machineLimits();
401409 const at = new Date(now).toISOString();
402410 const needs: string[] = [];
403411 const degraded: Degraded[] = [];
@@ -730,7 +738,8 @@ export function vitals(input: VitalsInput): Vitals {
730738 todo,
731739 degraded,
732740 disk,
733 memory: memoryReading(input.podMemory, undefined, hostFreeBytes),
741 memory: memoryReading(input.podMemory ?? { limitBytes: limits.memoryBytes, usedBytes: podMemoryUsedBytes() }, undefined, hostFreeBytes),
742 machine: { cpus: limits.cpus, memoryBytes: limits.memoryBytes, limitedBy: limits.limitedBy },
734743 queue,
735744 lane,
736745 stuck,
src/system/footprint.ts +10 −8
@@ -5,8 +5,8 @@
55 * it shows. The release proof records a reading (`release/prove.ts`) and
66 * /system shows the last one.
77 */
8 import { arch, availableParallelism } from "node:os";
9 import { memoryLimitBytes } from "./memory_limit.ts";
8 import { arch } from "node:os";
9 import { machineLimits, type MachineLimits } from "../core/capacity.ts";
1010 import { findChromium } from "../looks/capture.ts";
1111 import { treeBytes } from "./largest_dirs.ts";
1212 import { largePageCache } from "./cache_share.ts";
@@ -33,8 +33,11 @@ export interface Optional {
3333 export interface Footprint {
3434 readonly at: string;
3535 readonly arch: string;
36 /** The CPUs and memory this forge may use: its cgroup's limits where set, else the host's. */
3637 readonly cpus: number;
3738 readonly machineBytes: number;
39 /** Who set `cpus` and `machineBytes`; absent on a reading kept before #12518. */
40 readonly limitedBy?: "cgroup" | "host";
3841 /** Resident memory of this process, the one that serves pages and git. */
3942 readonly rssBytes: number;
4043 readonly heapBytes: number;
@@ -90,16 +93,15 @@ export function optionals(
9093 }
9194
9295 /** Reads the footprint now; the data directory's walk is asynchronous, off the loop. */
93 export async function readFootprint(dataDir: string, now: () => number = Date.now): Promise<Footprint> {
96 export async function readFootprint(dataDir: string, now: () => number = Date.now, limits: MachineLimits = machineLimits()): Promise<Footprint> {
9497 const mem = process.memoryUsage();
95 const cpus = availableParallelism();
96 const machineBytes = memoryLimitBytes();
9798 const diskBytes = await treeBytes(dataDir).catch(() => null);
9899 return {
99100 at: new Date(now()).toISOString(),
100101 arch: arch(),
101 cpus,
102 machineBytes,
102 cpus: limits.cpus,
103 machineBytes: limits.memoryBytes,
104 limitedBy: limits.limitedBy,
103105 rssBytes: mem.rss,
104106 heapBytes: mem.heapUsed,
105107 ageSeconds: Math.round(process.uptime()),
@@ -144,6 +146,6 @@ export function footprintFacts(f: Footprint): Fact[] {
144146 { label: "Memory", value: `${mib(f.rssBytes)} after ${f.ageSeconds} s running; budget ${mib(BUDGET.restBytes)}` },
145147 { label: "First page", value: f.readyMs === null ? "not up yet" : `${seconds(f.readyMs)} after start; budget ${seconds(BUDGET.readyMs)}` },
146148 { label: "Data on disk", value: f.diskBytes === null ? "not read" : `${mib(f.diskBytes)}; an empty forge's budget ${mib(BUDGET.emptyDiskBytes)}` },
147 { label: "Machine", value: `${f.arch}, ${f.cpus} CPU, ${mib(f.machineBytes)}` },
149 { label: "Machine", value: `${f.arch}, ${f.cpus} CPU, ${mib(f.machineBytes)}${f.limitedBy === "cgroup" ? ", limited by cgroup" : ""}` },
148150 ];
149151 }
src/system/view.ts +5
@@ -50,6 +50,8 @@ export type SystemView = {
5050 processes: number;
5151 groups: Group[];
5252 duplicates: Duplicate[];
53 /** Present when the pod's cgroup holds its cores or memory under the host's (#12518). */

Carry this to src/tui/ system screen too (both doors), with a test.

54 limitedBy?: "cgroup";
5355 };
5456
5557 /** Processes kept per group. */
@@ -162,7 +164,10 @@ export function buildSystem(prev: Raw | undefined, next: Raw, roots: Roots, self
162164 if (g.cpuCores !== null) g.cpuCores = Math.round(g.cpuCores * 100) / 100;
163165 }
164166 const mem = next.memory;
167 const limited =
168 limits !== undefined && ((cores.length > 0 && limits.cpus < cores.length) || (mem !== undefined && mem !== null && limits.memoryBytes < mem.total));
165169 return {
170 ...(limited ? { limitedBy: "cgroup" as const } : {}),
166171 at: new Date(next.at).toISOString(),
167172 seconds,
168173 cpu: {
src/tui/screens/system.ts +1 −1
@@ -116,7 +116,7 @@ export function systemLines(v: SystemView): string[] {
116116 (c, i) => `${pad(`cpu${i}`, 6)} ${pad(c === null ? "—" : `${Math.round(c)}%`, 4, "right")} ${spark(v.cpu.history[i] ?? [])}`,
117117 );
118118 return [
119 `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.limitCores ?? v.cpu.cores.length} cores · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`,
119 `CPU ${pct(v.cpu.busyPct)} over ${v.cpu.limitCores ?? v.cpu.cores.length} cores${v.limitedBy === "cgroup" ? " · limited by cgroup" : ""} · load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`,
120120 ...cores,
121121 ...(v.cpu.cores.length > CORES ? [`… and ${v.cpu.cores.length - CORES} more cores`] : []),
122122 `hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`} · memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"} · ${v.processes} processes`,
src/web/pages/system.ts +1
@@ -517,6 +517,7 @@ export function regions(v: SystemView): Record<string, string> {
517517 "sys-dup": dup,
518518 summary: html`${pill(`CPU ${pct(v.cpu.busyPct)}`, v.cpu.busyPct !== null && v.cpu.busyPct >= 90 ? "danger" : undefined)}
519519 ${pill(`${String(v.cpu.limitCores ?? v.cpu.cores.length)} cores`)}
520 ${v.limitedBy === "cgroup" ? pill("limited by cgroup") : null}
520521 ${pill(`load ${v.load ? v.load.map((l) => l.toFixed(2)).join(" ") : "—"}`)}
521522 ${pill(`hottest ${v.hottestC === null ? "—" : `${v.hottestC.toFixed(0)}°C`}`)}
522523 ${pill(`memory ${v.memory ? `${bytes(v.memory.usedBytes)} of ${bytes(v.memory.totalBytes)}` : "—"}`)}
test/footprint.test.ts +75 −1
@@ -3,7 +3,10 @@ import { mkdtempSync, writeFileSync } from "node:fs";
33 import { join } from "node:path";
44 import { tmpdir } from "node:os";
55 import { BUDGET, clearReadyMark, footprintFacts, footprintVerdict, markReady, optionals, readFootprint, type Footprint } from "../src/system/footprint.ts";
6 import { footprintSection } from "../src/web/pages/system.ts";
6 import { footprintSection, regions } from "../src/web/pages/system.ts";
7 import { cgroupLimits, machineLimits } from "../src/core/capacity.ts";
8 import { buildSystem } from "../src/system/view.ts";
9 import type { Raw } from "../src/system/proc.ts";
710 import { proveRelease, lastProof } from "../src/release/prove.ts";
811
912 const MIB = 1024 * 1024;
@@ -93,3 +96,74 @@ test("the release proof records the footprint it is handed once the version answ
9396 await proveRelease({ dataDir: dir, version: "1.0.0", url: "http://x.test", fetch: fetcher, giveUpMs: 1, sleep: async () => {}, footprint: async () => fp });
9497 expect(lastProof(dir)?.footprint?.rssBytes).toBe(123 * MIB);
9598 });
99
100 // ---- the cgroup's limits, not the host's (#12518) ----
101
102 const GIB = 1024 * MIB;
103 const host = { cpus: 80, memoryBytes: 64 * GIB };
104
105 test("cgroup v2 limits are read from the files' contents: 512 MiB and one CPU on an 80-core host", () => {
106 const l = cgroupLimits({ cpuMax: "100000 100000\n", memoryMax: "536870912\n" }, host);
107 expect(l).toEqual({ cpus: 1, memoryBytes: 512 * MIB, cpuLimited: true, memoryLimited: true, limitedBy: "cgroup" });
108 expect(cgroupLimits({ cpuMax: "50000 100000" }, host).cpus).toBe(1); // half a core is still one
109 expect(cgroupLimits({ cpuMax: "250000 100000" }, host).cpus).toBe(2);
110 });
111
112 test("unset, missing or malformed cgroup files fall back to the host", () => {
113 for (const files of [{}, { cpuMax: "max 100000", memoryMax: "max" }, { cpuMax: null, memoryMax: null }, { cpuMax: "junk", memoryMax: "12abc" }]) {
114 expect(cgroupLimits(files, host)).toEqual({ ...host, cpuLimited: false, memoryLimited: false, limitedBy: "host" });
115 }
116 // A limit at or over the host's own size is no limit.
117 expect(cgroupLimits({ memoryMax: String(128 * GIB) }, host).limitedBy).toBe("host");
118 });
119
120 test("a cgroup v1 host keeps its memory limit; v1's unset sentinel is the host", () => {
121 expect(cgroupLimits({ memoryMax: null, memoryV1: "1073741824" }, host).memoryBytes).toBe(GIB);
122 expect(cgroupLimits({ memoryMax: null, memoryV1: "9223372036854771712" }, host).limitedBy).toBe("host");
123 });
124
125 test("machineLimits reads a cgroup directory, and a missing one is the host", () => {
126 const dir = dataDir();
127 writeFileSync(join(dir, "cpu.max"), "200000 100000\n");
128 writeFileSync(join(dir, "memory.max"), "536870912\n");
129 const l = machineLimits(dir);
130 expect(l.memoryBytes).toBe(512 * MIB);
131 expect(l.limitedBy).toBe("cgroup");
132 expect(machineLimits(join(dir, "none")).limitedBy).toBe("host");
133 });
134
135 test("the footprint and its Machine fact carry the cgroup's limit and say so", async () => {
136 const f = await readFootprint(dataDir(), Date.now, cgroupLimits({ cpuMax: "100000 100000", memoryMax: "536870912" }, host));
137 expect([f.cpus, f.machineBytes, f.limitedBy]).toEqual([1, 512 * MIB, "cgroup"]);
138 expect(footprintFacts(f).find((x) => x.label === "Machine")!.value).toBe(`${f.arch}, 1 CPU, 512 MiB, limited by cgroup`);
139 expect(footprintFacts(reading()).find((x) => x.label === "Machine")!.value).not.toContain("cgroup");
140 });
141
142 test("health's vitals report the cgroup's memory, never the host's, when a limit is given", async () => {
143 const { createApp } = await import("../src/app.ts");
144 const { vitals } = await import("../src/kernel/vitals.ts");
145 const dir = dataDir();
146 const app = await createApp({ dataDir: dir });
147 try {
148 const v = vitals({ db: app.db, signals: app.signals, dataDir: dir, disk: () => ({ freeBytes: 95, totalBytes: 100 }),
149 limits: cgroupLimits({ cpuMax: "100000 100000", memoryMax: "536870912" }, host) });
150 expect(v.machine).toEqual({ cpus: 1, memoryBytes: 512 * MIB, limitedBy: "cgroup" });
151 expect(v.memory.totalBytes).toBe(512 * MIB);
152 expect(v.memory.freeBytes).toBeLessThanOrEqual(512 * MIB);
153 } finally {
154 app.close();
155 }
156 });
157
158 test("/system's summary says 'limited by cgroup' only under a limit", () => {
159 const raw: Raw = {
160 at: 1_000, cores: Array.from({ length: 80 }, () => ({ busy: 0, total: 0 })), temps: [], load: null,
161 memory: { total: 64 * GIB, available: 4 * GIB, swapTotal: 0, swapFree: 0 }, disks: [], net: [], io: [], procs: [],
162 };
163 const limited = buildSystem(undefined, raw, [], 1, { cpus: 1, memoryBytes: 512 * MIB, usedBytes: 100 * MIB });
164 expect(limited.limitedBy).toBe("cgroup");
165 expect(String(regions(limited)["summary"])).toContain("limited by cgroup");
166 const free = buildSystem(undefined, raw, [], 1, { cpus: 80, memoryBytes: 64 * GIB });
167 expect(free.limitedBy).toBeUndefined();
168 expect(String(regions(free)["summary"])).not.toContain("cgroup");
169 });
test/tui-system.test.ts +16 −1
@@ -16,8 +16,9 @@ import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
1616 import { setTelemetryGrant } from "../src/actions/telemetry.ts";
1717 import { hasGrant } from "../src/core/telemetry.ts";
1818 import { countRead, resetTelemetryReads } from "../src/system/telemetry_reads.ts";
19 import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemScreen } from "../src/tui/screens/system.ts";
19 import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts";
2020 import type { DbSize } from "../src/system/db_size.ts";
21 import type { SystemView } from "../src/system/view.ts";
2122 import { changeCapacity } from "../src/actions/capacity.ts";
2223 import { EGRESS_REPORTED } from "../src/system/egress_line.ts";
2324 import { keepFootprint } from "../src/system/footprint.ts";
@@ -468,3 +469,17 @@ describe("tui system footprint (#12092)", () => {
468469 tui.stop?.();
469470 });
470471 });
472
473 describe("the machine line says when a cgroup holds the forge under the host (#12518)", () => {
474 const view: SystemView = {
475 at: "", seconds: 0, cpu: { busyPct: null, cores: [10, 20, 30, 40], limitCores: 2, history: [] },
476 temps: [], hottestC: null, load: null, memory: null, disks: [], net: [], io: [],
477 processes: 0, groups: [], duplicates: [],
478 };
479 test("limited: the first line says so, as the page's pill does", () => {
480 expect(systemLines({ ...view, limitedBy: "cgroup" })[0]).toContain("over 2 cores · limited by cgroup");
481 });
482 test("the host's own size: no such words", () => {
483 expect(systemLines(view).join("\n")).not.toContain("cgroup");
484 });
485 });
test/vitals.test.ts +2 −1
@@ -95,7 +95,7 @@ test("a site admin with a read-only token gets the shape; others get the short o
9595 [
9696 "accessLog",
9797 "apps", "at", "broken", "checkedAt", "commit", "degraded", "disk", "draining", "errors", "flakes",
98 "incidents", "lane", "memory", "needs", "ok", "optional", "queue", "rollout", "schedules", "stale",
98 "incidents", "lane", "machine", "memory", "needs", "ok", "optional", "queue", "rollout", "schedules", "stale",
9999 "stuck",
100100 "todo", "version",
101101 ].sort(),
@@ -111,6 +111,7 @@ test("a site admin with a read-only token gets the shape; others get the short o
111111 const short = (await res2.json()) as Record<string, unknown>;
112112 expect(short["disk"]).toBeUndefined();
113113 expect(short["memory"]).toBeUndefined();
114 expect(short["machine"]).toBeUndefined();
114115 expect(short["rollout"]).toBeDefined();
115116 // A stranger is told whether it is well, and no more (#2917).
116117 const stranger = (await (await health()).json()) as Record<string, unknown>;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs