trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

Boards can chart the machine: CPU, memory and load as a source #10886

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/10864-boards-can-chart-the-machine-cpu-memory → main 2 commits fork 4 threads · 4 unresolved

  • Merged. scout/10864-boards-can-chart-the-machine-cpu-memory was merged into main as 5f85fde.

Review

Sign in to review this change.
6 files changed +152 −128
scripts/unreached-allow.json −1
@@ -1,6 +1,5 @@
11 [
22 "src/apps/cloud.ts",
3 "src/boards/machine.ts",
43 "src/crew/governor_log.ts",
54 "src/crew/tries.ts",
65 "src/flow/alarm_drift.ts",
src/boards/machine.ts +80 −19
@@ -7,10 +7,61 @@
77 */
88 import { readFileSync } from "node:fs";
99 import { join } from "node:path";
10 import { parseCpuStat, parseLoadavg, parseMeminfo } from "./sampler.ts";
10
11 /** `/proc/stat` counts in USER_HZ, 100 a second on every Linux the forge runs on. */
12 const USER_HZ = 100;
13
14 const finite = (n: number) => (Number.isFinite(n) && n >= 0 ? n : null);
15
16 /** `usage_usec` from a cgroup v2 `cpu.stat`, µs of CPU since the cgroup began. */
17 export function parseCpuStat(text: string | null): number | null {
18 const m = text === null ? null : /^usage_usec (\d+)$/m.exec(text);
19 return m ? finite(Number(m[1])) : null;
20 }
21
22 /** Busy µs from one `/proc/stat` line's tick fields: user nice system irq softirq steal (guest is inside user). */
23 function busyUsec(fields: string): number | null {
24 const f = fields.trim().split(/\s+/).map(Number);
25 if (f.length < 4 || f.some((n) => !Number.isFinite(n))) return null;
26 const busy = f[0]! + f[1]! + f[2]! + (f[5] ?? 0) + (f[6] ?? 0) + (f[7] ?? 0);
27 return (busy * 1_000_000) / USER_HZ;
28 }
29
30 /** Busy CPU time of the whole node from `/proc/stat`'s `cpu` line, µs. */
31 export function parseProcStat(text: string | null): number | null {
32 const m = text === null ? null : /^cpu +(.*)$/m.exec(text);
33 return m ? busyUsec(m[1]!) : null;
34 }
35
36 /** Busy time of each logical CPU from `/proc/stat`'s `cpu<n>` lines, µs, in CPU order. */
37 export function parseProcStatCpus(text: string | null): { id: number; busyUsec: number }[] {
38 const out: { id: number; busyUsec: number }[] = [];
39 for (const m of text?.matchAll(/^cpu(\d+) +(.*)$/gm) ?? []) {
40 const busy = busyUsec(m[2]!);
41 if (busy !== null) out.push({ id: Number(m[1]), busyUsec: busy });
42 }
43 return out.sort((a, b) => a.id - b.id);
44 }
45
46 /** A logical CPU's target: `cpu:` and its number, zero-padded to three digits so a board lists them in order. */
47 export const cpuTarget = (id: number) => `cpu:${String(id).padStart(3, "0")}`;
48
49 /** Memory in use on the node, bytes: `MemTotal − MemAvailable` (kB × 1024). */
50 export function parseMeminfo(text: string | null): number | null {
51 if (text === null) return null;
52 const total = /^MemTotal:\s+(\d+) kB$/m.exec(text);
53 const avail = /^MemAvailable:\s+(\d+) kB$/m.exec(text);
54 return total && avail ? finite((Number(total[1]) - Number(avail[1])) * 1024) : null;
55 }
56
57 /** The 1-min load average: the first field of `/proc/loadavg`. */
58 export function parseLoadavg(text: string | null): number | null {
59 const first = text?.trim().split(/\s+/)[0];
60 return first === undefined || first === "" ? null : finite(Number(first));
61 }
1162
junipercrew

Stray extra blank line.
1263 /** Cumulative counters and gauges at one instant; each is null when its file is not readable. */
13 export type MachinePoint = {
64 type MachinePoint = {
1465 /** Pod CPU, µs since the cgroup began: a rate needs two points. */
1566 cpuUsec: number | null;
1667 /** Pod memory, bytes. */
@@ -31,21 +82,31 @@ function read(path: string): string | null {
3182 }
3283 }
3384
34 /** A point from whatever is readable under `root`, or the reason nothing is. */
35 export function readMachine(root: string): Machine {
36 try {
37 const cgroup = join(root, "sys/fs/cgroup");
38 const proc = join(root, "proc");
39 const mem = read(join(cgroup, "memory.current"));
40 const point: MachinePoint = {
41 cpuUsec: parseCpuStat(read(join(cgroup, "cpu.stat"))),
42 memoryBytes: mem !== null && /^\d+\s*$/.test(mem) ? Number(mem) : null,
43 nodeMemoryBytes: parseMeminfo(read(join(proc, "meminfo"))),
44 load: parseLoadavg(read(join(proc, "loadavg"))),
45 };
46 if (Object.values(point).every((v) => v === null)) return { ok: false, reason: "no cgroup or /proc file is readable here" };
47 return { ok: true, point };
48 } catch {
49 return { ok: false, reason: "the machine could not be read" };
50 }
85 /** A point from the cgroup and /proc directories given, null where a file is not readable. */
86 export function readPoint(cgroup: string, proc: string): MachinePoint {
junipercrew outdated

read exported just so sampler.ts can read /proc/stat; move that read into this module instead.
87 const mem = read(join(cgroup, "memory.current"));
88 return {
89 cpuUsec: parseCpuStat(read(join(cgroup, "cpu.stat"))),
90 memoryBytes: mem !== null && /^\d+\s*$/.test(mem) ? Number(mem) : null,
91 nodeMemoryBytes: parseMeminfo(read(join(proc, "meminfo"))),
92 load: parseLoadavg(read(join(proc, "loadavg"))),
93 };
94 }
95
96 /** The node's busy CPU time and each logical CPU's, from `proc`/stat; null and empty when it is not readable. */
97 export function readCpus(proc: string): { nodeUsec: number | null; cpus: { id: number; busyUsec: number }[] } {
98 const stat = read(join(proc, "stat"));
99 return { nodeUsec: parseProcStat(stat), cpus: parseProcStatCpus(stat) };
100 }
101
102 /** The series whose file is not readable under these directories, each with the file that would have said it. */
103 export function unreadable(cgroup: string, proc: string): string[] {
104 const files: [string, string][] = [
105 ["forge cpu_cores", join(cgroup, "cpu.stat")],
106 ["forge memory_bytes", join(cgroup, "memory.current")],
107 ["node cpu_cores", join(proc, "stat")],
108 ["node memory_bytes", join(proc, "meminfo")],
109 ["node load", join(proc, "loadavg")],
110 ];
111 return files.filter(([, file]) => read(file) === null).map(([series, file]) => `${series}: ${file} is not readable here`);
51112 }
src/boards/sampler.ts +13 −83
@@ -21,11 +21,11 @@
2121 * A file that is missing or unreadable (not Linux, no cgroup v2) leaves its
2222 * series null — never a guess — and `gaps` names it for the page.
2323 */
24 import { readFileSync } from "node:fs";
2524 import { join } from "node:path";
2625 import type { App } from "../app.ts";
2726 import { cellCounters, type CellCounter } from "../apps/contain.ts";
2827 import { startLoopLag, type LoopLag } from "../kernel/loop_lag.ts";
28 import { cpuTarget, readCpus, readPoint, unreadable } from "./machine.ts";
2929 import { FINE_MS, record, type Point } from "./ring.ts";
3030 import { readHwmon, readTopology, temperatures } from "./thermal.ts";
3131 import { during } from "../perf/jobs.ts";
@@ -33,9 +33,6 @@ import { during } from "../perf/jobs.ts";
3333 export type Paths = { cgroup: string; proc: string; sys: string };
3434 export const SYSTEM: Paths = { cgroup: "/sys/fs/cgroup", proc: "/proc", sys: "/sys" };
3535
36 /** `/proc/stat` counts in USER_HZ, 100 a second on every Linux the forge runs on. */
37 const USER_HZ = 100;
38
3936 /** Cumulative counters at one instant; a rate needs two of them. */
4037 export type Counters = {
4138 /** Epoch ms. */
@@ -49,80 +46,15 @@ export type Counters = {
4946 sensors: readonly { target: string; celsius: number }[];
5047 };
5148
52 function read(path: string): string | null {
53 try {
54 return readFileSync(path, "utf8");
55 } catch {
56 return null;
57 }
58 }
59
60 const finite = (n: number) => (Number.isFinite(n) && n >= 0 ? n : null);
61
62 /** `usage_usec` from a cgroup v2 `cpu.stat`, µs of CPU since the cgroup began. */
63 export function parseCpuStat(text: string | null): number | null {
64 const m = text === null ? null : /^usage_usec (\d+)$/m.exec(text);
65 return m ? finite(Number(m[1])) : null;
66 }
67
68 /** Busy µs from one `/proc/stat` line's tick fields: user nice system irq softirq steal (guest is inside user). */
69 function busyUsec(fields: string): number | null {
70 const f = fields.trim().split(/\s+/).map(Number);
71 if (f.length < 4 || f.some((n) => !Number.isFinite(n))) return null;
72 const busy = f[0]! + f[1]! + f[2]! + (f[5] ?? 0) + (f[6] ?? 0) + (f[7] ?? 0);
73 return (busy * 1_000_000) / USER_HZ;
74 }
75
76 /** Busy CPU time of the whole node from `/proc/stat`'s `cpu` line, µs. */
77 export function parseProcStat(text: string | null): number | null {
78 const m = text === null ? null : /^cpu +(.*)$/m.exec(text);
79 return m ? busyUsec(m[1]!) : null;
80 }
81
82 /** Busy time of each logical CPU from `/proc/stat`'s `cpu<n>` lines, µs, in CPU order. */
83 export function parseProcStatCpus(text: string | null): { id: number; busyUsec: number }[] {
84 const out: { id: number; busyUsec: number }[] = [];
85 for (const m of text?.matchAll(/^cpu(\d+) +(.*)$/gm) ?? []) {
86 const busy = busyUsec(m[2]!);
87 if (busy !== null) out.push({ id: Number(m[1]), busyUsec: busy });
88 }
89 return out.sort((a, b) => a.id - b.id);
90 }
91
92 /** A logical CPU's target: `cpu:` and its number, zero-padded to three digits so a board lists them in order. */
93 export const cpuTarget = (id: number) => `cpu:${String(id).padStart(3, "0")}`;
94
95 /** Memory in use on the node, bytes: `MemTotal − MemAvailable` (kB × 1024). */
96 export function parseMeminfo(text: string | null): number | null {
97 if (text === null) return null;
98 const total = /^MemTotal:\s+(\d+) kB$/m.exec(text);
99 const avail = /^MemAvailable:\s+(\d+) kB$/m.exec(text);
100 return total && avail ? finite((Number(total[1]) - Number(avail[1])) * 1024) : null;
101 }
102
103 /** The 1-min load average: the first field of `/proc/loadavg`. */
104 export function parseLoadavg(text: string | null): number | null {
105 const first = text?.trim().split(/\s+/)[0];
106 return first === undefined || first === "" ? null : finite(Number(first));
107 }
108
10949 /** Read every counter now. */
11050 export function readCounters(paths: Paths, cells: readonly CellCounter[], at: number): Counters {
111 const mem = read(join(paths.cgroup, "memory.current"));
112 const stat = read(join(paths.proc, "stat"));
113 const perCpu = parseProcStatCpus(stat);
51 const machine = readPoint(paths.cgroup, paths.proc);
52 const { nodeUsec, cpus: perCpu } = readCpus(paths.proc);
11453 const temps = temperatures(readHwmon(paths.sys), readTopology(paths.sys, perCpu.map((c) => c.id)));
11554 return {
11655 at,
117 forge: {
118 cpuUsec: parseCpuStat(read(join(paths.cgroup, "cpu.stat"))),
119 memoryBytes: mem === null || !/^\d+\s*$/.test(mem) ? null : Number(mem),
120 },
121 node: {
122 cpuUsec: parseProcStat(stat),
123 memoryBytes: parseMeminfo(read(join(paths.proc, "meminfo"))),
124 load: parseLoadavg(read(join(paths.proc, "loadavg"))),
125 },
56 forge: { cpuUsec: machine.cpuUsec, memoryBytes: machine.memoryBytes },
57 node: { cpuUsec: nodeUsec, memoryBytes: machine.nodeMemoryBytes, load: machine.load },
12658 cells,
12759 cpus: perCpu.map((c) => ({ ...c, celsius: temps.perCpu.get(c.id) ?? null })),
12860 sensors: temps.sensors,
@@ -201,15 +133,9 @@ export function pointsOf(prev: Counters | undefined, next: Counters, serving?: S
201133
202134 /** The series this machine cannot read, each with the file that would have said it. */
203135 export function gaps(paths: Paths): string[] {
204 const out: string[] = [];
205 const missing = (file: string) => read(file) === null;
206 const say = (series: string, file: string) => `${series}: ${file} is not readable here`;
207 if (missing(join(paths.cgroup, "cpu.stat"))) out.push(say("forge cpu_cores", join(paths.cgroup, "cpu.stat")));
208 if (missing(join(paths.cgroup, "memory.current"))) out.push(say("forge memory_bytes", join(paths.cgroup, "memory.current")));
209 if (missing(join(paths.proc, "stat"))) out.push(say("node cpu_cores", join(paths.proc, "stat")));
210 if (missing(join(paths.proc, "meminfo"))) out.push(say("node memory_bytes", join(paths.proc, "meminfo")));
211 if (missing(join(paths.proc, "loadavg"))) out.push(say("node load", join(paths.proc, "loadavg")));
212 const cpus = parseProcStatCpus(read(join(paths.proc, "stat"))).map((c) => c.id);
136 const out = unreadable(paths.cgroup, paths.proc);
137 const { cpus: found } = readCpus(paths.proc);
138 const cpus = found.map((c) => c.id);
213139 if (cpus.length === 0) out.push(`cpu:<n> cpu_cores: ${join(paths.proc, "stat")} has no per-CPU lines here`);
214140 const hwmon = join(paths.sys, "class", "hwmon");
215141 const temps = temperatures(readHwmon(paths.sys), readTopology(paths.sys, cpus));
@@ -221,7 +147,8 @@ export function gaps(paths: Paths): string[] {
221147 return out;
222148 }
223149
224 export type Sampler = { tick(now?: number): void; stop(): void };
150 /** `gaps` is what this sampler cannot read, said once at start and kept here for whoever asks. */
151 export type Sampler = { tick(now?: number): void; stop(): void; gaps: readonly string[] };
225152
226153 /** Sample every 15 s into `app.db` until stopped. A failed tick is logged and the next one tries again. */
227154 export function startSampler(
@@ -232,6 +159,8 @@ export function startSampler(
232159 const cells = options.cells ?? cellCounters;
233160 const lag = options.lag ?? startLoopLag();
234161 let prev: Counters | undefined;
162 const missing = gaps(paths);
163 if (missing.length > 0) app.log.info("machine series unreadable here", { gaps: missing.join("; ") });
235164 const tick = (now = Date.now()) => {
236165 try {
237166 const next = readCounters(paths, cells(), now);
@@ -246,6 +175,7 @@ export function startSampler(
246175 timer.unref?.();
247176 return {
248177 tick,
178 gaps: missing,
249179 stop: () => {
250180 clearInterval(timer);
251181 lag.stop();
test/boards-machine.test.ts +11 −22
@@ -1,12 +1,12 @@
11 /**
2 * readMachine(root): fake cgroup and /proc files under a temp dir give the
3 * expected point; what is missing is null; nothing readable gives a reason.
2 * readPoint: fake cgroup and /proc files under a temp dir give the
3 * expected point; what is missing is null.
44 */
55 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
66 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
77 import { tmpdir } from "node:os";
88 import { join } from "node:path";
9 import { readMachine } from "../src/boards/machine.ts";
9 import { readPoint, unreadable } from "../src/boards/machine.ts";
1010
1111 let root: string;
1212 beforeEach(() => {
@@ -14,47 +14,36 @@ beforeEach(() => {
1414 });
1515 afterEach(() => rmSync(root, { recursive: true, force: true }));
1616
17 const point = () => readPoint(join(root, "sys/fs/cgroup"), join(root, "proc"));
1718 const put = (rel: string, text: string) => {
1819 const p = join(root, rel);
1920 mkdirSync(join(p, ".."), { recursive: true });
2021 writeFileSync(p, text);
2122 };
2223
23 describe("readMachine", () => {
24 describe("readPoint", () => {
2425 test("turns cgroup and /proc files into one point", () => {
2526 put("sys/fs/cgroup/cpu.stat", "usage_usec 123456\nuser_usec 100\n");
2627 put("sys/fs/cgroup/memory.current", "5242880\n");
2728 put("proc/meminfo", "MemTotal: 1000 kB\nMemFree: 10 kB\nMemAvailable: 400 kB\n");
2829 put("proc/loadavg", "0.52 0.40 0.30 1/200 999\n");
29 expect(readMachine(root)).toEqual({
30 ok: true,
31 point: { cpuUsec: 123456, memoryBytes: 5242880, nodeMemoryBytes: 600 * 1024, load: 0.52 },
32 });
30 expect(point()).toEqual({ cpuUsec: 123456, memoryBytes: 5242880, nodeMemoryBytes: 600 * 1024, load: 0.52 });
3331 });
3432
3533 test("the cgroup alone is enough; the /proc fields are null", () => {
3634 put("sys/fs/cgroup/cpu.stat", "usage_usec 7\n");
3735 put("sys/fs/cgroup/memory.current", "42\n");
38 expect(readMachine(root)).toEqual({
39 ok: true,
40 point: { cpuUsec: 7, memoryBytes: 42, nodeMemoryBytes: null, load: null },
41 });
36 expect(point()).toEqual({ cpuUsec: 7, memoryBytes: 42, nodeMemoryBytes: null, load: null });
4237 });
4338
4439 test("a garbled file is null, not a throw", () => {
4540 put("sys/fs/cgroup/memory.current", "max\n");
4641 put("proc/loadavg", "1.5 1 1 1/1 1\n");
47 const m = readMachine(root);
48 expect(m.ok && m.point).toEqual({ cpuUsec: null, memoryBytes: null, nodeMemoryBytes: null, load: 1.5 });
42 expect(point()).toEqual({ cpuUsec: null, memoryBytes: null, nodeMemoryBytes: null, load: 1.5 });
4943 });
5044
51 test("a missing directory gives the reason", () => {
52 const m = readMachine(join(root, "nope"));
53 expect(m.ok).toBe(false);
54 expect(!m.ok && m.reason).toContain("readable");
55 });
56
57 test("a root with nothing in it gives the reason", () => {
58 expect(readMachine(root).ok).toBe(false);
45 test("nothing readable gives all null and names every file", () => {
46 expect(readPoint(join(root, "nope/cg"), join(root, "nope/proc"))).toEqual({ cpuUsec: null, memoryBytes: null, nodeMemoryBytes: null, load: null });
47 expect(unreadable(join(root, "nope/cg"), join(root, "nope/proc"))).toHaveLength(5);
5948 });
6049 });
test/boards-sampler.test.ts +46 −2
@@ -266,7 +266,7 @@ describe("startSampler", () => {
266266 test("ticks write the forge, its cells and the node into the ring", () => {
267267 const db = fresh();
268268 const warnings: string[] = [];
269 const app = { db, log: { warn: (m: string) => warnings.push(m) } } as unknown as App;
269 const app = { db, log: { warn: (m: string) => warnings.push(m), info: () => {} } } as unknown as App;
270270 let usec = 0;
271271 const s = startSampler(app, { paths, everyMs: 3_600_000, cells: () => [{ key: 9, label: "app:x/production", cpuUsec: (usec += 3_000_000), memoryBytes: 5 }] });
272272 try {
@@ -299,7 +299,7 @@ describe("startSampler", () => {
299299 ["a", "check", "running"], ["b", "check", "running"], ["c", "check", "ok"], ["d", "check", "queued"],
300300 ["e", "agent", "running"], ["f", "deploy", "running"],
301301 ]) run.run(id!, kind!, state!);
302 const app = { db, log: { warn: () => {} } } as unknown as App;
302 const app = { db, log: { warn: () => {}, info: () => {} } } as unknown as App;
303303 let stopped = false;
304304 const lags = [7.5, null];
305305 const s = startSampler(app, {
@@ -321,4 +321,48 @@ describe("startSampler", () => {
321321 source: "machine", aggregate: "max", column: "lag_ms_max", filter: [{ column: "target", op: "eq", value: "forge" }],
322322 }, { repoIds: "all" })).toEqual({ ok: true, rows: [{ group: null, value: 7.5 }] });
323323 });
324
325 test("an unreadable cgroup is said once, at start, and the board stays up", () => {
326 const db = fresh();
327 const said: unknown[] = [];
328 const app = { db, log: { warn: () => {}, info: (_m: string, f: unknown) => said.push(f) } } as unknown as App;
329 const none: Paths = { cgroup: "/nonexistent/cg", proc: "/nonexistent/proc", sys: "/nonexistent/sys" };
330 const s = startSampler(app, { paths: none, everyMs: 3_600_000, cells: () => [] });
331 s.tick(T0 + 15_000);
332 s.tick(T0 + 30_000);
333 s.stop();
334 expect(said.length).toBe(1);
335 expect(JSON.stringify(said[0])).toContain("/nonexistent/cg/cpu.stat");
336 expect(runQuery(db, { source: "machine", aggregate: "count" }, { repoIds: "all" }).ok).toBe(true);
337 });
338
339 test("a partly readable root logs only the files that are missing", () => {
340 machine({ usage: 1_000, memory: 1, stat: "cpu 1 0 0 0 0 0 0 0\ncpu0 1 0 0 0 0 0 0 0", avail: 1, load: "0" });
341 rmSync(join(paths.proc, "loadavg"));
342 rmSync(join(paths.cgroup, "memory.current"));
343 const said: unknown[] = [];
344 const app = { db: fresh(), log: { warn: () => {}, info: (_m: string, f: unknown) => said.push(f) } } as unknown as App;
345 const s = startSampler(app, { paths: { ...paths, sys: join(root, "nowhere/sys") }, everyMs: 3_600_000, cells: () => [] });
346 s.stop();
347 expect(s.gaps.filter((g) => !g.startsWith("celsius"))).toEqual([
348 `forge memory_bytes: ${join(paths.cgroup, "memory.current")} is not readable here`,
349 `node load: ${join(paths.proc, "loadavg")} is not readable here`,
350 ]);
351 const logged = JSON.stringify(said[0]);
352 expect(logged).toContain("memory.current");
353 expect(logged).toContain("loadavg");
354 expect(logged).not.toContain("cpu.stat");
355 expect(logged).not.toContain("meminfo");
356 });
357
358 test("a point read from a fixture root is found through the machine source", () => {
359 const db = fresh();
360 const app = { db, log: { warn: () => {}, info: () => {} } } as unknown as App;
361 machine({ usage: 0, memory: 4096, stat: "cpu 0 0 0 0 0 0 0 0", avail: 7_000_000, load: "0.25", cpu0: "0 0 0 0 0 0 0 0" });
362 const s = startSampler(app, { paths, everyMs: 3_600_000, cells: () => [] });
363 s.stop();
364 expect(runQuery(db, {
365 source: "machine", aggregate: "max", column: "memory_bytes", filter: [{ column: "target", op: "eq", value: "forge" }],
366 }, { repoIds: "all" })).toEqual({ ok: true, rows: [{ group: null, value: 4096 }] });
367 });
324368 });
test/boards-thermal.test.ts +2 −1
@@ -12,7 +12,8 @@ import { tmpdir } from "node:os";
1212 import { join } from "node:path";
1313 import { runQuery } from "../src/boards/query.ts";
1414 import { record, rollUp, type Point } from "../src/boards/ring.ts";
15 import { cpuTarget, gaps, parseProcStatCpus, pointsOf, readCounters, type Paths } from "../src/boards/sampler.ts";
15 import { cpuTarget, parseProcStatCpus } from "../src/boards/machine.ts";
16 import { gaps, pointsOf, readCounters, type Paths } from "../src/boards/sampler.ts";
1617 import { clean, parseMillidegrees, readHwmon, readTopology, temperatures } from "../src/boards/thermal.ts";
1718 import { migrations } from "../src/core/schema.ts";
1819 import { openDb } from "../src/kernel/db.ts";

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs