Public: anyone can read this; the crew can change it.

Each app keeps a build cache across deploys, keyed by its lockfiles and pins, and its owner can clear it from the app page #12162

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/10526-each-app-keeps-a-build-cache-across-depl → main 11 commits fork 3 threads · 3 unresolved

  • Merged. scout/10526-each-app-keeps-a-build-cache-across-depl was merged into main as 8e3d756.

Review

Sign in to review this change.
14 files changed +683 −10
docs/features/build-cache.md added +11
@@ -0,0 +1,11 @@
1 # Build cache
2
3 Each environment of an app keeps one build cache under its own data (`apps/<id>/<env>/build-cache/<key>/`), keyed by a hash of `Cargo.lock`, `bun.lock` and the toolchain pins. `CARGO_TARGET_DIR` points into it and `~/.exact` in the build's home links to its `exact/`, so a second deploy of an unchanged Rust app skips the compile. A changed key starts a fresh directory and removes the old one.
4
5 - Previews have their own cache; they never read or write production's.
6 - The cache is writable in the build's cell, so it counts toward the step's disk limit. Uncontained hosts get `CARGO_TARGET_DIR` only (HOME is the forge's and is left alone).
7 - The app page shows its size to the repository's admins with a Clear button (`POST /:owner/:name/app/cache/clear`, same-origin form); after it the next deploy builds from scratch.
8 - The size is noted after each build, not measured on page view.
9 - The cache's bytes are added to the app's disk sample (`sampleStorage`), the figure the app page and the quota read.
10 - Clear is refused while any environment of the app is deploying (its build holds the cache); the page and the terminal say so.
11 - Terminal door: `src/tui/screens/buildcache.ts`, palette entry "Build cache" for a repository's admins (`c` clears, after a confirm), through the same `clearCache`. `app.cache.cleared` is an admins' event and redraws an open app page.
src/actions/build_cache.ts added +31
@@ -0,0 +1,31 @@
1 /** Clear an app's build cache (#10526): the repository's admins; the next deploy builds fresh. */
2 import type { App } from "../app.ts";
3 import { appCacheSize, clearBuildCache } from "../apps/build_cache.ts";
4 import { appsRuntime } from "../apps/runtime.ts";
5 import { getAppByRepo, type AppRecord } from "../core/apps.ts";
6 import type { Repo } from "../core/repos.ts";
7 import type { User } from "../core/users.ts";
8 import { err, ok, type Result } from "../kernel/result.ts";
9 import { requireRepoAdmin } from "./access.ts";
10
11 /** Whether any environment of the app is being deployed: its build holds the cache. */
12 export const deployInFlight = (app: App, record: AppRecord): boolean =>
13 appsRuntime(app)?.status(record).some((s) => s.state === "deploying") ?? false;
14
15 /** The page's and the terminal's one Clear: admins only, refused while a deploy is building. */
16 export function clearCache(
17 app: App,
outdated

Classify this event kind in the audience coverage test; add a test that Clear appends it. Also nothing stops Clear during a running build (see summary item 4).

18 actor: User | undefined,
19 repo: Repo,
20 busy: (app: App, record: AppRecord) => boolean = deployInFlight,
21 ): Result<{ freed: number }, "not_found" | "deploying"> {
22 const admin = requireRepoAdmin(app, actor, repo);
23 if (!admin.ok) return admin;
24 const record = getAppByRepo(app.db, repo.id);
25 if (!record) return err("not_found");
26 if (busy(app, record)) return err("deploying");
27 const freed = appCacheSize(app.config.dataDir, record.id);
28 clearBuildCache(app.config.dataDir, record.id);
29 app.bus.append({ kind: "app.cache.cleared", repoId: repo.id, actorId: admin.value.id, payload: { freed } });
30 return ok({ freed });
31 }
src/apps/build_cache.ts added +124
@@ -0,0 +1,124 @@
1 /**
2 * An environment's build cache (#10526): one directory under its own data,
3 * named by a hash of what a build depends on (its lockfiles and toolchain
4 * pins). Cargo's target dir and Exact's checkout live in it, so a second
5 * deploy of an unchanged app skips the compile. A changed key starts a fresh
6 * directory and the old one is removed.
7 *
8 * Every environment has its own: a preview never reads or writes
9 * production's, so a pull's build cannot poison what production reuses.
10 * The cache is writable in the build's cell, so its bytes count against the
11 * step's disk limit, and its size is noted beside it for the app's page.
12 */
13
14 import { createHash } from "node:crypto";
15 import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, lstatSync, writeFileSync } from "node:fs";
16 import { join } from "node:path";
17 import { diskBytes } from "./disk_walk.ts";
18 import { appsRoot, envDir, isEnvName, type EnvName } from "./data.ts";
19
20 /** The files whose content decides the key. */
21 export const LOCKFILES = ["Cargo.lock", "bun.lock"] as const;
22
23 const SIZE_FILE = ".size";
24
25 export const buildCacheRoot = (dataDir: string, appId: string, env: EnvName): string =>
26 join(envDir(dataDir, appId, env), "build-cache");
27
28 /** A short hash of the lockfiles in `checkout` and the toolchain pins. */
29 export function cacheKey(checkout: string, pins: Readonly<Record<string, string>> | undefined): string {
30 const hash = createHash("sha256");
31 for (const name of LOCKFILES) {
32 hash.update(`${name}\0`);
33 try {
34 hash.update(readFileSync(join(checkout, name)));
35 } catch {
36 hash.update("-");
37 }
38 hash.update("\0");
39 }
40 for (const [name, version] of Object.entries(pins ?? {}).sort(([a], [b]) => (a < b ? -1 : 1)))
41 hash.update(`pin:${name}=${version}\0`);
42 return hash.digest("hex").slice(0, 16);
43 }
44
45 export interface PreparedCache {
46 readonly dir: string;
47 /** The directory was there already: this build starts from what the last left. */
48 readonly reused: boolean;
49 }
50
51 /** The directory for `key` under `root`, created; every other key's is removed. */
52 export function prepareBuildCache(root: string, key: string): PreparedCache {
53 if (!/^[0-9a-f]{16}$/.test(key)) throw new Error("bad cache key");
54 const dir = join(root, key);
55 const reused = existsSync(dir);
56 if (existsSync(root))
57 for (const name of readdirSync(root)) if (name !== key && name !== SIZE_FILE) rmSync(join(root, name), { recursive: true, force: true });
58 mkdirSync(join(dir, "target"), { recursive: true });
59 mkdirSync(join(dir, "exact"), { recursive: true });
60 return { dir, reused };
61 }
62
63 /** What a build sees: cargo's target dir in the cache, `~/.exact` at its checkout store. */
64 export function buildCacheEnv(dir: string): Record<string, string> {
65 return { CARGO_TARGET_DIR: join(dir, "target") };
66 }
67
68 /** Make `<home>/.exact` the cache's `exact/` (home is the build's HOME); replaces an old link. */
69 export function linkExact(home: string, dir: string): void {
70 mkdirSync(home, { recursive: true });
71 const link = join(home, ".exact");
72 try {
73 lstatSync(link);
74 rmSync(link, { recursive: true, force: true });
75 } catch (error) {
76 // Nothing there yet is the usual case; anything else is the caller's to hear.
77 if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
78 }
79 symlinkSync(join(dir, "exact"), link);
80 }
81
82 /** Note the cache's size once a build has filled it. */
83 export async function noteSize(
84 root: string,
85 dir: string,
86 measure: (dirs: readonly string[]) => number | Promise<number> = diskBytes,
87 ): Promise<number> {
88 const bytes = await measure([dir]);
89 try {
90 writeFileSync(join(root, SIZE_FILE), String(bytes));
91 } catch {
92 /* the page shows the last note, or none */
93 }
94 return bytes;
95 }
96
97 /** The size noted at the last build, 0 when none: one small read, so a page can ask. */
98 export function cacheSize(root: string): number {
99 try {
100 const n = Number(readFileSync(join(root, SIZE_FILE), "utf8"));
101 return Number.isFinite(n) && n > 0 ? n : 0;
102 } catch {
103 return 0;
104 }
105 }
106
107 /** Remove every environment's cache of an app; the next deploy builds from scratch. */
108 export function clearBuildCache(dataDir: string, appId: string): void {
109 if (!/^[0-9a-z]+$/i.test(appId)) return;
110 const root = join(appsRoot(dataDir), appId);
111 if (!existsSync(root)) return;
112 for (const env of readdirSync(root))
113 if (isEnvName(env)) rmSync(buildCacheRoot(dataDir, appId, env), { recursive: true, force: true });
114 }
115
116 /** All environments' cache bytes for an app, for its page. */
117 export function appCacheSize(dataDir: string, appId: string): number {
118 if (!/^[0-9a-z]+$/i.test(appId)) return 0;
119 const root = join(appsRoot(dataDir), appId);
120 if (!existsSync(root)) return 0;
121 let total = 0;
122 for (const env of readdirSync(root)) if (isEnvName(env)) total += cacheSize(buildCacheRoot(dataDir, appId, env));
123 return total;
124 }
src/apps/process.ts +45 −8
@@ -16,6 +16,7 @@ import { envPull } from "./data.ts";
1616 import { forgeApiUrl } from "./token.ts";
1717 import { PROXY_VARS } from "./egress.ts";
1818 import { backgroundArgv } from "./background.ts";
19 import { buildCacheEnv, cacheKey, linkExact, noteSize, prepareBuildCache } from "./build_cache.ts";
1920 import { isReservedAppEnvName } from "../core/app_env.ts";
2021 import { isFeatureVarName } from "./feature_env.ts";
2122 import { formsOf, redactValues } from "../kernel/log.ts";
@@ -31,6 +32,8 @@ import { usedRefusal } from "./disk_guard.ts";
3132 import { cellInstall, meteredInstall, prefillInstallCache, withLock } from "../checks/install.ts";
3233 import {
3334 commandPid,
35 diskEntriesMax,
36 diskWalkOff,
3437 killTree,
3538 openCell,
3639 resolveLimits,
@@ -96,6 +99,13 @@ export interface StartInput {
9699 * fill will not trust (no `bun.lock`) installs as before, into its home.
97100 */
98101 readonly installCache?: string | undefined;
102 /**
103 * This environment's build-cache root (`apps/build_cache.ts`): cargo's
104 * target dir and Exact's checkout live in a directory under it, keyed by
105 * the lockfiles and pins, writable in the build's cell. Never production's
106 * for a preview. Unset, the build keeps its files in the checkout and home.
107 */
108 readonly buildCache?: string | undefined;
99109 }
100110
101111 /** What a caller knows about a cell; the rest comes from the manifest. */
@@ -482,13 +492,13 @@ export function stepLimits(input: Pick<StartInput, "manifest">): Limits {
482492 }
483493
484494 /** The cell a step of this deploy runs in: the checkout and the home, the registry and what the app declared. */
485 export function stepCell(input: StartInput, use?: ToolchainUse): CellSpec | undefined {
495 export function stepCell(input: StartInput, use?: ToolchainUse, cache?: string): CellSpec | undefined {
486496 const base = input.cell;
487497 if (base === undefined) return undefined;
488498 return {
489499 ...(use === undefined ? {} : { reveal: use.reveal }),
490500 dir: input.dir,
491 writable: [],
501 writable: cache === undefined ? [] : [cache],
492502 home: base.home,
493503 hide: base.hide,
494504 egress: [...STEP_HOSTS, ...(input.manifest.egress ?? [])],
@@ -530,9 +540,10 @@ export function installPlan(
530540 env: Record<string, string>,
531541 filled: boolean,
532542 use?: ToolchainUse,
543 built?: string,
533544 ): InstallPlan {
534545 const cache = input.installCache;
535 const cell = stepCell(input, use);
546 const cell = stepCell(input, use, built);
536547 if (cache === undefined || !filled) return { command, env, cell, lock: undefined };
537548 if (cell !== undefined)
538549 return { ...cellInstall(command, env, cache), cell: { ...cell, reveal: [...(cell.reveal ?? []), cache] }, lock: undefined };
@@ -549,6 +560,7 @@ async function install(
549560 input: StartInput,
550561 env: Record<string, string>,
551562 use?: ToolchainUse,
563 built?: string,
552564 ): Promise<Result<void, string>> {
553565 let filled = false;
554566 if (input.installCache !== undefined) {
@@ -563,7 +575,7 @@ async function install(
563575 if (fill.ok) filled = fill.value;
564576 else input.log(`cache not filled: ${fill.error}`);
565577 }
566 const plan = installPlan(command, input, env, filled, use);
578 const plan = installPlan(command, input, env, filled, use, built);
567579 const step = (): Promise<Result<void, string>> =>
568580 runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell, build: stepLimits(input) });
569581 if (plan.lock !== undefined) {
@@ -597,8 +609,25 @@ export async function startProcess(
597609 const used = toolchainUse(manifest.toolchain, scratch);
598610 if (!used.ok) return err({ stage: "install", message: used.error });
599611 const use = used.value;
612 // The build cache: cargo's target dir and Exact's checkout, kept across
613 // deploys while the lockfiles and pins are the same (`apps/build_cache.ts`).
614 // It is an optimisation: one that cannot be made is logged and skipped.
615 let cacheDir: string | undefined;
616 if (input.buildCache !== undefined) {
617 try {
618 const prepared = prepareBuildCache(input.buildCache, cacheKey(input.dir, manifest.toolchain));
619 cacheDir = prepared.dir;
620 // Uncontained, HOME is the forge's own and is not touched.
621 if (input.cell !== undefined) linkExact(input.cell.home, cacheDir);
622 log(prepared.reused ? "build cache: reusing the last build's" : "build cache: starting fresh");
623 } catch (error) {
624 input.log(`build cache skipped: ${error instanceof Error ? error.message : String(error)}`);
625 }
626 }
627 const cacheEnv = cacheDir === undefined ? {} : buildCacheEnv(cacheDir);
600628 const stepEnv = childEnv({
601629 ...use?.env,
630 ...cacheEnv,
602631 CI: "1",
603632 FORGE_APP_ENV: input.env === "production" ? "production" : "preview",
604633 // A mirror the forge itself is pointed at is the one the fill and the
@@ -607,12 +636,17 @@ export async function startProcess(
607636 });
608637
609638 if (manifest.install !== null && input.warm !== true) {
610 const installed = await install(manifest.install, input, stepEnv, use);
639 const installed = await install(manifest.install, input, stepEnv, use, cacheDir);
611640 if (!installed.ok)
612641 return err({ stage: "install", message: installed.error });
613642 }
614643 if (input.signal.aborted)
615644 return err({ stage: "start", message: "cancelled" });
645 const noted = (): void => {
646 if (cacheDir !== undefined && input.buildCache !== undefined)
647 void noteSize(input.buildCache, cacheDir, async (dirs) => (await diskWalkOff(dirs, diskEntriesMax(0))).bytes).catch(() => undefined);
648 };
649 noted();
616650
617651 // `test` starts here, alongside the app below, and is aborted on its own
618652 // signal — a start failure must not leave it running past its run, and
@@ -631,8 +665,11 @@ export async function startProcess(
631665 env: stepEnv,
632666 log,
633667 signal: testAbort.signal,
634 cell: stepCell(input, use),
668 cell: stepCell(input, use, cacheDir),
635669 build: stepLimits(input),
670 }).then((result) => {
671 noted();
672 return result;
636673 });
637674
638675 // The port is chosen, then released, then bound by the app: another
@@ -651,7 +688,7 @@ export async function startProcess(
651688 : openCell(
652689 {
653690 dir: input.dir,
654 writable: [input.dataDir],
691 writable: cacheDir === undefined ? [input.dataDir] : [input.dataDir, cacheDir],
655692 home: input.cell.home,
656693 hide: input.cell.hide,
657694 ...(use === undefined ? {} : { reveal: use.reveal }),
@@ -670,7 +707,7 @@ export async function startProcess(
670707 );
671708 const command = shellCommand(manifest.run);
672709 const env = appEnv({
673 toolchain: use?.env,
710 toolchain: { ...use?.env, ...cacheEnv },
674711 port,
675712 dataDir: input.dataDir,
676713 env: input.env,
src/apps/runtime.ts +5
@@ -81,6 +81,7 @@ import {
8181 SAMPLE_SPAN_MS,
8282 } from "../core/meter.ts";
8383 import { dirBytes } from "./bytes.ts";
84 import { appCacheSize, buildCacheRoot } from "./build_cache.ts";
8485 import { BUSY_WORDS, busyReason, type BusyReason } from "./busy.ts";
8586 import { exportTree, readFileAt } from "./checkout.ts";
8687 import { seconds, Stopwatch } from "./timing.ts";
@@ -1611,6 +1612,8 @@ export class AppsRuntime {
16111612 // Every environment, production too, installs from the cache the
16121613 // repository's checks fill, so a new preview fetches nothing it has.
16131614 installCache: checkInstallCache(this.app.config.dataDir, repo.id),
1615 // Its own, per environment: a preview never touches production's.
1616 buildCache: buildCacheRoot(this.app.config.dataDir, state.app.id, state.env),
16141617 };
16151618 this.noteCell(run.id, write);
16161619 clock.lap("setup");
@@ -2394,6 +2397,8 @@ export class AppsRuntime {
23942397 dataBytes += await dirBytes(
23952398 envDataDir(this.app.config.dataDir, record.id, name),
23962399 );
2400 // The build cache is the app's disk too (#10526).
2401 dataBytes += appCacheSize(this.app.config.dataDir, record.id);
23972402 const repoBytes = await dirBytes(
23982403 repoDir(this.app.reposRoot, repo.ownerName, repo.name),
23992404 );
src/core/event_audience.ts +2
@@ -212,6 +212,8 @@ export function classifiedAudience(kind: string): EventAudience | undefined {
212212 kind.startsWith("app.domain.") ||
213213 // A private address and port, and who may reach it (#3543).
214214 kind.startsWith("app.egress.") ||
215 // Clearing the build cache is an admin's act on the app's disk (#10526).
216 kind === "app.cache.cleared" ||
215217 kind.startsWith("prompt.") ||
216218 kind.startsWith("secret.") ||
217219 // The forge's mail ceiling is its owner's to hear (`actions/mail.ts`).
src/http/server.ts +2
@@ -356,6 +356,7 @@ import {
356356 appsOpen,
357357 appPage,
358358 appRedeploy,
359 appCacheClear,
359360 appsList,
360361 appStop,
361362 newAppPage,
@@ -1452,6 +1453,7 @@ export function buildRouter(): Router<Handler> {
14521453 router.add("GET", "/:owner/:name/app/jobs", jobsPage);
14531454 router.add("POST", "/:owner/:name/app/jobs/:id/:action", steerJobPage);
14541455 router.add("POST", "/:owner/:name/app/schedules/:schedule/:action", steerSchedulePage);
1456 router.add("POST", "/:owner/:name/app/cache/clear", appCacheClear);
14551457 router.add("POST", "/:owner/:name/app/:env/redeploy", appRedeploy);
14561458 router.add("POST", "/:owner/:name/app/:env/stop", appStop);
14571459 router.add("GET", "/:owner/:name/events", eventsList);
src/tui/screens/buildcache.ts added +93
@@ -0,0 +1,93 @@
1 /**
2 * The build cache, in the terminal (#10526): the app page's size and Clear
3 * (`web/studio/previews.ts`) through the same action (`actions/build_cache.ts`),
4 * gated the same way: a repository's admins. `c` clears, after a confirm. The
5 * write is an event on the bus, so an open app page redraws and this screen
6 * follows a clear made there.
7 */
8
9 import { clearCache } from "../../actions/build_cache.ts";
10 import { appCacheSize } from "../../apps/build_cache.ts";
11 import { getAppByRepo } from "../../core/apps.ts";
12 import { canAdmin, type Repo } from "../../core/repos.ts";
13 import type { Ctx, Screen } from "../app/app.ts";
14 import { binding } from "../app/keys.ts";
15 import type { Command } from "../app/palette.ts";
16 import type { Session } from "../session.ts";
17 import { style } from "../term/style.ts";
18 import { truncate } from "../term/width.ts";
19 import { confirmScreen, later, live } from "./common.ts";
20
21 export const mayClearCache = (f: Session, repo: Repo): boolean => f.user !== undefined && canAdmin(f.db, repo, f.user);
22
23 /** The cache's bytes, for an admin; 0 for anyone else and for an app without one. */
24 export function cacheBytes(f: Session, repo: Repo): number {
25 if (!mayClearCache(f, repo)) return 0;
26 const record = getAppByRepo(f.db, repo.id);
27 return record === undefined ? 0 : appCacheSize(f.app.config.dataDir, record.id);
28 }
29
30 const size = (n: number): string =>
31 n >= 1 << 30 ? `${(n / (1 << 30)).toFixed(1)} GiB` : n >= 1 << 20 ? `${(n / (1 << 20)).toFixed(1)} MiB` : `${Math.max(1, Math.round(n / 1024))} KiB`;
32
33 export function buildCacheScreen(f: Session, ctx: Ctx, repo: Repo): Screen {
34 const where = `${repo.ownerName}/${repo.name}`;
35 const screen: Screen = {
36 title: `${where} — Build cache`,
37 keys: [binding(["c"], "clear the build cache")],
38 async load(signal) {
39 live(f, signal, { repoId: repo.id, where: (e) => e.kind === "app.cache.cleared" || e.kind === "run.finished" }, () => ctx.invalidate());
40 },
41 render(area, s) {
42 const c = ctx.theme.c;
43 const bytes = cacheBytes(f, repo);
44 const said = !mayClearCache(f, repo)
45 ? "An admin of this repository manages its build cache."
46 : bytes === 0
47 ? "No build cache yet: the next build fills it."
48 : `Build cache ${size(bytes)}. c clears it; the next deploy builds from scratch.`;
49 s.text(area.x, area.y, truncate(said, area.w), style(c.mutedFg, c.bg), area.x + area.w);
50 },
51 key(e) {
52 if (e.ctrl || e.alt || e.key !== "c") return false;
53 if (!mayClearCache(f, repo)) {
54 ctx.toast("An admin of this repository manages its build cache.", "danger");
55 return true;
56 }
57 void later(ctx, () =>
58 confirmScreen(ctx, screen, {
59 title: "Clear build cache",
60 body: ["The next deploy builds from scratch."],
61 actions: [{ label: "Cancel", variant: "ghost" }, { label: "Clear" }],
62 selected: 0,
63 onChoose: (i) => {
64 if (i !== 1) return;
65 const done = clearCache(f.app, f.user, repo);
66 ctx.toast(
67 done.ok ? "Build cache cleared." : done.error === "deploying" ? "A deploy is building now; clear it when it is done." : "No such app.",
68 done.ok ? undefined : "danger",
69 );
70 ctx.invalidate();
71 },
72 }),
73 );
74 return true;
75 },
76 };
77 return screen;
78 }
79
80 /** The palette's command while an app's repository the viewer administers is open. */
81 export function buildCacheCommands(f: Session, ctx: Ctx, repo: Repo | undefined): Command[] {
82 if (repo === undefined || !mayClearCache(f, repo) || getAppByRepo(f.db, repo.id) === undefined) return [];
83 return [
84 {
85 id: "build-cache",
86 title: `${where(repo)} — Build cache`,
87 hint: "see the build cache's size and clear it",
88 run: () => ctx.push(buildCacheScreen(f, ctx, repo)),
89 },
90 ];
91 }
92
93 const where = (repo: Repo): string => `${repo.ownerName}/${repo.name}`;
src/tui/screens/workspace.ts +2
@@ -172,6 +172,7 @@ import { autonomyCommands } from "./autonomy.ts";
172172 import { trafficCommands } from "./trafficview.ts";
173173 import { chaptersCommands } from "./chapters.ts";
174174 import { outsideAgentsCommands } from "./outsideagents.ts";
175 import { buildCacheCommands } from "./buildcache.ts";
175176 import { schedulesCommands } from "./schedules.ts";
176177 import { landCommands } from "./landasis.ts";
177178 import { shiftCommands } from "./subtitle_shift.ts";
@@ -2005,6 +2006,7 @@ export function workspaceScreen(
20052006 ...trafficCommands(f, ctx, repo),
20062007 ...schedulesCommands(f, ctx, repo),
20072008 ...outsideAgentsCommands(f, ctx, repo),
2009 ...buildCacheCommands(f, ctx, repo),
20082010 ...landCommands(f, ctx, repo),
20092011 ...shiftCommands(f, ctx, repo, () => screen),
20102012 ...dataImportCommands(f, ctx, repo),
src/web/live.ts +1 −1
@@ -487,7 +487,7 @@ export const liveClientJs = /* js */ `
487487 'branch.created', 'branch.deleted',
488488 'label.created', 'label.updated', 'label.deleted',
489489 'run.queued', 'run.started', 'run.finished', 'app.serving',
490 'schedule.steered', 'app.created', 'app.updated', 'app.deleted', 'app.stopped',
490 'schedule.steered', 'app.created', 'app.updated', 'app.deleted', 'app.stopped', 'app.cache.cleared',
491491 'repo.upstream_checked', 'repo.links_probed', 'repo.source_added', 'repo.source_removed', 'repo.upstream_follow', 'repo.following', 'limb.moved', 'limb.replan', 'limb.replan_taken_back', 'issue.assigned', 'pull.assigned', 'crew.added', 'crew.removed',
492492 ];
493493 var extra = (cfg.getAttribute('data-repos') || '').split(' ').filter(Boolean);
src/web/pages/apps.ts +13
@@ -12,6 +12,7 @@
1212 * 404 for everyone.
1313 */
1414
15 import { clearCache } from "../../actions/build_cache.ts";
1516 import { returnTo, withReturn } from "../return_to.ts";
1617 import { crewKeyCard } from "../studio/crew_key.ts";
1718 import { storageLimitMessage } from "../../core/storage.ts";
@@ -914,3 +915,15 @@ export async function appStop(ctx: Ctx): Promise<Response> {
914915 message: stoppedText(env as EnvName),
915916 });
916917 }
918
919 // POST /:owner/:name/app/cache/clear
920 export async function appCacheClear(ctx: Ctx): Promise<Response> {
921 const gate = await envForm(ctx);
922 if (gate instanceof Response) return gate;
923 const { repo, user } = gate;
924 const cleared = clearCache(ctx.app, user, repo);
925 if (!cleared.ok && cleared.error === "deploying")
926 return flashTo(ctx, envBack(ctx, repo), { message: "A deploy is building now; clear the cache when it is done." });
927 if (!cleared.ok) return notFoundPage(ctx);
928 return flashTo(ctx, envBack(ctx, repo), { message: "Build cache cleared. The next deploy builds from scratch." });
929 }
src/web/studio/previews.ts +15 −1
@@ -7,6 +7,7 @@
77 * deploy event refreshes (`appStateFragment`) share.
88 */
99
10 import { appCacheSize } from "../../apps/build_cache.ts";
1011 import { HELP_BUILD } from "../../core/contribute.ts";
1112 import { lacks } from "../../core/roles.ts";
1213 import { notePhase, takeSessionTime, timePhase } from "../../core/route_timing.ts";
@@ -20,7 +21,7 @@ import {
2021 type EnvStatus, envsFor } from "../../apps/runtime.ts";
2122 import { getAppByRepo, type AppRecord } from "../../core/apps.ts";
2223 import { listPulls, type Pull } from "../../core/pulls.ts";
23 import { canSeeOwnerFacts, type Repo } from "../../core/repos.ts";
24 import { canAdmin, canSeeOwnerFacts, type Repo } from "../../core/repos.ts";
2425 import { latestRunsForIssues, listRuns, type Run } from "../../core/runs.ts";
2526 import type { Ctx } from "../../http/context.ts";
2627 import { readRunLogTail } from "../../runs/log.ts";
@@ -38,6 +39,7 @@ import {
3839 envFacts,
3940 envLine,
4041 envPhrase,
42 fileSize,
4143 icon,
4244 listCard,
4345 previewDataNote,
@@ -343,6 +345,7 @@ export function appPanels(
343345 ctx.app.config.sshHostKey,
344346 ),
345347 )}
348 ${buildCacheLine(ctx, repo, record, base)}
346349 ${tail.box ?? null}
347350 </div>
348351 <span class="spacer"></span>
@@ -362,6 +365,17 @@ export function appPanels(
362365 };
363366 }
364367
368 /** The build cache's size and its Clear, for the repository's admins; none until one is filled. */

Page-only door: add the terminal screen in src/tui/ for the same size and Clear (summary item 1). Also needs a route-level test (admin vs guest, cross-origin POST).

369 function buildCacheLine(ctx: Ctx, repo: Repo, record: AppRecord, base: string): Html | null {
370 if (!canAdmin(ctx.app.db, repo, ctx.auth.user)) return null;
371 const bytes = appCacheSize(ctx.app.config.dataDir, record.id);
372 if (bytes === 0) return null;
373 return html`<form method="post" action="${base}/app/cache/clear" class="row-fields">
374 <span class="muted small">Build cache ${fileSize(bytes)}</span>
375 ${btn("Clear", { size: "sm", title: "Remove the build cache; the next deploy builds from scratch" })}
376 </form>`;
377 }
378
365379 /**
366380 * GET /:owner/:name/app/state — the head card and previews list, bare. The
367381 * studio page's own live client (`live.ts`) fetches this on a deploy run
test/build-cache-doors.test.ts added +138
@@ -0,0 +1,138 @@
1 /** The build cache's doors (#10526): the app page, the terminal screen, the event, the refusals. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { clearCache } from "../src/actions/build_cache.ts";
7 import { createApp, type App } from "../src/app.ts";
8 import { AppsRuntime } from "../src/apps/runtime.ts";
9 import { buildCacheRoot, prepareBuildCache } from "../src/apps/build_cache.ts";
10 import { createApp as makeAppRecord } from "../src/core/apps.ts";
11 import { eventAudience } from "../src/core/event_audience.ts";
12 import { listEvents } from "../src/core/events.ts";
13 import { createRepo, type Repo } from "../src/core/repos.ts";
14 import { createSession } from "../src/core/sessions.ts";
15 import { createUser, getUserByName, type User } from "../src/core/users.ts";
16 import { serve, type Servers } from "../src/http/server.ts";
17 import { createForgeApp } from "../src/tui/forge.ts";
18 import { buildCacheCommands, buildCacheScreen } from "../src/tui/screens/buildcache.ts";
19 import { headlessTerminal } from "../src/tui/term/terminal.ts";
20 import { adminCookie } from "./helpers/admin.ts";
21 import { settleApp } from "./helpers/tui.ts";
22
23 let dir: string;
24 let app: App;
25 let servers: Servers;
26 let admin: User;
27 let bob: User;
28 let repo: Repo;
29 let appId: string;
30 let adminHeader: string;
31 let bobHeader: string;
32 let cacheDir: string;
33
34 beforeAll(async () => {
35 dir = mkdtempSync(join(tmpdir(), "bc-doors-"));
36 app = createApp({ dataDir: dir, port: 0 });
37 servers = await serve(app);
38 adminHeader = await adminCookie(app, { username: "trevato", email: "[email protected]", password: "a-good-passphrase" });
39 admin = getUserByName(app.db, "trevato")!;
40 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
41 if (!made.ok) throw new Error(made.error);
42 bob = made.value;
43 bobHeader = `forge_session=${createSession(app.db, bob.id).token}`;
44 const r = await createRepo(app.db, app.reposRoot, { ownerId: admin.id, name: "lib", visibility: "public", description: "", defaultBranch: "main" });
45 if (!r.ok) throw new Error(r.error);
46 repo = r.value;
47 const rec = makeAppRecord(app.db, { repoId: repo.id, slug: "lib", access: "public" });
48 if (!rec.ok) throw new Error(rec.error);
49 appId = rec.value.id;
50 });
51 afterAll(async () => {
52 await servers.stop();
53 app.close();
54 rmSync(dir, { recursive: true, force: true });
55 });
56
57 function fill(): void {
58 const root = buildCacheRoot(dir, appId, "production");
59 cacheDir = prepareBuildCache(root, "a".repeat(16)).dir;
60 writeFileSync(join(cacheDir, "target", "f"), "x");
61 writeFileSync(join(root, ".size"), "4096");
62 }
63 const get = async (cookie?: string) =>
64 await (await fetch(`${servers.url}/trevato/lib/app`, { headers: cookie ? { cookie } : {} })).text();
65 const post = (cookie: string, headers: Record<string, string> = {}) =>
66 fetch(`${servers.url}/trevato/lib/app/cache/clear`, { method: "POST", redirect: "manual", headers: { cookie, ...headers } });
67 const key = (k: string) => ({ type: "key" as const, key: k, ctrl: false, alt: false, shift: false });
68
69 test("the page shows size and Clear to an admin only; a cross-origin POST is refused; a same-origin one clears and writes the event", async () => {
70 fill();
71 expect(await get(adminHeader)).toContain("/app/cache/clear");
72 expect(await get(bobHeader)).not.toContain("/app/cache/clear");
73 expect(await get()).not.toContain("/app/cache/clear");
74
75 expect((await post(adminHeader, { origin: "http://evil.example" })).status).toBe(403);
76 expect(existsSync(cacheDir)).toBe(true);
77 expect((await post(bobHeader, { origin: servers.url })).status).toBe(404);
78 expect(existsSync(cacheDir)).toBe(true);
79
80 const done = await post(adminHeader, { origin: servers.url });
81 expect(done.status).toBe(303);
82 expect(existsSync(cacheDir)).toBe(false);
83 const event = listEvents(app.db, { repoId: repo.id }).find((e) => e.kind === "app.cache.cleared");
84 expect(event?.payload).toEqual({ freed: 4096 });
85 expect(eventAudience("app.cache.cleared")).toBe("admins");
86 });
87
88 test("Clear is refused while a deploy is building, and works after", () => {
89 fill();
90 expect(clearCache(app, admin, repo, () => true)).toEqual({ ok: false, error: "deploying" });
91 expect(existsSync(cacheDir)).toBe(true);
92 expect(clearCache(app, admin, repo, () => false).ok).toBe(true);
93 expect(existsSync(cacheDir)).toBe(false);
94 });
95
96 test("the terminal screen shows the size and clears through the same action, for an admin only", async () => {
97 fill();
98 const open = (user: User) => {
99 const { app: tui, session } = createForgeApp(
100 app,
101 { sshUsername: user.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user },
102 { terminal: headlessTerminal(120, 40), exitHooks: false },
103 );
104 tui.start();
105 tui.ctx.push(buildCacheScreen(session, tui.ctx, repo));
106 return { tui, session };
107 };
108 const a = open(admin);
109 await settleApp(a.tui);
110 expect(a.tui.screen.toText()).toContain("Build cache 4 KiB");
111 expect(buildCacheCommands(a.session, a.tui.ctx, repo).length).toBe(1);
112 a.tui.dispatch(key("c"));
113 await settleApp(a.tui);
114 a.tui.dispatch(key("right"));
115 a.tui.dispatch(key("enter"));
116 await settleApp(a.tui);
117 expect(existsSync(cacheDir)).toBe(false);
118 expect(a.tui.screen.toText()).toContain("No build cache yet");
119 a.tui.stop?.();
120
121 fill();
122 const b = open(bob);
123 await settleApp(b.tui);
124 expect(b.tui.screen.toText()).toContain("An admin of this repository");
125 expect(buildCacheCommands(b.session, b.tui.ctx, repo)).toEqual([]);
126 b.tui.dispatch(key("c"));
127 await settleApp(b.tui);
128 expect(existsSync(cacheDir)).toBe(true);
129 b.tui.stop?.();
130 });
131
132 test("the cache counts in the app's disk sample, the quota's figure", async () => {
133 fill();
134 const runtime = new AppsRuntime(app, { domain: "apps.test", scheme: "http", forgeUrl: servers.url, storageEveryMs: 1 });
135 await (runtime as unknown as { sampleStorage(): Promise<void> }).sampleStorage();
136 const row = app.db.query<{ data_bytes: number }, [string]>("SELECT data_bytes FROM storage WHERE repo_id = ?1").get(repo.id);
137 expect(row?.data_bytes).toBe(4096);
138 });
test/build-cache.test.ts added +201
@@ -0,0 +1,201 @@
1 /** The build cache (#10526): keyed by lockfiles and pins, reused by the next deploy, clearable. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs";
4 import { createRepo } from "../src/actions/repos.ts";
outdated

Duplicate bun:test import; merge with line 2. Also drop the unused extra param of deploy().

5 import { clearCache } from "../src/actions/build_cache.ts";
6 import { createApp, type App } from "../src/app.ts";
7 import { createApp as makeAppRecord } from "../src/core/apps.ts";
8 import { createUser } from "../src/core/users.ts";
9 import { proveAll } from "./helpers/verified.ts";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import {
13 appCacheSize,
14 buildCacheEnv,
15 buildCacheRoot,
16 cacheKey,
17 cacheSize,
18 clearBuildCache,
19 linkExact,
20 noteSize,
21 prepareBuildCache,
22 } from "../src/apps/build_cache.ts";
23 import type { AppManifest } from "../src/apps/manifest.ts";
24 import { startProcess } from "../src/apps/process.ts";
25
26 const tmp = (): string => mkdtempSync(join(tmpdir(), "bc-"));
27
28 describe("cacheKey", () => {
29 test("changes with a lockfile or a pin, not with other files", () => {
30 const dir = tmp();
31 const none = cacheKey(dir, undefined);
32 writeFileSync(join(dir, "Cargo.lock"), "a");
33 const a = cacheKey(dir, undefined);
34 expect(a).not.toBe(none);
35 expect(cacheKey(dir, undefined)).toBe(a);
36 writeFileSync(join(dir, "README.md"), "x");
37 expect(cacheKey(dir, undefined)).toBe(a);
38 writeFileSync(join(dir, "Cargo.lock"), "b");
39 expect(cacheKey(dir, undefined)).not.toBe(a);
40 expect(cacheKey(dir, { rust: "1.82" })).not.toBe(cacheKey(dir, undefined));
41 expect(cacheKey(dir, undefined)).toMatch(/^[0-9a-f]{16}$/);
42 });
43 });
44
45 describe("prepareBuildCache", () => {
46 test("a changed key starts fresh and removes the old directory", () => {
47 const root = join(tmp(), "bc");
48 const first = prepareBuildCache(root, "a".repeat(16));
49 expect(first.reused).toBe(false);
50 writeFileSync(join(first.dir, "target", "artifact"), "x");
51 expect(prepareBuildCache(root, "a".repeat(16)).reused).toBe(true);
52 const next = prepareBuildCache(root, "b".repeat(16));
53 expect(next.reused).toBe(false);
54 expect(existsSync(first.dir)).toBe(false);
55 });
56 test("refuses a key that is not a hash", () => {
57 expect(() => prepareBuildCache(join(tmp(), "bc"), "../x")).toThrow();
58 });
59 test("the size note survives a key change", async () => {
60 const root = join(tmp(), "bc");
61 const { dir } = prepareBuildCache(root, "a".repeat(16));
62 writeFileSync(join(dir, "target", "f"), "x".repeat(10_000));
63 expect(await noteSize(root, dir)).toBeGreaterThan(0);
64 prepareBuildCache(root, "b".repeat(16));
65 expect(cacheSize(root)).toBeGreaterThan(0);
66 });
67 });
68
69 describe("env and home", () => {
70 test("cargo's target dir and Exact's checkout are inside the cache", () => {
71 const dir = tmp();
72 expect(buildCacheEnv(dir)).toEqual({ CARGO_TARGET_DIR: join(dir, "target") });
73 const home = join(tmp(), "home");
74 mkdirSync(dir, { recursive: true });
75 linkExact(home, dir);
76 linkExact(home, dir);
77 expect(lstatSync(join(home, ".exact")).isSymbolicLink()).toBe(true);
78 expect(readlinkSync(join(home, ".exact"))).toBe(join(dir, "exact"));
79 });
80 });
81
82 const manifest = (install: string): AppManifest => ({
83 run: `${process.execPath} server.ts`,
84 install,
85 test: null,
86 health: "/",
87 ready: 10,
88 tty: null,
89 service: null,
90 looks: ["/"],
91 });
92
93 describe("a second deploy", () => {
94 async function deploy(data: string, co: string, install: string) {
95 const lines: string[] = [];
96 const started = await startProcess({
97 dir: co,
98 dataDir: join(data, "d"),
99 manifest: manifest(install),
100 env: "production",
101 url: "http://x.test",
102 commitOid: "c".repeat(40),
103 forgeUrl: "http://forge.test",
104 identityKey: "k",
105 log: (l) => lines.push(l),
106 signal: new AbortController().signal,
107 buildCache: join(data, "build-cache"),
108 });
109 if (!started.ok) throw new Error(started.error.message);
110 await started.value.handle.stop();
111 return lines;
112 }
113
114 test("reuses the cache, and a cleared cache builds from scratch", async () => {
115 const data = tmp();
116 const co = tmp();
117 writeFileSync(join(co, "server.ts"), `Bun.serve({ port: Number(process.env.PORT), fetch: () => new Response("ok") });\n`);
118 writeFileSync(join(co, "Cargo.lock"), "lock-1");
119 // "Compile": only when the target dir has no artifact; count compiles.
120 const compile = `test -f "$CARGO_TARGET_DIR/artifact" || { echo built >> "$CARGO_TARGET_DIR/../compiles"; echo x > "$CARGO_TARGET_DIR/artifact"; }`;
121 const compiles = (): number => {
122 const key = cacheKey(co, undefined);
123 const file = join(data, "build-cache", key, "compiles");
124 return existsSync(file) ? readFileSync(file, "utf8").trim().split("\n").length : 0;
125 };
126 const first = await deploy(data, co, compile);
127 expect(first.join("\n")).toContain("build cache: starting fresh");
128 expect(compiles()).toBe(1);
129 const second = await deploy(data, co, compile);
130 expect(second.join("\n")).toContain("reusing");
131 expect(compiles()).toBe(1);
132 expect(appCacheSize(data, "nope")).toBe(0);
133
134 // A changed lockfile starts a fresh cache and removes the old one.
135 const old = join(data, "build-cache", cacheKey(co, undefined));
136 writeFileSync(join(co, "Cargo.lock"), "lock-2");
137 await deploy(data, co, compile);
138 expect(existsSync(old)).toBe(false);
139 expect(compiles()).toBe(1);
140
141 rmSync(join(data, "build-cache"), { recursive: true });
142 await deploy(data, co, compile);
143 expect(compiles()).toBe(1);
144 });
145 });
146
147 describe("clearBuildCache", () => {
148 test("removes every environment's cache of the app and nothing else", () => {
149 const data = tmp();
150 for (const env of ["production", "pull-3"] as const) {
151 const root = buildCacheRoot(data, "app1", env);
152 const { dir } = prepareBuildCache(root, "a".repeat(16));
153 writeFileSync(join(dir, "target", "f"), "x");
154 writeFileSync(join(root, ".size"), "4096");
155 }
156 mkdirSync(join(data, "apps", "app1", "production", "data"), { recursive: true });
157 writeFileSync(join(data, "apps", "app1", "production", "data", "db"), "keep");
158 expect(appCacheSize(data, "app1")).toBe(8192);
159 clearBuildCache(data, "app1");
160 expect(appCacheSize(data, "app1")).toBe(0);
161 expect(existsSync(buildCacheRoot(data, "app1", "pull-3"))).toBe(false);
162 expect(readFileSync(join(data, "apps", "app1", "production", "data", "db"), "utf8")).toBe("keep");
163 });
164 });
165
166 describe("clearCache, the Clear button's action", () => {
167 let root = "";
168 let app: App;
169 beforeAll(() => {
170 root = tmp();
171 app = createApp({ dataDir: root, port: 0 });
172 });
173 afterAll(() => {
174 app.close();
175 rmSync(root, { recursive: true, force: true });
176 });
177 test("an admin clears it; a stranger and a signed-out visitor are told nothing", async () => {
178 const mk = async (username: string, first = false) => {
179 const u = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst: first });
180 if (!u.ok) throw new Error(u.error);
181 return u.value;
182 };
183 const alice = await mk("alice", true);
184 const bob = await mk("bob");
185 proveAll(app);
186 const made = await createRepo(app, alice, { name: "rusty" });
187 if (!made.ok) throw new Error(made.error);
188 const rec = makeAppRecord(app.db, { repoId: made.value.id, slug: "rusty", access: "public" });
189 if (!rec.ok) throw new Error(rec.error);
190 const cacheRoot = buildCacheRoot(root, rec.value.id, "production");
191 const { dir } = prepareBuildCache(cacheRoot, "a".repeat(16));
192 writeFileSync(join(dir, "target", "f"), "x");
193 writeFileSync(join(cacheRoot, ".size"), "4096");
194
195 expect(clearCache(app, bob, made.value)).toEqual({ ok: false, error: "not_found" });
196 expect(clearCache(app, undefined, made.value)).toEqual({ ok: false, error: "not_found" });
197 expect(existsSync(dir)).toBe(true);
198 expect(clearCache(app, alice, made.value)).toEqual({ ok: true, value: { freed: 4096 } });
199 expect(existsSync(dir)).toBe(false);
200 });
201 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs