Public: anyone can read this; the crew can change it.

App builds can name a pinned toolchain (Rust first), with a per-app build cache #10483

Parked opened this in answer to
Closed without merging

scout/forge:scout/10426-app-builds-can-name-a-pinned-toolchain-r → main 8 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
15 files changed +455 −14
docs/REFERENCE.md +1
@@ -500,6 +500,7 @@ This page is the front door. Everything else is a file of its own under
500500 **Using it**
501501
502502 - [`docs/VISION.md`](VISION.md) — what the forge is for, and the order of work.
503 - [`docs/TOOLCHAINS.md`](TOOLCHAINS.md) — pinned toolchains for app builds (Rust first) and the per-app build cache.
503504 - [`docs/STUDIO.md`](STUDIO.md) — the front door in full: the five words and where each is decided, the box, the three lists, a tell, a run and a change each on its own page, and the board.
504505 - [`docs/TOUR.md`](TOUR.md) — every other page and screen, with each of its doors: the terminal over SSH, the home and its figures, review, signing in, the page-view beacon, the board's notes, long threads, who can see a repository, archive, where a repository came from, big directories, search, sharing to the forge, link-preview cards and choosing among variants.
505506 - [`docs/ASKS.md`](ASKS.md) — what waits on you and how to answer it: waiting pulls, mutes, wave proposals, feed chips and the decider.
docs/TOOLCHAINS.md added +27
@@ -0,0 +1,27 @@
1 # Pinned toolchains and the build cache
2
3 An app's `forge.json` may name tools its install and test steps need:
4
5 ```json
6 { "app": { "run": "./server", "install": "cargo build --release" },
7 "toolchain": { "rust": "1.85.0" } }
8 ```
9
10 - Only exact versions of known tools (`rust`) are accepted.
11 - The forge never downloads a toolchain at build time. The operator installs
12 one once at `<FORGE_TOOLCHAINS or $FORGE_DATA/toolchains>/rust/1.85.0/bin`
13 (a `rustup` or nix-built tree); a build only reads it, read-only, and puts its
14 `bin` first on `PATH`. A version the forge lacks is refused with one line:
15 `this forge does not provide rust 1.99.0: ask its operator to install it, or name a version it has`.
16 - **Build cache** (`src/apps/build_cache.ts`): `<data>/build-cache/<app>/<key>`,
17 key = hash of `Cargo.lock`, `bun.lock`, `package-lock.json` and the pins. Steps
18 get `FORGE_BUILD_CACHE`, `CARGO_HOME` and `CARGO_TARGET_DIR` inside it, so cargo's
19 registry and target dir (and a framework checkout kept under `$FORGE_BUILD_CACHE`)
20 survive deploys. The two newest keys are kept. The app's admin clears it from
21 the app page (**Build cache → Clear**).
22 - **Limits**: a build with a toolchain gets 15 minutes for each of install and test;
23 when it hits them the failure reads "the build used its 15 minutes".
24
25 Left out: CPU and memory caps beyond the cell's existing limits, and the
26 cache is not yet shown on the page when the time limit is hit beyond the
27 deploy's failure line.
scripts/unreached-allow.json −1
@@ -15,7 +15,6 @@
1515 "src/flow/review_rounds.ts",
1616 "src/flow/shared_red.ts",
1717 "src/flow/stale_fold.ts",
18 "src/kernel/why_now.ts",
1918 "src/spawn/pod.ts",
2019 "src/tui/gallery.ts",
2120 "src/tui/ui/qr.ts",
src/actions/build_cache.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * An app's build cache, from the owner's side (`apps/build_cache.ts`): how
3 * much is kept, and clearing it. Admin of the repository only; a reader or
4 * a stranger is told `not_found`, as everywhere.
5 */
6 import type { App } from "../app.ts";
7 import { clearBuildCache, buildCacheBytes } from "../apps/build_cache.ts";
8 import { getAppByRepo } from "../core/apps.ts";
9 import type { Repo } from "../core/repos.ts";
10 import type { User } from "../core/users.ts";
11 import { err, ok, type Result } from "../kernel/result.ts";
12 import { requireRepoAdmin } from "./access.ts";
13
14 /** Bytes the app's build cache holds; `not_found` for anyone who may not clear it. */
15 export function buildCacheSize(app: App, actor: User | undefined, repo: Repo): Result<number, "not_found"> {
16 const admin = requireRepoAdmin(app, actor, repo);
17 if (!admin.ok) return admin;
18 const record = getAppByRepo(app.db, repo.id);
19 if (!record) return err("not_found");
20 return ok(buildCacheBytes(app.config.dataDir, record.id));
21 }
22
23 /** Clear the app's build cache; the next build starts cold. */
24 export function clearAppBuildCache(app: App, actor: User | undefined, repo: Repo): Result<boolean, "not_found"> {
25 const admin = requireRepoAdmin(app, actor, repo);
26 if (!admin.ok) return admin;
27 const record = getAppByRepo(app.db, repo.id);
28 if (!record) return err("not_found");
29 return ok(clearBuildCache(app.config.dataDir, record.id));
30 }
src/apps/build_cache.ts added +83
@@ -0,0 +1,83 @@
1 /**
2 * An app's build cache: what a build would otherwise redo every deploy —
3 * cargo's registry and target directory, a framework's checkout. One per
4 * app and per key, `<data>/build-cache/<app>/<key>`, where the key is a
5 * hash of the lockfiles and the toolchain pins: change either and the build
6 * starts on a fresh directory, the old one kept until it is the third
7 * oldest. Kept across deploys, cleared by the app's admin from its page.
8 */
9
10 import { createHash } from "node:crypto";
11 import { existsSync, mkdirSync, readdirSync, rmSync, statSync, utimesSync } from "node:fs";
12 import { join } from "node:path";
13 import type { ToolchainSpec } from "./toolchain.ts";
14
15 /** The lockfiles a key reads, in this order. */
16 export const LOCKFILES = ["Cargo.lock", "bun.lock", "package-lock.json"] as const;
17
18 /** How many keys an app keeps; older ones are removed when a new one is made. */
19 export const KEYS_KEPT = 2;
20
21 const safe = (id: string): string => id.replace(/[^A-Za-z0-9_-]/g, "_");
22
23 export function buildCacheRoot(dataDir: string, appId: string): string {
24 return join(dataDir, "build-cache", safe(appId));
25 }
26
27 /** The key for these lockfile contents (`undefined` for an absent file) and pins. */
28 export function buildCacheKey(
29 files: Readonly<Record<string, string | undefined>>,
30 pins: ToolchainSpec,
31 ): string {
32 const hash = createHash("sha256");
33 for (const name of LOCKFILES) hash.update(`${name}\0${files[name] ?? ""}\0`);
34 for (const [tool, version] of Object.entries(pins).sort()) hash.update(`${tool}=${version}\0`);
35 return hash.digest("hex").slice(0, 16);
36 }
37
38 /** Make (or reuse) the directory for a key, drop the oldest beyond {@link KEYS_KEPT}, return it. */
39 export function openBuildCache(dataDir: string, appId: string, key: string): string {
40 const root = buildCacheRoot(dataDir, appId);
41 const dir = join(root, key);
42 mkdirSync(dir, { recursive: true });
43 const now = new Date();
44 utimesSync(dir, now, now);
45 const others = readdirSync(root)
46 .filter((name) => name !== key)
47 .map((name) => ({ name, at: statSync(join(root, name)).mtimeMs }))
48 .sort((a, b) => b.at - a.at);
49 for (const old of others.slice(KEYS_KEPT - 1)) rmSync(join(root, old.name), { recursive: true, force: true });
50 return dir;
51 }
52
53 /** What a step is told about its cache. */
54 export function buildCacheEnv(dir: string): Record<string, string> {
55 return {
56 FORGE_BUILD_CACHE: dir,
57 CARGO_HOME: join(dir, "cargo"),
58 CARGO_TARGET_DIR: join(dir, "target"),
59 };
60 }
61
62 /** Remove everything cached for an app. Returns whether there was anything. */
63 export function clearBuildCache(dataDir: string, appId: string): boolean {
64 const root = buildCacheRoot(dataDir, appId);
65 if (!existsSync(root)) return false;
66 rmSync(root, { recursive: true, force: true });
67 return true;
68 }
69
70 /** Bytes on disk under an app's cache (files only, no follow of links). */
71 export function buildCacheBytes(dataDir: string, appId: string): number {
72 const walk = (dir: string): number => {
73 let total = 0;
74 for (const entry of readdirSync(dir, { withFileTypes: true })) {
75 const path = join(dir, entry.name);
76 if (entry.isDirectory()) total += walk(path);
77 else if (entry.isFile()) total += statSync(path).size;
78 }
79 return total;
80 };
81 const root = buildCacheRoot(dataDir, appId);
82 return existsSync(root) ? walk(root) : 0;
83 }
src/apps/manifest.ts +9
@@ -17,6 +17,7 @@ import type { DeclaredLimits } from "./contain.ts";
1717 import { isPublicAddress } from "./egress.ts";
1818 import { isLaunch, LAUNCHES, type Launch } from "../git/builds.ts";
1919 import { parseAdapter, type AdapterSpec } from "./adapter.ts";
20 import { parseToolchain, type ToolchainSpec } from "./toolchain.ts";
2021
2122 export interface AppManifest {
2223 /** The command that serves on `$PORT`. Run through `sh -c`, `exec`-ed. */
@@ -82,6 +83,11 @@ export interface AppManifest {
8283 * schedule (`apps/adapter.ts`). Unset is not an adapter.
8384 */
8485 readonly adapter?: AdapterSpec | undefined;
86 /**
87 * `toolchain` (top level of `forge.json`): pinned tools the install and
88 * test steps get, with a build cache (`apps/toolchain.ts`). Unset is none.
89 */
90 readonly toolchain?: ToolchainSpec | undefined;
8591 }
8692
8793 export type ManifestError =
@@ -185,6 +191,8 @@ export function parseManifest(
185191 return invalid("forge.json: toolbar must be true or false");
186192 const adapter = parseAdapter(parsed.value["adapter"]);
187193 if (!adapter.ok) return invalid(adapter.error);
194 const toolchain = parseToolchain(parsed.value["toolchain"]);
195 if (!toolchain.ok) return invalid(toolchain.error);
188196 // Only `egress` opens a host (docs/ADAPTERS.md): a source host it does
189197 // not list would be refused by the proxy, so the block is invalid.
190198 const unopened = adapter.value?.source?.hosts.find(
@@ -226,6 +234,7 @@ export function parseManifest(
226234 limits: limits.value,
227235 toolbar: toolbar ?? true,
228236 ...(adapter.value === undefined ? {} : { adapter: adapter.value }),
237 ...(toolchain.value === undefined ? {} : { toolchain: toolchain.value }),
229238 });
230239 }
231240
src/apps/process.ts +79 −9
@@ -22,6 +22,9 @@ import { spellings, stripForgeDirs } from "../runs/paths.ts";
2222 import { seconds } from "./timing.ts";
2323 import { notFoundHint, shellCommand } from "./run_command.ts";
2424 import { cellInstall, meteredInstall, prefillInstallCache, withLock } from "../checks/install.ts";
25 import { existsSync, readFileSync } from "node:fs";
26 import { LOCKFILES, buildCacheEnv, buildCacheKey, openBuildCache } from "./build_cache.ts";
27 import { resolveToolchain, toolchainRoot, withToolchainPath } from "./toolchain.ts";
2528 import {
2629 commandPid,
2730 killTree,
@@ -86,6 +89,44 @@ export interface StartInput {
8689 * fill will not trust (no `bun.lock`) installs as before, into its home.
8790 */
8891 readonly installCache?: string | undefined;
92 /**
93 * The forge's data directory and the app's id, which a pinned toolchain
94 * and its build cache are found by (`apps/toolchain.ts`, `build_cache.ts`).
95 */
96 readonly forgeData?: string | undefined;
97 readonly appId?: string | undefined;
98 }
99
100 /** What a build with a pinned toolchain adds to its steps. */
101 export interface BuildKit {
102 readonly env: Record<string, string>;
103 readonly reveal: readonly string[];
104 readonly writable: readonly string[];
105 }
106
107 /** A toolchain build's time: named in its failure, so the app page can say it. */
108 export const BUILD_MINUTES = 15;
109
110 /**
111 * The toolchain and cache an app's steps get, `undefined` when it names no
112 * toolchain, or the one line saying the forge does not provide it.
113 */
114 export function buildKit(input: StartInput, path: string): Result<BuildKit | undefined, string> {
115 const spec = input.manifest.toolchain;
116 if (spec === undefined || input.forgeData === undefined || input.appId === undefined) return ok(undefined);
117 const provided = resolveToolchain(toolchainRoot(input.forgeData), spec);
118 if (!provided.ok) return provided;
119 const files: Record<string, string | undefined> = {};
120 for (const name of LOCKFILES) {
121 const at = `${input.dir}/${name}`;
122 files[name] = existsSync(at) ? readFileSync(at, "utf8") : undefined;
123 }
124 const cache = openBuildCache(input.forgeData, input.appId, buildCacheKey(files, spec));
125 return ok({
126 env: { ...buildCacheEnv(cache), PATH: withToolchainPath(path, provided.value) },
127 reveal: provided.value.dirs,
128 writable: [cache],
129 });
89130 }
90131
91132 /** What a caller knows about a cell; the rest comes from the manifest. */
@@ -314,6 +355,8 @@ export async function runStep(
314355 timeoutMs?: number;
315356 /** Run inside a cell (`apps/contain.ts`) instead of as a plain child. */
316357 cell?: CellSpec | undefined;
358 /** What a timeout says it used, instead of `took longer than Ns`. */
359 timeoutWords?: string | undefined;
317360 },
318361 ): Promise<Result<void, string>> {
319362 options.log(`$ ${command}`);
@@ -357,7 +400,7 @@ export async function runStep(
357400 // is sent, so a step in flight never outlives its own cancellation.
358401 const stopReading = new AbortController();
359402 const timer = setTimeout(() => {
360 why = `${name} took longer than ${Math.round(timeoutMs / 1000)}s`;
403 why = options.timeoutWords ?? `${name} took longer than ${Math.round(timeoutMs / 1000)}s`;
361404 kill();
362405 stopReading.abort();
363406 }, timeoutMs);
@@ -398,13 +441,24 @@ export async function runStep(
398441 }
399442 }
400443
444 /** A toolchain build gets {@link BUILD_MINUTES}, and says so when it uses them. */
445 function limitsFor(kit: BuildKit | undefined): { timeoutMs?: number; timeoutWords?: string } {
446 return kit === undefined
447 ? {}
448 : {
449 timeoutMs: BUILD_MINUTES * 60_000,
450 timeoutWords: `the build used its ${BUILD_MINUTES} minutes (the time limit for a build with a toolchain)`,
451 };
452 }
453
401454 /** The cell a step of this deploy runs in: the checkout and the home, the registry and what the app declared. */
402 export function stepCell(input: StartInput): CellSpec | undefined {
455 export function stepCell(input: StartInput, kit?: BuildKit): CellSpec | undefined {
403456 const base = input.cell;
404457 if (base === undefined) return undefined;
405458 return {
406459 dir: input.dir,
407 writable: [],
460 writable: kit?.writable ?? [],
461 ...(kit === undefined ? {} : { reveal: kit.reveal }),
408462 home: base.home,
409463 hide: base.hide,
410464 egress: [...STEP_HOSTS, ...(input.manifest.egress ?? [])],
@@ -443,9 +497,10 @@ export function installPlan(
443497 input: StartInput,
444498 env: Record<string, string>,
445499 filled: boolean,
500 kit?: BuildKit,
446501 ): InstallPlan {
447502 const cache = input.installCache;
448 const cell = stepCell(input);
503 const cell = stepCell(input, kit);
449504 if (cache === undefined || !filled) return { command, env, cell, lock: undefined };
450505 if (cell !== undefined)
451506 return { ...cellInstall(command, env, cache), cell: { ...cell, reveal: [...(cell.reveal ?? []), cache] }, lock: undefined };
@@ -461,6 +516,7 @@ async function install(
461516 command: string,
462517 input: StartInput,
463518 env: Record<string, string>,
519 kit?: BuildKit,
464520 ): Promise<Result<void, string>> {
465521 let filled = false;
466522 if (input.installCache !== undefined) {
@@ -475,9 +531,16 @@ async function install(
475531 if (fill.ok) filled = fill.value;
476532 else input.log(`cache not filled: ${fill.error}`);
477533 }
478 const plan = installPlan(command, input, env, filled);
534 const plan = installPlan(command, input, env, filled, kit);
479535 const step = (): Promise<Result<void, string>> =>
480 runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell });
536 runStep("install", plan.command, {
537 cwd: input.dir,
538 env: plan.env,
539 log: input.log,
540 signal: input.signal,
541 cell: plan.cell,
542 ...limitsFor(kit),
543 });
481544 if (plan.lock !== undefined) {
482545 const held = await withLock(plan.lock, step, { signal: input.signal });
483546 return held.ok ? held.value : held;
@@ -507,8 +570,14 @@ export async function startProcess(
507570 ...forgeRegistry(),
508571 });
509572
573 // A pinned toolchain and its cache, or the one line that refuses the build.
574 const provided = input.warm === true ? ok(undefined) : buildKit(input, stepEnv["PATH"] ?? "");
575 if (!provided.ok) return err({ stage: "install", message: provided.error });
576 const kit = provided.value;
577 const stepEnvKit = kit === undefined ? stepEnv : { ...stepEnv, ...kit.env };
578
510579 if (manifest.install !== null && input.warm !== true) {
511 const installed = await install(manifest.install, input, stepEnv);
580 const installed = await install(manifest.install, input, stepEnvKit, kit);
512581 if (!installed.ok)
513582 return err({ stage: "install", message: installed.error });
514583 }
@@ -529,10 +598,11 @@ export async function startProcess(
529598 ? Promise.resolve(ok())
530599 : runStep("test", manifest.test, {
531600 cwd: input.dir,
532 env: stepEnv,
601 env: stepEnvKit,
533602 log,
534603 signal: testAbort.signal,
535 cell: stepCell(input),
604 cell: stepCell(input, kit),
605 ...limitsFor(kit),
536606 });
537607
538608 // The port is chosen, then released, then bound by the app: another
src/apps/runtime.ts +2
@@ -1510,6 +1510,8 @@ export class AppsRuntime {
15101510 // Every environment, production too, installs from the cache the
15111511 // repository's checks fill, so a new preview fetches nothing it has.
15121512 installCache: checkInstallCache(this.app.config.dataDir, repo.id),
1513 forgeData: this.app.config.dataDir,
1514 appId: state.app.id,
15131515 };
15141516 this.noteCell(run.id, write);
15151517 clock.lap("setup");
src/apps/toolchain.ts added +78
@@ -0,0 +1,78 @@
1 /**
2 * A pinned toolchain for an app's build and test steps: `toolchain` at the
3 * top level of `forge.json`, `{"rust": "1.85.0"}`. The forge never fetches
4 * one at build time. A toolchain is a directory its operator installed once
5 * — `<toolchains>/<tool>/<version>/bin` — and a build only reads it, so a
6 * version the forge does not hold is refused with one plain line, not
7 * downloaded. `<toolchains>` is `FORGE_TOOLCHAINS`, else `<data>/toolchains`.
8 */
9
10 import { existsSync, statSync } from "node:fs";
11 import { join } from "node:path";
12 import { err, ok, type Result } from "../kernel/result.ts";
13
14 /** The tools a manifest may name; a closed set. */
15 export const TOOLCHAINS = ["rust"] as const;
16 export type Tool = (typeof TOOLCHAINS)[number];
17
18 /** `{ rust: "1.85.0" }`: each named tool with its exact version. */
19 export type ToolchainSpec = { readonly [T in Tool]?: string };
20
21 const VERSION_RE = /^\d{1,3}\.\d{1,3}(\.\d{1,3})?$/;
22
23 /** The manifest's `toolchain` value, validated; `undefined` when absent. */
24 export function parseToolchain(
25 value: unknown,
26 ): Result<ToolchainSpec | undefined, string> {
27 if (value === undefined) return ok(undefined);
28 if (typeof value !== "object" || value === null || Array.isArray(value))
29 return err('forge.json: toolchain must be an object like {"rust": "1.85.0"}');
30 const out: { [T in Tool]?: string } = {};
31 for (const [tool, version] of Object.entries(value)) {
32 if (!(TOOLCHAINS as readonly string[]).includes(tool))
33 return err(`forge.json: toolchain takes ${TOOLCHAINS.join(", ")}, not ${tool.slice(0, 40)}`);
34 if (typeof version !== "string" || !VERSION_RE.test(version))
35 return err(`forge.json: toolchain.${tool} must be an exact version like "1.85.0"`);
36 out[tool as Tool] = version;
37 }
38 return ok(Object.keys(out).length === 0 ? undefined : out);
39 }
40
41 /** Where the operator's pinned toolchains live. */
42 export function toolchainRoot(dataDir: string): string {
43 const named = process.env["FORGE_TOOLCHAINS"];
44 return named !== undefined && named !== "" ? named : join(dataDir, "toolchains");
45 }
46
47 /** What a step is given for the toolchain: directories to read, and the PATH prefix. */
48 export interface Provided {
49 /** Each toolchain's directory, shown read-only inside the cell. */
50 readonly dirs: readonly string[];
51 /** `bin` directories put in front of PATH. */
52 readonly bins: readonly string[];
53 }
54
55 /** The toolchain directories for a spec, or the one line saying which is missing. */
56 export function resolveToolchain(
57 root: string,
58 spec: ToolchainSpec,
59 ): Result<Provided, string> {
60 const dirs: string[] = [];
61 const bins: string[] = [];
62 for (const tool of TOOLCHAINS) {
63 const version = spec[tool];
64 if (version === undefined) continue;
65 const dir = join(root, tool, version);
66 const bin = join(dir, "bin");
67 if (!existsSync(bin) || !statSync(bin).isDirectory())
68 return err(`this forge does not provide ${tool} ${version}: ask its operator to install it, or name a version it has`);
69 dirs.push(dir);
70 bins.push(bin);
71 }
72 return ok({ dirs, bins });
73 }
74
75 /** `PATH` with the toolchain's bins first. */
76 export function withToolchainPath(path: string, provided: Provided): string {
77 return [...provided.bins, path].filter((p) => p !== "").join(":");
78 }
src/http/server.ts +2
@@ -225,6 +225,7 @@ import {
225225 newIssueSubmit,
226226 } from "../web/pages/issues.ts";
227227 import { adapterPull } from "../web/studio/adapter.ts";
228 import { buildCacheClear } from "../web/studio/build_cache.ts";
228229 import { noteModerate } from "../web/studio/notes.ts";
229230 import { tellPage } from "../web/studio/tell.ts";
230231 import { issuePicture } from "../web/studio/picture.ts";
@@ -1310,6 +1311,7 @@ export function buildRouter(): Router<Handler> {
13101311 router.add("GET", "/apps/open", appsOpen);
13111312 router.add("POST", "/:owner/:name/app/notes", noteModerate);
13121313 router.add("POST", "/:owner/:name/app/pull", adapterPull);
1314 router.add("POST", "/:owner/:name/app/build-cache/clear", buildCacheClear);
13131315 router.add("POST", "/:owner/:name/app/:env/redeploy", appRedeploy);
13141316 router.add("POST", "/:owner/:name/app/:env/stop", appStop);
13151317 router.add("GET", "/:owner/:name/events", eventsList);
src/web/studio/app.ts +2 −1
@@ -57,6 +57,7 @@ import { nowCard, nowRegion } from "./now.ts";
5757 import { repoAsks } from "./asks.ts";
5858 import { notesCard } from "./notes.ts";
5959 import { adapterCard } from "./adapter.ts";
60 import { buildCacheCard } from "./build_cache.ts";
6061 import { appPanels } from "./previews.ts";
6162 import { noCrewLine, pausedWords, servingHistory, type WordCtx } from "./words.ts";
6263 import { crewKeyNote } from "./crew.ts";
@@ -515,7 +516,7 @@ export async function studioApp(
515516 ${notesCard(ctx, repo, record)}
516517 </div>
517518 <aside class="stack">
518 ${adapter} ${crewCard} ${weekCard} ${ownCard} ${more}
519 ${adapter} ${ownerFacts ? buildCacheCard(ctx, repo) : null} ${crewCard} ${weekCard} ${ownCard} ${more}
519520 </aside>
520521 </div>
521522 ${liveMount({ repo: base, app: true })}`;
src/web/studio/build_cache.ts added +36
@@ -0,0 +1,36 @@
1 /**
2 * The build cache card on an app's page: what a toolchain build keeps
3 * between deploys, and the owner's button to clear it. Shown to the app's
4 * admins when there is something cached.
5 */
6 import { buildCacheSize, clearAppBuildCache } from "../../actions/build_cache.ts";
7 import type { Repo } from "../../core/repos.ts";
8 import { formatBytes } from "../../core/storage.ts";
9 import type { Ctx } from "../../http/context.ts";
10 import { loadRepo, notFoundPage, repoBase } from "../chrome.ts";
11 import { readForm } from "../form.ts";
12 import { html, type Html } from "../html.ts";
13 import { flashTo } from "../layout.ts";
14 import { btn } from "../ui.ts";
15
16 export function buildCacheCard(ctx: Ctx, repo: Repo): Html | null {
17 const size = buildCacheSize(ctx.app, ctx.auth.user, repo);
18 if (!size.ok || size.value === 0) return null;
19 return html`<section class="build-cache">
20 <h2>Build cache</h2>
21 <p class="muted">Kept across deploys, keyed by the lockfiles and the toolchain pins: ${formatBytes(size.value)}.</p>
22 <form method="post" action="${repoBase(repo)}/app/build-cache/clear">${btn("Clear build cache", { size: "sm", variant: "outline" })}</form>
23 </section>`;
24 }
25
26 // POST /:owner/:name/app/build-cache/clear
27 export async function buildCacheClear(ctx: Ctx): Promise<Response> {
28 const repo = loadRepo(ctx);
29 if (!repo) return notFoundPage(ctx);
30 const size = buildCacheSize(ctx.app, ctx.auth.user, repo);
31 if (!size.ok) return notFoundPage(ctx);
32 const form = await readForm(ctx, "rw");
33 if (form instanceof Response) return form;
34 clearAppBuildCache(ctx.app, ctx.auth.user, repo);
35 return flashTo(ctx, repoBase(repo), { message: "Build cache cleared: the next build starts cold." });
36 }
test/build-cart.test.ts +8 −2
@@ -77,8 +77,14 @@ describe("a pin's cart", () => {
7777 expect(res.status).toBe(202);
7878 expect(await res.json()).toMatchObject({ demo: { name: "fiesta-kart", version: "1", launch: "file", page: "/", showcase: true } });
7979 // karts is no app, so there is no build to capture from: the state says why.
80 await Bun.sleep(50);
81 expect(await (await api("karts/builds/v1/cart")).json()).toMatchObject({ capture: { state: "failed", error: "not_an_app" } });
80 // Wait for the capture to settle, so it cannot land after the next test's import.
81 let state: unknown = {};
82 for (let i = 0; i < 100; i++) {
83 state = await (await api("karts/builds/v1/cart")).json();
84 if ((state as { capture: { state: string } }).capture.state === "failed") break;
85 await Bun.sleep(50);
86 }
87 expect(state).toMatchObject({ capture: { state: "failed", error: "not_an_app" } });
8288 });
8389
8490 test("a captured cart plays sandboxed, with its picture, and the builds page links it", async () => {
test/child-quota.test.ts +1 −1
@@ -101,7 +101,7 @@ describe("child quota", () => {
101101
102102 test("room for small but not large is refused only for large", async () => {
103103 clear();
104 process.env["FORGE_CHILD_MEMORY_MIB"] = "2000";
104 process.env["FORGE_CHILD_MEMORY_MIB"] = "4096";
105105 expect(quotaRefusal(dir, "small")).toBeUndefined();
106106 expect(quotaRefusal(dir, "large")).toContain("memory limit");
107107 });
test/toolchain.test.ts added +97
@@ -0,0 +1,97 @@
1 import { expect, test } from "bun:test";
2 import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
3 import { existsSync } from "node:fs";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { startProcess } from "../src/apps/process.ts";
7 import { parseManifest } from "../src/apps/manifest.ts";
8 import { parseToolchain } from "../src/apps/toolchain.ts";
9 import { buildCacheKey, clearBuildCache, openBuildCache, buildCacheBytes } from "../src/apps/build_cache.ts";
10
11 test("forge.json names a toolchain as exact versions of known tools", () => {
12 expect(parseToolchain(undefined)).toEqual({ ok: true, value: undefined });
13 expect(parseToolchain({ rust: "1.85.0" })).toEqual({ ok: true, value: { rust: "1.85.0" } });
14 for (const bad of [{ rust: "latest" }, { rust: 1 }, { go: "1.2.3" }, "rust", [], { rust: "1.85; rm" }])
15 expect(parseToolchain(bad).ok).toBe(false);
16 const m = parseManifest(JSON.stringify({ app: { run: "bun x.ts" }, toolchain: { rust: "1.85.0" } }), undefined);
17 expect(m.ok && m.value.toolchain).toEqual({ rust: "1.85.0" });
18 });
19
20 test("the cache key follows lockfiles and pins", () => {
21 const a = buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.85.0" });
22 expect(buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.85.0" })).toBe(a);
23 expect(buildCacheKey({ "Cargo.lock": "b" }, { rust: "1.85.0" })).not.toBe(a);
24 expect(buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.86.0" })).not.toBe(a);
25 });
26
27 test("the build cache is kept per key, bounded, and cleared", async () => {
28 const data = await mkdtemp(join(tmpdir(), "forge-bc-"));
29 try {
30 const one = openBuildCache(data, "app1", "k1");
31 await writeFile(join(one, "f"), "12345");
32 expect(buildCacheBytes(data, "app1")).toBe(5);
33 await Bun.sleep(15);
34 openBuildCache(data, "app1", "k2");
35 await Bun.sleep(15);
36 openBuildCache(data, "app1", "k3");
37 expect(existsSync(one)).toBe(false);
38 expect(clearBuildCache(data, "app1")).toBe(true);
39 expect(buildCacheBytes(data, "app1")).toBe(0);
40 } finally {
41 await rm(data, { recursive: true, force: true });
42 }
43 });
44
45 const rustApp = async (version: string, install: string) => {
46 const data = await mkdtemp(join(tmpdir(), "forge-tc-"));
47 const dir = join(data, "checkout");
48 await mkdir(dir);
49 // A stand-in cargo in a pinned toolchain: writes into the cache's target dir.
50 const bin = join(data, "toolchains", "rust", "1.85.0", "bin");
51 await mkdir(bin, { recursive: true });
52 await writeFile(join(bin, "cargo"), '#!/bin/sh\nmkdir -p "$CARGO_TARGET_DIR" && echo hello-rust > "$CARGO_TARGET_DIR/out" && cp "$CARGO_TARGET_DIR/out" built.txt\n');
53 await chmod(join(bin, "cargo"), 0o755);
54 await writeFile(join(dir, "server.ts"), "Bun.serve({port:Number(process.env.PORT),hostname:'127.0.0.1',fetch:()=>new Response(require('fs').readFileSync('built.txt','utf8'))})");
55 const lines: string[] = [];
56 const started = await startProcess({
57 dir,
58 dataDir: data,
59 forgeData: data,
60 appId: "app9",
61 manifest: { run: "bun server.ts", install, health: "/", ready: 8, toolchain: { rust: version } } as never,
62 env: "production",
63 url: "http://app.test",
64 commitOid: "0".repeat(40),
65 forgeUrl: "http://forge.test",
66 identityKey: "key",
67 log: (l) => void lines.push(l),
68 signal: new AbortController().signal,
69 });
70 return { data, started, lines };
71 };
72
73 test("a Rust app builds with its pinned toolchain, caches, and serves", async () => {
74 const { data, started } = await rustApp("1.85.0", "cargo build");
75 try {
76 expect(started.ok).toBe(true);
77 if (!started.ok) return;
78 const res = await fetch(`http://127.0.0.1:${started.value.handle.port}/`);
79 expect(await res.text()).toContain("hello-rust");
80 expect(buildCacheBytes(data, "app9")).toBeGreaterThan(0);
81 await started.value.handle.stop();
82 await started.value.handle.exited;
83 } finally {
84 await rm(data, { recursive: true, force: true });
85 }
86 }, 30_000);
87
88 test("an app naming a toolchain the forge lacks is refused with one line", async () => {
89 const { data, started } = await rustApp("1.99.0", "cargo build");
90 try {
91 expect(started.ok).toBe(false);
92 if (started.ok) return;
93 expect(started.error.message).toBe("this forge does not provide rust 1.99.0: ask its operator to install it, or name a version it has");
94 } finally {
95 await rm(data, { recursive: true, force: true });
96 }
97 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs