Public: anyone can read this; the crew can change it.

P0 efficiency: every contained check, gate and preview re-downloads npm (~30 GiB/day on the vxrail): share the repository's bun cache with cells #7568

Parked opened this
Closed without merging

torque/forge:torque/7541-p0-efficiency-every-contained-check-gate → main 8 commits fork 1 thread · 1 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
8 files changed +278 −12
docs/APPS.md +21 −6
@@ -691,12 +691,27 @@ queued again, and the forge that carries on runs it from the start.
691691 or `ECONNRESET` is tried once more after 2 s. If it fails that way again,
692692 the check is `failed: infrastructure`. It is queued again, and the merge
693693 queue counts it as cut off, not red. If the re-run fails the same way, it
694 fails and says `infrastructure`. A check in a cell installs into its own
695 home, so its bun cache is shared with nothing. An uncontained check uses
696 its repository's cache (`<data>/check-install/<repo>/bun`), and installs
697 into it take turns under a lock file, so two never extract one package
698 into the same directory. A `job` check installs in its own pod and is not
699 retried.
694 fails and says `infrastructure`. An uncontained check uses its
695 repository's cache (`<data>/check-install/<repo>/bun`), and installs into
696 it take turns under a lock file, so two never extract one package into the
697 same directory. A `job` check installs in its own pod with its own cache
698 and is not retried.
699
700 **A contained check installs from the shared cache**
701 (`src/checks/install.ts`, #7543). A check in a cell reads its repository's
702 cache and never writes it: before the install, the forge fills it on the
703 host from a scratch copy of the commit's checkout (no `bunfig.toml` or
704 `.npmrc` of the pull's, `--frozen-lockfile --ignore-scripts`, under the
705 same lock), and binds it read-only into the cell, which installs from it
706 copying. A warm install downloads nothing. The forge fills only a lockfile
707 whose every package is a plain `name@version` from the pinned registry
708 with a sha512 integrity (`src/checks/fillable.ts`); anything else
709 (workspaces, git, file or tarball URLs, patches, an override or
710 resolution to any of those, a binary `bun.lockb`)
711 installs into the cell's own cache as before, and the log says why.
712 Nothing evicts the shared cache yet: it grows by every package version any
713 checked commit of the repository has locked, forks' pulls included,
714 bounded only by the disk (eviction is #7574).
700715
701716 **Placement.** `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check
702717 in a cell on the forge's machine. `job` runs install and check as one
docs/THREATS.md +11
@@ -94,6 +94,17 @@ person approved.
9494 and only this repository's directory re-allowed, so a cache made later is
9595 denied too, and deleting the repository removes it.
9696 Test: `test/crew-install-cache.test.ts`
97 - A check's cell reads its repository's shared bun cache
98 (`check-install/<repo id>/bun`) through a read-only bind and cannot write
99 it, so one pull cannot plant a package the gate on main installs. The
100 forge fills it on the host with `--ignore-scripts`, and reads no
101 `bunfig.toml` or `.npmrc` of the pull's.
102 Test: `test/check-install.test.ts`
103 - The forge fills that cache only from a lockfile whose every package is a
104 registry `name@version` with a sha512 integrity (`checks/fillable.ts`):
105 the host never fetches a URL a pull chose and reads no file, link or
106 workspace a pull points at, an override's or resolution's included.
107 Test: `test/check-fillable.test.ts`
97108 - The same policy holds against a real bubblewrap, not only as a string.
98109 Test: `test/crew-sandbox-real.test.ts`
99110 - An agent cannot push to the default branch or merge; a person merges.
scripts/rehearse-roll.ts +15 −3
@@ -19,7 +19,7 @@
1919 * environment, never this shell's.
2020 */
2121
22 import { cpSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
22 import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
2323 import { tmpdir } from "node:os";
2424 import { join, resolve } from "node:path";
2525 import {
@@ -84,6 +84,18 @@ function start(
8484 return { name, url: `http://127.0.0.1:${port}`, proc, log };
8585 }
8686
87 /**
88 * The last lines a forge wrote to stderr, for the error that names its log:
89 * a gate's temp directory is gone by the time its log is read, so the log
90 * would otherwise point at a file nobody can open (#7632).
91 */
92 export function errTail(forge: Pick<Forge, "log">, lines = 20): string {
93 const path = `${forge.log}.err`;
94 if (!existsSync(path)) return "";
95 const tail = readFileSync(path, "utf8").trimEnd().split("\n").slice(-lines).join("\n");
96 return tail === "" ? "" : `\n${tail}`;
97 }
98
8799 /** One readiness probe: ready, not ready (a 503, or nothing listening), or silent past its timeout. */
88100 async function probe(forge: Forge, timeoutMs: number): Promise<ProbeAnswer> {
89101 try {
@@ -145,7 +157,7 @@ export async function rehearse(
145157 const bootBy = Date.now() + 60_000;
146158 while (!(await ready(old, 1000))) {
147159 if (old.proc.exitCode !== null || Date.now() > bootBy)
148 throw new Error(`the old forge did not come up; see ${old.log}.err`);
160 throw new Error(`the old forge did not come up; see ${old.log}.err${errTail(old)}`);
149161 await Bun.sleep(100);
150162 }
151163 rotation.saw(old, true);
@@ -208,7 +220,7 @@ export async function rehearse(
208220 }
209221 }
210222 if (next.proc.exitCode !== null)
211 throw new Error(`the new forge exited; see ${next.log}.err`);
223 throw new Error(`the new forge exited; see ${next.log}.err${errTail(next)}`);
212224 await Bun.sleep(args.probeMs);
213225 }
214226 };
src/checks/fillable.ts added +96
@@ -0,0 +1,96 @@
1 /**
2 * Whether the forge may fill a repository's shared bun cache from a commit's
3 * lockfile (#7543, #7541). The fill runs on the host, outside a cell's
4 * egress (`checks/install.ts`, `prefillInstallCache`), so a lockfile is
5 * filled only when every package in it is a plain `name@version` from the
6 * pinned registry with a sha512 integrity bun checks: no tarball URL, git,
7 * file, link or workspace dependency, and no patches. Anything else installs
8 * as before, into the cell's own cache.
9 */
10
11 import { err, ok, type Result } from "../kernel/result.ts";
12
13 /** `[email protected]` or `@scope/[email protected]+build`: a registry package at one version. */
14 const REGISTRY_ID = /^(@[a-z0-9][\w.-]*\/)?[a-z0-9][\w.-]*@\d+\.\d+\.\d+[\w.+-]*$/i;
15
16 /** A dependency range or tag, optionally an `npm:` alias: never a path, URL, git or workspace spec. */
17 const REGISTRY_SPEC = /^(npm:(@[\w.-]+\/)?[\w.-]+@)?[\w\s.^~<>=|*+-]*$/;
18
19 const DEPENDENCY_FIELDS = ["dependencies", "devDependencies", "optionalDependencies", "peerDependencies"];
20
21 /**
22 * Whether the forge may fill the shared cache from this `package.json` and
23 * `bun.lock`: an Err names what keeps it out. Every package must come from
24 * `registry` by name and version, with a sha512 integrity bun checks.
25 */
26 export function fillable(packageJson: string, lock: string, registry: string): Result<void, string> {
27 let manifest: unknown;
28 let parsed: unknown;
29 try {
30 manifest = JSON.parse(packageJson);
31 parsed = Bun.JSONC.parse(lock);
32 } catch {
33 return err("package.json or bun.lock does not parse");
34 }
35 if (!isRecord(manifest) || !isRecord(parsed)) return err("package.json or bun.lock is not an object");
36 if (manifest["workspaces"] !== undefined) return err("workspaces");
37 for (const field of DEPENDENCY_FIELDS) {
38 const deps = manifest[field];
39 if (deps === undefined) continue;
40 if (!isRecord(deps)) return err(`${field} is not an object`);
41 for (const [name, spec] of Object.entries(deps))
42 if (typeof spec !== "string" || !REGISTRY_SPEC.test(spec))
43 return err(`${name} is not a registry dependency`);
44 }
45 if (parsed["patchedDependencies"] !== undefined) return err("patched dependencies");
46 // An override replaces a package wherever it is locked: a path, URL or git
47 // spec in one is as much the host's fetch as a dependency's would be.
48 for (const [where, field] of [[manifest, "overrides"], [manifest, "resolutions"], [parsed, "overrides"]] as const) {
49 const bad = field in where ? nonRegistry(where[field]) : undefined;
50 if (bad !== undefined) return err(`${field}: ${bad} is not a registry version`);
51 }
52 const workspaces = parsed["workspaces"];
53 if (!isRecord(workspaces) || Object.keys(workspaces).some((key) => key !== ""))
54 return err("workspaces");
55 const packages = parsed["packages"] ?? {};
56 if (!isRecord(packages)) return err("bun.lock has no packages");
57 let origin: string;
58 try {
59 origin = new URL(registry).origin;
60 } catch {
61 return err("the registry is not a URL");
62 }
63 for (const [key, entry] of Object.entries(packages)) {
64 if (!Array.isArray(entry) || entry.length !== 4) return err(`${key} is not a registry package`);
65 const [id, url, , integrity] = entry as unknown[];
66 if (typeof id !== "string" || !REGISTRY_ID.test(id)) return err(`${key} is not a registry package`);
67 if (url !== "" && !(typeof url === "string" && sameOrigin(url, origin)))
68 return err(`${key} comes from another registry`);
69 if (typeof integrity !== "string" || !integrity.startsWith("sha512-"))
70 return err(`${key} has no sha512 integrity`);
71 }
72 return ok(undefined);
73 }
74
75 /** The first override spec that is not a registry range, nested ones included; undefined when all are. */
76 function nonRegistry(value: unknown, name = "overrides"): string | undefined {
77 if (typeof value === "string") return REGISTRY_SPEC.test(value) ? undefined : name;
78 if (!isRecord(value)) return name;
79 for (const [key, inner] of Object.entries(value)) {
80 const bad = nonRegistry(inner, key);
81 if (bad !== undefined) return bad;
82 }
83 return undefined;
84 }
85
86 function sameOrigin(url: string, origin: string): boolean {
87 try {
88 return new URL(url).origin === origin;
89 } catch {
90 return false;
91 }
92 }
93
94 function isRecord(value: unknown): value is Record<string, unknown> {
95 return typeof value === "object" && value !== null && !Array.isArray(value);
96 }
src/checks/install.ts +6 −1
@@ -20,6 +20,7 @@ import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rm
2020 import { tmpdir } from "node:os";
2121 import { dirname, join } from "node:path";
2222 import { err, ok, type Result } from "../kernel/result.ts";
23 import { fillable } from "./fillable.ts";
2324
2425 /** What an install prints when the machine, not the code, failed it. */
2526 export const TRANSIENT_INSTALL = /Fail extracting tarball|EINTEGRITY|ECONNRESET/;
@@ -98,7 +99,11 @@ export function lockfileRefusal(dir: string, registry: string): string | undefin
9899 if (!url.startsWith(registry)) return `the lockfile names ${url.slice(0, 80)}, not the registry`;
99100 if (/[@"'/](?:github|gitlab|bitbucket|git|ssh|git\+\w+):/i.test(text))
100101 return "the lockfile has a git dependency";
101 return undefined;
102 // Every package by name and version with a sha512 integrity, or no fill:
103 // a file, link or workspace dependency, or a patch, is read on the host.
104 const manifest = existsSync(join(dir, "package.json")) ? readFileSync(join(dir, "package.json"), "utf8") : "{}";
105 const strict = fillable(manifest, text, registry);
106 return strict.ok ? undefined : strict.error;
102107 }
103108
104109 /**
test/check-fillable.test.ts added +105
@@ -0,0 +1,105 @@
1 /**
2 * What the forge fills its shared bun cache from, on the host (#7543,
3 * #7541): only a lockfile whose every package is a registry `name@version`
4 * with a sha512 integrity; anything else installs into the cell's own cache.
5 */
6 import { describe, expect, test } from "bun:test";
7 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { fillable } from "../src/checks/fillable.ts";
11 import { lockfileRefusal } from "../src/checks/install.ts";

Non-blocking: two blank lines after the imports; drop one.

12
13 const lockOf = (packages: Record<string, unknown>, workspaces: Record<string, unknown> = { "": {} }): string =>
14 JSON.stringify({ lockfileVersion: 1, workspaces, packages });
15 const SHA = "sha512-AAAA";
16
17 describe("fillable: what the forge fills on the host", () => {
18 const registry = "https://registry.npmjs.org/";
19
20 test("plain registry packages by name and version are filled, scoped and aliased too", () => {
21 const lock = `{
22 "lockfileVersion": 1,
23 "workspaces": { "": { "name": "p", "dependencies": { "a": "^1.0.0" }, }, },
24 "packages": {
25 "a": ["[email protected]", "", {}, "${SHA}"],
26 "@s/b": ["@s/[email protected]", "", { "os": "linux" }, "${SHA}"],
27 "ts-api": ["[email protected]", "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", {}, "${SHA}"],
28 },
29 }`;
30 const manifest = '{"dependencies":{"a":"^1.0.0","@s/b":"2.0.0-rc.1"},"devDependencies":{"ts-api":"npm:typescript@^5.9.0","c":"latest"}}';
31 expect(fillable(manifest, lock, registry)).toEqual({ ok: true, value: undefined });
32 });
33
34 test("anything the registry does not serve by name keeps the fill off the host", () => {
35 const deny = (manifest: string, lock: string): string => {
36 const result = fillable(manifest, lock, registry);
37 if (result.ok) throw new Error("filled");
38 return result.error;
39 };
40 const plain = "{}";
41 expect(deny(plain, lockOf({ x: ["[email protected]", "http://10.0.0.1/x.tgz", {}, SHA] }))).toContain("another registry");
42 expect(deny(plain, lockOf({ x: ["x@github:o/r#abc", {}, "o-r-abc"] }))).toContain("not a registry package");
43 expect(deny(plain, lockOf({ x: ["x@file:../../etc", {}] }))).toContain("not a registry package");
44 expect(deny(plain, lockOf({ x: ["[email protected]", "", {}, "sha1-abc"] }))).toContain("sha512");
45 expect(deny(plain, lockOf({}, { "": {}, "packages/a": {} }))).toBe("workspaces");
46 expect(deny('{"workspaces":["packages/*"]}', lockOf({}))).toBe("workspaces");
47 expect(deny('{"dependencies":{"x":"file:../x"}}', lockOf({}))).toContain("x is not a registry dependency");
48 expect(deny('{"dependencies":{"x":"o/r"}}', lockOf({}))).toContain("x is not a registry dependency");
49 expect(deny('{"dependencies":{"x":"git+https://h/r.git"}}', lockOf({}))).toContain("x is not");
50 expect(deny(plain, "not json")).toContain("does not parse");
51 expect(deny(plain, JSON.stringify({ lockfileVersion: 1, workspaces: { "": {} }, patchedDependencies: {} }))).toBe(
52 "patched dependencies",
53 );
54 });
55
56 test("an override or resolution that is not a registry version keeps the fill off the host", () => {
57 const deny = (manifest: string, lock: string): string => {
58 const result = fillable(manifest, lock, registry);
59 if (result.ok) throw new Error("filled");
60 return result.error;
61 };
62 expect(deny('{"overrides":{"x":"file:../../etc"}}', lockOf({}))).toBe("overrides: x is not a registry version");
63 expect(deny('{"overrides":{"a":{"x":"github:o/r"}}}', lockOf({}))).toBe("overrides: x is not a registry version");
64 expect(deny('{"resolutions":{"x":"https://h/x.tgz"}}', lockOf({}))).toBe("resolutions: x is not a registry version");
65 const locked = JSON.stringify({ lockfileVersion: 1, workspaces: { "": {} }, overrides: { x: "link:x" }, packages: {} });
66 expect(deny("{}", locked)).toBe("overrides: x is not a registry version");
67 const pinned = '{"overrides":{"x":"1.2.3","a":{"y":"^2.0.0"}},"resolutions":{"z":"~3.1.0"}}';
68 expect(fillable(pinned, lockOf({}), registry)).toEqual({ ok: true, value: undefined });
69 });
70 });
71
72 describe("lockfileRefusal: the fill asks fillable too", () => {
73 const registry = "https://registry.npmjs.org/";
74 const checkout = (manifest: string, lock: string): string => {
75 const dir = mkdtempSync(join(tmpdir(), "fillable-"));
76 writeFileSync(join(dir, "package.json"), manifest);
77 writeFileSync(join(dir, "bun.lock"), lock);
78 return dir;
79 };
80
81 test("a registry lockfile with sha512 integrities is filled", () => {
82 const dir = checkout('{"dependencies":{"a":"1.2.3"}}', lockOf({ a: ["[email protected]", "", {}, SHA] }));
83 try {
84 expect(lockfileRefusal(dir, registry)).toBeUndefined();
85 } finally {
86 rmSync(dir, { recursive: true, force: true });
87 }
88 });
89
90 test("no URL and no git, yet a file dependency, a workspace or a weak integrity is still refused", () => {
91 const cases: [string, string, string][] = [
92 ['{"dependencies":{"x":"file:../../etc"}}', lockOf({ x: ["x@file:../../etc", {}] }), "not a registry"],
93 ['{"workspaces":["packages/*"]}', lockOf({}), "workspaces"],
94 ["{}", lockOf({ x: ["[email protected]", "", {}, "sha1-abc"] }), "sha512"],
95 ];
96 for (const [manifest, lock, why] of cases) {
97 const dir = checkout(manifest, lock);
98 try {
99 expect(lockfileRefusal(dir, registry)).toContain(why);
100 } finally {
101 rmSync(dir, { recursive: true, force: true });
102 }
103 }
104 });
105 });
test/check-install.test.ts +3 −1
@@ -246,6 +246,8 @@ describe("prefillInstallCache (#7543)", () => {
246246 const made = Bun.spawnSync(["tar", "-czf", join(dir, "fixture.tgz"), "-C", join(dir, "pkg"), "package"]);
247247 if (made.exitCode !== 0) throw new Error("tar is needed for the fixture registry");
248248 const tarball = new Uint8Array(await Bun.file(join(dir, "fixture.tgz")).arrayBuffer());
249 // As npm serves it: the fill trusts only a lockfile with sha512 integrities (`checks/fillable.ts`).
250 const integrity = `sha512-${new Bun.CryptoHasher("sha512").update(tarball).digest("base64")}`;
249251 const hits: string[] = [];
250252 const server: ReturnType<typeof Bun.serve> = Bun.serve({
251253 port: 0,
@@ -262,7 +264,7 @@ describe("prefillInstallCache (#7543)", () => {
262264 "1.0.0": {
263265 name: "fixture-pkg",
264266 version: "1.0.0",
265 dist: { tarball: `http://127.0.0.1:${server.port}/fixture-pkg/-/fixture-pkg-1.0.0.tgz` },
267 dist: { tarball: `http://127.0.0.1:${server.port}/fixture-pkg/-/fixture-pkg-1.0.0.tgz`, integrity },
266268 },
267269 },
268270 });
test/rehearsal.test.ts +21 −1
@@ -6,7 +6,10 @@
66 */
77
88 import { describe, expect, test } from "bun:test";
9 import { rehearse } from "../scripts/rehearse-roll.ts";
9 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { errTail, rehearse } from "../scripts/rehearse-roll.ts";
1013 import {
1114 count,
1215 DEFAULTS,
@@ -148,6 +151,23 @@ describe("the verdict", () => {
148151 });
149152 });
150153
154 describe("a forge that will not start", () => {
155 test("the error carries the tail of its stderr, not only the path to it (#7632)", () => {
156 const dir = mkdtempSync(join(tmpdir(), "rehearsal-err-"));
157 try {
158 const log = join(dir, "old.log");
159 expect(errTail({ log })).toBe("");
160 writeFileSync(`${log}.err`, `${Array.from({ length: 30 }, (_, i) => `line ${i}`).join("\n")}\n`);
161 expect(errTail({ log }, 2)).toBe("\nline 28\nline 29");
162 expect(errTail({ log }).split("\n")).toHaveLength(21);
163 writeFileSync(`${log}.err`, "");
164 expect(errTail({ log })).toBe("");
165 } finally {
166 rmSync(dir, { recursive: true, force: true });
167 }
168 });
169 });
170
151171 describe("a real roll of two forge processes on one data directory", () => {
152172 const short = (lingerS: number): RehearsalArgs => ({
153173 ...parseArgs([], `${import.meta.dir}/..`),

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs