trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns4g rEventsFlows

Repo transfer, step two: git over HTTPS and SSH, running runs, previews and forks follow a moved repository #10204

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/570-repo-transfer-step-two-git-over-https-an → main 2 commits fork

  • Merged. torque/570-repo-transfer-step-two-git-over-https-an was merged into main as e8ed808.

Review

Sign in to review this change.
6 files changed +360 −8
docs/features/repo-transfer.md +16 −3
@@ -43,10 +43,23 @@ answers `308` to the new owner for someone who may read the repository, so
4343 redirect on its first request). A push to the old URL is never applied: it is
4444 refused with `403` and a sentence naming the new path and the `git remote
4545 set-url` to run. Someone who cannot read it gets what a missing path answers.
46 Over SSH the old path is not redirected (SSH has no redirect): the remote must
47 be updated to the new path.
46 Over SSH, which has no redirect, a fetch or clone of the old path is served
47 from the new place, with one line on stderr naming the new path and the `git
48 remote set-url` to run. A push to the old path is refused with the same line
49 and never applied — it does not make a new repository there either. Someone
50 who cannot read it gets "repository not found", as for any missing path.
4851
49 Not yet: runs and previews follow in the next step.
52 Runs and apps follow the move. A deploy reads the repository when its turn
53 comes, not when it was asked for, so one queued or waiting for a permit at
54 the move runs against the new place. Production and every running preview
55 start again at the commit they serve (a run with reason `moved`), so the
56 process holds the new owner's read token; an asleep preview wakes as the new
57 owner's. Hostnames do not change: they hang off the app's slug, not its owner.
58 A crew run already under way keeps working: its token is bound to the
59 repository by id, and the run gate lets a request under the old path through
60 to the `308` that names the new one. Forks keep their parent (it is held by
61 id) and their link to its objects is rewritten; repository-scoped tokens are
62 held by id too.
5063
5164 For 30 days after a move, the repository's page says where it came from: one
5265 muted line, "Moved from alice/demo on 2026-11-09", under the head, for someone
src/apps/runtime.ts +30 −3
@@ -298,6 +298,13 @@ interface Live {
298298 */
299299 const PUSH_REASONS: readonly string[] = ["push", "opened", "merge"];
300300
301 /**
302 * The deploy reasons that start the same commit again, even while it serves
303 * or is coming up: someone pressed redeploy, or the repository moved to
304 * another owner (#570) and what runs must start again as the new owner's.
305 */
306 const AGAIN: ReadonlySet<string> = new Set(["redeploy", "moved"]);
307
301308 /** How long between two instants, in seconds to one decimal place: `3.4s`. */
302309 function elapsedSeconds(fromIso: string, toIso: string): string {
303310 return seconds(Date.parse(toIso) - Date.parse(fromIso));
@@ -985,6 +992,20 @@ export class AppsRuntime {
985992 );
986993 return;
987994 }
995 // A move (#570): a process started before it holds the old owner's
996 // token and limits, and a deploy in flight may have read the old
997 // place on disk. Each starts again at the commit it serves, under the
998 // new owner; an asleep one wakes as the new owner's anyway, and a
999 // queued one reads the repository when its turn comes.
1000 case "repo.transferred": {
1001 for (const state of this.envs.values()) {
1002 if (state.app.id !== record.id) continue;
1003 const commit = state.deploying?.run.commitOid ?? state.live?.commitOid;
1004 if (commit === undefined || state.queued !== undefined) continue;
1005 this.requestDeploy(record, repo, state.env, commit, "moved", event.actorId ?? undefined);
1006 }
1007 return;
1008 }
9881009 case "pull.closed":
9891010 case "pull.merged": {
9901011 const number = event.payload["number"];
@@ -1050,11 +1071,11 @@ export class AppsRuntime {
10501071 state.queued === undefined &&
10511072 reason !== "restart" &&
10521073 reason !== "oom" &&
1053 reason !== "redeploy"
1074 !AGAIN.has(reason)
10541075 ) {
10551076 return undefined;
10561077 }
1057 if (state.deploying?.run.commitOid === commitOid && reason !== "redeploy") {
1078 if (state.deploying?.run.commitOid === commitOid && !AGAIN.has(reason)) {
10581079 return state.deploying.run;
10591080 }
10601081 const issue =
@@ -1166,9 +1187,13 @@ export class AppsRuntime {
11661187
11671188 private async execute(
11681189 state: Environment,
1169 repo: Repo,
1190 queued: Repo,
11701191 run: Run,
11711192 ): Promise<void> {
1193 // The row as it is now, not as it was when the deploy was asked for: a
1194 // move (#570) between the two gives the repository a new owner and a
1195 // new place on disk, and a queued run follows it there.
1196 let repo = getRepoById(this.app.db, queued.id) ?? queued;
11721197 const claimed = claimRun(this.app.db, run.id);
11731198 if (!claimed) return; // cancelled meanwhile
11741199 const lease = holdLease(this.app.db, run.id);
@@ -1215,6 +1240,8 @@ export class AppsRuntime {
12151240 .run(
12161241 () => {
12171242 clock.lap("turn");
1243 // Read again after the wait for a permit: a move may land in it.
1244 repo = getRepoById(this.app.db, repo.id) ?? repo;
12181245 return this.bringUp(state, repo, run, write, abort.signal, clock);
12191246 },
12201247 { first: state.env === "production" },
src/http/moved.ts +7 −1
@@ -8,7 +8,7 @@
88 */
99
1010 import type { Database } from "bun:sqlite";
11 import { canRead } from "../core/repos.ts";
11 import { canRead, type Repo } from "../core/repos.ts";
1212 import { redirectFor } from "../core/transfer.ts";
1313 import type { User } from "../core/users.ts";
1414 import { isSafeName } from "../git/guard.ts";
@@ -32,6 +32,12 @@ function isPush(url: URL, rest: string): boolean {
3232 return rest === "git-receive-pack" || (rest === "info/refs" && url.searchParams.get("service") === "git-receive-pack");
3333 }
3434
35 /** The repository an old path names, when a move left that path behind; whoever asks. */
36 export function movedTarget(db: Database, url: URL): Repo | undefined {
37 const at = repoPath(url.pathname);
38 return at === undefined ? undefined : redirectFor(db, at.owner, at.name);
39 }
40
3541 export function movedResponse(db: Database, url: URL, user: User | undefined): Response | undefined {
3642 const at = repoPath(url.pathname);
3743 if (at === undefined) return undefined;
src/http/run_gate.ts +7
@@ -23,6 +23,7 @@
2323 import type { App } from "../app.ts";
2424 import { getRepoById, type Repo } from "../core/repos.ts";
2525 import type { Auth } from "./context.ts";
26 import { movedTarget } from "./moved.ts";
2627 import { apiError } from "./respond.ts";
2728
2829 const hidden = (): Response => apiError("not_found", 404, "not found");
@@ -103,5 +104,11 @@ export function runGate(
103104 return undefined;
104105 if (gitRead(method, url, parent)) return undefined;
105106 }
107 // A path one of these repositories moved from (#570): the run was briefed
108 // before the move, so it passes on to the 308 that names the new path,
109 // which this gate then judges like any other. A push is refused there.
110 const moved = movedTarget(app.db, url);
111 if (moved !== undefined && (moved.id === repo.id || moved.id === fork.id || moved.id === parent?.id))
112 return undefined;
106113 return hidden();
107114 }
src/tui/ssh.ts +31 −1
@@ -15,7 +15,9 @@ import { holdOpen } from "../http/caller.ts";
1515 import { join } from "node:path";
1616 import type { App as Forge } from "../app.ts";
1717 import { findKeyByFingerprint, touchKey } from "../core/keys.ts";
18 import { canRead, canWrite, getRepo } from "../core/repos.ts";
18 import { canRead, canWrite, getRepo, type Repo } from "../core/repos.ts";
19 import { redirectFor } from "../core/transfer.ts";
20 import { sshCloneUrl } from "../ssh/command.ts";
1921 import { getUserById, isActive, type User } from "../core/users.ts";
2022 import { createRepo, repositoryLimitMessage, resolveOwner } from "../actions/repos.ts";
2123 import { hookEnv } from "../git/hooks.ts";
@@ -289,6 +291,20 @@ export async function serveGitCommand(
289291
290292 const write = parsed.service === "git-receive-pack";
291293 let repo = getRepo(forge.db, parsed.owner, parsed.name);
294 // A path a move left behind (#570): SSH has no redirect, so a fetch is
295 // served from the new place with one line naming the new remote, and a
296 // push is refused with that line — never applied, never push-to-create
297 // taking the old path back by accident. Someone who cannot read the
298 // repository gets the "not found" below, as for any missing path.
299 const moved = repo === undefined ? redirectFor(forge.db, parsed.owner, parsed.name) : undefined;
300 if (moved !== undefined && canRead(forge.db, moved, user)) {
301 session.writeStderr(`forge: ${movedLine(forge, parsed, moved, write)}${CRLF}`);
302 if (write) {
303 session.exit(1);
304 return;
305 }
306 repo = moved;
307 }
292308 if (!repo && write && user) {
293309 // Push-to-create, the same rule as smart HTTP: an authed owner (or org
294310 // writer) pushing to a repo that does not exist makes it.
@@ -363,6 +379,20 @@ export async function serveGitCommand(
363379 if (write) storageChanged(forge, repo.ownerId);
364380 }
365381
382 /** The one stderr line a moved repository's old SSH path answers with. */
383 export function movedLine(
384 forge: Forge,
385 from: { owner: string; name: string },
386 to: Repo,
387 push: boolean,
388 ): string {
389 const path = `${to.ownerName}/${to.name}`;
390 const remote = sshCloneUrl(forge, to.ownerName, to.name) ?? path;
391 return push
392 ? `${from.owner}/${from.name} moved to ${path}. Nothing was pushed: run git remote set-url origin ${remote} and push again.`
393 : `${from.owner}/${from.name} moved to ${path}; fetching from there. Point the remote at it: git remote set-url origin ${remote}`;
394 }
395
366396 export interface ForgeSshOptions {
367397 /** 0 for an ephemeral port; the real one lands in `app.config.sshPort`. */
368398 port: number;
test/repo-transfer-follow.test.ts added +269
@@ -0,0 +1,269 @@
1 /**
2 * Repo transfer, step two (#570): a repository moves with a fork, a running
3 * preview and a deploy waiting its turn, and each follows it. The waiting
4 * deploy runs against the new place on disk; the preview and production
5 * start again holding the new owner's token; the fork keeps its parent and
6 * reads its objects; a run token bound to the repository reaches it through
7 * the old path; git over HTTPS fetches the old URL; git over SSH serves the
8 * old path with one line naming the new remote, and refuses a push there.
9 */
10
11 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
12 import { mkdtemp, rm } from "node:fs/promises";
13 import { tmpdir } from "node:os";
14 import { join } from "node:path";
15 import { enableApp } from "../src/actions/apps.ts";
16 import { openPull } from "../src/actions/pulls.ts";
17 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
18 import { transferRepository } from "../src/actions/transfer.ts";
19 import { createApp, type App } from "../src/app.ts";
20 import { AppsRuntime } from "../src/apps/runtime.ts";
21 import type { AppRecord } from "../src/core/apps.ts";
22 import { forkRepo, getRepo, getRepoById, type Repo } from "../src/core/repos.ts";
23 import { mintRunToken } from "../src/core/run_token.ts";
24 import { listRuns, type Run } from "../src/core/runs.ts";
25 import { verifyToken } from "../src/core/tokens.ts";
26 import { createUser, type User } from "../src/core/users.ts";
27 import { repoDir } from "../src/git/repos.ts";
28 import { serve, type Servers } from "../src/http/server.ts";
29 import type { SshSession } from "../src/ssh/channels.ts";
30 import { serveGitCommand } from "../src/tui/ssh.ts";
31 import { quietLogs } from "./helpers/globals.ts";
32
33 quietLogs();
34
35 const DOMAIN = "apps.test";
36 /** The app answers with the forge token it was started with, so a test can ask whose it is. */
37 const SERVER = `Bun.serve({ port: Number(process.env.PORT), hostname: "127.0.0.1",
38 fetch: () => new Response(process.env.FORGE_TOKEN ?? "none") });
39 `;
40 const MANIFEST = `${JSON.stringify({ app: { run: "bun run server.ts" } })}\n`;
41
42 const git = async (cwd: string, ...args: string[]) => {
43 const p = Bun.spawn(["git", ...args], {
44 cwd,
45 stdout: "pipe",
46 stderr: "pipe",
47 env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_SYSTEM: "/dev/null" },
48 });
49 const [out, err] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text()]);
50 return { code: await p.exited, out: out.trim(), err };
51 };
52
53 /** An exec channel driven by hand: the client sends a flush (it wants nothing) and hangs up. */
54 function sshSession(command: string) {
55 const out: string[] = [];
56 const err: string[] = [];
57 let code: number | undefined;
58 const text = (d: string | Uint8Array) => (typeof d === "string" ? d : new TextDecoder().decode(d));
59 const session = {
60 command,
61 username: "git",
62 env: {},
63 remoteAddress: "127.0.0.1",
64 write: (d: string | Uint8Array) => void out.push(text(d)),
65 writeStderr: (d: string | Uint8Array) => void err.push(text(d)),
66 exit: (c: number) => void (code = c),
67 onData: (cb: (d: Uint8Array) => void) => setTimeout(() => cb(new TextEncoder().encode("0000")), 50),
68 onEof: (cb: () => void) => setTimeout(cb, 100),
69 onClose: () => {},
70 drain: async () => {},
71 } as unknown as SshSession;
72 return { session, out, err, code: () => code };
73 }
74
75 describe("a moved repository's fork, preview, queued deploy and remotes follow it", () => {
76 let dataDir = "";
77 let work = "";
78 let app: App;
79 let servers: Servers;
80 let runtime: AppsRuntime;
81 let alice: User;
82 let carol: User;
83 let before: Repo;
84 let record: AppRecord;
85 let fork: Repo;
86 let pullNumber = 0;
87 let queued: Run;
88 let tip = "";
89
90 async function until<T>(read: () => T | undefined, what: string, ms = 60_000): Promise<T> {
91 const deadline = Date.now() + ms;
92 while (Date.now() < deadline) {
93 const value = read();
94 if (value !== undefined) return value;
95 await Bun.sleep(25);
96 }
97 throw new Error(`${what} did not happen in time`);
98 }
99 const runs = () => listRuns(app.db, before.id, { kind: "deploy", limit: 50 });
100 const settled = (where: (r: Run) => boolean, what: string) =>
101 until(() => runs().find((r) => where(r) && r.state !== "queued" && r.state !== "running"), what);
102 const viaHost = async (host: string) =>
103 (await fetch(`${servers.url}/`, { headers: { host }, redirect: "manual" })).text();
104 const ownerOfServed = async (host: string) => {
105 const held = verifyToken(app.db, await viaHost(host));
106 return held.ok ? held.value.user.username : "none";
107 };
108
109 beforeAll(async () => {
110 dataDir = await mkdtemp(join(tmpdir(), "forge-transfer-follow-"));
111 work = await mkdtemp(join(tmpdir(), "forge-transfer-follow-wc-"));
112 app = createApp({ dataDir, port: 0 });
113 servers = await serve(app);
114 runtime = new AppsRuntime(app, {
115 domain: DOMAIN,
116 scheme: "http",
117 publicPort: servers.port,
118 forgeUrl: servers.url,
119 concurrency: 1,
120 });
121 await runtime.start();
122 const mk = async (username: string): Promise<User> => {
123 const u = await createUser(app.db, {
124 origin: "seed",
125 trusted: true,
126 username,
127 email: `${username}@forge.local`,
128 password: "a-long-follow-passphrase",
129 adminIfFirst: username === "alice",
130 });
131 if (!u.ok) throw new Error(JSON.stringify(u.error));
132 return u.value;
133 };
134 alice = await mk("alice");
135 await mk("bob");
136 carol = await mk("carol");
137 const made = await createRepo(app, alice, { name: "lab", visibility: "public" });
138 if (!made.ok) throw new Error(String(made.error));
139 before = made.value;
140 const first = await commitToBranch(app, alice, before, {
141 message: "first",
142 files: [
143 { path: "forge.json", content: MANIFEST },
144 { path: "server.ts", content: SERVER },
145 ],
146 });
147 if (!first.ok) throw new Error(JSON.stringify(first.error));
148 const enabled = enableApp(app, alice, before, { slug: "lab" });
149 if (!enabled.ok) throw new Error(String(enabled.error));
150 record = enabled.value;
151 const prod = await settled((r) => r.env === "production", "production's first deploy");
152 if (prod.state !== "ok") throw new Error(`production: ${prod.summary}`);
153
154 const branch = await commitToBranch(app, alice, before, {
155 branch: "feat",
156 base: "main",
157 message: "feat",
158 files: [{ path: "feat.txt", content: "feat\n" }],
159 });
160 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
161 const opened = await openPull(app, alice, before, { title: "feat", baseBranch: "main", headBranch: "feat" });
162 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
163 pullNumber = opened.value.number;
164 const preview = await settled((r) => r.env === `pull-${pullNumber}`, "the preview");
165 if (preview.state !== "ok") throw new Error(`preview: ${preview.summary}`);
166 expect(await ownerOfServed(`lab--${pullNumber}.${DOMAIN}`)).toBe("alice");
167
168 const forked = await forkRepo(app.db, app.reposRoot, before, carol.id);
169 if (!forked.ok) throw new Error(String(forked.error));
170 fork = forked.value;
171
172 // Every permit taken: the next deploy is claimed and waits its turn,
173 // holding the repository as it was asked for — at alice/lab.
174 const release = await runtime.lane.acquire();
175 const second = await commitToBranch(app, alice, before, {
176 message: "second",
177 files: [{ path: "second.txt", content: "second\n" }],
178 });
179 if (!second.ok) throw new Error(JSON.stringify(second.error));
180 tip = second.value.commitOid;
181 queued = await until(
182 () => runs().find((r) => r.env === "production" && r.commitOid === tip && r.state === "running"),
183 "the waiting deploy",
184 );
185 const moved = await transferRepository(app, alice, before, "bob");
186 if (!moved.ok) throw new Error(moved.error);
187 release();
188 }, 120_000);
189
190 afterAll(async () => {
191 await runtime.stop();
192 servers.stop();
193 app.close();
194 await rm(dataDir, { recursive: true, force: true });
195 await rm(work, { recursive: true, force: true });
196 });
197
198 test("the deploy that waited runs against the new place and serves", async () => {
199 const done = await settled((r) => r.id === queued.id, "the waiting deploy");
200 expect(done.summary).not.toContain("not a git repository");
201 expect(done.state).toBe("ok");
202 }, 60_000);
203
204 test("production and the running preview start again as the new owner's", async () => {
205 for (const env of ["production", `pull-${pullNumber}`]) {
206 const again = await settled((r) => r.env === env && r.reason === "moved", `${env} starting again`);
207 expect(again.state).toBe("ok");
208 }
209 await until(
210 () => runtime.status(record).find((e) => e.env === "production" && e.commitOid === tip && e.state === "running"),
211 "production serving the new tip",
212 );
213 expect(await ownerOfServed(`lab.${DOMAIN}`)).toBe("bob");
214 expect(await ownerOfServed(`lab--${pullNumber}.${DOMAIN}`)).toBe("bob");
215 }, 60_000);
216
217 test("the fork keeps its parent and still reads the objects it shares", async () => {
218 const now = getRepoById(app.db, fork.id)!;
219 expect(now.forkParentId).toBe(before.id);
220 const read = await git(repoDir(app.reposRoot, "carol", "lab"), "cat-file", "-t", `${tip}~1`);
221 expect(read.out).toBe("commit");
222 });
223
224 test("a run token bound to the repository follows the old API path to the new one", async () => {
225 const token = mintRunToken(app.db, carol.id, "r1", { repoId: before.id, forkId: fork.id }, "2999-01-01T00:00:00Z").token;
226 const headers = { authorization: `Bearer ${token}` };
227 const old = await fetch(`${servers.url}/api/v1/repos/alice/lab`, { headers, redirect: "manual" });
228 expect(old.status).toBe(308);
229 expect(old.headers.get("location")).toBe("/api/v1/repos/bob/lab");
230 const there = await fetch(`${servers.url}/api/v1/repos/bob/lab`, { headers });
231 expect(there.status).toBe(200);
232 });
233
234 test("git fetch against the old HTTPS remote works", async () => {
235 expect((await git(work, "clone", `${servers.url}/alice/lab.git`, "wc")).code).toBe(0);
236 expect((await git(join(work, "wc"), "fetch", "origin")).code).toBe(0);
237 expect((await git(join(work, "wc"), "rev-parse", "origin/main")).out).toBe(tip);
238 });
239
240 test("ssh under the old path serves the repository and names the new remote once", async () => {
241 const s = sshSession("git-upload-pack '/alice/lab.git'");
242 await serveGitCommand(app, s.session, carol);
243 expect(s.code()).toBe(0);
244 expect(s.out.join("")).toContain(`${tip} HEAD`);
245 const said = s.err.join("").split("\n").filter((l) => l.includes("moved to bob/lab"));
246 expect(said).toHaveLength(1);
247 expect(said[0]).toContain("git remote set-url origin");
248 });
249
250 test("an ssh push to the old path is refused, and does not take the path back", async () => {
251 const s = sshSession("git-receive-pack '/alice/lab.git'");
252 await serveGitCommand(app, s.session, alice);
253 expect(s.code()).toBe(1);
254 expect(s.err.join("")).toContain("Nothing was pushed");
255 expect(getRepo(app.db, "alice", "lab")).toBeUndefined();
256 });
257
258 test("someone who cannot read it gets what a missing path answers", async () => {
259 const hidden = await createRepo(app, alice, { name: "vault", visibility: "private" });
260 if (!hidden.ok) throw new Error(String(hidden.error));
261 const moved = await transferRepository(app, alice, hidden.value, "bob");
262 expect(moved.ok).toBe(true);
263 const s = sshSession("git-upload-pack '/alice/vault.git'");
264 await serveGitCommand(app, s.session, carol);
265 expect(s.code()).toBe(1);
266 expect(s.err.join("")).toContain("repository not found: alice/vault");
267 expect(s.err.join("")).not.toContain("bob");
268 });
269 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs