Public: anyone can read this; the crew can change it.
/oembed?url= answers a public app or repository link with a rich card for other tools #15079
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14455-oembed-url-answers-a-public-app-or-repos → main 15 commits fork 3 threads · 3 unresolved
12 files
changed +177
−2
docs/STANDARDS.md +1
| @@ -12,6 +12,7 @@ its own mark. Nothing here is configured, and nothing here is optional. | ||
| 12 | 12 | | `/.well-known/change-password` | where a password manager sends a person | `/settings/password` | |
| 13 | 13 | | `/robots.txt` | crawlers stay out of accounts, search and the API; the sitemap | — | |
| 14 | 14 | | `/sitemap.xml` | the front page and every public repository, never a private one | `listReposVisibleTo(db, nobody)` | |
| 15 | | `/oembed?url=<page>&format=json` | the oEmbed 1.0 `link` card of a public repository or app page (title, author, provider, thumbnail); 404 alike for private and unknown, 400 for another host, 501 for `xml`; public pages advertise it with `<link rel="alternate" type="application/json+oembed">` | `getRepo` + `canRead(db, repo, undefined)`; the url is parsed, never fetched | | |
| 15 | 16 | | `/manifest.webmanifest` | the forge as an installable app | — | |
| 16 | 17 | | `/llms.txt` | the forge for an agent (llmstxt.org): how to report, how to offer a change, the API and MCP, the public repositories | `listReposVisibleTo(db, nobody)`, the report limits | |
| 17 | 18 | | `/icons/forge-<size>.png` | the mark at 192, 512, 512-maskable and 180 (iOS), painted at boot | `design/mark.ts`, the favicon's two colours | |
src/contract/openapi/routes/getOembed.ts added +3
| @@ -0,0 +1,3 @@ | ||
| 1 | import type { Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { method: "get", path: "/oembed", summary: "The oEmbed 1.0 link card of a public repository or app page", detail: "oEmbed 1.0 `link` answer for the page `url` (a link on this forge; only its first two path segments, owner and name, are read, and it is never fetched) as { version, type, title, author_name, author_url, provider_name, provider_url, cache_age, thumbnail_url }. `format=json` (the default) is the only format: `xml` is a 501. A private repository and an unknown one are the same 404; a url on another host, or none, is a 400. Counted by the anonymous per-source limit; cacheable for five minutes. Public pages advertise it as `<link rel=\"alternate\" type=\"application/json+oembed\">`", auth: false, role: "anonymous", query: { url: "string", format: "string" }, errorStatus: { 400: "url is missing, too long, or on another host" }, refusals: { bad_request: "400, url is missing, too long, or on another host", not_found: "404: the page names no repository you can read, anonymously", not_implemented: "501, format is not json", rate_limited: "429, too many reads from one source without a token" } }; |
src/contract/openapi/routes/index.ts +1
| @@ -262,6 +262,7 @@ export { route as getApiV1Version } from "./getApiV1Version.ts"; | ||
| 262 | 262 | export { route as getContractOpenapiJson } from "./getContractOpenapiJson.ts"; |
| 263 | 263 | export { route as getHealth } from "./getHealth.ts"; |
| 264 | 264 | export { route as getMetrics } from "./getMetrics.ts"; |
| 265 | export { route as getOembed } from "./getOembed.ts"; | |
| 265 | 266 | export { route as getOwnerNamePullsNumberCheckTxt } from "./getOwnerNamePullsNumberCheckTxt.ts"; |
| 266 | 267 | export { route as patchApiV1AdminSettings } from "./patchApiV1AdminSettings.ts"; |
| 267 | 268 | export { route as patchApiV1AgentsUsername } from "./patchApiV1AgentsUsername.ts"; |
src/contract/tags.ts +1
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | |
| 7 | 7 | /** A path segment that names a resource under another tag. */ |
| 8 | 8 | const ALIAS: Readonly<Record<string, string>> = { |
| 9 | oembed: "meta", | |
| 9 | 10 | app: "apps", |
| 10 | 11 | timers: "apps", |
| 11 | 12 | schedules: "apps", |
src/core/reserved.ts +1
| @@ -129,6 +129,7 @@ export const SYSTEM_NAMES: readonly string[] = [ | ||
| 129 | 129 | "signin", |
| 130 | 130 | "signup", |
| 131 | 131 | "sitemap.xml", |
| 132 | "oembed", | |
| 132 | 133 | "releases", |
| 133 | 134 | "releases.json", |
| 134 | 135 | "templates.json", |
src/http/marks/oembed.json added +4
| @@ -0,0 +1,4 @@ | ||
| 1 | { | |
| 2 | "head": "/oembed", | |
| 3 | "mark": "open" | |
| 4 | } |
src/http/server.ts +3
| @@ -231,6 +231,7 @@ import { | ||
| 231 | 231 | agentCardJson, |
| 232 | 232 | forgeJson, |
| 233 | 233 | sitemapXml, |
| 234 | oembed, | |
| 234 | 235 | webManifest, |
| 235 | 236 | } from "../web/standards.ts"; |
| 236 | 237 | import { |
| @@ -630,6 +631,7 @@ import { isProbe } from "../kernel/probe.ts"; | ||
| 630 | 631 | import { strangerCode } from "./stranger_code.ts"; |
| 631 | 632 | import { sourceArchive } from "./archive.ts"; |
| 632 | 633 | import { agentsMd } from "./agents_md.ts"; |
| 634 | ||
| 633 | 635 | import { takeStrangerRead } from "./stranger_reads.ts"; |
Stray blank line added in the import block; remove it. |
||
| 634 | 636 | import { parseCidrs } from "../kernel/cidr.ts"; |
| 635 | 637 | import { |
| @@ -898,6 +900,7 @@ export function buildRouter(): Router<Handler> { | ||
| 898 | 900 | router.add("GET", "/agents", agentsPage); |
| 899 | 901 | router.add("GET", "/.well-known/change-password", changePassword); |
| 900 | 902 | router.add("GET", "/robots.txt", robotsTxt); |
| 903 | router.add("GET", "/oembed", oembed); | |
| 901 | 904 | router.add("GET", "/releases.json", releasesJson); |
| 902 | 905 | router.add("GET", "/sitemap.xml", sitemapXml); |
| 903 | 906 | router.add("GET", "/manifest.webmanifest", webManifest); |
src/web/standards.ts +78 −1
| @@ -13,6 +13,7 @@ | ||
| 13 | 13 | * /.well-known/change-password where a password manager sends someone |
| 14 | 14 | * /robots.txt what a crawler may index, and the sitemap |
| 15 | 15 | * /sitemap.xml every public repository |
| 16 | * /oembed?url= the oEmbed link card of a public repository page | |
| 16 | 17 | * /releases.json the releases this forge has shipped, newest |
| 17 | 18 | * first, for a child forge to read |
| 18 | 19 | * /manifest.webmanifest the forge as an installable app |
| @@ -47,7 +48,8 @@ import { llmsRepos, newestLanding, sitemapRepos, type LlmsListing } from "../cor | ||
| 47 | 48 | import { canRead, getRepo, type Repo } from "../core/repos.ts"; |
| 48 | 49 | import { publicForgeRepo } from "../core/forge_repo.ts"; |
| 49 | 50 | import type { Ctx } from "../http/context.ts"; |
| 50 | import { notFound } from "../http/respond.ts"; | |
| 51 | import { apiError, json, notFound } from "../http/respond.ts"; | |
| 52 | import { takeStrangerRead } from "../http/stranger_reads.ts"; | |
| 51 | 53 | import { SHARE_TARGET } from "./share_target.ts"; |
| 52 | 54 | import { offerTable } from "./offer.ts"; |
| 53 | 55 | import { getPlans } from "../settings/plans.ts"; |
| @@ -719,6 +721,13 @@ export function pageLinks(ctx: Ctx): PageLink[] { | ||
| 719 | 721 | href: `/${repo.ownerName}/${repo.name}/report`, |
| 720 | 722 | title: REPORT_LINK_TITLE, |
| 721 | 723 | }); |
| 724 | // A public page unfurls elsewhere through oEmbed; a private one advertises nothing. | |
| 725 | links.push({ | |
| 726 | rel: "alternate", | |
| 727 | href: oembedHref(ctx.url.origin, ctx.url.pathname), | |
| 728 | title: "oEmbed", | |
| 729 | type: OEMBED_TYPE, | |
| 730 | }); | |
| 722 | 731 | // The same door for an agent: the endpoint that takes the JSON. |
| 723 | 732 | links.push({ |
| 724 | 733 | rel: "help", |
| @@ -746,3 +755,71 @@ export function linkHeader(links: readonly PageLink[]): string { | ||
| 746 | 755 | ) |
| 747 | 756 | .join(", "); |
| 748 | 757 | } |
| 758 | ||
| 759 | // ---- /oembed: the oEmbed 1.0 link card of a public repository or app page (never fetches the url) | |
| 760 | ||
| 761 | export const OEMBED_TYPE = "application/json+oembed"; | |
| 762 | const URL_MAX = 2048; | |
| 763 | ||
| 764 | /** The `<link>` a public page advertises: where its oEmbed answer is. */ | |
| 765 | export function oembedHref(origin: string, pathname: string): string { | |
| 766 | return `${origin}/oembed?url=${encodeURIComponent(`${origin}${pathname}`)}&format=json`; | |
| 767 | } | |
| 768 | ||
| 769 | /** | |
| 770 | * `[owner, name]` from a url on `origin`; undefined when it is another host or names no repository. | |
The comment says undefined for another host, but the function returns "foreign" for that and undefined for no repository. Fix the comment and the return type description. |
||
| 771 | */ | |
| 772 | export function repoOfUrl(raw: string, origin: string): readonly [string, string] | "foreign" | undefined { | |
| 773 | let u: URL; | |
| 774 | try { | |
| 775 | u = new URL(raw); | |
| 776 | } catch { | |
| 777 | return "foreign"; | |
| 778 | } | |
| 779 | if (u.origin !== origin) return "foreign"; | |
| 780 | const parts = u.pathname.split("/").filter((s) => s !== ""); | |
| 781 | if (parts.length < 2) return undefined; | |
| 782 | try { | |
| 783 | return [decodeURIComponent(parts[0]!), decodeURIComponent(parts[1]!)]; | |
| 784 | } catch { | |
| 785 | return undefined; | |
| 786 | } | |
| 787 | } | |
| 788 | ||
| 789 | export function oembed(ctx: Ctx): Response { | |
| 790 | const verdict = takeStrangerRead(ctx.app.db, ctx.request, false); | |
| 791 | if (verdict !== undefined && !verdict.allowed) | |
| 792 | return json( | |
| 793 | { error: "rate_limited", message: `too many requests: ${verdict.limit} per minute from one source — retry in ${verdict.retryAfter}s` }, | |
| 794 | 429, | |
| 795 | { "retry-after": String(verdict.retryAfter) }, | |
| 796 | ); | |
| 797 | const format = ctx.url.searchParams.get("format") ?? "json"; | |
| 798 | if (format !== "json") return apiError("not_implemented", 501, "only format=json is served"); | |
| 799 | const raw = ctx.url.searchParams.get("url") ?? ""; | |
| 800 | if (raw === "" || raw.length > URL_MAX) return apiError("bad_request", 400, "url must be a link to a page on this forge"); | |
| 801 | const at = repoOfUrl(raw, ctx.url.origin); | |
| 802 | if (at === "foreign") return apiError("bad_request", 400, "url must be a link to a page on this forge"); | |
| 803 | if (at === undefined) return notFound(); | |
| 804 | const repo = getRepo(ctx.app.db, at[0], at[1]); | |
| 805 | if (repo === undefined || !canRead(ctx.app.db, repo, undefined)) return notFound(); | |
| 806 | const origin = ctx.url.origin; | |
| 807 | const slug = `${repo.ownerName}/${repo.name}`; | |
| 808 | return json( | |
| 809 | { | |
| 810 | version: "1.0", | |
| 811 | type: "link", | |
| 812 | title: slug, | |
| 813 | author_name: repo.ownerName, | |
| 814 | author_url: `${origin}/${encodeURIComponent(repo.ownerName)}`, | |
| 815 | provider_name: "Open Platform Forge", | |
| 816 | provider_url: origin, | |
| 817 | cache_age: 300, | |
| 818 | thumbnail_url: `${origin}/og/repo/${encodeURIComponent(repo.ownerName)}~${encodeURIComponent(repo.name)}.png`, | |
| 819 | thumbnail_width: 1200, | |
| 820 | thumbnail_height: 630, | |
| 821 | }, | |
| 822 | 200, | |
| 823 | { "cache-control": "public, max-age=300" }, | |
| 824 | ); | |
| 825 | } |
test/design-audit.test.ts +1 −1
| @@ -17,7 +17,7 @@ const doc = readFileSync("docs/design/audit.md", "utf8"); | ||
| 17 | 17 | |
| 18 | 18 | /** Routes that answer a file, a fragment, a redirect hop or a callback. */ |
| 19 | 19 | const NOT_PAGES = |
| 20 | /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|templates\.json$|maintenance\.json$|changelog\.atom$|explore\.atom$|feed\.json$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|forges\/export\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/; | |
| 20 | /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|oembed$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|templates\.json$|maintenance\.json$|changelog\.atom$|explore\.atom$|feed\.json$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|forges\/export\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/; | |
| 21 | 21 | |
| 22 | 22 | const getRoutes = [ |
| 23 | 23 | ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g), |
test/forge-looks.test.ts +1
| @@ -53,6 +53,7 @@ const NOT_A_PAGE: Record<string, string> = { | ||
| 53 | 53 | "/explore.atom": "Atom feed of /explore", |
| 54 | 54 | "/feed.json": "JSON Feed of /feed", |
| 55 | 55 | "/sitemap.xml": "XML for crawlers", |
| 56 | "/oembed": "JSON for other sites to build a link card; covered by test/oembed.test.ts", | |
| 56 | 57 | "/manifest.webmanifest": "the install manifest, JSON", |
| 57 | 58 | "/opensearch.xml": "the browser search description, XML", |
| 58 | 59 | "/sw.js": "the service worker script", |
test/oembed.test.ts added +82
| @@ -0,0 +1,82 @@ | ||
| 1 | /** /oembed: a public repository unfurls; a private one and an unknown one are the same 404. */ | |
| 2 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { createUser } from "../src/core/users.ts"; | |
| 9 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 10 | ||
| 11 | let dataDir: string; | |
| 12 | let app: App; | |
| 13 | let servers: Servers; | |
| 14 | ||
| 15 | const ask = (url: string, extra = "&format=json") => fetch(`${servers.url}/oembed?url=${encodeURIComponent(url)}${extra}`); | |
| 16 | ||
| 17 | beforeAll(async () => { | |
| 18 | dataDir = await mkdtemp(join(tmpdir(), "forge-oembed-")); | |
| 19 | app = createApp({ dataDir, port: 0 }); | |
| 20 | servers = await serve(app); | |
| 21 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 22 | if (!u.ok) throw new Error(u.error); | |
| 23 | for (const [name, visibility] of [["open", "public"], ["secret", "private"]] as const) { | |
| 24 | const r = await createRepo(app, u.value, { name, visibility }); | |
| 25 | if (!r.ok) throw new Error(String(r.error)); | |
| 26 | } | |
| 27 | }); | |
| 28 | ||
| 29 | afterAll(async () => { | |
| 30 | servers.stop(); | |
| 31 | app.close(); | |
| 32 | await rm(dataDir, { recursive: true, force: true }); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a public repository answers the oEmbed link type, with the security floor", async () => { | |
| 36 | const res = await ask(`${servers.url}/alice/open`); | |
| 37 | expect(res.status).toBe(200); | |
| 38 | expect(res.headers.get("content-type")).toContain("application/json"); | |
| 39 | expect(res.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 40 | expect(res.headers.get("referrer-policy")).not.toBeNull(); | |
| 41 | expect(res.headers.get("content-security-policy")).not.toBeNull(); | |
| 42 | const body = (await res.json()) as Record<string, unknown>; | |
| 43 | expect(body.version).toBe("1.0"); | |
| 44 | expect(body.type).toBe("link"); | |
| 45 | expect(body.title).toBe("alice/open"); | |
| 46 | expect(body.author_name).toBe("alice"); | |
| 47 | expect(body.provider_name).toBe("Open Platform Forge"); | |
| 48 | expect(body.thumbnail_url).toBe(`${servers.url}/og/repo/alice~open.png`); | |
| 49 | // A deeper page of the same repository is the same card; format may be left out. | |
| 50 | expect((await ask(`${servers.url}/alice/open/issues`, "")).status).toBe(200); | |
| 51 | }); | |
| 52 | ||
| 53 | test("a private repository answers the same 404 bytes as an unknown one", async () => { | |
| 54 | const priv = await ask(`${servers.url}/alice/secret`); | |
| 55 | const none = await ask(`${servers.url}/alice/nothing-here`); | |
| 56 | expect(priv.status).toBe(404); | |
| 57 | expect(none.status).toBe(404); | |
| 58 | const a = await priv.text(); | |
| 59 | expect(a).toBe(await none.text()); | |
| 60 | expect(a).not.toContain("secret"); | |
| 61 | }); | |
| 62 | ||
| 63 | test("another host is a 400, xml a 501, and a hostile url is never echoed", async () => { | |
| 64 | expect((await ask("https://example.com/alice/open")).status).toBe(400); | |
| 65 | expect((await ask("not a url")).status).toBe(400); | |
| 66 | expect((await fetch(`${servers.url}/oembed`)).status).toBe(400); | |
| 67 | expect((await ask(`${servers.url}/alice/open`, "&format=xml")).status).toBe(501); | |
| 68 | const evil = await ask(`${servers.url}/alice/<script>alert(1)</script>`); | |
| 69 | expect(evil.status).toBe(404); | |
| 70 | expect(await evil.text()).not.toContain("<script>"); | |
| 71 | const foreign = await ask("https://evil.test/<script>"); | |
| 72 | expect(await foreign.text()).not.toContain("<script>"); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a public page advertises the card; a private page advertises none", async () => { | |
| 76 | const open = await (await fetch(`${servers.url}/alice/open`)).text(); | |
| 77 | expect(open).toContain('type="application/json+oembed"'); | |
| 78 | expect(open).toContain(encodeURIComponent(`${servers.url}/alice/open`)); | |
| 79 | const secret = await fetch(`${servers.url}/alice/secret`); | |
| 80 | expect(await secret.text()).not.toContain("oembed"); | |
| 81 | expect(secret.headers.get("link") ?? "").not.toContain("oembed"); | |
| 82 | }); |
test/openapi-drift.test.ts +1
| @@ -13,6 +13,7 @@ const NOT_ROUTED: Readonly<Record<string, string>> = { | ||
| 13 | 13 | "GET /health": "routed outside /api/v1 (src/http/api.ts), the monitor's alias of /api/v1/health (#13191)", |
| 14 | 14 | "GET /{}/{}/pulls/{}/check.txt": "routed outside /api/v1 (src/http/server.ts, check_text.ts); test/openapi-check-txt.test.ts pins it to server.ts", |
| 15 | 15 | "POST /mcp": "routed outside /api/v1 (src/http/mcp.ts); the MCP has its own card and tool list", |
| 16 | "GET /oembed": "routed outside /api/v1 (src/http/server.ts, web/standards.ts): the oEmbed endpoint other tools discover from a page's head", | |
| 16 | 17 | }; |
| 17 | 18 | const NOT_DESCRIBED: Readonly<Record<string, string>> = {}; |
| 18 | 19 |
new above
offline
Keep the list in its existing order: place "oembed" alphabetically, not after sitemap.xml.