Public: anyone can read this; the crew can change it.
Keep media pages (/media) for strangers at a full commit id (#9405) #9640
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/9505-keep-media-pages-media-for-strangers-at → main 2 commits fork
3 files
changed +53
−5
src/web/pages/media.ts +20 −5
| @@ -23,6 +23,7 @@ import type { Ctx } from "../../http/context.ts"; | ||
| 23 | 23 | import { holds } from "../../http/etag.ts"; |
| 24 | 24 | import { parseRange } from "../../http/range.ts"; |
| 25 | 25 | import { loadRepo, notFoundPage } from "../chrome.ts"; |
| 26 | import { cachedMedia, keepMedia } from "./tree_cache.ts"; | |
| 26 | 27 | import { HEAD_BYTES, mediaType } from "../render/kinds.ts"; |
| 27 | 28 | |
| 28 | 29 | /** The policy every media answer leaves with. */ |
| @@ -57,10 +58,20 @@ export async function mediaBlob(ctx: Ctx): Promise<Response> { | ||
| 57 | 58 | |
| 58 | 59 | const dir = repoDir(ctx.app.reposRoot, repo.ownerName, repo.name); |
| 59 | 60 | const reader = readerFor(dir); |
| 60 | const stat = await reader.stat(`${ref}:${path}`).catch(() => undefined); | |
| 61 | if (stat === undefined || !stat.ok || stat.value.type !== "blob") | |
| 62 | return notFoundPage(ctx); | |
| 63 | const { oid, size } = stat.value; | |
| 61 | // At a full commit id a small file never changes: a repeat read spawns no git (#9505). | |
| 62 | const pinned = /^[0-9a-f]{40}$/.test(ref); | |
| 63 | const held = pinned ? cachedMedia(repo.id, ref, path) : undefined; | |
| 64 | let oid: string; | |
| 65 | let size: number; | |
| 66 | if (held !== undefined) { | |
| 67 | oid = held.oid; | |
| 68 | size = held.body.length; | |
| 69 | } else { | |
| 70 | const stat = await reader.stat(`${ref}:${path}`).catch(() => undefined); | |
| 71 | if (stat === undefined || !stat.ok || stat.value.type !== "blob") | |
| 72 | return notFoundPage(ctx); | |
| 73 | ({ oid, size } = stat.value); | |
| 74 | } | |
| 64 | 75 | |
| 65 | 76 | const download = ctx.url.searchParams.get("download") === "1"; |
| 66 | 77 | const etag = `"${oid}"`; |
| @@ -90,11 +101,15 @@ export async function mediaBlob(ctx: Ctx): Promise<Response> { | ||
| 90 | 101 | // its first bytes and streamed, so its size never becomes memory. |
| 91 | 102 | let body: Uint8Array | undefined; |
| 92 | 103 | let head: Uint8Array; |
| 93 | if (size <= SMALL_BYTES) { | |
| 104 | if (held !== undefined) { | |
| 105 | body = held.body; | |
| 106 | head = body.subarray(0, HEAD_BYTES); | |
| 107 | } else if (size <= SMALL_BYTES) { | |
| 94 | 108 | const object = await reader.object(oid).catch(() => undefined); |
| 95 | 109 | if (object === undefined || !object.ok) return notFoundPage(ctx); |
| 96 | 110 | body = object.value.body; |
| 97 | 111 | head = body.subarray(0, HEAD_BYTES); |
| 112 | if (pinned) keepMedia(repo.id, ref, path, { oid, body }); | |
| 98 | 113 | } else { |
| 99 | 114 | const sniff = await blobStream(dir, oid, 0, HEAD_BYTES - 1, { signal: ctx.request.signal }); |
| 100 | 115 | if (sniff === "busy" || sniff === "gone") return busy(); |
src/web/pages/tree_cache.ts +16
| @@ -54,6 +54,7 @@ export const treeCacheStats = () => cache.stats(); | ||
| 54 | 54 | export const clearTreeCache = (): void => { |
| 55 | 55 | cache.clear(); |
| 56 | 56 | rawCache.clear(); |
| 57 | mediaCache.clear(); | |
| 57 | 58 | parts.clear(); |
| 58 | 59 | clearKeptPages(); |
| 59 | 60 | }; |
| @@ -136,3 +137,18 @@ export function keepRaw(repoId: string, commit: string, path: string, blob: Blob | ||
| 136 | 137 | if (blob.kind === "missing" || blob.kind === "too_large") return; |
| 137 | 138 | if (blob.size <= RAW_MAX) rawCache.set(rawKey(repoId, commit, path), blob, TTL_MS, blob.size); |
| 138 | 139 | } |
| 140 | ||
| 141 | /** A small media file at a full commit id (#9505): its blob id and bytes, so a repeat read spawns no git. */ | |
| 142 | export interface KeptMedia { | |
| 143 | readonly oid: string; | |
| 144 | readonly body: Uint8Array; | |
| 145 | } | |
| 146 | const MEDIA_MAX = 1024 * 1024; | |
| 147 | const mediaCache = new ViewCache({ maxEntries: 512, maxBytes: 32 * 1024 * 1024 }); | |
| 148 | const mediaKey = (repoId: string, commit: string, path: string): string => ["media", repoId, commit, path].join("\0"); | |
| 149 | ||
| 150 | export const cachedMedia = (repoId: string, commit: string, path: string) => mediaCache.get<KeptMedia>(mediaKey(repoId, commit, path)); | |
| 151 | ||
| 152 | export function keepMedia(repoId: string, commit: string, path: string, media: KeptMedia): void { | |
| 153 | if (media.body.length <= MEDIA_MAX) mediaCache.set(mediaKey(repoId, commit, path), media, TTL_MS, media.body.length + 128); | |
| 154 | } |
test/media.test.ts +17
| @@ -17,6 +17,7 @@ import { mediaGate } from "../src/git/env.ts"; | ||
| 17 | 17 | import { blobStream, MEDIA_QUEUE_MAX, refTip, readerFor } from "../src/git/read.ts"; |
| 18 | 18 | import { repoDir } from "../src/git/repos.ts"; |
| 19 | 19 | import { serve, type Servers } from "../src/http/server.ts"; |
| 20 | import { cachedMedia } from "../src/web/pages/tree_cache.ts"; | |
| 20 | 21 | import { MEDIA_POLICY, PDF_POLICY } from "../src/web/pages/media.ts"; |
| 21 | 22 | import { seedAdmin } from "./helpers/admin.ts"; |
| 22 | 23 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -89,6 +90,22 @@ afterAll(async () => { | ||
| 89 | 90 | }); |
| 90 | 91 | |
| 91 | 92 | describe("the media route", () => { |
| 93 | test("a small file at a full commit id is kept, a branch's and a large one are not", async () => { | |
| 94 | const repoId = app.db.query<{ id: string }, []>("select id from repo where name = 'gallery'").get()!.id; | |
| 95 | expect(cachedMedia(repoId, sha, "art/logo.png")).toBeUndefined(); | |
| 96 | const first = await get(`/alice/gallery/media/${sha}/art/logo.png`); | |
| 97 | const bytes = new Uint8Array(await first.arrayBuffer()); | |
| 98 | const held = cachedMedia(repoId, sha, "art/logo.png"); | |
| 99 | expect(held?.body).toEqual(PNG); | |
| 100 | const again = await get(`/alice/gallery/media/${sha}/art/logo.png`); | |
| 101 | expect(again.headers.get("etag")).toBe(first.headers.get("etag")); | |
| 102 | expect(new Uint8Array(await again.arrayBuffer())).toEqual(bytes); | |
| 103 | await get("/alice/gallery/media/main/art/logo.png"); | |
| 104 | expect(cachedMedia(repoId, "main", "art/logo.png")).toBeUndefined(); | |
| 105 | await (await get(`/alice/gallery/media/${sha}/clip.mp4`)).arrayBuffer(); | |
| 106 | expect(cachedMedia(repoId, sha, "clip.mp4")).toBeUndefined(); | |
| 107 | }); | |
| 108 | ||
| 92 | 109 | test("a confirmed PNG is served as image/png with its bytes", async () => { |
| 93 | 110 | const res = await get("/alice/gallery/media/main/art/logo.png"); |
| 94 | 111 | expect(res.status).toBe(200); |
new above
offline