feat(files): the files layer — schema, router, media route (SUPER-2077)

.env.example

@@ -121,6 +121,8 @@ R2_PRIVATE_BUCKET=
 # Where pages are served from, one subdomain per page under it (apps/usercontent):
 # https://frame.supersetusercontent.com
 USERCONTENT_URL=
+# App-referenced files (apps/usercontent, same Worker): https://media.supersetusercontent.com
+MEDIA_URL=
 # Shared with the usercontent Worker (wrangler secret); 32+ characters.
 USERCONTENT_TOKEN_SECRET=
 # Optional: page thumbnails are skipped while unset.

.env.local.example

@@ -106,6 +106,7 @@ R2_PRIVATE_BUCKET=superset-private-dev
 # Setup's port allocator writes USERCONTENT_DEV_PORT (base+14) and this URL
 # into the shared .env; 8787 is the bare fallback used without setup.
 USERCONTENT_URL=http://frame.usercontent.localhost:8787
+MEDIA_URL=http://media.usercontent.localhost:8787
 USERCONTENT_TOKEN_SECRET=local-dev-usercontent-secret-change-me-32
 # Deliberately empty: unset means page thumbnails are skipped, which is the
 # right local behavior. A fake value would make every publish enqueue a

.github/workflows/deploy-preview.yml

@@ -130,6 +130,7 @@ jobs:
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_PRIVATE_BUCKET: superset-private-dev
           USERCONTENT_URL: ${{ vars.USERCONTENT_URL_DEV }}
+          MEDIA_URL: ${{ vars.MEDIA_URL_DEV }}
           USERCONTENT_TOKEN_SECRET: ${{ secrets.USERCONTENT_TOKEN_SECRET_DEV }}
           CLOUDFLARE_BROWSER_RENDERING_TOKEN: ${{ secrets.CLOUDFLARE_BROWSER_RENDERING_TOKEN }}
           NEXT_PUBLIC_API_URL: https://${{ env.API_ALIAS }}
@@ -194,6 +195,7 @@ jobs:
             --env R2_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY \
             --env R2_PRIVATE_BUCKET=$R2_PRIVATE_BUCKET \
             --env USERCONTENT_URL=$USERCONTENT_URL \
+            --env MEDIA_URL=$MEDIA_URL \
             --env USERCONTENT_TOKEN_SECRET=$USERCONTENT_TOKEN_SECRET \
             --env CLOUDFLARE_BROWSER_RENDERING_TOKEN=$CLOUDFLARE_BROWSER_RENDERING_TOKEN \
             --env NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL \
@@ -318,6 +320,7 @@ jobs:
           NEXT_PUBLIC_DOCS_URL: https://${{ env.DOCS_ALIAS }}
           NEXT_PUBLIC_RELAY_URL: ${{ secrets.RELAY_URL }}
           USERCONTENT_URL: ${{ vars.USERCONTENT_URL_DEV }}
+          MEDIA_URL: ${{ vars.MEDIA_URL_DEV }}
           BETTER_AUTH_SECRET: ${{ secrets.BETTER_AUTH_SECRET }}
           NEXT_PUBLIC_COOKIE_DOMAIN: ${{ secrets.NEXT_PUBLIC_COOKIE_DOMAIN }}
           NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}
@@ -352,6 +355,7 @@ jobs:
             --env NEXT_PUBLIC_DOCS_URL=$NEXT_PUBLIC_DOCS_URL \
             --env NEXT_PUBLIC_RELAY_URL=$NEXT_PUBLIC_RELAY_URL \
             --env USERCONTENT_URL=$USERCONTENT_URL \
+            --env MEDIA_URL=$MEDIA_URL \
             --env NEXT_PUBLIC_COOKIE_DOMAIN=$NEXT_PUBLIC_COOKIE_DOMAIN \
             --env NEXT_PUBLIC_POSTHOG_KEY=$NEXT_PUBLIC_POSTHOG_KEY \
             --env NEXT_PUBLIC_POSTHOG_HOST=$NEXT_PUBLIC_POSTHOG_HOST \

.github/workflows/deploy-production.yml

@@ -87,6 +87,7 @@ jobs:
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_PRIVATE_BUCKET: ${{ vars.R2_PRIVATE_BUCKET }}
           USERCONTENT_URL: ${{ vars.USERCONTENT_URL }}
+          MEDIA_URL: ${{ vars.MEDIA_URL }}
           USERCONTENT_TOKEN_SECRET: ${{ secrets.USERCONTENT_TOKEN_SECRET }}
           CLOUDFLARE_BROWSER_RENDERING_TOKEN: ${{ secrets.CLOUDFLARE_BROWSER_RENDERING_TOKEN }}
           GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
@@ -157,6 +158,7 @@ jobs:
             --env R2_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY \
             --env R2_PRIVATE_BUCKET=$R2_PRIVATE_BUCKET \
             --env USERCONTENT_URL=$USERCONTENT_URL \
+            --env MEDIA_URL=$MEDIA_URL \
             --env USERCONTENT_TOKEN_SECRET=$USERCONTENT_TOKEN_SECRET \
             --env CLOUDFLARE_BROWSER_RENDERING_TOKEN=$CLOUDFLARE_BROWSER_RENDERING_TOKEN \
             --env NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL \
@@ -263,6 +265,7 @@ jobs:
           NEXT_PUBLIC_DOCS_URL: ${{ secrets.NEXT_PUBLIC_DOCS_URL }}
           NEXT_PUBLIC_RELAY_URL: ${{ secrets.RELAY_URL }}
           USERCONTENT_URL: ${{ vars.USERCONTENT_URL }}
+          MEDIA_URL: ${{ vars.MEDIA_URL }}
           BETTER_AUTH_SECRET: ${{ secrets.BETTER_AUTH_SECRET }}
           NEXT_PUBLIC_COOKIE_DOMAIN: ${{ secrets.NEXT_PUBLIC_COOKIE_DOMAIN }}
           NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}
@@ -297,6 +300,7 @@ jobs:
             --env NEXT_PUBLIC_DOCS_URL=$NEXT_PUBLIC_DOCS_URL \
             --env NEXT_PUBLIC_RELAY_URL=$NEXT_PUBLIC_RELAY_URL \
             --env USERCONTENT_URL=$USERCONTENT_URL \
+            --env MEDIA_URL=$MEDIA_URL \
             --env NEXT_PUBLIC_COOKIE_DOMAIN=$NEXT_PUBLIC_COOKIE_DOMAIN \
             --env NEXT_PUBLIC_POSTHOG_KEY=$NEXT_PUBLIC_POSTHOG_KEY \
             --env NEXT_PUBLIC_POSTHOG_HOST=$NEXT_PUBLIC_POSTHOG_HOST \
@@ -526,12 +530,10 @@ jobs:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
         run: bunx wrangler deploy
 
-  deploy-usercontent:
-    name: Deploy Usercontent Worker to Cloudflare
+  deploy-content:
+    name: Deploy Content Worker to Cloudflare
     runs-on: ubuntu-latest
     environment: production
-    permissions:
-      contents: read
 
     steps:
       - name: Checkout repository

apps/api/src/app/api/files/jobs/sweep/route.ts

@@ -0,0 +1,26 @@
+import {
+	FILE_SWEEP_JOB_PATH,
+	sweepPendingFiles,
+} from "@superset/trpc/file-sweep";
+import { verifyQstashRequest } from "@/lib/verifyQstash";
+
+export const maxDuration = 300;
+export const dynamic = "force-dynamic";
+
+/**
+ * Daily: a `pending` row is an upload whose `complete` never came — the
+ * client died mid-PUT, or verification failed after the object landed.
+ * After a day it is never completing; drop the row and the object.
+ */
+export async function POST(request: Request) {
+	const body = await request.text();
+	const rejected = await verifyQstashRequest(
+		request,
+		body,
+		FILE_SWEEP_JOB_PATH,
+	);
+	if (rejected) return rejected;
+
+	const result = await sweepPendingFiles();
+	return Response.json(result);
+}

apps/usercontent/scripts/dev.ts

@@ -39,6 +39,7 @@ const generated = [
 	"# Generated by scripts/dev.ts from the worktree's .env — do not edit.",
 	...Object.entries(secrets).map(([key, value]) => `${key}=${value}`),
 	`USERCONTENT_URL=http://frame.usercontent.localhost:${port}`,
+	`MEDIA_URL=http://media.usercontent.localhost:${port}`,
 	`APP_URL=http://localhost:${webPort}`,
 	"FRAME_ANCESTORS=http://localhost:* file:",
 	"",

apps/usercontent/src/env.ts

@@ -10,6 +10,8 @@ import { z } from "zod";
 const envSchema = z.object({
 	/** Base URL framed content hangs off, e.g. https://frame.supersetusercontent.com */
 	USERCONTENT_URL: z.string().url(),
+	/** Base URL for app-referenced files, e.g. https://media.supersetusercontent.com */
+	MEDIA_URL: z.string().url(),
 	/** Where a reader without a ticket is sent to sign in and come back. */
 	APP_URL: z.string().url(),
 	/** Space-separated CSP sources allowed to frame a page. */

apps/usercontent/src/index.ts

@@ -1,6 +1,9 @@
 import * as Sentry from "@sentry/cloudflare";
 import { PAGE_COMMENTS_RUNTIME_SOURCE } from "@superset/shared/page-comments-runtime";
 import {
+	FILE_CONTENT_SECURITY_POLICY,
+	fileOriginalKey,
+	fileResponsePolicy,
 	injectScriptTag,
 	type PageManifest,
 	type PageTicketClaims,
@@ -13,6 +16,7 @@ import {
 	servedVersionOf,
 	THUMBNAIL_FILENAME,
 	TICKET_QUERY_PARAM,
+	verifyFileTicket,
 	verifyPageTicket,
 } from "@superset/shared/usercontent";
 import { type Context, Hono } from "hono";
@@ -175,14 +179,131 @@ async function serveThumbnail(c: Context<AppContext>): Promise<Response> {
 	});
 }
 
+const FILE_ID =
+	/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+function parseRange(
+	header: string | undefined,
+): { offset: number; length?: number } | { suffix: number } | undefined {
+	if (!header) return undefined;
+	const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
+	if (!match) return undefined;
+	const [, startRaw = "", endRaw = ""] = match;
+	if (startRaw === "" && endRaw !== "") return { suffix: Number(endRaw) };
+	if (startRaw === "") return undefined;
+	const offset = Number(startRaw);
+	if (endRaw === "") return { offset };
+	const end = Number(endRaw);
+	if (end < offset) return undefined;
+	return { offset, length: end - offset + 1 };
+}
+
+function contentDisposition(
+	disposition: "inline" | "attachment",
+	filename: string,
+): string {
+	const fallback =
+		filename.replace(/[^\x20-\x7e]+/g, "_").replace(/["\\]/g, "_") || "file";
+	return `${disposition}; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
+}
+
+/**
+ * `/files/<fileId>` on the media host: app-referenced attachments. The
+ * ticket carries the server-sniffed content type, so serving needs no
+ * database; the policy decides what a browser may do with it, and `Range`
+ * makes video seek.
+ */
+async function serveFile(c: Context<AppContext>): Promise<Response> {
+	if (requestHost(c) !== new URL(c.env.MEDIA_URL).host) return notFound();
+	const fileId = c.req.param("fileId") ?? "";
+	if (!FILE_ID.test(fileId)) return notFound();
+	const ticket = c.req.query(TICKET_QUERY_PARAM);
+	if (!ticket) return notFound();
+	const claims = await verifyFileTicket(
+		[
+			c.env.USERCONTENT_TOKEN_SECRET,
+			c.env.USERCONTENT_TOKEN_SECRET_PREVIOUS ?? "",
+		],
+		ticket,
+	);
+	if (!claims || claims.fileId !== fileId) return notFound();
+
+	const key = fileOriginalKey(fileId);
+	const range = parseRange(c.req.header("range"));
+	let object: R2ObjectBody | null;
+	try {
+		object = await c.env.PRIVATE.get(key, range ? { range } : undefined);
+	} catch {
+		return new Response("Range not satisfiable", {
+			status: 416,
+			headers: { "Cache-Control": "no-store" },
+		});
+	}
+	if (!object) return notFound();
+
+	const policy = fileResponsePolicy({
+		contentType: claims.contentType,
+		fetchDest: c.req.header("sec-fetch-dest"),
+	});
+	const filenameParam = c.req.param("filename");
+	let filename = fileId;
+	if (filenameParam) {
+		try {
+			filename = decodeURIComponent(filenameParam);
+		} catch {
+			filename = filenameParam;
+		}
+	}
+
+	const headers = new Headers({
+		"Content-Type": policy.contentType,
+		"Content-Disposition": contentDisposition(policy.disposition, filename),
+		"Content-Security-Policy": FILE_CONTENT_SECURITY_POLICY,
+		"Superset-Storage-Key": key,
+		"X-Content-Type-Options": "nosniff",
+		"Referrer-Policy": "no-referrer",
+		"X-Robots-Tag": "noindex, nofollow",
+		"Accept-Ranges": "bytes",
+		// Like thumbnails: the browser may keep the bytes only as long as the
+		// ticket that fetched them, so revocation is bounded by ticket life.
+		"Cache-Control": `private, max-age=${Math.max(
+			0,
+			Math.min(claims.exp - Math.floor(Date.now() / 1000), 86400),
+		)}, immutable`,
+	});
+	if (range && object.range) {
+		const offset =
+			"offset" in object.range && object.range.offset !== undefined
+				? object.range.offset
+				: Math.max(
+						object.size - (object.range as { suffix: number }).suffix,
+						0,
+					);
+		const length =
+			"length" in object.range && object.range.length !== undefined
+				? object.range.length
+				: object.size - offset;
+		headers.set(
+			"Content-Range",
+			`bytes ${offset}-${offset + length - 1}/${object.size}`,
+		);
+		headers.set("Content-Length", String(length));
+		return new Response(object.body, { status: 206, headers });
+	}
+	headers.set("Content-Length", String(object.size));
+	return new Response(object.body, { headers });
+}
+
 // Pages hang off `frame.<zone>`; the zone apex and `frame.` itself have
 // nothing to serve, so readers arriving there belong in the app.
 app.use("*", async (c, next) => {
 	assertEnv(c.env);
 	const host = requestHost(c);
 	const base = baseHost(c);
 	const apex = base.slice(base.indexOf(".") + 1);
-	if (host !== base && host !== apex) return next();
+	const media = new URL(c.env.MEDIA_URL).host;
+	if (host !== base && host !== apex && host !== media) return next();
+	if (host === media && c.req.path.startsWith("/files/")) return next();
 	if (c.req.path === "/health") return c.json({ ok: true });
 	return c.redirect(c.env.APP_URL, 302);
 });
@@ -212,6 +333,8 @@ app.get("/versions/:version/", servePage);
 app.get("/versions/:version/:ticket{~[^/]+}", addTrailingSlash);
 app.get("/versions/:version/:ticket{~[^/]+}/", servePage);
 app.get(`/versions/:version/${THUMBNAIL_FILENAME}`, serveThumbnail);
+app.get("/files/:fileId", serveFile);
+app.get("/files/:fileId/:filename", serveFile);
 app.notFound(() => notFound());
 
 // Exceptions only; no-op until SENTRY_DSN is set.

apps/usercontent/wrangler.jsonc

@@ -15,6 +15,10 @@
 			"pattern": "frame.supersetusercontent.com/*",
 			"zone_name": "supersetusercontent.com"
 		},
+		{
+			"pattern": "media.supersetusercontent.com/*",
+			"zone_name": "supersetusercontent.com"
+		},
 		{
 			"pattern": "supersetusercontent.com/*",
 			"zone_name": "supersetusercontent.com"
@@ -24,6 +28,7 @@
 	"compatibility_flags": ["nodejs_als"],
 	"vars": {
 		"USERCONTENT_URL": "https://frame.supersetusercontent.com",
+		"MEDIA_URL": "https://media.supersetusercontent.com",
 		"APP_URL": "https://app.superset.sh",
 		// Who may frame a page. `file:` is the packaged desktop renderer; a
 		// frame still needs a valid ticket, so this admits embedding, not reading.

packages/db/drizzle/0101_add_files_and_attachments.sql

@@ -0,0 +1,32 @@
+CREATE TYPE "public"."attachment_parent_kind" AS ENUM('page_version', 'issue', 'doc', 'chat_session', 'comment');--> statement-breakpoint
+CREATE TYPE "public"."file_status" AS ENUM('pending', 'ready');--> statement-breakpoint
+CREATE TABLE "attachments" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"file_id" uuid NOT NULL,
+	"parent_kind" "attachment_parent_kind" NOT NULL,
+	"parent_id" uuid NOT NULL,
+	"path" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "files" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"organization_id" uuid NOT NULL,
+	"name" text NOT NULL,
+	"content_type" text NOT NULL,
+	"size_bytes" bigint NOT NULL,
+	"sha256" text NOT NULL,
+	"status" "file_status" DEFAULT 'pending' NOT NULL,
+	"created_by_user_id" uuid,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "attachments" ADD CONSTRAINT "attachments_file_id_files_id_fk" FOREIGN KEY ("file_id") REFERENCES "public"."files"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "files" ADD CONSTRAINT "files_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "auth"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "files" ADD CONSTRAINT "files_created_by_user_id_users_id_fk" FOREIGN KEY ("created_by_user_id") REFERENCES "auth"."users"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "attachments_file_id_idx" ON "attachments" USING btree ("file_id");--> statement-breakpoint
+CREATE INDEX "attachments_parent_kind_parent_id_idx" ON "attachments" USING btree ("parent_kind","parent_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "attachments_parent_path_unique" ON "attachments" USING btree ("parent_kind","parent_id","path") WHERE "attachments"."path" is not null;--> statement-breakpoint
+CREATE INDEX "files_organization_id_created_at_idx" ON "files" USING btree ("organization_id","created_at" desc);--> statement-breakpoint
+CREATE INDEX "files_status_created_at_idx" ON "files" USING btree ("status","created_at");