diff --git a/bun.lock b/bun.lock index acf47bca0815..62dbf794652c 100644 --- a/bun.lock +++ b/bun.lock @@ -814,11 +814,8 @@ "diff": "catalog:", "dompurify": "3.3.1", "fuzzysort": "catalog:", - "katex": "0.16.27", "luxon": "catalog:", "marked": "catalog:", - "marked-katex-extension": "5.1.6", - "marked-shiki": "catalog:", "morphdom": "2.7.8", "motion": "12.34.5", "remeda": "catalog:", @@ -831,7 +828,6 @@ "devDependencies": { "@tsconfig/node22": "catalog:", "@types/bun": "catalog:", - "@types/katex": "0.16.7", "@types/luxon": "catalog:", "@typescript/native-preview": "catalog:", "typescript": "catalog:", @@ -991,7 +987,6 @@ "katex": "0.16.27", "luxon": "catalog:", "marked": "catalog:", - "marked-katex-extension": "5.1.6", "marked-shiki": "catalog:", "morphdom": "2.7.8", "motion": "12.34.5", @@ -1137,7 +1132,7 @@ "hono": "4.10.7", "hono-openapi": "1.1.2", "luxon": "3.6.1", - "marked": "17.0.6", + "marked": "18.0.7", "marked-shiki": "1.2.1", "opentui-spinner": "0.0.7", "remeda": "2.26.0", @@ -4318,9 +4313,7 @@ "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], - "marked": ["marked@17.0.6", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA=="], - - "marked-katex-extension": ["marked-katex-extension@5.1.6", "", { "peerDependencies": { "katex": ">=0.16 <0.17", "marked": ">=4 <18" } }, "sha512-vYpLXwmlIDKILIhJtiRTgdyZRn5sEYdFBuTmbpjD7lbCIzg0/DWyK3HXIntN3Tp8zV6hvOUgpZNLWRCgWVc24A=="], + "marked": ["marked@18.0.7", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-iDVQ5ldaiKXn6b2JroX5kgRfmwgqolW7NpaEzTl1k/2Zh1njIEN9yniyLV/mOvWwtsE8OGgkjsCYvijuPk1dtA=="], "marked-shiki": ["marked-shiki@1.2.1", "", { "peerDependencies": { "marked": ">=7.0.0", "shiki": ">=1.0.0" } }, "sha512-yHxYQhPY5oYaIRnROn98foKhuClark7M373/VpLxiy5TrDu9Jd/LsMwo8w+U91Up4oDb9IXFrP0N1MFRz8W/DQ=="], diff --git a/package.json b/package.json index 2a8eef635687..15725c865fc2 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "hono-openapi": "1.1.2", "fuzzysort": "3.1.0", "luxon": "3.6.1", - "marked": "17.0.6", + "marked": "18.0.7", "marked-shiki": "1.2.1", "remend": "1.3.0", "@playwright/test": "1.59.1", diff --git a/packages/app/src/app.tsx b/packages/app/src/app.tsx index f47c432e4206..647cb55e5b4e 100644 --- a/packages/app/src/app.tsx +++ b/packages/app/src/app.tsx @@ -3,7 +3,6 @@ import * as Sentry from "@sentry/solid" import { I18nProvider } from "@opencode-ai/ui/context" import { DialogProvider } from "@opencode-ai/ui/context/dialog" import { FileComponentProvider } from "@opencode-ai/ui/context/file" -import { MarkedProvider } from "@opencode-ai/ui/context/marked" import { File } from "@opencode-ai/session-ui/file" import { Font } from "@opencode-ai/ui/font" import { Splash } from "@opencode-ai/ui/logo" @@ -405,9 +404,7 @@ export function AppBaseProviders(props: ParentProps<{ locale?: Locale }>) { - - {props.children} - + {props.children} diff --git a/packages/app/src/pages/home/home-sessions-controller.tsx b/packages/app/src/pages/home/home-sessions-controller.tsx index a12e9800e9fe..c0ff0c88ae5b 100644 --- a/packages/app/src/pages/home/home-sessions-controller.tsx +++ b/packages/app/src/pages/home/home-sessions-controller.tsx @@ -1,7 +1,6 @@ import type { Session } from "@opencode-ai/sdk/v2/client" import { preloadMarkdown } from "@opencode-ai/session-ui/markdown-cache" import { useDialog } from "@opencode-ai/ui/context/dialog" -import { useMarked } from "@opencode-ai/ui/context/marked" import { useQuery } from "@tanstack/solid-query" import { DateTime } from "luxon" import { type Accessor, createEffect, createMemo, createRoot, type JSX, startTransition } from "solid-js" @@ -44,7 +43,6 @@ export function createHomeSessionsController(home: HomeController) { const command = useCommand() const dialog = useDialog() const language = useLanguage() - const marked = useMarked() const projectDirectories = createMemo(() => { const project = home.project.selected() if (!project) return home.project.list().flatMap(directories) @@ -119,7 +117,7 @@ export function createHomeSessionsController(home: HomeController) { (ctx.sync.session.data.message[record.session.id] ?? []).flatMap((message) => (ctx.sync.session.data.part[message.id] ?? []).flatMap((part) => { if (part.type !== "text" || !part.text) return [] - return preloadMarkdown(part.text, part.id, marked) + return preloadMarkdown(part.text, part.id) }), ), ), diff --git a/packages/session-ui/package.json b/packages/session-ui/package.json index cec400af420c..da940b879e59 100644 --- a/packages/session-ui/package.json +++ b/packages/session-ui/package.json @@ -31,7 +31,6 @@ "devDependencies": { "@tsconfig/node22": "catalog:", "@types/bun": "catalog:", - "@types/katex": "0.16.7", "@types/luxon": "catalog:", "@typescript/native-preview": "catalog:", "typescript": "catalog:", @@ -54,11 +53,8 @@ "diff": "catalog:", "dompurify": "3.3.1", "fuzzysort": "catalog:", - "katex": "0.16.27", "luxon": "catalog:", "marked": "catalog:", - "marked-katex-extension": "5.1.6", - "marked-shiki": "catalog:", "morphdom": "2.7.8", "motion": "12.34.5", "remeda": "catalog:", diff --git a/packages/session-ui/src/components/markdown-cache.tsx b/packages/session-ui/src/components/markdown-cache.tsx index 3f8f0259f32c..ee9efef05607 100644 --- a/packages/session-ui/src/components/markdown-cache.tsx +++ b/packages/session-ui/src/components/markdown-cache.tsx @@ -1,6 +1,6 @@ import { checksum } from "@opencode-ai/core/util/encode" import DOMPurify from "dompurify" -import { project } from "./markdown-stream" +import { parseMarkdown } from "./markdown-worker" export type MarkdownCacheEntry = { raw: string @@ -52,27 +52,18 @@ export function touchCachedMarkdown(key: string, value: MarkdownCacheEntry) { cache.delete(first) } -export async function preloadMarkdown( - text: string, - cacheKey: string, - parser: { parse(text: string): string | Promise }, -) { - await Promise.all( - project(undefined, text, false).blocks.map(async (block, index) => { - if (block.mode === "code") return - const key = `${cacheKey}:${index}:${block.mode}` - const cached = getCachedMarkdown(key) - if (cached?.raw === block.raw) { - touchCachedMarkdown(key, cached) - return - } - const hash = checksum(block.raw) - if (!hash) return - touchCachedMarkdown(key, { - raw: block.raw, - hash, - html: sanitizeMarkdown(await Promise.resolve(parser.parse(block.src))), - }) - }), - ) +export async function preloadMarkdown(text: string, cacheKey: string) { + const key = `${cacheKey}:0:full` + const cached = getCachedMarkdown(key) + if (cached?.raw === text) { + touchCachedMarkdown(key, cached) + return + } + const hash = checksum(text) + if (!hash) return + touchCachedMarkdown(key, { + raw: text, + hash, + html: sanitizeMarkdown(await parseMarkdown(text)), + }) } diff --git a/packages/session-ui/src/components/markdown-preload.test.ts b/packages/session-ui/src/components/markdown-preload.test.ts deleted file mode 100644 index cba107def2a6..000000000000 --- a/packages/session-ui/src/components/markdown-preload.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, test } from "bun:test" -import { preloadMarkdown } from "./markdown-cache" - -test("preloads completed markdown into the render cache", async () => { - const parsed: string[] = [] - const parser = { - parse(text: string) { - parsed.push(text) - return `

${text}

` - }, - } - const key = `markdown-preload-${crypto.randomUUID()}` - - await preloadMarkdown("prepared response", key, parser) - await preloadMarkdown("prepared response", key, parser) - - expect(parsed).toEqual(["prepared response"]) -}) diff --git a/packages/session-ui/src/components/markdown-projection.ts b/packages/session-ui/src/components/markdown-projection.ts new file mode 100644 index 000000000000..459583d41191 --- /dev/null +++ b/packages/session-ui/src/components/markdown-projection.ts @@ -0,0 +1,11 @@ +import type { Block, Projection } from "./markdown-stream" + +export function completedProjection(text: string): Projection { + return { text, blocks: [{ raw: text, src: text, mode: "full" }] } +} + +export function canReusePendingBlock(current: Pick | undefined, next: Block) { + if (!current || current.mode !== next.mode) return false + if (next.mode === "code" || next.mode === "live") return next.raw.startsWith(current.raw) + return current.raw === next.raw +} diff --git a/packages/session-ui/src/components/markdown-stream.test.ts b/packages/session-ui/src/components/markdown-stream.test.ts index e792a3383968..baa0803f44b8 100644 --- a/packages/session-ui/src/components/markdown-stream.test.ts +++ b/packages/session-ui/src/components/markdown-stream.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" -import { canReusePendingBlock, project, stream } from "./markdown-stream" +import { canReusePendingBlock } from "./markdown-projection" +import { project, stream } from "./markdown-stream" describe("markdown stream", () => { test("heals incomplete emphasis while streaming", () => { @@ -129,6 +130,9 @@ describe("markdown stream", () => { expect( canReusePendingBlock({ mode: "code", raw: "```ts\none" }, { mode: "code", raw: "```ts\none two", src: "" }), ).toBe(true) + expect(canReusePendingBlock({ mode: "live", raw: "partial" }, { mode: "live", raw: "partial text", src: "" })).toBe( + true, + ) expect(canReusePendingBlock({ mode: "code", raw: "```ts\none" }, { mode: "live", raw: "one", src: "" })).toBe(false) }) @@ -145,6 +149,39 @@ describe("markdown stream", () => { }) }) + test("finalizes only the live tail when streaming stops", () => { + const live = project(undefined, "# Plan\n\nFinished paragraph.\n\n- final item", true) + const final = project(live, live.text, false) + + expect(final.blocks[0]).toBe(live.blocks[0]) + expect(final.blocks[1]).toBe(live.blocks[1]) + expect(final.blocks[2]).toEqual({ raw: "- final item", src: "- final item", mode: "full" }) + }) + + test("catches up paced text before finalizing", () => { + const live = project(undefined, "# Plan\n\nFinished paragraph.\n\n- final", true) + const final = project(live, `${live.text} item`, false) + + expect(canReusePendingBlock(live.blocks[0], final.blocks[0]!)).toBe(true) + expect(canReusePendingBlock(live.blocks[1], final.blocks[1]!)).toBe(true) + expect(final.blocks[2]).toEqual({ raw: "- final item", src: "- final item", mode: "full" }) + }) + + test("completes an open code block when streaming stops", () => { + const live = project(undefined, "```ts\nconst value = 1", true) + const final = project(live, live.text, false) + + expect(final.blocks).toEqual([ + { + raw: "```ts\nconst value = 1", + src: "const value = 1", + mode: "code", + language: "ts", + complete: true, + }, + ]) + }) + test("does not add a blank line before the first streamed code", () => { const previous = project(undefined, "```ts\n", true) const next = project(previous, `${previous.text}const x = 1`, true) diff --git a/packages/session-ui/src/components/markdown-stream.ts b/packages/session-ui/src/components/markdown-stream.ts index 3b1c25266037..18cf4157bebd 100644 --- a/packages/session-ui/src/components/markdown-stream.ts +++ b/packages/session-ui/src/components/markdown-stream.ts @@ -1,5 +1,6 @@ import { marked, type Tokens } from "marked" import remend from "remend" +import { completedProjection } from "./markdown-projection" export type Block = { raw: string @@ -50,7 +51,7 @@ function heal(text: string) { } export function stream(text: string, live: boolean): Block[] { - if (!live) return [{ raw: text, src: text, mode: "full" }] satisfies Block[] + if (!live) return completedProjection(text).blocks if (refs(text)) return [{ raw: text, src: heal(text), mode: "live" }] satisfies Block[] const tokens = marked.lexer(text) const tail = tokens.findLastIndex((token) => token.type !== "space") @@ -84,14 +85,25 @@ export function stream(text: string, live: boolean): Block[] { return [...result, { raw, src: openCode(code.raw), mode: "code", language: language(code.lang) }] } -export function canReusePendingBlock(current: Pick | undefined, next: Block) { - if (!current || current.mode !== next.mode) return false - if (next.mode === "code") return next.raw.startsWith(current.raw) - return current.raw === next.raw -} - export function project(previous: Projection | undefined, text: string, live: boolean): Projection { - if (!live || !previous || !text.startsWith(previous.text)) return { text, blocks: stream(text, live) } + if (!live) { + const current = + previous?.text === text + ? previous + : previous && text.startsWith(previous.text) + ? project(previous, text, true) + : undefined + if (!current) return completedProjection(text) + return { + text, + blocks: current.blocks.map((block) => { + if (block.mode === "live") return { raw: block.raw, src: block.raw, mode: "full" } + if (block.mode === "code" && !block.complete) return { ...block, complete: true } + return block + }), + } + } + if (!previous || !text.startsWith(previous.text)) return { text, blocks: stream(text, live) } const tail = previous.blocks.at(-1) const suffix = text.slice(previous.text.length) if (!suffix || tail?.mode !== "code" || tail.complete || closesFence(tail.raw, suffix)) diff --git a/packages/session-ui/src/components/markdown-worker-protocol.test.ts b/packages/session-ui/src/components/markdown-worker-protocol.test.ts index 45a169c6a286..ecd2967285cd 100644 --- a/packages/session-ui/src/components/markdown-worker-protocol.test.ts +++ b/packages/session-ui/src/components/markdown-worker-protocol.test.ts @@ -10,6 +10,7 @@ const response = (id: number, reset: boolean, stable: [string, string][], unstab type: "highlight" as const, id, key: "code", + language: "typescript", reset, stable, unstable, @@ -20,6 +21,7 @@ test("accumulates stable worker tokens and replaces the unstable tail", () => { type: "highlight", id: 1, key: "code", + language: "typescript", reset: true, stable: [token("one\n")], unstable: [token("tw")], @@ -28,6 +30,7 @@ test("accumulates stable worker tokens and replaces the unstable tail", () => { type: "highlight", id: 2, key: "code", + language: "typescript", reset: false, stable: [token("two\n")], unstable: [token("three")], @@ -35,6 +38,7 @@ test("accumulates stable worker tokens and replaces the unstable tail", () => { expect(second.stable.map((item) => item[0])).toEqual(["one\n", "two\n"]) expect(second.unstable.map((item) => item[0])).toEqual(["three"]) + expect(second.language).toBe("typescript") }) test("increments generation only when the worker resets token identity", () => { @@ -45,12 +49,13 @@ test("increments generation only when the worker resets token identity", () => { }) test("ignores stale worker responses and resets replacement streams", () => { - const current = { id: 2, generation: 1, stable: [token("current")], unstable: [] } + const current = { id: 2, generation: 1, language: "typescript", stable: [token("current")], unstable: [] } expect( applyMarkdownWorkerResponse(current, { type: "highlight", id: 1, key: "code", + language: "typescript", reset: false, stable: [token("stale")], unstable: [], @@ -62,6 +67,7 @@ test("ignores stale worker responses and resets replacement streams", () => { type: "highlight", id: 3, key: "code", + language: "typescript", reset: true, stable: [token("replacement")], unstable: [], diff --git a/packages/session-ui/src/components/markdown-worker-protocol.ts b/packages/session-ui/src/components/markdown-worker-protocol.ts index e0b59e38df20..0fb7e1e7459c 100644 --- a/packages/session-ui/src/components/markdown-worker-protocol.ts +++ b/packages/session-ui/src/components/markdown-worker-protocol.ts @@ -1,27 +1,32 @@ -import type { ThemeRegistrationResolved } from "shiki" +import type { Projection } from "./markdown-stream" export type MarkdownToken = [content: string, style: string] export type MarkdownWorkerRequest = - | { type: "init"; theme: ThemeRegistrationResolved } + | { type: "parse"; id: number; text: string } + | { type: "project"; id: number; key: string; text: string; live: boolean } | { type: "highlight"; id: number; key: string; text: string; language: string; complete?: boolean } | { type: "dispose"; key: string } export type MarkdownWorkerResponse = + | { type: "parse"; id: number; html: string } + | { type: "project"; id: number; key: string; projection: Projection } | { type: "highlight" id: number key: string + language: string reset: boolean stable: MarkdownToken[] unstable: MarkdownToken[] } - | { type: "error"; id: number; key: string; message: string } + | { type: "error"; id: number; key?: string; message: string } | { type: "superseded"; id: number; key: string } export type MarkdownWorkerState = { id: number generation: number + language: string stable: MarkdownToken[] unstable: MarkdownToken[] } @@ -42,6 +47,7 @@ export function applyMarkdownWorkerResponse( return { id: response.id, generation: (state?.generation ?? 0) + (response.reset ? 1 : 0), + language: response.language, stable: response.reset ? response.stable : [...(state?.stable ?? []), ...response.stable], unstable: response.unstable, } diff --git a/packages/session-ui/src/components/markdown-worker.ts b/packages/session-ui/src/components/markdown-worker.ts index 1ca87467bb0c..abb0e3c9caad 100644 --- a/packages/session-ui/src/components/markdown-worker.ts +++ b/packages/session-ui/src/components/markdown-worker.ts @@ -1,5 +1,4 @@ -import MarkdownShikiWorkerUrl from "./markdown-shiki.worker.ts?worker&url" -import { OpenCodeTheme } from "@opencode-ai/ui/context/marked" +import MarkdownWorkerUrl from "./markdown.worker.ts?worker&url" import { applyMarkdownWorkerResponse, shouldReleaseMarkdownWorkerState, @@ -8,18 +7,32 @@ import { type MarkdownWorkerState, } from "./markdown-worker-protocol" import { createWorkerTransport } from "./markdown-worker-transport" +import type { Projection } from "./markdown-stream" -type Pending = { +type HighlightPending = { key: string complete: boolean resolve: (state: MarkdownWorkerState) => void reject: (error: Error) => void } +type ProjectPending = { + key: string + resolve: (projection: Projection) => void + reject: (error: Error) => void +} + +type ParsePending = { + resolve: (html: string) => void + reject: (error: Error) => void +} + let worker: Worker | undefined let disabled: Error | undefined let nextID = 0 -const pending = new Map() +const pending = new Map() +const projects = new Map() +const parses = new Map() const states = new Map() const keys = new Set() const latest = new Map() @@ -32,6 +45,43 @@ const transport = createWorkerTransport>({ + post: (request) => worker!.postMessage(request), + supersede: (request) => { + const result = projects.get(request.id) + if (!result) return + projects.delete(request.id) + result.reject(new MarkdownWorkerSupersededError()) + }, +}) + +export function parseMarkdown(text: string) { + const instance = getWorker() + const id = ++nextID + return new Promise((resolve, reject) => { + parses.set(id, { resolve, reject }) + instance.postMessage({ type: "parse", id, text } satisfies MarkdownWorkerRequest) + }) +} + +export function projectMarkdown(key: string, text: string, live: boolean) { + getWorker() + const id = ++nextID + return new Promise((resolve, reject) => { + projects.set(id, { key, resolve, reject }) + projectTransport.send({ type: "project", id, key, text, live }) + }) +} + +export function disposeMarkdownProjection(key: string) { + projectTransport.dispose(key) + projects.forEach((request, id) => { + if (request.key !== key) return + projects.delete(id) + request.reject(new MarkdownWorkerDisposedError()) + }) + worker?.postMessage({ type: "dispose", key } satisfies MarkdownWorkerRequest) +} export function highlightStreamingCode(key: string, text: string, language: string, complete = false) { const instance = getWorker() @@ -67,48 +117,97 @@ function getWorker() { if (worker) return worker if (disabled) throw new MarkdownWorkerUnavailableError(disabled.message) try { - worker = new Worker(MarkdownShikiWorkerUrl, { type: "module" }) + worker = new Worker(MarkdownWorkerUrl, { type: "module" }) } catch (error) { disabled = error instanceof Error ? error : new Error(String(error)) throw new MarkdownWorkerUnavailableError(disabled.message) } worker.onmessage = (event: MessageEvent) => { + if (event.data.type === "parse") { + const result = parses.get(event.data.id) + if (!result) return + parses.delete(event.data.id) + result.resolve(event.data.html) + return + } + if (event.data.type === "project") { + const result = projects.get(event.data.id) + if (!result) { + projectTransport.complete(event.data.key, event.data.id) + return + } + projects.delete(event.data.id) + result.resolve(event.data.projection) + projectTransport.complete(event.data.key, event.data.id) + return + } + if (event.data.type === "error") { + const parsed = parses.get(event.data.id) + if (parsed) { + parses.delete(event.data.id) + parsed.reject(new Error(event.data.message)) + return + } + const projected = projects.get(event.data.id) + if (projected) { + projects.delete(event.data.id) + projected.reject(new Error(event.data.message)) + projectTransport.complete(projected.key, event.data.id) + return + } + } + if (event.data.type === "superseded") { + const projected = projects.get(event.data.id) + if (projected) { + projects.delete(event.data.id) + projected.reject(new MarkdownWorkerSupersededError()) + projectTransport.complete(projected.key, event.data.id) + return + } + } + const key = event.data.key + if (!key) return const result = pending.get(event.data.id) if (!result) { - transport.complete(event.data.key, event.data.id) + transport.complete(key, event.data.id) return } pending.delete(event.data.id) - if (!keys.has(event.data.key)) { + if (!keys.has(key)) { result.reject(new MarkdownWorkerDisposedError()) - transport.complete(event.data.key, event.data.id) + transport.complete(key, event.data.id) return } if (event.data.type === "superseded") { result.reject(new MarkdownWorkerSupersededError()) - transport.complete(event.data.key, event.data.id) + transport.complete(key, event.data.id) return } if (event.data.type === "error") { result.reject(new Error(event.data.message)) - transport.complete(event.data.key, event.data.id) + transport.complete(key, event.data.id) return } - const state = applyMarkdownWorkerResponse(states.get(event.data.key), event.data) - if (shouldReleaseMarkdownWorkerState(result.complete, latest.get(event.data.key), event.data.id)) { - states.delete(event.data.key) - keys.delete(event.data.key) - latest.delete(event.data.key) - } else states.set(event.data.key, state) + const state = applyMarkdownWorkerResponse(states.get(key), event.data) + if (shouldReleaseMarkdownWorkerState(result.complete, latest.get(key), event.data.id)) { + states.delete(key) + keys.delete(key) + latest.delete(key) + } else states.set(key, state) result.resolve(state) - transport.complete(event.data.key, event.data.id) + transport.complete(key, event.data.id) } const fail = (message: string) => { const error = new Error(message) disabled = error transport.reset() + projectTransport.reset() pending.forEach((request) => request.reject(error)) + projects.forEach((request) => request.reject(error)) + parses.forEach((request) => request.reject(error)) pending.clear() + projects.clear() + parses.clear() states.clear() keys.clear() latest.clear() @@ -117,6 +216,5 @@ function getWorker() { } worker.onerror = (event) => fail(event.message || "Markdown highlighting worker failed") worker.onmessageerror = () => fail("Markdown worker response failed") - worker.postMessage({ type: "init", theme: OpenCodeTheme } satisfies MarkdownWorkerRequest) return worker } diff --git a/packages/session-ui/src/components/markdown.tsx b/packages/session-ui/src/components/markdown.tsx index 02361f1db950..bd8d5a2dddfa 100644 --- a/packages/session-ui/src/components/markdown.tsx +++ b/packages/session-ui/src/components/markdown.tsx @@ -1,4 +1,3 @@ -import { useMarked } from "@opencode-ai/ui/context/marked" import { useI18n } from "@opencode-ai/ui/context/i18n" import morphdom from "morphdom" import { checksum } from "@opencode-ai/core/util/encode" @@ -6,7 +5,6 @@ import { type Accessor, type ComponentProps, createEffect, - createMemo, createResource, createSignal, createUniqueId, @@ -18,14 +16,17 @@ import { isServer, render } from "solid-js/web" import { Icon as IconV2 } from "@opencode-ai/ui/v2/icon" import { IconButtonV2 } from "@opencode-ai/ui/v2/icon-button-v2" import { TooltipV2 } from "@opencode-ai/ui/v2/tooltip-v2" -import { bundledLanguages } from "shiki" -import { canReusePendingBlock, project, type Block, type Projection } from "./markdown-stream" +import { canReusePendingBlock, completedProjection } from "./markdown-projection" +import type { Block, Projection } from "./markdown-stream" import { + disposeMarkdownProjection, disposeStreamingCode, highlightStreamingCode, MarkdownWorkerDisposedError, MarkdownWorkerSupersededError, MarkdownWorkerUnavailableError, + parseMarkdown, + projectMarkdown, } from "./markdown-worker" import { markdownBlockKey, type MarkdownToken } from "./markdown-worker-protocol" import { shouldResetCodeTokens, type RenderedCodeState } from "./markdown-code-state" @@ -67,10 +68,14 @@ function fallback(markdown: string) { } async function code(text: string, language: string | undefined, key: string, complete = false) { - const name = language && language in bundledLanguages ? language : "text" try { - const result = await highlightStreamingCode(key, text, name, complete) - return { language: name, generation: result.generation, stable: result.stable, unstable: result.unstable } + const result = await highlightStreamingCode(key, text, language ?? "text", complete) + return { + language: result.language, + generation: result.generation, + stable: result.stable, + unstable: result.unstable, + } } catch (error) { if ( !(error instanceof MarkdownWorkerDisposedError) && @@ -78,7 +83,7 @@ async function code(text: string, language: string | undefined, key: string, com !(error instanceof MarkdownWorkerUnavailableError) ) console.error("Markdown highlighting worker failed", error) - return { language: name, generation: 0, stable: [], unstable: [[text, ""] as MarkdownToken] } + return { language: language ?? "text", generation: 0, stable: [], unstable: [[text, ""] as MarkdownToken] } } } @@ -352,6 +357,10 @@ function initialResult(text: string, key: string | undefined, projection: Projec } } +function pendingProjection(text: string): Projection { + return { text, blocks: text ? [{ raw: text, src: text, mode: "live" }] : [] } +} + export function Markdown( props: ComponentProps<"div"> & { text: string @@ -362,21 +371,44 @@ export function Markdown( }, ) { const [local, others] = splitProps(props, ["text", "cacheKey", "streaming", "class", "classList"]) - const marked = useMarked() const i18n = useI18n() const [root, setRoot] = createSignal() const owner = createUniqueId() const activeCodeKeys = new Set() const completedCode = new Map>() - const projection = createMemo((previous: Projection | undefined) => - project(previous, local.text, local.streaming ?? false), + let streamed = false + const [projection] = createResource( + () => { + if (isServer) return + const live = local.streaming ?? false + if (live) streamed = true + if (!live && !streamed) return + return { key: owner, text: local.text, live } + }, + (src) => projectMarkdown(src.key, src.text, src.live), + { initialValue: pendingProjection("") }, ) + const currentProjection = () => { + if (!(local.streaming ?? false) && !streamed) return completedProjection(local.text) + const value = projection.latest + if (value?.text === local.text) return value + if (value?.text) return value + return pendingProjection(local.text) + } const [html] = createResource( () => { + if (isServer) + return { + text: local.text, + key: local.cacheKey, + projection: pendingProjection(local.text), + } + const value = !(local.streaming ?? false) && !streamed ? completedProjection(local.text) : projection.latest + if (!value || value.text !== local.text) return return { text: local.text, key: local.cacheKey, - projection: projection(), + projection: value, } }, async (src) => { @@ -426,7 +458,7 @@ export function Markdown( } const hash = checksum(block.raw) - const safe = sanitizeMarkdown(await Promise.resolve(marked.parse(block.src))) + const safe = sanitizeMarkdown(await parseMarkdown(block.src)) if (key && hash) touchCachedMarkdown(key, { raw: block.raw, hash, html: safe }) return { key: blockKey, mode: block.mode, raw: block.raw, hash: hash ?? "", html: safe } }), @@ -449,7 +481,12 @@ export function Markdown( ) }, { - initialValue: initialResult(local.text, local.cacheKey, projection(), owner), + initialValue: initialResult( + local.text, + local.cacheKey, + local.streaming ? pendingProjection(local.text) : completedProjection(local.text), + owner, + ), }, ) @@ -458,7 +495,7 @@ export function Markdown( createEffect(() => { const container = root() const result = html.latest ?? html() - const projected = projection() + const projected = currentProjection() const content = local.text ? pendingBlocks(result, projected, local.cacheKey, owner) : [] if (!container) return if (isServer) return @@ -497,6 +534,7 @@ export function Markdown( onCleanup(() => { if (copyCleanup) copyCleanup() + disposeMarkdownProjection(owner) activeCodeKeys.forEach(disposeCode) completedCode.clear() }) diff --git a/packages/session-ui/src/components/markdown-shiki.worker.ts b/packages/session-ui/src/components/markdown.worker.ts similarity index 56% rename from packages/session-ui/src/components/markdown-shiki.worker.ts rename to packages/session-ui/src/components/markdown.worker.ts index d7acabee7b6b..fe7187cfd590 100644 --- a/packages/session-ui/src/components/markdown-shiki.worker.ts +++ b/packages/session-ui/src/components/markdown.worker.ts @@ -1,6 +1,8 @@ /// import { ShikiStreamTokenizer } from "@shikijs/stream" +import { createMarkdownParser } from "@opencode-ai/ui/context/marked-parser" +import { OpenCodeTheme } from "@opencode-ai/ui/context/marked-theme" import { bundledLanguages, createHighlighter, @@ -11,6 +13,7 @@ import { } from "shiki" import type { MarkdownToken, MarkdownWorkerRequest, MarkdownWorkerResponse } from "./markdown-worker-protocol" import { createLatestWorkerQueue } from "./markdown-worker-queue" +import { project, type Projection } from "./markdown-stream" type Stream = { language: string @@ -19,30 +22,70 @@ type Stream = { } const streams = new Map() +const projections = new Map() let highlighter: ReturnType | undefined -const queue = createLatestWorkerQueue>({ +const highlightQueue = createLatestWorkerQueue>({ run: highlight, supersede: (request) => post({ type: "superseded", id: request.id, key: request.key }), dispose: (key) => void streams.delete(key), }) +const projectQueue = createLatestWorkerQueue>({ + run: runProject, + supersede: (request) => post({ type: "superseded", id: request.id, key: request.key }), + dispose: (key) => void projections.delete(key), +}) +const parser = createMarkdownParser(async (code, language) => { + const instance = await getHighlighter() + const name = language in bundledLanguages ? language : "text" + if (!instance.getLoadedLanguages().includes(name)) + await instance.loadLanguage(bundledLanguages[name as BundledLanguage]) + return instance.codeToHtml(code, { lang: name as BundledLanguage, theme: "OpenCode", tabindex: false }) +}) self.onmessage = (event: MessageEvent) => { - if (event.data.type === "init") { - highlighter ??= createHighlighter({ themes: [event.data.theme], langs: [] }) + if (event.data.type === "dispose") { + highlightQueue.dispose(event.data.key) + projectQueue.dispose(event.data.key) return } - if (event.data.type === "dispose") { - queue.dispose(event.data.key) + if (event.data.type === "parse") { + void parse(event.data) + return + } + if (event.data.type === "project") { + projectQueue.highlight(event.data) return } - queue.highlight(event.data) + highlightQueue.highlight(event.data) +} + +async function parse(request: Extract) { + try { + post({ type: "parse", id: request.id, html: await parser.parse(request.text) }) + } catch (error) { + post({ type: "error", id: request.id, message: error instanceof Error ? error.message : String(error) }) + } +} + +async function runProject(request: Extract) { + try { + const projection = project(projections.get(request.key), request.text, request.live) + projections.set(request.key, projection) + post({ type: "project", id: request.id, key: request.key, projection }) + } catch (error) { + post({ + type: "error", + id: request.id, + key: request.key, + message: error instanceof Error ? error.message : String(error), + }) + } } async function highlight(request: Extract) { try { - const instance = await highlighter - if (!instance) throw new Error("Shiki worker is not initialized") + const instance = await getHighlighter() const language = request.language in bundledLanguages ? request.language : "text" if (!instance.getLoadedLanguages().includes(language)) await instance.loadLanguage(bundledLanguages[language as BundledLanguage]) @@ -54,6 +97,7 @@ async function highlight(request: Extract @@ -81,6 +125,7 @@ async function highlight(request: Extract token.content.length > 0).map(token), unstable: result.unstable.filter((token) => token.content.length > 0).map(token), @@ -95,6 +140,10 @@ async function highlight(request: Extract
- }> - - +
@@ -1776,9 +1774,7 @@ PART_MAPPING["reasoning"] = function ReasoningPartDisplay(props) { return (
- }> - - +
) diff --git a/packages/session-ui/src/pierre/worker.ts b/packages/session-ui/src/pierre/worker.ts index d25dee4d9dcd..1c1bebb40b55 100644 --- a/packages/session-ui/src/pierre/worker.ts +++ b/packages/session-ui/src/pierre/worker.ts @@ -1,5 +1,8 @@ import { WorkerPoolManager } from "@pierre/diffs/worker" import ShikiWorkerUrl from "@pierre/diffs/worker/worker.js?worker&url" +import { registerOpenCodeTheme } from "@opencode-ai/ui/context/marked-theme-register" + +registerOpenCodeTheme() export type WorkerPoolStyle = "unified" | "split" diff --git a/packages/ui/package.json b/packages/ui/package.json index b7aa5ce794af..7409562b7e6c 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -94,7 +94,6 @@ "katex": "0.16.27", "luxon": "catalog:", "marked": "catalog:", - "marked-katex-extension": "5.1.6", "marked-shiki": "catalog:", "morphdom": "2.7.8", "motion": "12.34.5", diff --git a/packages/ui/src/context/marked-code-span.ts b/packages/ui/src/context/marked-code-span.ts deleted file mode 100644 index 895935c3116d..000000000000 --- a/packages/ui/src/context/marked-code-span.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { MarkedExtension } from "marked" - -// Keep adjacent tilde and backtick runs separate until markedjs/marked#4011 is released. -export const markedCodeSpanBoundary = { - tokenizer: { - inlineText(src) { - const match = /^(`+(?=~)|~+(?=`))/.exec(src) - if (!match) return false - return { - type: "text", - raw: match[0], - text: match[0], - escaped: this.lexer.state.inRawBlock, - } - }, - }, -} satisfies MarkedExtension diff --git a/packages/ui/src/context/marked-parser.test.ts b/packages/ui/src/context/marked-parser.test.ts new file mode 100644 index 000000000000..3c79e0afa712 --- /dev/null +++ b/packages/ui/src/context/marked-parser.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test" +import { createMarkdownParser } from "./marked-parser" + +const parser = createMarkdownParser((code, language) => `
${code}
`) + +test("renders links with application attributes", async () => { + expect(await parser.parse("[OpenCode](https://opencode.ai)")).toBe( + '

OpenCode

\n', + ) +}) + +test("renders inline and block math", async () => { + expect(await parser.parse("\\(x^2\\)")).toContain('') + expect(await parser.parse("$$\nx^2\n$$\n")).toContain('') +}) + +test("uses the configured code highlighter", async () => { + expect(await parser.parse("```ts\nconst value = 1\n```\n")).toBe('
const value = 1
\n') +}) diff --git a/packages/ui/src/context/marked-parser.tsx b/packages/ui/src/context/marked-parser.tsx new file mode 100644 index 000000000000..71e48351ca53 --- /dev/null +++ b/packages/ui/src/context/marked-parser.tsx @@ -0,0 +1,68 @@ +import katex from "katex" +import { Marked, type MarkedExtension, type Tokens } from "marked" +import markedShiki from "marked-shiki" + +export function createMarkdownParser(highlight: (code: string, language: string) => string | Promise) { + return new Marked( + { + renderer: { + link({ href, title, text }) { + const titleAttr = title ? ` title="${title}"` : "" + return `${text}` + }, + }, + }, + katexExtension, + markedShiki({ highlight }), + ) +} + +const inlineMathRegex = /^\\\(((?:\\.|[^\\\n])*?)\\\)/ +const blockMathRegex = /^\$\$\n([\s\S]+?)\n\$\$(?:\n|$)/ + +const katexExtension: MarkedExtension = { + extensions: [ + { + name: "inlineKatex", + level: "inline", + start(src) { + const index = src.indexOf("\\(") + if (index === -1) return + return index + }, + tokenizer(src) { + const match = src.match(inlineMathRegex) + if (!match) return + return { + type: "inlineKatex", + raw: match[0], + text: match[1].trim(), + displayMode: false, + } + }, + renderer: renderKatexToken, + }, + { + name: "blockKatex", + level: "block", + tokenizer(src) { + const match = src.match(blockMathRegex) + if (!match) return + return { + type: "blockKatex", + raw: match[0], + text: match[1].trim(), + displayMode: true, + } + }, + renderer: renderKatexToken, + }, + ], +} + +function renderKatexToken(token: Tokens.Generic) { + return katex.renderToString(typeof token.text === "string" ? token.text : "", { + displayMode: token.displayMode === true, + throwOnError: false, + }) +} diff --git a/packages/ui/src/context/marked-code-span.test.ts b/packages/ui/src/context/marked-regression.test.ts similarity index 82% rename from packages/ui/src/context/marked-code-span.test.ts rename to packages/ui/src/context/marked-regression.test.ts index ba67b2f67888..8861b930a073 100644 --- a/packages/ui/src/context/marked-code-span.test.ts +++ b/packages/ui/src/context/marked-regression.test.ts @@ -1,9 +1,8 @@ import { expect, test } from "bun:test" import { Marked } from "marked" -import { markedCodeSpanBoundary } from "./marked-code-span" test("preserves code spans adjacent to tildes", async () => { - const marked = new Marked(markedCodeSpanBoundary) + const marked = new Marked() expect(await marked.parse("~`0.1576` to measurement-window-only `0.00092`")).toBe( "

~0.1576 to measurement-window-only 0.00092

\n", diff --git a/packages/ui/src/context/marked-theme-register.tsx b/packages/ui/src/context/marked-theme-register.tsx new file mode 100644 index 000000000000..135381eab8d2 --- /dev/null +++ b/packages/ui/src/context/marked-theme-register.tsx @@ -0,0 +1,10 @@ +import { registerCustomTheme } from "@pierre/diffs" +import { OpenCodeTheme } from "./marked-theme" + +let registered = false + +export function registerOpenCodeTheme() { + if (registered) return + registered = true + registerCustomTheme("OpenCode", () => Promise.resolve(OpenCodeTheme)) +} diff --git a/packages/ui/src/context/marked-theme.tsx b/packages/ui/src/context/marked-theme.tsx new file mode 100644 index 000000000000..527a35e678f0 --- /dev/null +++ b/packages/ui/src/context/marked-theme.tsx @@ -0,0 +1,372 @@ +import type { ThemeRegistrationResolved } from "@pierre/diffs" + +export const OpenCodeTheme = { + name: "OpenCode", + bg: "var(--color-background-stronger)", + fg: "var(--text-base)", + colors: { + "editor.background": "var(--color-background-stronger)", + "editor.foreground": "var(--text-base)", + "gitDecoration.addedResourceForeground": "var(--syntax-diff-add)", + "gitDecoration.deletedResourceForeground": "var(--syntax-diff-delete)", + "gitDecoration.modifiedResourceForeground": "var(--syntax-diff-unknown)", + // "gitDecoration.conflictingResourceForeground": "#ffca00", + // "gitDecoration.modifiedResourceForeground": "#1a76d4", + // "gitDecoration.untrackedResourceForeground": "#00cab1", + // "gitDecoration.ignoredResourceForeground": "#84848A", + // "terminal.titleForeground": "#adadb1", + // "terminal.titleInactiveForeground": "#84848A", + // "terminal.background": "#141415", + // "terminal.foreground": "#adadb1", + // "terminal.ansiBlack": "#141415", + // "terminal.ansiRed": "#ff2e3f", + // "terminal.ansiGreen": "#0dbe4e", + // "terminal.ansiYellow": "#ffca00", + // "terminal.ansiBlue": "#008cff", + // "terminal.ansiMagenta": "#c635e4", + // "terminal.ansiCyan": "#08c0ef", + // "terminal.ansiWhite": "#c6c6c8", + // "terminal.ansiBrightBlack": "#141415", + // "terminal.ansiBrightRed": "#ff2e3f", + // "terminal.ansiBrightGreen": "#0dbe4e", + // "terminal.ansiBrightYellow": "#ffca00", + // "terminal.ansiBrightBlue": "#008cff", + // "terminal.ansiBrightMagenta": "#c635e4", + // "terminal.ansiBrightCyan": "#08c0ef", + // "terminal.ansiBrightWhite": "#c6c6c8", + }, + tokenColors: [ + { + scope: ["comment", "punctuation.definition.comment", "string.comment"], + settings: { + foreground: "var(--syntax-comment)", + }, + }, + { + scope: ["entity.other.attribute-name"], + settings: { + foreground: "var(--syntax-property)", // maybe attribute + }, + }, + { + scope: ["constant", "entity.name.constant", "variable.other.constant", "variable.language", "entity"], + settings: { + foreground: "var(--syntax-constant)", + }, + }, + { + scope: ["entity.name", "meta.export.default", "meta.definition.variable"], + settings: { + foreground: "var(--syntax-type)", + }, + }, + { + scope: ["meta.object.member"], + settings: { + foreground: "var(--syntax-primitive)", + }, + }, + { + scope: [ + "variable.parameter.function", + "meta.jsx.children", + "meta.block", + "meta.tag.attributes", + "entity.name.constant", + "meta.embedded.expression", + "meta.template.expression", + "string.other.begin.yaml", + "string.other.end.yaml", + ], + settings: { + foreground: "var(--syntax-punctuation)", + }, + }, + { + scope: ["entity.name.function", "support.type.primitive"], + settings: { + foreground: "var(--syntax-primitive)", + }, + }, + { + scope: ["support.class.component"], + settings: { + foreground: "var(--syntax-type)", + }, + }, + { + scope: "keyword", + settings: { + foreground: "var(--syntax-keyword)", + }, + }, + { + scope: [ + "keyword.operator", + "storage.type.function.arrow", + "punctuation.separator.key-value.css", + "entity.name.tag.yaml", + "punctuation.separator.key-value.mapping.yaml", + ], + settings: { + foreground: "var(--syntax-operator)", + }, + }, + { + scope: ["storage", "storage.type"], + settings: { + foreground: "var(--syntax-keyword)", + }, + }, + { + scope: ["storage.modifier.package", "storage.modifier.import", "storage.type.java"], + settings: { + foreground: "var(--syntax-primitive)", + }, + }, + { + scope: [ + "string", + "punctuation.definition.string", + "string punctuation.section.embedded source", + "entity.name.tag", + ], + settings: { + foreground: "var(--syntax-string)", + }, + }, + { + scope: "support", + settings: { + foreground: "var(--syntax-primitive)", + }, + }, + { + scope: ["support.type.object.module", "variable.other.object", "support.type.property-name.css"], + settings: { + foreground: "var(--syntax-object)", + }, + }, + { + scope: "meta.property-name", + settings: { + foreground: "var(--syntax-property)", + }, + }, + { + scope: "variable", + settings: { + foreground: "var(--syntax-variable)", + }, + }, + { + scope: "variable.other", + settings: { + foreground: "var(--syntax-variable)", + }, + }, + { + scope: [ + "invalid.broken", + "invalid.illegal", + "invalid.unimplemented", + "invalid.deprecated", + "message.error", + "markup.deleted", + "meta.diff.header.from-file", + "punctuation.definition.deleted", + "brackethighlighter.unmatched", + "token.error-token", + ], + settings: { + foreground: "var(--syntax-critical)", + }, + }, + { + scope: "carriage-return", + settings: { + foreground: "var(--syntax-keyword)", + }, + }, + { + scope: "string source", + settings: { + foreground: "var(--syntax-variable)", + }, + }, + { + scope: "string variable", + settings: { + foreground: "var(--syntax-constant)", + }, + }, + { + scope: [ + "source.regexp", + "string.regexp", + "string.regexp.character-class", + "string.regexp constant.character.escape", + "string.regexp source.ruby.embedded", + "string.regexp string.regexp.arbitrary-repitition", + "string.regexp constant.character.escape", + ], + settings: { + foreground: "var(--syntax-regexp)", + }, + }, + { + scope: "support.constant", + settings: { + foreground: "var(--syntax-primitive)", + }, + }, + { + scope: "support.variable", + settings: { + foreground: "var(--syntax-variable)", + }, + }, + { + scope: "meta.module-reference", + settings: { + foreground: "var(--syntax-info)", + }, + }, + { + scope: "punctuation.definition.list.begin.markdown", + settings: { + foreground: "var(--syntax-punctuation)", + }, + }, + { + scope: ["markup.heading", "markup.heading entity.name"], + settings: { + fontStyle: "bold", + foreground: "var(--syntax-info)", + }, + }, + { + scope: "markup.quote", + settings: { + foreground: "var(--syntax-info)", + }, + }, + { + scope: "markup.italic", + settings: { + fontStyle: "italic", + // foreground: "", + }, + }, + { + scope: "markup.bold", + settings: { + fontStyle: "bold", + foreground: "var(--text-strong)", + }, + }, + { + scope: [ + "markup.raw", + "markup.inserted", + "meta.diff.header.to-file", + "punctuation.definition.inserted", + "markup.changed", + "punctuation.definition.changed", + "markup.ignored", + "markup.untracked", + ], + settings: { + foreground: "var(--text-base)", + }, + }, + { + scope: "meta.diff.range", + settings: { + fontStyle: "bold", + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: "meta.diff.header", + settings: { + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: "meta.separator", + settings: { + fontStyle: "bold", + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: "meta.output", + settings: { + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: "meta.export.default", + settings: { + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: [ + "brackethighlighter.tag", + "brackethighlighter.curly", + "brackethighlighter.round", + "brackethighlighter.square", + "brackethighlighter.angle", + "brackethighlighter.quote", + ], + settings: { + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: ["constant.other.reference.link", "string.other.link"], + settings: { + fontStyle: "underline", + foreground: "var(--syntax-unknown)", + }, + }, + { + scope: "token.info-token", + settings: { + foreground: "var(--syntax-info)", + }, + }, + { + scope: "token.warn-token", + settings: { + foreground: "var(--syntax-warning)", + }, + }, + { + scope: "token.debug-token", + settings: { + foreground: "var(--syntax-info)", + }, + }, + ], + semanticTokenColors: { + comment: "var(--syntax-comment)", + string: "var(--syntax-string)", + number: "var(--syntax-constant)", + regexp: "var(--syntax-regexp)", + keyword: "var(--syntax-keyword)", + variable: "var(--syntax-variable)", + parameter: "var(--syntax-variable)", + property: "var(--syntax-property)", + function: "var(--syntax-primitive)", + method: "var(--syntax-primitive)", + type: "var(--syntax-type)", + class: "var(--syntax-type)", + namespace: "var(--syntax-type)", + enumMember: "var(--syntax-primitive)", + "variable.constant": "var(--syntax-constant)", + "variable.defaultLibrary": "var(--syntax-unknown)", + }, +} as unknown as ThemeRegistrationResolved diff --git a/packages/ui/src/context/marked.tsx b/packages/ui/src/context/marked.tsx index cead61ba6090..155682f718ae 100644 --- a/packages/ui/src/context/marked.tsx +++ b/packages/ui/src/context/marked.tsx @@ -1,570 +1,28 @@ -import { marked, type MarkedExtension, type Tokens } from "marked" -import markedShiki from "marked-shiki" -import katex from "katex" +import { getSharedHighlighter } from "@pierre/diffs" import { bundledLanguages, type BundledLanguage } from "shiki" import { createSimpleContext } from "./helper" -import { markedCodeSpanBoundary } from "./marked-code-span" -import { getSharedHighlighter, registerCustomTheme, ThemeRegistrationResolved } from "@pierre/diffs" +import { createMarkdownParser } from "./marked-parser" +import { registerOpenCodeTheme } from "./marked-theme-register" -export const OpenCodeTheme = { - name: "OpenCode", - bg: "var(--color-background-stronger)", - fg: "var(--text-base)", - colors: { - "editor.background": "var(--color-background-stronger)", - "editor.foreground": "var(--text-base)", - "gitDecoration.addedResourceForeground": "var(--syntax-diff-add)", - "gitDecoration.deletedResourceForeground": "var(--syntax-diff-delete)", - "gitDecoration.modifiedResourceForeground": "var(--syntax-diff-unknown)", - // "gitDecoration.conflictingResourceForeground": "#ffca00", - // "gitDecoration.modifiedResourceForeground": "#1a76d4", - // "gitDecoration.untrackedResourceForeground": "#00cab1", - // "gitDecoration.ignoredResourceForeground": "#84848A", - // "terminal.titleForeground": "#adadb1", - // "terminal.titleInactiveForeground": "#84848A", - // "terminal.background": "#141415", - // "terminal.foreground": "#adadb1", - // "terminal.ansiBlack": "#141415", - // "terminal.ansiRed": "#ff2e3f", - // "terminal.ansiGreen": "#0dbe4e", - // "terminal.ansiYellow": "#ffca00", - // "terminal.ansiBlue": "#008cff", - // "terminal.ansiMagenta": "#c635e4", - // "terminal.ansiCyan": "#08c0ef", - // "terminal.ansiWhite": "#c6c6c8", - // "terminal.ansiBrightBlack": "#141415", - // "terminal.ansiBrightRed": "#ff2e3f", - // "terminal.ansiBrightGreen": "#0dbe4e", - // "terminal.ansiBrightYellow": "#ffca00", - // "terminal.ansiBrightBlue": "#008cff", - // "terminal.ansiBrightMagenta": "#c635e4", - // "terminal.ansiBrightCyan": "#08c0ef", - // "terminal.ansiBrightWhite": "#c6c6c8", - }, - tokenColors: [ - { - scope: ["comment", "punctuation.definition.comment", "string.comment"], - settings: { - foreground: "var(--syntax-comment)", - }, - }, - { - scope: ["entity.other.attribute-name"], - settings: { - foreground: "var(--syntax-property)", // maybe attribute - }, - }, - { - scope: ["constant", "entity.name.constant", "variable.other.constant", "variable.language", "entity"], - settings: { - foreground: "var(--syntax-constant)", - }, - }, - { - scope: ["entity.name", "meta.export.default", "meta.definition.variable"], - settings: { - foreground: "var(--syntax-type)", - }, - }, - { - scope: ["meta.object.member"], - settings: { - foreground: "var(--syntax-primitive)", - }, - }, - { - scope: [ - "variable.parameter.function", - "meta.jsx.children", - "meta.block", - "meta.tag.attributes", - "entity.name.constant", - "meta.embedded.expression", - "meta.template.expression", - "string.other.begin.yaml", - "string.other.end.yaml", - ], - settings: { - foreground: "var(--syntax-punctuation)", - }, - }, - { - scope: ["entity.name.function", "support.type.primitive"], - settings: { - foreground: "var(--syntax-primitive)", - }, - }, - { - scope: ["support.class.component"], - settings: { - foreground: "var(--syntax-type)", - }, - }, - { - scope: "keyword", - settings: { - foreground: "var(--syntax-keyword)", - }, - }, - { - scope: [ - "keyword.operator", - "storage.type.function.arrow", - "punctuation.separator.key-value.css", - "entity.name.tag.yaml", - "punctuation.separator.key-value.mapping.yaml", - ], - settings: { - foreground: "var(--syntax-operator)", - }, - }, - { - scope: ["storage", "storage.type"], - settings: { - foreground: "var(--syntax-keyword)", - }, - }, - { - scope: ["storage.modifier.package", "storage.modifier.import", "storage.type.java"], - settings: { - foreground: "var(--syntax-primitive)", - }, - }, - { - scope: [ - "string", - "punctuation.definition.string", - "string punctuation.section.embedded source", - "entity.name.tag", - ], - settings: { - foreground: "var(--syntax-string)", - }, - }, - { - scope: "support", - settings: { - foreground: "var(--syntax-primitive)", - }, - }, - { - scope: ["support.type.object.module", "variable.other.object", "support.type.property-name.css"], - settings: { - foreground: "var(--syntax-object)", - }, - }, - { - scope: "meta.property-name", - settings: { - foreground: "var(--syntax-property)", - }, - }, - { - scope: "variable", - settings: { - foreground: "var(--syntax-variable)", - }, - }, - { - scope: "variable.other", - settings: { - foreground: "var(--syntax-variable)", - }, - }, - { - scope: [ - "invalid.broken", - "invalid.illegal", - "invalid.unimplemented", - "invalid.deprecated", - "message.error", - "markup.deleted", - "meta.diff.header.from-file", - "punctuation.definition.deleted", - "brackethighlighter.unmatched", - "token.error-token", - ], - settings: { - foreground: "var(--syntax-critical)", - }, - }, - { - scope: "carriage-return", - settings: { - foreground: "var(--syntax-keyword)", - }, - }, - { - scope: "string source", - settings: { - foreground: "var(--syntax-variable)", - }, - }, - { - scope: "string variable", - settings: { - foreground: "var(--syntax-constant)", - }, - }, - { - scope: [ - "source.regexp", - "string.regexp", - "string.regexp.character-class", - "string.regexp constant.character.escape", - "string.regexp source.ruby.embedded", - "string.regexp string.regexp.arbitrary-repitition", - "string.regexp constant.character.escape", - ], - settings: { - foreground: "var(--syntax-regexp)", - }, - }, - { - scope: "support.constant", - settings: { - foreground: "var(--syntax-primitive)", - }, - }, - { - scope: "support.variable", - settings: { - foreground: "var(--syntax-variable)", - }, - }, - { - scope: "meta.module-reference", - settings: { - foreground: "var(--syntax-info)", - }, - }, - { - scope: "punctuation.definition.list.begin.markdown", - settings: { - foreground: "var(--syntax-punctuation)", - }, - }, - { - scope: ["markup.heading", "markup.heading entity.name"], - settings: { - fontStyle: "bold", - foreground: "var(--syntax-info)", - }, - }, - { - scope: "markup.quote", - settings: { - foreground: "var(--syntax-info)", - }, - }, - { - scope: "markup.italic", - settings: { - fontStyle: "italic", - // foreground: "", - }, - }, - { - scope: "markup.bold", - settings: { - fontStyle: "bold", - foreground: "var(--text-strong)", - }, - }, - { - scope: [ - "markup.raw", - "markup.inserted", - "meta.diff.header.to-file", - "punctuation.definition.inserted", - "markup.changed", - "punctuation.definition.changed", - "markup.ignored", - "markup.untracked", - ], - settings: { - foreground: "var(--text-base)", - }, - }, - { - scope: "meta.diff.range", - settings: { - fontStyle: "bold", - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: "meta.diff.header", - settings: { - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: "meta.separator", - settings: { - fontStyle: "bold", - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: "meta.output", - settings: { - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: "meta.export.default", - settings: { - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: [ - "brackethighlighter.tag", - "brackethighlighter.curly", - "brackethighlighter.round", - "brackethighlighter.square", - "brackethighlighter.angle", - "brackethighlighter.quote", - ], - settings: { - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: ["constant.other.reference.link", "string.other.link"], - settings: { - fontStyle: "underline", - foreground: "var(--syntax-unknown)", - }, - }, - { - scope: "token.info-token", - settings: { - foreground: "var(--syntax-info)", - }, - }, - { - scope: "token.warn-token", - settings: { - foreground: "var(--syntax-warning)", - }, - }, - { - scope: "token.debug-token", - settings: { - foreground: "var(--syntax-info)", - }, - }, - ], - semanticTokenColors: { - comment: "var(--syntax-comment)", - string: "var(--syntax-string)", - number: "var(--syntax-constant)", - regexp: "var(--syntax-regexp)", - keyword: "var(--syntax-keyword)", - variable: "var(--syntax-variable)", - parameter: "var(--syntax-variable)", - property: "var(--syntax-property)", - function: "var(--syntax-primitive)", - method: "var(--syntax-primitive)", - type: "var(--syntax-type)", - class: "var(--syntax-type)", - namespace: "var(--syntax-type)", - enumMember: "var(--syntax-primitive)", - "variable.constant": "var(--syntax-constant)", - "variable.defaultLibrary": "var(--syntax-unknown)", - }, -} as unknown as ThemeRegistrationResolved +export { OpenCodeTheme } from "./marked-theme" -registerCustomTheme("OpenCode", () => Promise.resolve(OpenCodeTheme)) - -function renderMathInText(text: string): string { - let result = text - - // Display math: $$...$$ - const displayMathRegex = /\$\$([\s\S]*?)\$\$/g - result = result.replace(displayMathRegex, (_, math) => { - try { - return katex.renderToString(math, { - displayMode: true, - throwOnError: false, - }) - } catch { - return `$$${math}$$` - } - }) - - // Inline math: \(...\) - const inlineMathRegex = /\\\(((?:\\.|[^\\\n])*?)\\\)/g - result = result.replace(inlineMathRegex, (_, math) => { - try { - return katex.renderToString(math, { - displayMode: false, - throwOnError: false, - }) - } catch { - return `\\(${math}\\)` - } - }) - - return result -} - -const inlineMathRegex = /^\\\(((?:\\.|[^\\\n])*?)\\\)/ -const blockMathRegex = /^\$\$\n([\s\S]+?)\n\$\$(?:\n|$)/ - -const katexExtension: MarkedExtension = { - extensions: [ - { - name: "inlineKatex", - level: "inline", - start(src) { - const index = src.indexOf("\\(") - if (index === -1) return - return index - }, - tokenizer(src) { - const match = src.match(inlineMathRegex) - if (!match) return - return { - type: "inlineKatex", - raw: match[0], - text: match[1].trim(), - displayMode: false, - } - }, - renderer: renderKatexToken, - }, - { - name: "blockKatex", - level: "block", - tokenizer(src) { - const match = src.match(blockMathRegex) - if (!match) return - return { - type: "blockKatex", - raw: match[0], - text: match[1].trim(), - displayMode: true, - } - }, - renderer: renderKatexToken, - }, - ], -} - -function renderKatexToken(token: Tokens.Generic) { - return katex.renderToString(typeof token.text === "string" ? token.text : "", { - displayMode: token.displayMode === true, - throwOnError: false, - }) -} - -function renderMathExpressions(html: string): string { - // Split on code/pre/kbd tags to avoid processing their contents - const codeBlockPattern = /(<(?:pre|code|kbd)[^>]*>[\s\S]*?<\/(?:pre|code|kbd)>)/gi - const parts = html.split(codeBlockPattern) - - return parts - .map((part, i) => { - // Odd indices are the captured code blocks - leave them alone - if (i % 2 === 1) return part - // Process math only in non-code parts - return renderMathInText(part) - }) - .join("") -} - -async function highlightCodeBlocks(html: string): Promise { - const codeBlockRegex = /
([\s\S]*?)<\/code><\/pre>/g
-  const matches = [...html.matchAll(codeBlockRegex)]
-  if (matches.length === 0) return html
-
-  const highlighter = await getSharedHighlighter({
-    themes: ["OpenCode"],
-    langs: [],
-    preferredHighlighter: "shiki-wasm",
-  })
-
-  let result = html
-  for (const match of matches) {
-    const [fullMatch, lang, escapedCode] = match
-    const code = escapedCode
-      .replace(/</g, "<")
-      .replace(/>/g, ">")
-      .replace(/&/g, "&")
-      .replace(/"/g, '"')
-      .replace(/'/g, "'")
-
-    let language = lang || "text"
-    if (!(language in bundledLanguages)) {
-      language = "text"
-    }
-    if (!highlighter.getLoadedLanguages().includes(language)) {
-      await highlighter.loadLanguage(language as BundledLanguage)
-    }
-
-    const highlighted = highlighter.codeToHtml(code, {
-      lang: language,
-      theme: "OpenCode",
-      tabindex: false,
-    })
-    result = result.replace(fullMatch, () => highlighted)
-  }
-
-  return result
-}
-
-export type NativeMarkdownParser = (markdown: string) => Promise
+registerOpenCodeTheme()
 
 export const { use: useMarked, provider: MarkedProvider } = createSimpleContext({
   name: "Marked",
-  init: (props: { nativeParser?: NativeMarkdownParser }) => {
-    const jsParser = marked.use(
-      markedCodeSpanBoundary,
-      {
-        renderer: {
-          link({ href, title, text }) {
-            const titleAttr = title ? ` title="${title}"` : ""
-            return `${text}`
-          },
-        },
-      },
-      katexExtension,
-      markedShiki({
-        async highlight(code, lang) {
-          const highlighter = await getSharedHighlighter({
-            themes: ["OpenCode"],
-            langs: [],
-            preferredHighlighter: "shiki-wasm",
-          })
-          if (!(lang in bundledLanguages)) {
-            lang = "text"
-          }
-          if (!highlighter.getLoadedLanguages().includes(lang)) {
-            await highlighter.loadLanguage(lang as BundledLanguage)
-          }
-          return highlighter.codeToHtml(code, {
-            lang: lang || "text",
-            theme: "OpenCode",
-            tabindex: false,
-          })
-        },
-      }),
-    )
-
-    if (props.nativeParser) {
-      const nativeParser = props.nativeParser
-      return {
-        async parse(markdown: string): Promise {
-          const html = await nativeParser(markdown)
-          const withMath = renderMathExpressions(html)
-          return highlightCodeBlocks(withMath)
-        },
-      }
-    }
-
-    return jsParser
-  },
+  init: () =>
+    createMarkdownParser(async (code, language) => {
+      const highlighter = await getSharedHighlighter({
+        themes: ["OpenCode"],
+        langs: [],
+        preferredHighlighter: "shiki-wasm",
+      })
+      const name = language in bundledLanguages ? language : "text"
+      if (!highlighter.getLoadedLanguages().includes(name)) await highlighter.loadLanguage(name as BundledLanguage)
+      return highlighter.codeToHtml(code, {
+        lang: name,
+        theme: "OpenCode",
+        tabindex: false,
+      })
+    }),
 })