diff --git a/.env.local.example b/.env.local.example index 7bf8ed6f9..70d5ca27d 100644 --- a/.env.local.example +++ b/.env.local.example @@ -3,48 +3,53 @@ # cp .env.local.example .env.local # +# ----------------------------------------------------------------------------- +# Mintlify analytics (pnpm analytics:fetch*) +# Dashboard: https://app.mintlify.com/settings/organization/api-keys +# Script docs: .github/scripts/analytics/README.md +# +# MINTLIFY_API_KEY — Admin API key (mint_…). Not the Assistant key (mint_dsc_). +# MINTLIFY_PROJECT_ID — Project ID for this docs deployment (docs.comfy.org). +# ANALYTICS_PAGE_LIMIT — optional, rows per page (1–1000, default 200) +# ANALYTICS_PAGE_DELAY_MS — optional, ms between pages (default 36000 ≈ 100 req/h) +# ----------------------------------------------------------------------------- + MINTLIFY_API_KEY= +MINTLIFY_PROJECT_ID= +# ANALYTICS_PAGE_LIMIT=200 +# ANALYTICS_PAGE_DELAY_MS=36000 -# Used by: npm run translate, npm run cms:sync, etc. -# Requires Bun: https://bun.sh +# Used by: pnpm translate, pnpm cms:sync, etc. Requires Bun: https://bun.sh # ----------------------------------------------------------------------------- -# Translation API (translate-i18n.ts) -# OpenAI-compatible endpoint. Works with OpenRouter, DeepSeek, DashScope Qwen-MT, etc. +# Translation API (pnpm translate, pnpm cms:prepare) +# OpenAI-compatible endpoint — OpenRouter, DeepSeek, DashScope Qwen-MT, etc. # ----------------------------------------------------------------------------- # --- OpenRouter --- -# API keys: https://openrouter.ai/keys -# Docs: https://openrouter.ai/docs -# Models: any OpenRouter model id, e.g. deepseek/deepseek-chat, anthropic/claude-sonnet-4 +# https://openrouter.ai/keys # TRANSLATE_API_KEY= # TRANSLATE_API_BASE_URL=https://openrouter.ai/api/v1 # TRANSLATE_API_MODEL=deepseek/deepseek-chat # --- DeepSeek --- -# API keys: https://platform.deepseek.com/api_keys -# Docs: https://api-docs.deepseek.com/ -# Models: deepseek-v4-pro (quality) | deepseek-v4-flash (faster/cheaper) -# Note: deepseek-chat / deepseek-reasoner are deprecated after 2026-07-24. +# https://platform.deepseek.com/api_keys # TRANSLATE_API_KEY= # TRANSLATE_API_BASE_URL=https://api.deepseek.com # TRANSLATE_API_MODEL=deepseek-v4-pro -# TRANSLATE_API_MODEL=deepseek-v4-flash # TRANSLATE_CONCURRENCY=5 -# --- DashScope Qwen-MT (alternative) --- +# --- DashScope Qwen-MT --- # TRANSLATE_API_KEY= # TRANSLATE_API_BASE_URL=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 # TRANSLATE_API_MODEL=qwen-mt-plus -# --- Other fallbacks --- # TRANSLATE_CJK_API_KEY= # DASHSCOPE_API_KEY= # ----------------------------------------------------------------------------- -# Translation quality review (review-i18n.ts / npm run translate:review) — optional -# Independent LLM-as-a-judge that scores translations. Use a CHEAP/FAST model — -# evaluation is lighter than translation. Falls back to TRANSLATE_* if unset. +# Translation review (pnpm translate:review) — optional +# Falls back to TRANSLATE_* when unset. Prefer a cheap/fast model. # ----------------------------------------------------------------------------- # REVIEW_API_KEY= @@ -53,30 +58,23 @@ MINTLIFY_API_KEY= # REVIEW_CONCURRENCY=5 # ----------------------------------------------------------------------------- -# Glossary sync (sync-glossary.mjs) — optional -# Path to the ComfyUI frontend locales. Defaults to ../ComfyUI_frontend/src/locales; -# also settable via frontend_locales_path in translation-config.json. +# Glossary sync (pnpm glossary:sync) — optional # ----------------------------------------------------------------------------- # FRONTEND_LOCALES_PATH=../ComfyUI_frontend/src/locales # ----------------------------------------------------------------------------- -# Optional — external link tracking (track-external-links.py) -# Usually set in GitHub Actions; only needed for local runs +# External link tracking (track-external-links.py) — optional, usually CI only # ----------------------------------------------------------------------------- # GITHUB_TOKEN= # ----------------------------------------------------------------------------- -# Strapi CMS — changelog sync (see .github/scripts/cms/README.md) -# Create token: Strapi Admin → Settings → API Tokens (find/create/update release-note) +# Strapi CMS (pnpm cms:sync) — see .github/scripts/cms/README.md +# Create token: Strapi Admin → Settings → API Tokens (release-note permissions) +# CMS_SYNC_ALL=1 — optional full backfill (see cms README) # ----------------------------------------------------------------------------- # CMS_BASE_URL=https://cms.example.com # CMS_API_TOKEN= # CMS_PROJECT=comfyui - -# Translation (cms:prepare — same keys as pnpm translate) -# TRANSLATE_API_KEY= -# TRANSLATE_API_BASE_URL= -# TRANSLATE_API_MODEL=qwen-mt-plus diff --git a/.github/scripts/analytics/README.md b/.github/scripts/analytics/README.md new file mode 100644 index 000000000..d64b77e23 --- /dev/null +++ b/.github/scripts/analytics/README.md @@ -0,0 +1,107 @@ +# Mintlify analytics cache + +Local cache of Mintlify AI assistant, search, and feedback data for docs gap analysis. + +**Credentials:** [`.env.local.example`](../../../.env.local.example) + +## Design + +### Goal + +Find where the docs AI assistant fails (`unanswered`), what users search for, and negative page feedback — before editing content. + +### Data sources (Mintlify Admin API) + +Three endpoints, fetched in order: + +| Phase | API | What you get | +|-------|-----|--------------| +| **assistant** | `/v1/analytics/{projectId}/assistant` | User question, response, sources, `resolutionStatus` (`answered` / `unanswered`) | +| **searches** | `/v1/analytics/{projectId}/searches` | Search terms, hit counts, CTR, top clicked page | +| **feedback** | `/v1/analytics/{projectId}/feedback` | Page ratings and comments | + +There is no CSV export API — only paginated JSON. The dashboard “Export to CSV” is email-based and not scriptable. This CLI paginates, merges, and writes lean local reports. + +### Fetch model + +``` +CLI → 7-day date chunks (configurable) → paginated API pages → store/ merge → by-day/ + summary files +``` + +- **Incremental:** if `manifest.json` exists, only fetch since last `dateTo` (1-day overlap). +- **Checkpoint:** `checkpoint.json` + `store/` survive Ctrl+C, 504, or 429; re-run the same command to resume. +- **Flush:** every 10 API pages and after each chunk; assistant reports are written before searches start. +- **Rate limit:** 100 requests/org/hour shared across all analytics endpoints. Default 36s between pages. + +### Output layout (gitignored: `tmp/analytics-cache/`) + +| Path | Purpose | +|------|---------| +| `assistant-summary.md` | **Start here** — index linking to daily files | +| `by-day/YYYY-MM-DD.md` | That day's conversations (unanswered first) | +| `by-day/YYYY-MM-DD.json` | Slim JSON per day | +| `unanswered-index.json` | Days with unanswered questions | +| `searches-top.json` | Top 100 search terms (lean mode) | +| `feedback-negative.json` | Negative feedback only (lean mode) | +| `store/` | Raw merge state for resume/incremental | +| `checkpoint.json` | In-progress run state (removed on success) | +| `manifest.json` | Last completed run metadata | + +Use `--full` for monolithic JSON exports. Use `--assistant-only` to skip searches and feedback. + +### Recommended workflows + +| Task | Command | +|------|---------| +| Regular docs tuning | `pnpm analytics:fetch` (30 days, incremental) | +| AI Q&A only | `pnpm analytics:fetch:assistant` | +| One year of history | `pnpm analytics:fetch:all` | +| Custom dates | `pnpm analytics:fetch -- --date-from YYYY-MM-DD --date-to YYYY-MM-DD --fresh` | + +After a run, read `assistant-summary.md` → `by-day/YYYY-MM-DD.md` → `unanswered-index.json`. + +--- + +## Setup + +```bash +cp .env.local.example .env.local +# Fill MINTLIFY_API_KEY + MINTLIFY_PROJECT_ID — see .env.local.example +``` + +## Commands + +```bash +pnpm analytics:fetch # incremental if cache exists, else last 30 days +pnpm analytics:fetch:assistant # AI Q&A only (30 days; add --all for 1 year) +pnpm analytics:fetch:all # ~1 year, all three datasets; auto-resume +pnpm analytics:fetch -- --fresh # ignore cache, refetch window +pnpm analytics:fetch -- --resume # resume interrupted run only +pnpm analytics:fetch -- --days 14 +pnpm analytics:fetch -- --date-from 2025-01-01 --date-to 2025-12-31 +pnpm analytics:fetch -- --assistant-only +pnpm analytics:fetch -- --full +pnpm analytics:fetch:dry-run +``` + +### Date range + +| Flag | Meaning | +|------|---------| +| (default) | Last **30 days** | +| `--all` | Last **365 days** (1 year) | +| `--days N` | Last **N days** | +| `--date-from` + `--date-to` | Custom range; **both required**; last day is **inclusive** | + +Use `--fresh` when changing the date window so old checkpoint/store does not mix with the new range. + +### Checkpoint resume + +1. Progress in `tmp/analytics-cache/checkpoint.json` +2. Re-run the same command — finished chunks are skipped +3. `pnpm analytics:fetch -- --assistant-only --resume` — stop after assistant if stuck in searches phase + +### Resilience + +- **504 / 414:** 7-day chunks; page 2+ sends cursor only; auto-bisect on 414 +- **429:** backoff + resume; override throttle via `ANALYTICS_PAGE_LIMIT` / `ANALYTICS_PAGE_DELAY_MS` (see `.env.local.example`) diff --git a/.github/scripts/analytics/analytics-checkpoint.ts b/.github/scripts/analytics/analytics-checkpoint.ts new file mode 100644 index 000000000..67f17c4a6 --- /dev/null +++ b/.github/scripts/analytics/analytics-checkpoint.ts @@ -0,0 +1,205 @@ +import { mkdir, readFile, rename, unlink, writeFile } from "fs/promises"; +import { randomBytes } from "crypto"; +import { dirname, join } from "path"; +import type { + AssistantConversation, + DateRange, + FeedbackEntry, + SearchRow, +} from "./mintlify-analytics.ts"; +import { chunkKey, isoDateOnly } from "./mintlify-analytics.ts"; + +export const CHECKPOINT_FILE = "checkpoint.json"; +export const STORE_DIR = "store"; + +export interface FetchCheckpoint { + version: 1; + status: "in_progress" | "failed" | "complete"; + projectId: string; + targetRange: DateRange; + chunkDays: number; + mode: "lean" | "full"; + assistantOnly: boolean; + startedAt: string; + updatedAt: string; + phase: "assistant" | "searches" | "feedback" | "finalize"; + completedChunks: { + assistant: string[]; + searches: string[]; + feedback: string[]; + }; + error?: string; +} + +export interface ConversationStore { + range: DateRange; + conversations: AssistantConversation[]; +} + +export interface SearchStore { + range: DateRange; + totalSearches: number; + searches: SearchRow[]; +} + +export interface FeedbackStore { + range: DateRange; + feedback: FeedbackEntry[]; +} + +export interface ManifestSnapshot { + fetchedAt: string; + projectId: string; + dateFrom: string; + dateTo: string; +} + +export function storePath(cacheDir: string, name: string): string { + return join(cacheDir, STORE_DIR, name); +} + +export function checkpointPath(cacheDir: string): string { + return join(cacheDir, CHECKPOINT_FILE); +} + +export async function readJsonFile(path: string): Promise { + try { + return JSON.parse(await readFile(path, "utf-8")) as T; + } catch { + return null; + } +} + +export async function writeJsonFile(path: string, data: unknown): Promise { + await mkdir(dirname(path), { recursive: true }); + const tmp = `${path}.${randomBytes(8).toString("hex")}.tmp`; + const content = `${JSON.stringify(data, null, 2)}\n`; + try { + await writeFile(tmp, content, "utf-8"); + await rename(tmp, path); + } catch (error) { + await unlink(tmp).catch(() => {}); + throw error; + } +} + +export async function loadCheckpoint(cacheDir: string): Promise { + return readJsonFile(checkpointPath(cacheDir)); +} + +export async function saveCheckpoint(cacheDir: string, checkpoint: FetchCheckpoint): Promise { + await mkdir(cacheDir, { recursive: true }); + checkpoint.updatedAt = new Date().toISOString(); + await writeJsonFile(checkpointPath(cacheDir), checkpoint); +} + +export async function clearCheckpoint(cacheDir: string): Promise { + try { + await unlink(checkpointPath(cacheDir)); + } catch { + // ignore missing file + } +} + +export function completedChunkSet(checkpoint: FetchCheckpoint, resource: keyof FetchCheckpoint["completedChunks"]): Set { + return new Set(checkpoint.completedChunks[resource]); +} + +export async function markChunkComplete( + cacheDir: string, + checkpoint: FetchCheckpoint, + resource: keyof FetchCheckpoint["completedChunks"], + range: DateRange +): Promise { + const key = chunkKey(range); + if (!checkpoint.completedChunks[resource].includes(key)) { + checkpoint.completedChunks[resource].push(key); + } + await saveCheckpoint(cacheDir, checkpoint); +} + +export async function loadConversationStore(cacheDir: string): Promise { + return readJsonFile(storePath(cacheDir, "conversations.json")); +} + +export async function saveConversationStore(cacheDir: string, store: ConversationStore): Promise { + await writeJsonFile(storePath(cacheDir, "conversations.json"), store); +} + +export async function loadSearchStore(cacheDir: string): Promise { + return readJsonFile(storePath(cacheDir, "searches.json")); +} + +export async function saveSearchStore(cacheDir: string, store: SearchStore): Promise { + await writeJsonFile(storePath(cacheDir, "searches.json"), store); +} + +export async function loadFeedbackStore(cacheDir: string): Promise { + return readJsonFile(storePath(cacheDir, "feedback.json")); +} + +export async function saveFeedbackStore(cacheDir: string, store: FeedbackStore): Promise { + await writeJsonFile(storePath(cacheDir, "feedback.json"), store); +} + +export function mergeConversations( + existing: AssistantConversation[], + incoming: AssistantConversation[] +): AssistantConversation[] { + const byId = new Map(); + for (const row of existing) byId.set(row.id, row); + for (const row of incoming) byId.set(row.id, row); + return [...byId.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp)); +} + +export function mergeSearches(existing: SearchRow[], incoming: SearchRow[]): SearchRow[] { + const byQuery = new Map(); + for (const row of existing) byQuery.set(row.searchQuery, { ...row }); + for (const row of incoming) { + const prev = byQuery.get(row.searchQuery); + if (!prev) { + byQuery.set(row.searchQuery, { ...row }); + continue; + } + // Overlapping incremental windows re-fetch the same range — keep the fresher row, don't sum hits. + if (row.lastSearchedAt >= prev.lastSearchedAt) { + prev.hits = row.hits; + prev.lastSearchedAt = row.lastSearchedAt; + prev.topClickedPage = row.topClickedPage; + prev.ctr = row.ctr; + } + } + return [...byQuery.values()].sort((a, b) => b.hits - a.hits); +} + +export function mergeFeedback(existing: FeedbackEntry[], incoming: FeedbackEntry[]): FeedbackEntry[] { + const byId = new Map(); + for (const row of existing) byId.set(row.id, row); + for (const row of incoming) byId.set(row.id, row); + return [...byId.values()].sort((a, b) => + (a.createdAt ?? "").localeCompare(b.createdAt ?? "") + ); +} + +export function unionRange(a: DateRange, b: DateRange): DateRange { + return { + dateFrom: a.dateFrom < b.dateFrom ? a.dateFrom : b.dateFrom, + dateTo: a.dateTo > b.dateTo ? a.dateTo : b.dateTo, + }; +} + +/** Incremental fetch window: from last manifest dateTo (with 1-day overlap) through tomorrow. */ +export function incrementalRangeFromManifest(manifest: ManifestSnapshot): DateRange | null { + const dateTo = new Date(); + dateTo.setUTCDate(dateTo.getUTCDate() + 1); + const from = new Date(`${manifest.dateTo}T00:00:00.000Z`); + from.setUTCDate(from.getUTCDate() - 1); + const dateFrom = isoDateOnly(from); + const nextDateTo = isoDateOnly(dateTo); + if (dateFrom >= nextDateTo) return null; + return { dateFrom, dateTo: nextDateTo }; +} + +export function shouldResume(checkpoint: FetchCheckpoint | null): checkpoint is FetchCheckpoint { + return !!checkpoint && (checkpoint.status === "in_progress" || checkpoint.status === "failed"); +} diff --git a/.github/scripts/analytics/analytics-writers.ts b/.github/scripts/analytics/analytics-writers.ts new file mode 100644 index 000000000..74494e075 --- /dev/null +++ b/.github/scripts/analytics/analytics-writers.ts @@ -0,0 +1,267 @@ +import { mkdir, writeFile } from "fs/promises"; +import { join } from "path"; +import type { AssistantConversation, DateRange, FeedbackEntry, SearchRow } from "./mintlify-analytics.ts"; +import { writeJsonFile } from "./analytics-checkpoint.ts"; + +const BY_DAY_DIR = "by-day"; + +export interface SlimConversation { + id: string; + timestamp: string; + query: string; + resolutionStatus: string; + queryCategory: string | null; + pageUrl: string | null; + sourceCount: number; + sources: string[]; + responsePreview: string; +} + +function truncate(text: string, max = 240): string { + const normalized = text.replace(/\s+/g, " ").trim(); + if (normalized.length <= max) return normalized; + return `${normalized.slice(0, max - 1)}…`; +} + +export function toSlim(conversation: AssistantConversation, previewLen = 200): SlimConversation { + return { + id: conversation.id, + timestamp: conversation.timestamp, + query: truncate(conversation.query, previewLen), + resolutionStatus: conversation.resolutionStatus, + queryCategory: conversation.queryCategory, + pageUrl: conversation.pageUrl, + sourceCount: conversation.sources.length, + sources: conversation.sources.map((s) => s.title), + responsePreview: truncate(conversation.response, previewLen), + }; +} + +export function dayFromTimestamp(timestamp: string): string { + return timestamp.slice(0, 10); +} + +export function groupConversationsByDay( + conversations: AssistantConversation[] +): Map { + const map = new Map(); + for (const row of conversations) { + const day = dayFromTimestamp(row.timestamp); + const list = map.get(day) ?? []; + list.push(row); + map.set(day, list); + } + for (const list of map.values()) { + list.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + } + return map; +} + +export interface DayStats { + date: string; + conversations: number; + unanswered: number; + noSources: number; +} + +export function statsByDay(conversations: AssistantConversation[]): DayStats[] { + const byDay = groupConversationsByDay(conversations); + return [...byDay.entries()] + .map(([date, rows]) => ({ + date, + conversations: rows.length, + unanswered: rows.filter((r) => r.resolutionStatus === "unanswered").length, + noSources: rows.filter((r) => r.sources.length === 0).length, + })) + .sort((a, b) => b.date.localeCompare(a.date)); +} + +function buildDayMarkdown(day: string, conversations: AssistantConversation[]): string { + const unanswered = conversations.filter((c) => c.resolutionStatus === "unanswered"); + const lines: string[] = [ + `# Assistant insights — ${day}`, + "", + `- Conversations: **${conversations.length}**`, + `- Unanswered: **${unanswered.length}**`, + "", + ]; + + if (unanswered.length > 0) { + lines.push("## Unanswered", ""); + for (const item of unanswered) { + lines.push( + `- **${truncate(item.query, 200)}**`, + ` - Page: ${item.pageUrl ?? "(unknown)"}`, + ` - Category: ${item.queryCategory ?? "(none)"}`, + ` - Response: ${truncate(item.response, 180)}`, + "" + ); + } + } + + lines.push("## All conversations", ""); + for (const item of conversations) { + lines.push( + `- **${truncate(item.query, 160)}** (${item.resolutionStatus})`, + ` - ${item.timestamp}${item.pageUrl ? ` · ${item.pageUrl}` : ""}`, + item.sources.length > 0 + ? ` - Sources: ${item.sources.map((s) => s.title).join("; ")}` + : " - Sources: (none)", + "" + ); + } + + return lines.join("\n"); +} + +export function buildIndexMarkdown( + range: DateRange, + conversations: AssistantConversation[], + searches: SearchRow[], + feedback: FeedbackEntry[], + dayStats: DayStats[] +): string { + const unanswered = conversations.filter((c) => c.resolutionStatus === "unanswered"); + const negativeFeedback = feedback.filter((f) => f.helpful === false); + + const lines: string[] = [ + "# Mintlify assistant insights (index)", + "", + `Date range: ${range.dateFrom} → ${range.dateTo}`, + "", + "## Snapshot", + "", + `- Assistant conversations: **${conversations.length}**`, + `- Unanswered: **${unanswered.length}**`, + `- Search terms tracked: **${searches.length}**`, + `- Negative feedback: **${negativeFeedback.length}**`, + "", + "## Daily reports (newest first)", + "", + "Open a day file for full detail. Large exports are split under `by-day/`.", + "", + "| Date | Conversations | Unanswered | Report |", + "|------|---------------|------------|--------|", + ]; + + for (const row of dayStats) { + lines.push( + `| ${row.date} | ${row.conversations} | ${row.unanswered} | [${row.date}.md](by-day/${row.date}.md) |` + ); + } + + lines.push("", "## Unanswered days", ""); + const unansweredDays = dayStats.filter((d) => d.unanswered > 0); + if (unansweredDays.length === 0) { + lines.push("_None in this range._", ""); + } else { + for (const row of unansweredDays) { + lines.push(`- **${row.date}** — ${row.unanswered} unanswered → [by-day/${row.date}.md](by-day/${row.date}.md)`); + } + lines.push(""); + } + + if (searches.length > 0) { + lines.push("## Top search terms (all time in range)", ""); + for (const row of searches.slice(0, 15)) { + lines.push(`- **${row.searchQuery}** — ${row.hits} searches`); + } + lines.push(""); + } + + lines.push( + "## Other files", + "", + "- `by-day/YYYY-MM-DD.json` — slim JSON per day", + "- `searches-top.json` / `feedback-negative.json` — aggregated analytics", + "- `store/` — raw data for resume/incremental fetch", + "" + ); + + return lines.join("\n"); +} + +export async function writeDayPartition( + cacheDir: string, + day: string, + conversations: AssistantConversation[] +): Promise { + const dir = join(cacheDir, BY_DAY_DIR); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, `${day}.md`), buildDayMarkdown(day, conversations), "utf-8"); + await writeJsonFile(join(dir, `${day}.json`), { + date: day, + total: conversations.length, + unanswered: conversations.filter((c) => c.resolutionStatus === "unanswered").length, + conversations: conversations.map((c) => toSlim(c)), + }); +} + +/** Rewrite daily MD/JSON for given days using the full conversation store. */ +export async function writeDayPartitions( + cacheDir: string, + allConversations: AssistantConversation[], + daysToUpdate?: Set +): Promise { + const byDay = groupConversationsByDay(allConversations); + const days = daysToUpdate ?? new Set(byDay.keys()); + const written: string[] = []; + + for (const day of [...days].sort()) { + const rows = byDay.get(day) ?? []; + if (rows.length === 0) continue; + await writeDayPartition(cacheDir, day, rows); + written.push(`${BY_DAY_DIR}/${day}.md`); + } + + return written; +} + +export async function writeSplitSummaries( + cacheDir: string, + range: DateRange, + conversations: AssistantConversation[], + searches: SearchRow[], + feedback: FeedbackEntry[], + daysToUpdate?: Set +): Promise<{ dayFiles: string[]; dayStats: DayStats[] }> { + const dayFiles = await writeDayPartitions(cacheDir, conversations, daysToUpdate); + const dayStats = statsByDay(conversations); + await writeFile( + join(cacheDir, "assistant-summary.md"), + buildIndexMarkdown(range, conversations, searches, feedback, dayStats), + "utf-8" + ); + await writeJsonFile(join(cacheDir, "unanswered-index.json"), { + range, + days: dayStats + .filter((d) => d.unanswered > 0) + .map((d) => ({ date: d.date, unanswered: d.unanswered, file: `${BY_DAY_DIR}/${d.date}.md` })), + }); + return { dayFiles, dayStats }; +} + +export async function writeIndexOnly( + cacheDir: string, + range: DateRange, + conversations: AssistantConversation[], + searches: SearchRow[], + feedback: FeedbackEntry[] +): Promise { + const dayStats = statsByDay(conversations); + await writeFile( + join(cacheDir, "assistant-summary.md"), + buildIndexMarkdown(range, conversations, searches, feedback, dayStats), + "utf-8" + ); + await writeJsonFile(join(cacheDir, "unanswered-index.json"), { + range, + days: dayStats + .filter((d) => d.unanswered > 0) + .map((d) => ({ date: d.date, unanswered: d.unanswered, file: `${BY_DAY_DIR}/${d.date}.md` })), + }); +} + +export function daysInBatch(batch: AssistantConversation[]): Set { + return new Set(batch.map((c) => dayFromTimestamp(c.timestamp))); +} diff --git a/.github/scripts/analytics/fetch-assistant-insights.ts b/.github/scripts/analytics/fetch-assistant-insights.ts new file mode 100644 index 000000000..1282037f0 --- /dev/null +++ b/.github/scripts/analytics/fetch-assistant-insights.ts @@ -0,0 +1,722 @@ +#!/usr/bin/env bun +/** + * Pull Mintlify assistant/search/feedback analytics into a local cache for docs work. + * + * Usage: + * pnpm analytics:fetch # incremental if cache exists, else last 30 days + * pnpm analytics:fetch:all # ~1 year, chunked + checkpoint resume + * pnpm analytics:fetch:assistant # AI Q&A only (default 30d; use --all for 1y) + * pnpm analytics:fetch -- --fresh # ignore cache, refetch window + * pnpm analytics:fetch -- --resume # resume interrupted run only + * pnpm analytics:fetch -- --days 14 + * pnpm analytics:fetch -- --date-from 2025-01-01 --date-to 2025-12-31 + * pnpm analytics:fetch -- --assistant-only # skip searches & feedback + * pnpm analytics:fetch -- --full + * pnpm analytics:fetch:dry-run + * + * Checkpoint / incremental: + * - Each chunk writes immediately to tmp/analytics-cache/ (summary + JSON) + * - store/ holds raw merge state; checkpoint.json tracks progress for resume + * + * Output: tmp/analytics-cache/ (gitignored) + * assistant-summary.md — short index + links to by-day/ + * by-day/YYYY-MM-DD.md — one readable file per day + * by-day/YYYY-MM-DD.json — slim JSON per day + */ + +import { mkdir } from "fs/promises"; +import { join } from "path"; +import { loadEnvLocal, ROOT } from "../cms/cms-env.ts"; +import { + type FetchCheckpoint, + clearCheckpoint, + completedChunkSet, + incrementalRangeFromManifest, + loadCheckpoint, + loadConversationStore, + loadFeedbackStore, + loadSearchStore, + markChunkComplete, + mergeConversations, + mergeFeedback, + mergeSearches, + readJsonFile, + saveCheckpoint, + saveConversationStore, + saveFeedbackStore, + saveSearchStore, + shouldResume, + unionRange, + writeJsonFile, +} from "./analytics-checkpoint.ts"; +import { + type AssistantConversation, + type DateRange, + type FeedbackEntry, + type SearchRow, + dateRangeExplicit, + dateRangeForDays, + fetchAssistantConversationsChunked, + fetchFeedbackChunked, + fetchSearchQueriesChunked, + DEFAULT_PAGE_DELAY_MS, + DEFAULT_PAGE_LIMIT, + MINTLIFY_ANALYTICS_HOURLY_LIMIT, + daysInRange, +} from "./mintlify-analytics.ts"; +import { + daysInBatch, + statsByDay, + writeDayPartitions, + writeIndexOnly, +} from "./analytics-writers.ts"; + +const CACHE_DIR = join(ROOT, "tmp/analytics-cache"); +const DEFAULT_DAYS = 30; +const ALL_DAYS = 365; +const DEFAULT_CHUNK_DAYS = 7; +const TOP_SEARCHES = 100; +/** Persist store + summary to disk every N API pages (searches can be 100+ pages per chunk). */ +const SAVE_EVERY_PAGES = 10; + +type RunMode = "fresh" | "incremental" | "resume"; + +interface CliOptions { + days: number; + chunkDays: number; + pageLimit: number; + pageDelayMs: number; + dateFrom?: string; + dateTo?: string; + assistantOnly: boolean; + dryRun: boolean; + all: boolean; + full: boolean; + fresh: boolean; + incremental: boolean; + resume: boolean; +} + +interface Manifest { + fetchedAt: string; + status: "in_progress" | "complete"; + projectId: string; + dateFrom: string; + dateTo: string; + mode: "lean" | "full"; + chunkDays: number; + lastRun: RunMode; + outputDir: string; + counts: { + assistantConversations: number; + unansweredAssistant: number; + assistantWithoutSources: number; + searches: number; + feedback: number; + negativeFeedback: number; + }; + files: string[]; + dayFileCount?: number; +} + +function clampPageLimit(value: number): number { + return Math.min(1000, Math.max(1, value)); +} + +function parseArgs(argv: string[]): CliOptions { + const pageLimitEnv = Number(process.env.ANALYTICS_PAGE_LIMIT); + const pageDelayEnv = Number(process.env.ANALYTICS_PAGE_DELAY_MS); + const options: CliOptions = { + days: DEFAULT_DAYS, + chunkDays: DEFAULT_CHUNK_DAYS, + pageLimit: + Number.isFinite(pageLimitEnv) && pageLimitEnv > 0 + ? clampPageLimit(pageLimitEnv) + : DEFAULT_PAGE_LIMIT, + pageDelayMs: + Number.isFinite(pageDelayEnv) && pageDelayEnv >= 0 ? pageDelayEnv : DEFAULT_PAGE_DELAY_MS, + assistantOnly: false, + dryRun: false, + all: false, + full: false, + fresh: false, + incremental: false, + resume: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--dry-run") options.dryRun = true; + else if (arg === "--assistant-only") options.assistantOnly = true; + else if (arg === "--all") options.all = true; + else if (arg === "--full") options.full = true; + else if (arg === "--fresh") options.fresh = true; + else if (arg === "--incremental") options.incremental = true; + else if (arg === "--resume") options.resume = true; + else if (arg === "--page-limit") { + const value = Number(argv[++i]); + if (!Number.isFinite(value) || value < 1 || value > 1000) { + throw new Error("--page-limit must be between 1 and 1000"); + } + options.pageLimit = clampPageLimit(value); + } else if (arg.startsWith("--page-limit=")) { + const value = Number(arg.slice("--page-limit=".length)); + if (!Number.isFinite(value) || value < 1 || value > 1000) { + throw new Error("--page-limit must be between 1 and 1000"); + } + options.pageLimit = clampPageLimit(value); + } else if (arg === "--page-delay-ms") { + const value = Number(argv[++i]); + if (!Number.isFinite(value) || value < 0) { + throw new Error("--page-delay-ms must be >= 0"); + } + options.pageDelayMs = value; + } else if (arg.startsWith("--page-delay-ms=")) { + const value = Number(arg.slice("--page-delay-ms=".length)); + if (!Number.isFinite(value) || value < 0) { + throw new Error("--page-delay-ms must be >= 0"); + } + options.pageDelayMs = value; + } + else if (arg === "--days") { + const value = Number(argv[++i]); + if (!Number.isFinite(value) || value < 1) throw new Error("--days must be a positive number"); + options.days = value; + } else if (arg.startsWith("--days=")) { + const value = Number(arg.slice("--days=".length)); + if (!Number.isFinite(value) || value < 1) throw new Error("--days must be a positive number"); + options.days = value; + } else if (arg === "--chunk-days") { + const value = Number(argv[++i]); + if (!Number.isFinite(value) || value < 1) { + throw new Error("--chunk-days must be a positive number"); + } + options.chunkDays = value; + } else if (arg.startsWith("--chunk-days=")) { + const value = Number(arg.slice("--chunk-days=".length)); + if (!Number.isFinite(value) || value < 1) { + throw new Error("--chunk-days must be a positive number"); + } + options.chunkDays = value; + } else if (arg === "--date-from") { + options.dateFrom = argv[++i]; + } else if (arg.startsWith("--date-from=")) { + options.dateFrom = arg.slice("--date-from=".length); + } else if (arg === "--date-to") { + options.dateTo = argv[++i]; + } else if (arg.startsWith("--date-to=")) { + options.dateTo = arg.slice("--date-to=".length); + } + } + + if (options.dateFrom && !options.dateTo) { + throw new Error("--date-from requires --date-to (both ends of the range)"); + } + if (options.dateTo && !options.dateFrom) { + throw new Error("--date-to requires --date-from"); + } + + return options; +} + +function resolveFetchRange(options: CliOptions): DateRange { + if (options.dateFrom && options.dateTo) { + if (options.all) { + console.log("[analytics] Using --date-from/--date-to (--all ignored)"); + } + return dateRangeExplicit(options.dateFrom, options.dateTo); + } + const daySpan = options.all ? ALL_DAYS : options.days; + return dateRangeForDays(daySpan); +} + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`Missing ${name}. Copy .env.local.example → .env.local and fill it in.`); + } + return value; +} + +function createCheckpoint( + projectId: string, + range: DateRange, + chunkDays: number, + mode: "lean" | "full", + assistantOnly: boolean +): FetchCheckpoint { + return { + version: 1, + status: "in_progress", + projectId, + targetRange: range, + chunkDays, + mode, + assistantOnly, + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + phase: "assistant", + completedChunks: { assistant: [], searches: [], feedback: [] }, + }; +} + +function resolveRunMode( + options: CliOptions, + checkpoint: FetchCheckpoint | null, + manifest: Manifest | null, + projectId: string +): RunMode { + if (options.resume) { + if (!shouldResume(checkpoint)) { + throw new Error("No interrupted fetch to resume (checkpoint.json missing or complete)."); + } + return "resume"; + } + if (options.fresh) return "fresh"; + if (shouldResume(checkpoint)) return "resume"; + if ((options.incremental || manifest) && manifest?.projectId === projectId && !options.all) { + return "incremental"; + } + return "fresh"; +} + +async function writeOutputs( + options: CliOptions, + runMode: RunMode, + projectId: string, + range: DateRange, + chunkDays: number, + conversations: AssistantConversation[], + searches: SearchRow[], + totalSearches: number, + feedback: FeedbackEntry[], + status: Manifest["status"] = "complete", + logWrite = false, + daysToUpdate?: Set +): Promise { + const unanswered = conversations.filter((c) => c.resolutionStatus === "unanswered"); + const noSources = conversations.filter((c) => c.sources.length === 0); + const negativeFeedback = feedback.filter((f) => f.helpful === false); + const topSearches = searches.slice(0, TOP_SEARCHES); + const dayStats = statsByDay(conversations); + + if (daysToUpdate && daysToUpdate.size > 0) { + await writeDayPartitions(CACHE_DIR, conversations, daysToUpdate); + } + + await writeIndexOnly(CACHE_DIR, range, conversations, topSearches, negativeFeedback); + + const files = [ + "manifest.json", + "assistant-summary.md", + "unanswered-index.json", + "by-day/YYYY-MM-DD.md", + "by-day/YYYY-MM-DD.json", + ...(options.full ? ["assistant-conversations.json"] : []), + ...(options.assistantOnly + ? [] + : [ + options.full ? "searches.json" : "searches-top.json", + options.full ? "feedback.json" : "feedback-negative.json", + ]), + ]; + + const manifest: Manifest = { + fetchedAt: new Date().toISOString(), + status, + projectId, + dateFrom: range.dateFrom, + dateTo: range.dateTo, + mode: options.full ? "full" : "lean", + chunkDays, + lastRun: runMode, + outputDir: CACHE_DIR, + dayFileCount: dayStats.length, + counts: { + assistantConversations: conversations.length, + unansweredAssistant: unanswered.length, + assistantWithoutSources: noSources.length, + searches: searches.length, + feedback: feedback.length, + negativeFeedback: negativeFeedback.length, + }, + files, + }; + + await writeJsonFile(join(CACHE_DIR, "manifest.json"), manifest); + + if (options.full) { + await writeJsonFile(join(CACHE_DIR, "assistant-conversations.json"), { + range, + total: conversations.length, + conversations, + }); + } + + if (!options.assistantOnly) { + if (options.full) { + await writeJsonFile(join(CACHE_DIR, "searches.json"), { range, totalSearches, searches }); + await writeJsonFile(join(CACHE_DIR, "feedback.json"), { range, total: feedback.length, feedback }); + } else { + await writeJsonFile(join(CACHE_DIR, "searches-top.json"), { + range, + totalSearches, + top: TOP_SEARCHES, + searches: topSearches, + }); + await writeJsonFile(join(CACHE_DIR, "feedback-negative.json"), { + range, + total: negativeFeedback.length, + feedback: negativeFeedback, + }); + } + } + + if (logWrite) { + const dayHint = + daysToUpdate && daysToUpdate.size > 0 + ? `, updated days: ${[...daysToUpdate].sort().join(", ")}` + : ""; + console.log( + `[analytics] Updated ${CACHE_DIR} (${conversations.length} conversations, ${unanswered.length} unanswered${dayHint})` + ); + } +} + +/** Write user-facing cache files from current in-memory stores (called after each chunk). */ +async function flushOutputs( + options: CliOptions, + runMode: RunMode, + projectId: string, + outputRange: DateRange, + chunkDays: number, + conversationStore: { conversations: AssistantConversation[] }, + searchStore: { searches: SearchRow[]; totalSearches: number }, + feedbackStore: { feedback: FeedbackEntry[] }, + status: Manifest["status"], + logWrite: boolean, + daysToUpdate?: Set +): Promise { + await writeOutputs( + options, + runMode, + projectId, + outputRange, + chunkDays, + conversationStore.conversations, + searchStore.searches, + searchStore.totalSearches, + feedbackStore.feedback, + status, + logWrite, + daysToUpdate + ); +} + +async function main(): Promise { + await loadEnvLocal(); + const options = parseArgs(process.argv.slice(2)); + const apiKey = requireEnv("MINTLIFY_API_KEY"); + const projectId = requireEnv("MINTLIFY_PROJECT_ID"); + const config = { apiKey, projectId }; + + await mkdir(CACHE_DIR, { recursive: true }); + console.log(`[analytics] Output directory: ${CACHE_DIR}`); + + const existingManifest = await readJsonFile(join(CACHE_DIR, "manifest.json")); + let checkpoint = await loadCheckpoint(CACHE_DIR); + const runMode = resolveRunMode(options, checkpoint, existingManifest, projectId); + + let fetchRange: DateRange; + if (runMode === "resume" && checkpoint) { + fetchRange = checkpoint.targetRange; + console.log(`[analytics] Resuming interrupted fetch (${checkpoint.phase} phase)`); + } else if (runMode === "incremental" && existingManifest) { + const inc = incrementalRangeFromManifest(existingManifest); + if (!inc) { + console.log("[analytics] Already up to date — refreshing summary from store"); + const conversations = (await loadConversationStore(CACHE_DIR))?.conversations ?? []; + const searchStoreSnapshot = await loadSearchStore(CACHE_DIR); + const searches = searchStoreSnapshot?.searches ?? []; + const totalSearches = searchStoreSnapshot?.totalSearches ?? 0; + const feedback = (await loadFeedbackStore(CACHE_DIR))?.feedback ?? []; + const range = { + dateFrom: existingManifest.dateFrom, + dateTo: existingManifest.dateTo, + }; + if (!options.dryRun) { + await writeOutputs( + options, + "incremental", + projectId, + range, + existingManifest.chunkDays, + conversations, + searches, + totalSearches, + feedback + ); + } + return; + } + fetchRange = inc; + console.log(`[analytics] Incremental update: ${fetchRange.dateFrom} → ${fetchRange.dateTo}`); + } else { + fetchRange = resolveFetchRange(options); + const windowLabel = options.all ? "Full fetch (1 year)" : "Fetch window"; + console.log(`[analytics] ${windowLabel}: ${fetchRange.dateFrom} → ${fetchRange.dateTo}`); + } + + const chunkDays = Math.min(options.chunkDays, daysInRange(fetchRange)); + + let conversationStore = + runMode === "fresh" + ? { range: fetchRange, conversations: [] as AssistantConversation[] } + : (await loadConversationStore(CACHE_DIR)) ?? { range: fetchRange, conversations: [] }; + let searchStore = + runMode === "fresh" + ? { range: fetchRange, totalSearches: 0, searches: [] as SearchRow[] } + : (await loadSearchStore(CACHE_DIR)) ?? { range: fetchRange, totalSearches: 0, searches: [] }; + let feedbackStore = + runMode === "fresh" + ? { range: fetchRange, feedback: [] as FeedbackEntry[] } + : (await loadFeedbackStore(CACHE_DIR)) ?? { range: fetchRange, feedback: [] }; + + if (runMode === "fresh") { + await clearCheckpoint(CACHE_DIR); + checkpoint = createCheckpoint( + projectId, + fetchRange, + chunkDays, + options.full ? "full" : "lean", + options.assistantOnly + ); + await saveCheckpoint(CACHE_DIR, checkpoint); + } else if (runMode === "incremental") { + checkpoint = createCheckpoint( + projectId, + fetchRange, + chunkDays, + options.full ? "full" : "lean", + options.assistantOnly + ); + await saveCheckpoint(CACHE_DIR, checkpoint); + } else if (!checkpoint) { + throw new Error("Resume requested but checkpoint.json is missing."); + } + + console.log( + `[analytics] ${projectId} | mode ${runMode} | chunk ${chunkDays}d | output ${options.full ? "full" : "lean"}${options.assistantOnly ? " | assistant-only" : ""}` + ); + const reqPerHour = + options.pageDelayMs > 0 + ? Math.floor(3_600_000 / options.pageDelayMs) + : MINTLIFY_ANALYTICS_HOURLY_LIMIT; + console.log( + `[analytics] Rate: ${options.pageLimit}/page, ${options.pageDelayMs}ms delay (~${reqPerHour} req/h; Mintlify org limit ${MINTLIFY_ANALYTICS_HOURLY_LIMIT}/h)` + ); + if (reqPerHour > MINTLIFY_ANALYTICS_HOURLY_LIMIT) { + console.warn( + `[analytics] Delay may be too short for ${MINTLIFY_ANALYTICS_HOURLY_LIMIT}/h limit — try --page-delay-ms 36000` + ); + } + + const fetchOpts = { + limit: options.pageLimit, + pageDelayMs: options.pageDelayMs, + }; + + if (options.dryRun) { + console.log("[analytics] Dry run — would fetch and write cache"); + return; + } + + const cp = checkpoint!; + + const outputRange = (): DateRange => + runMode === "incremental" && existingManifest + ? unionRange( + { dateFrom: existingManifest.dateFrom, dateTo: existingManifest.dateTo }, + conversationStore.range + ) + : conversationStore.range; + + let pagesSinceDiskSave = 0; + + async function persistToDisk(label: string, daysToUpdate?: Set): Promise { + await saveConversationStore(CACHE_DIR, conversationStore); + await saveSearchStore(CACHE_DIR, searchStore); + await saveFeedbackStore(CACHE_DIR, feedbackStore); + await flushOutputs( + options, + runMode, + projectId, + outputRange(), + chunkDays, + conversationStore, + searchStore, + feedbackStore, + "in_progress", + false, + daysToUpdate + ); + cp.updatedAt = new Date().toISOString(); + await saveCheckpoint(CACHE_DIR, cp); + pagesSinceDiskSave = 0; + console.log(`\n[analytics] 💾 saved ${label}`); + console.log(`[analytics] ${join(CACHE_DIR, "store/searches.json")} (${searchStore.searches.length} unique search terms)`); + console.log(`[analytics] ${join(CACHE_DIR, "store/conversations.json")} (${conversationStore.conversations.length} conversations)`); + console.log(`[analytics] ${join(CACHE_DIR, "searches-top.json")}, manifest.json, checkpoint.json`); + console.log( + `[analytics] (updates existing files — tmp/ is gitignored, use Cmd+P to open or enable "Show Excluded Files")` + ); + } + + function trackPagePersist( + label: string, + mergeFn: () => Set | undefined + ): (items: unknown[]) => Promise { + return async (items) => { + pagesSinceDiskSave += 1; + if (pagesSinceDiskSave >= SAVE_EVERY_PAGES) { + await persistToDisk(label, mergeFn()); + } + }; + } + + try { + if (cp.phase === "assistant") { + await fetchAssistantConversationsChunked(config, fetchRange, chunkDays, { + ...fetchOpts, + skipChunks: completedChunkSet(cp, "assistant"), + onPageComplete: async (items) => { + conversationStore.conversations = mergeConversations( + conversationStore.conversations, + items as AssistantConversation[] + ); + await trackPagePersist( + `assistant progress (${conversationStore.conversations.length} conversations)`, + () => daysInBatch(items as AssistantConversation[]) + )(items); + }, + onChunkComplete: async (chunk, batch) => { + conversationStore.conversations = mergeConversations( + conversationStore.conversations, + batch as AssistantConversation[] + ); + conversationStore.range = unionRange(conversationStore.range, chunk); + await markChunkComplete(CACHE_DIR, cp, "assistant", chunk); + await persistToDisk( + `assistant chunk ${chunk.dateFrom} → ${chunk.dateTo}`, + daysInBatch(batch as AssistantConversation[]) + ); + }, + }); + cp.phase = options.assistantOnly ? "finalize" : "searches"; + await saveCheckpoint(CACHE_DIR, cp); + console.log(`[analytics] Assistant total in store: ${conversationStore.conversations.length}`); + await persistToDisk("assistant phase complete"); + console.log( + `[analytics] ✓ AI Q&A ready → assistant-summary.md, by-day/, unanswered-index.json` + ); + if (!options.assistantOnly) { + console.log(`[analytics] Continuing with searches, then feedback…`); + } + } + + if (!options.assistantOnly && (cp.phase === "searches" || cp.phase === "assistant")) { + if (cp.phase === "assistant") cp.phase = "searches"; + await fetchSearchQueriesChunked(config, fetchRange, chunkDays, { + ...fetchOpts, + skipChunks: completedChunkSet(cp, "searches"), + onPageComplete: async (items) => { + searchStore.searches = mergeSearches(searchStore.searches, items as SearchRow[]); + await trackPagePersist( + `searches progress (${searchStore.searches.length} terms)`, + () => undefined + )(items); + }, + onChunkComplete: async (chunk, batch) => { + searchStore.searches = mergeSearches(searchStore.searches, batch as SearchRow[]); + searchStore.range = unionRange(searchStore.range, chunk); + await markChunkComplete(CACHE_DIR, cp, "searches", chunk); + await persistToDisk( + `searches chunk ${chunk.dateFrom} → ${chunk.dateTo} (${searchStore.searches.length} terms)` + ); + }, + }); + cp.phase = "feedback"; + await saveCheckpoint(CACHE_DIR, cp); + console.log(`[analytics] Search terms in store: ${searchStore.searches.length}`); + } + + if (!options.assistantOnly && (cp.phase === "feedback" || cp.phase === "searches")) { + if (cp.phase === "searches") cp.phase = "feedback"; + await fetchFeedbackChunked(config, fetchRange, chunkDays, { + ...fetchOpts, + skipChunks: completedChunkSet(cp, "feedback"), + onPageComplete: async (items) => { + feedbackStore.feedback = mergeFeedback(feedbackStore.feedback, items as FeedbackEntry[]); + await trackPagePersist( + `feedback progress (${feedbackStore.feedback.length} entries)`, + () => undefined + )(items); + }, + onChunkComplete: async (chunk, batch) => { + feedbackStore.feedback = mergeFeedback(feedbackStore.feedback, batch as FeedbackEntry[]); + feedbackStore.range = unionRange(feedbackStore.range, chunk); + await markChunkComplete(CACHE_DIR, cp, "feedback", chunk); + await persistToDisk( + `feedback chunk ${chunk.dateFrom} → ${chunk.dateTo} (${feedbackStore.feedback.length} entries)` + ); + }, + }); + cp.phase = "finalize"; + await saveCheckpoint(CACHE_DIR, cp); + console.log(`[analytics] Feedback entries in store: ${feedbackStore.feedback.length}`); + } + + await writeOutputs( + options, + runMode, + projectId, + outputRange(), + chunkDays, + conversationStore.conversations, + searchStore.searches, + searchStore.totalSearches, + feedbackStore.feedback, + "complete", + false + ); + + cp.status = "complete"; + cp.phase = "finalize"; + await saveCheckpoint(CACHE_DIR, cp); + await clearCheckpoint(CACHE_DIR); + + console.log(`[analytics] Done → tmp/analytics-cache/`); + console.log( + `[analytics] Unanswered: ${conversationStore.conversations.filter((c) => c.resolutionStatus === "unanswered").length} | Start with assistant-summary.md` + ); + } catch (error) { + cp.status = "failed"; + cp.error = error instanceof Error ? error.message : String(error); + await saveCheckpoint(CACHE_DIR, cp); + if ( + conversationStore.conversations.length > 0 || + searchStore.searches.length > 0 || + feedbackStore.feedback.length > 0 + ) { + await persistToDisk("partial progress after error"); + console.error(`[analytics] Partial results saved under ${CACHE_DIR}`); + } + throw error; + } +} + +main().catch((error) => { + console.error(`[analytics] ${error instanceof Error ? error.message : String(error)}`); + console.error("[analytics] Re-run the same command to resume from checkpoint.json"); + process.exit(1); +}); diff --git a/.github/scripts/analytics/mintlify-analytics.ts b/.github/scripts/analytics/mintlify-analytics.ts new file mode 100644 index 000000000..94040b6ff --- /dev/null +++ b/.github/scripts/analytics/mintlify-analytics.ts @@ -0,0 +1,619 @@ +const API_BASE = "https://api.mintlify.com/v1"; +const DEFAULT_TIMEOUT_MS = 60_000; +const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); +/** Mintlify analytics: 100 requests/org/hour shared across all export endpoints. */ +export const MINTLIFY_ANALYTICS_HOURLY_LIMIT = 100; +export const DEFAULT_PAGE_LIMIT = 200; +/** ~100 requests/hour org limit → 36s between pages when unset. */ +export const DEFAULT_PAGE_DELAY_MS = 36_000; +const MAX_PAGINATION_PAGES = 10_000; + +export type ResolutionStatus = "answered" | "unanswered"; + +export interface AssistantSource { + title: string; + url: string; +} + +export interface AssistantConversation { + id: string; + timestamp: string; + query: string; + response: string; + sources: AssistantSource[]; + resolutionStatus: ResolutionStatus; + queryCategory: string | null; + pageUrl: string | null; +} + +export interface AssistantPage { + conversations: AssistantConversation[]; + nextCursor: string | null; + hasMore: boolean; +} + +export interface SearchRow { + searchQuery: string; + hits: number; + ctr: number; + topClickedPage: string | null; + lastSearchedAt: string; +} + +export interface SearchPage { + searches: SearchRow[]; + totalSearches: number; + nextCursor: string | null; +} + +export interface FeedbackEntry { + id: string; + path: string; + comment: string | null; + createdAt: string | null; + source: "code_snippet" | "contextual" | "agent" | "thumbs_only"; + status: "pending" | "in_progress" | "resolved" | "dismissed"; + helpful?: boolean; + contact?: string | null; + code?: string; + filename?: string | null; + lang?: string | null; +} + +export interface FeedbackPage { + feedback: FeedbackEntry[]; + nextCursor: string | null; + hasMore: boolean; +} + +export interface DateRange { + dateFrom: string; + dateTo: string; +} + +export interface MintlifyAnalyticsConfig { + apiKey: string; + projectId: string; +} + +export interface FetchProgress { + resource: "assistant" | "searches" | "feedback"; + range: DateRange; + page: number; + pageSize: number; + total: number; +} + +export interface PaginatedFetchOptions { + limit?: number; + /** Pause after each page (ms). Use ~36000 to stay under 100 req/hour org limit. */ + pageDelayMs?: number; + timeoutMs?: number; + maxRetries?: number; + onProgress?: (progress: FetchProgress) => void; + skipChunks?: Set; + onChunkComplete?: (chunk: DateRange, items: unknown[]) => void | Promise; + /** Called after each API page (use for incremental disk saves). */ + onPageComplete?: ( + items: unknown[], + meta: { page: number; chunkTotal: number; range: DateRange } + ) => void | Promise; +} + +export function chunkKey(range: DateRange): string { + return `${range.dateFrom}_${range.dateTo}`; +} + +function buildQuery(params: Record): string { + const search = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value === undefined || value === "") continue; + search.set(key, String(value)); + } + const query = search.toString(); + return query ? `?${query}` : ""; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Mintlify sometimes returns a full URL; only pass the cursor token in the next request. */ +function normalizeCursor(raw: string | null | undefined): string | undefined { + if (!raw) return undefined; + const trimmed = raw.trim(); + if (!trimmed) return undefined; + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + try { + const parsed = new URL(trimmed); + return parsed.searchParams.get("cursor") ?? trimmed; + } catch { + return trimmed; + } + } + return trimmed; +} + +function isMintlifyApiError(error: unknown): error is Error { + return error instanceof Error && error.message.startsWith("Mintlify API "); +} + +export function daysInRange(range: DateRange): number { + const from = new Date(`${range.dateFrom}T00:00:00.000Z`); + const to = new Date(`${range.dateTo}T00:00:00.000Z`); + return Math.max(1, Math.round((to.getTime() - from.getTime()) / 86_400_000)); +} + +function bisectRange(range: DateRange): [DateRange, DateRange] { + const from = new Date(`${range.dateFrom}T00:00:00.000Z`); + const to = new Date(`${range.dateTo}T00:00:00.000Z`); + const mid = new Date(from.getTime() + (to.getTime() - from.getTime()) / 2); + const midDay = isoDateOnly(mid); + return [ + { dateFrom: range.dateFrom, dateTo: midDay }, + { dateFrom: midDay, dateTo: range.dateTo }, + ]; +} + +async function mintlifyGet( + config: MintlifyAnalyticsConfig, + path: string, + params: Record = {}, + options: PaginatedFetchOptions = {} +): Promise { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const maxRetries = options.maxRetries ?? 4; + const url = `${API_BASE}${path}${buildQuery(params)}`; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch(url, { + signal: controller.signal, + headers: { + Authorization: `Bearer ${config.apiKey}`, + Accept: "application/json", + }, + }); + + if (!response.ok) { + let detail = response.statusText; + try { + const body = (await response.json()) as { error?: string }; + if (body.error) detail = body.error; + } catch { + // ignore parse errors + } + + if (RETRYABLE_STATUSES.has(response.status) && attempt < maxRetries) { + const retryAfterHeader = response.headers.get("Retry-After"); + const retryAfterSec = retryAfterHeader ? Number(retryAfterHeader) : NaN; + const waitMs = Number.isFinite(retryAfterSec) + ? Math.max(1000, retryAfterSec * 1000) + : response.status === 429 + ? Math.max(30_000, 1000 * 2 ** attempt) + : 1000 * 2 ** attempt; + console.warn( + `[analytics] ${response.status} ${detail} — retry ${attempt + 1}/${maxRetries} in ${Math.round(waitMs / 1000)}s` + ); + await sleep(waitMs); + continue; + } + + if (response.status === 403) { + throw new Error( + `Mintlify API 403: ${detail}. Use an Admin API key (mint_…) from the same organization as the project ID.` + ); + } + if (response.status === 414) { + throw new Error( + `Mintlify API 414: ${detail}. Pagination URL exceeded server limit — use smaller --chunk-days (e.g. 3 or 1).` + ); + } + throw new Error(`Mintlify API ${response.status}: ${detail}`); + } + + return (await response.json()) as T; + } catch (error) { + if (isMintlifyApiError(error)) throw error; + const isAbort = error instanceof Error && error.name === "AbortError"; + const message = isAbort ? `request timed out after ${timeoutMs}ms` : String(error); + if (attempt < maxRetries) { + const waitMs = 1000 * 2 ** attempt; + console.warn(`[analytics] ${message} — retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`); + await sleep(waitMs); + continue; + } + throw new Error(isAbort ? `Mintlify API timeout: ${message}` : message); + } finally { + clearTimeout(timer); + } + } + + throw new Error("Mintlify API request failed after retries"); +} + +async function fetchPaginated( + config: MintlifyAnalyticsConfig, + resource: FetchProgress["resource"], + path: string, + range: DateRange, + readItems: (page: unknown) => TItem[], + hasMore: (page: unknown, items: TItem[]) => { continue: boolean; cursor?: string }, + options: PaginatedFetchOptions = {} +): Promise { + const limit = Math.min(1000, Math.max(1, options.limit ?? DEFAULT_PAGE_LIMIT)); + const pageDelayMs = Math.max(0, options.pageDelayMs ?? DEFAULT_PAGE_DELAY_MS); + const all: TItem[] = []; + let cursor: string | undefined; + let previousCursor: string | undefined; + let page = 0; + + for (;;) { + if (page > 0 && pageDelayMs > 0) { + await sleep(pageDelayMs); + } + page += 1; + if (page > MAX_PAGINATION_PAGES) { + console.warn( + `[analytics] Stopping pagination after ${MAX_PAGINATION_PAGES} pages (${resource})` + ); + break; + } + // After page 1, cursor encodes the window — omit dateFrom/dateTo to keep the URL short. + const queryParams = cursor + ? { limit, cursor } + : { dateFrom: range.dateFrom, dateTo: range.dateTo, limit }; + + const raw = await mintlifyGet(config, path, queryParams, options); + const items = readItems(raw); + all.push(...items); + options.onProgress?.({ + resource, + range, + page, + pageSize: items.length, + total: all.length, + }); + await options.onPageComplete?.(items, { page, chunkTotal: all.length, range }); + + const next = hasMore(raw, items); + if (!next.continue || !next.cursor) break; + const nextCursor = normalizeCursor(next.cursor); + if (!nextCursor || nextCursor === cursor || nextCursor === previousCursor) { + console.warn( + `[analytics] Pagination cursor did not advance (${resource}, page ${page}); stopping` + ); + break; + } + previousCursor = cursor; + cursor = nextCursor; + } + + return all; +} + +async function fetchPaginatedWithBisect( + config: MintlifyAnalyticsConfig, + resource: FetchProgress["resource"], + path: string, + range: DateRange, + readItems: (page: unknown) => TItem[], + hasMore: (page: unknown, items: TItem[]) => { continue: boolean; cursor?: string }, + options: PaginatedFetchOptions = {} +): Promise { + try { + return await fetchPaginated(config, resource, path, range, readItems, hasMore, options); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes("414") || daysInRange(range) <= 1) throw error; + const [left, right] = bisectRange(range); + console.warn( + `[analytics] 414 for ${range.dateFrom} → ${range.dateTo}; splitting into ${left.dateFrom}..${left.dateTo} and ${right.dateFrom}..${right.dateTo}` + ); + const merged = [ + ...(await fetchPaginatedWithBisect(config, resource, path, left, readItems, hasMore, options)), + ...(await fetchPaginatedWithBisect(config, resource, path, right, readItems, hasMore, options)), + ]; + return merged; + } +} + +export async function fetchAllAssistantConversations( + config: MintlifyAnalyticsConfig, + range: DateRange, + options: PaginatedFetchOptions = {} +): Promise { + return fetchPaginatedWithBisect( + config, + "assistant", + `/analytics/${config.projectId}/assistant`, + range, + (page) => (page as AssistantPage).conversations, + (page) => { + const p = page as AssistantPage; + return { continue: p.hasMore && !!p.nextCursor, cursor: p.nextCursor ?? undefined }; + }, + { limit: DEFAULT_PAGE_LIMIT, ...options } + ); +} + +export async function fetchAllSearchQueries( + config: MintlifyAnalyticsConfig, + range: DateRange, + options: PaginatedFetchOptions = {} +): Promise<{ searches: SearchRow[]; totalSearches: number }> { + const path = `/analytics/${config.projectId}/searches`; + + async function fetchForRange(subRange: DateRange): Promise<{ searches: SearchRow[]; totalSearches: number }> { + let totalSearches = 0; + let capturedTotal = false; + try { + const searches = await fetchPaginated( + config, + "searches", + path, + subRange, + (page) => { + const p = page as SearchPage; + if (!capturedTotal) { + totalSearches = p.totalSearches; + capturedTotal = true; + } + return p.searches; + }, + (page) => { + const p = page as SearchPage; + return { continue: !!p.nextCursor, cursor: p.nextCursor ?? undefined }; + }, + { limit: DEFAULT_PAGE_LIMIT, ...options } + ); + return { searches, totalSearches }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes("414") || daysInRange(subRange) <= 1) throw error; + const [left, right] = bisectRange(subRange); + console.warn( + `[analytics] 414 for ${subRange.dateFrom} → ${subRange.dateTo}; splitting into ${left.dateFrom}..${left.dateTo} and ${right.dateFrom}..${right.dateTo}` + ); + const leftResult = await fetchForRange(left); + const rightResult = await fetchForRange(right); + return { + searches: [...leftResult.searches, ...rightResult.searches], + totalSearches: leftResult.totalSearches + rightResult.totalSearches, + }; + } + } + + return fetchForRange(range); +} + +export async function fetchAllFeedback( + config: MintlifyAnalyticsConfig, + range: DateRange, + options: PaginatedFetchOptions = {} +): Promise { + return fetchPaginatedWithBisect( + config, + "feedback", + `/analytics/${config.projectId}/feedback`, + range, + (page) => (page as FeedbackPage).feedback, + (page) => { + const p = page as FeedbackPage; + return { continue: p.hasMore && !!p.nextCursor, cursor: p.nextCursor ?? undefined }; + }, + { limit: DEFAULT_PAGE_LIMIT, ...options } + ); +} + +export function isoDateOnly(date: Date): string { + return date.toISOString().slice(0, 10); +} + +export function dateRangeForDays(days: number): DateRange { + const dateTo = new Date(); + dateTo.setUTCDate(dateTo.getUTCDate() + 1); + const dateFrom = new Date(dateTo); + dateFrom.setUTCDate(dateFrom.getUTCDate() - Math.max(1, days)); + return { + dateFrom: isoDateOnly(dateFrom), + dateTo: isoDateOnly(dateTo), + }; +} + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Validate YYYY-MM-DD (UTC calendar date). */ +export function parseIsoDateOnly(value: string, label: string): string { + const trimmed = value.trim(); + if (!ISO_DATE_RE.test(trimmed)) { + throw new Error(`${label} must be YYYY-MM-DD (got "${value}")`); + } + const parsed = new Date(`${trimmed}T00:00:00.000Z`); + if (Number.isNaN(parsed.getTime()) || isoDateOnly(parsed) !== trimmed) { + throw new Error(`${label} is not a valid calendar date (got "${value}")`); + } + return trimmed; +} + +/** Inclusive last day — converted to exclusive `dateTo` for the API. */ +export function dateRangeExplicit(dateFrom: string, dateToInclusive: string): DateRange { + const from = parseIsoDateOnly(dateFrom, "--date-from"); + const through = parseIsoDateOnly(dateToInclusive, "--date-to"); + const exclusiveEnd = new Date(`${through}T00:00:00.000Z`); + exclusiveEnd.setUTCDate(exclusiveEnd.getUTCDate() + 1); + const dateTo = isoDateOnly(exclusiveEnd); + if (from >= dateTo) { + throw new Error(`--date-from (${from}) must be before --date-to (${through})`); + } + return { dateFrom: from, dateTo }; +} + +/** Split [dateFrom, dateTo) into smaller windows (dateTo is exclusive). */ +export function splitDateRange(range: DateRange, chunkDays: number): DateRange[] { + const chunks: DateRange[] = []; + let cursor = new Date(`${range.dateFrom}T00:00:00.000Z`); + const end = new Date(`${range.dateTo}T00:00:00.000Z`); + + while (cursor < end) { + const chunkEnd = new Date(cursor); + chunkEnd.setUTCDate(chunkEnd.getUTCDate() + chunkDays); + if (chunkEnd > end) chunkEnd.setTime(end.getTime()); + chunks.push({ + dateFrom: isoDateOnly(cursor), + dateTo: isoDateOnly(chunkEnd), + }); + cursor = chunkEnd; + } + + return chunks; +} + +export async function fetchAssistantConversationsChunked( + config: MintlifyAnalyticsConfig, + range: DateRange, + chunkDays: number, + options: PaginatedFetchOptions = {} +): Promise { + const chunks = splitDateRange(range, chunkDays); + const merged: AssistantConversation[] = []; + const seen = new Set(); + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + if (options.skipChunks?.has(chunkKey(chunk))) { + console.log( + `[analytics] assistant chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo} (skipped, cached)` + ); + continue; + } + + console.log( + `[analytics] assistant chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo}` + ); + const batch = await fetchAllAssistantConversations(config, chunk, { + ...options, + onProgress: (progress) => { + options.onProgress?.(progress); + process.stdout.write( + `\r[analytics] page ${progress.page}, +${progress.pageSize} (chunk total ${progress.total}, all ${merged.length + progress.total}) ` + ); + }, + onPageComplete: options.onPageComplete, + }); + process.stdout.write("\n"); + + for (const row of batch) { + if (seen.has(row.id)) continue; + seen.add(row.id); + merged.push(row); + } + await options.onChunkComplete?.(chunk, batch); + console.log(`[analytics] chunk done: ${batch.length} rows (${merged.length} unique this run)`); + } + + return merged; +} + +export async function fetchSearchQueriesChunked( + config: MintlifyAnalyticsConfig, + range: DateRange, + chunkDays: number, + options: PaginatedFetchOptions = {} +): Promise<{ searches: SearchRow[]; totalSearches: number }> { + const chunks = splitDateRange(range, chunkDays); + const byQuery = new Map(); + let totalSearches = 0; + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + if (options.skipChunks?.has(chunkKey(chunk))) { + console.log( + `[analytics] searches chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo} (skipped, cached)` + ); + continue; + } + + console.log( + `[analytics] searches chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo}` + ); + const { searches, totalSearches: chunkTotal } = await fetchAllSearchQueries(config, chunk, { + ...options, + onProgress: (progress) => { + options.onProgress?.(progress); + process.stdout.write( + `\r[analytics] page ${progress.page}, +${progress.pageSize} (chunk ${progress.total}) ` + ); + }, + onPageComplete: options.onPageComplete, + }); + process.stdout.write("\n"); + totalSearches += chunkTotal; + + for (const row of searches) { + const existing = byQuery.get(row.searchQuery); + if (!existing) { + byQuery.set(row.searchQuery, { ...row }); + continue; + } + existing.hits += row.hits; + if (row.lastSearchedAt > existing.lastSearchedAt) { + existing.lastSearchedAt = row.lastSearchedAt; + existing.topClickedPage = row.topClickedPage; + existing.ctr = row.ctr; + } + } + await options.onChunkComplete?.(chunk, searches); + } + + const searches = [...byQuery.values()].sort((a, b) => b.hits - a.hits); + return { searches, totalSearches }; +} + +export async function fetchFeedbackChunked( + config: MintlifyAnalyticsConfig, + range: DateRange, + chunkDays: number, + options: PaginatedFetchOptions = {} +): Promise { + const chunks = splitDateRange(range, chunkDays); + const merged: FeedbackEntry[] = []; + const seen = new Set(); + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + if (options.skipChunks?.has(chunkKey(chunk))) { + console.log( + `[analytics] feedback chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo} (skipped, cached)` + ); + continue; + } + + console.log( + `[analytics] feedback chunk ${i + 1}/${chunks.length}: ${chunk.dateFrom} → ${chunk.dateTo}` + ); + const batch = await fetchAllFeedback(config, chunk, { + ...options, + onProgress: (progress) => { + options.onProgress?.(progress); + process.stdout.write( + `\r[analytics] page ${progress.page}, +${progress.pageSize} (chunk ${progress.total}) ` + ); + }, + onPageComplete: options.onPageComplete, + }); + process.stdout.write("\n"); + + for (const row of batch) { + if (seen.has(row.id)) continue; + seen.add(row.id); + merged.push(row); + } + await options.onChunkComplete?.(chunk, batch); + } + + return merged; +} diff --git a/.github/scripts/cms/README.md b/.github/scripts/cms/README.md index 5a35695ef..5b19540f1 100644 --- a/.github/scripts/cms/README.md +++ b/.github/scripts/cms/README.md @@ -2,6 +2,25 @@ Push **draft** release notes to Strapi CMS. Content is **simplified for end users** in staging, separate from full docs changelog. +## Relationship to docs translation + +`changelog/index.mdx` is the single English source. **Do not shorten it for CMS** — Step 1 (LLM simplify) produces popup copy in `staging/en/`. + +| | Docs translation ([i18n/README.md](../i18n/README.md)) | CMS sync (this doc) | +|--|--|--| +| Command | `pnpm translate` | `pnpm cms:prepare:en` / `cms:prepare:locales` / `cms:sync` | +| Output | `{lang}/**/*.mdx` on Mintlify | `.github/scripts/cms/staging/` → Strapi | +| Input | Full docs MDX | LLM-simplified popup copy | +| Locales | ja, zh, ko | en, zh, ja, ko, fr, ru, es | + +## Agent rules + +- Do **not** use `pnpm translate` to fill CMS staging — use `pnpm cms:prepare:en` then `cms:prepare:locales`. +- Get user approval on **staging EN** before `cms:prepare:locales`; on **all staging** before `cms:sync`. +- Sync and publish **comfyui only** by default (`--project comfyui`). Use `--project cloud` only after explicit user confirmation. +- Strapi publish is **manual by default** — run `pnpm cms:publish` after review (not automatic on sync). +- Do commit `.github/scripts/cms/staging/` and `published-versions.json` after Strapi publish. + ## Architecture Three separate steps — review between each; sync only after confirmation: @@ -66,7 +85,7 @@ pnpm cms:sync -- v0.25.1 # Step 3: push drafts (after staging pnpm cms:publish -- v0.25.1 # publish + refresh published-versions.json ``` -Default: **comfyui + cloud** on prepare / sync / publish. Single project: `--project cloud`. +Default: **comfyui + cloud** on prepare. **Sync/publish default for agents: comfyui only** — add `--project cloud` only when the user confirms. Local default (no args): **all unpublished EN versions** (from `published-versions.json`). Full backfill: `CMS_SYNC_ALL=1`. @@ -99,7 +118,7 @@ Configured in `cms-config.json` → `simplify`: ## API tokens -See `.env.local.example`: `TRANSLATE_API_KEY` (prepare), `CMS_API_TOKEN` (sync). +See [`.env.local.example`](../../../.env.local.example) (`TRANSLATE_*` for prepare, `CMS_*` for sync). ## Publish (draft → live) diff --git a/.github/scripts/i18n/README.md b/.github/scripts/i18n/README.md index 8d4f5a943..a960be843 100644 --- a/.github/scripts/i18n/README.md +++ b/.github/scripts/i18n/README.md @@ -4,6 +4,27 @@ Tooling that translates the English MDX docs into the languages listed in [`translation-config.json`](./translation-config.json) (currently ja / zh / ko). English is the single source of truth; every other language is generated. +## Relationship to CMS changelog + +`changelog/index.mdx` is the full English release notes for the **Mintlify docs site**. +This pipeline produces `{lang}/**/*.mdx` via `pnpm translate`. + +**CMS popup release notes** are a separate pipeline — simplified copy in +`.github/scripts/cms/staging/` → Strapi. See [cms/README.md](../cms/README.md). +Do **not** use `pnpm translate` to fill CMS staging. + +``` +changelog/index.mdx + ├─► pnpm translate → zh/ ja/ ko/ (Mintlify docs) + └─► pnpm cms:prepare → staging/ → Strapi (in-app popup) +``` + +## Agent rules + +- Do **not** commit `.github/i18n-logs/`. +- Do commit translated docs (`zh/`, `ja/`, `ko/`) after a translation run. +- Prose style for English MDX: see [AGENTS.md](../../../AGENTS.md#prose-style-english-mdx). + ## How translation works `translate-i18n.ts` is the entry point. It is **incremental**: each translated @@ -92,9 +113,8 @@ pnpm translate:review -- --sample 20 # N pending files per language pnpm translate:review -- --min-score 4 # report files scoring below 4/5 ``` -Configure a dedicated (cheap) judge model via `REVIEW_API_KEY` / -`REVIEW_API_BASE_URL` / `REVIEW_API_MODEL` in `.env.local`; falls back to the -`TRANSLATE_*` model when unset. Use a fast model — evaluation is lighter than +Configure a dedicated (cheap) judge model via `REVIEW_API_*` — see +[`.env.local.example`](../../../.env.local.example); falls back to the `TRANSLATE_*` model when unset. Use a fast model — evaluation is lighter than translation, and reasoning-heavy models are slow and can drop connections under concurrency (lower `REVIEW_CONCURRENCY` if you see socket errors). diff --git a/.gitignore b/.gitignore index e1ea91ca2..9c9cd0850 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ docs.bak !.env.local.example # Translation run logs (translate-i18n.ts) .github/i18n-logs/ -# CMS staging translations (generated by cms:prepare) +# Local scratch + analytics cache (fetch-assistant-insights.ts → tmp/analytics-cache/) tmp/ # Python bytecode __pycache__/ diff --git a/AGENTS.md b/AGENTS.md index 191c60fb2..58f9feb48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,96 +1,36 @@ # Agent guide — ComfyUI docs repo -English is the source of truth for documentation. This repo has **three separate pipelines** for content that leaves the Mintlify site — do not mix them. +English is the source of truth. **Docs translation**, **CMS changelog sync**, and **Mintlify analytics** are separate pipelines — do not mix them. -## Skills (read when relevant) +## Skills | Skill | Path | Use when | |-------|------|----------| -| **docs-i18n-translate** | [.cursor/skills/docs-i18n-translate/SKILL.md](.cursor/skills/docs-i18n-translate/SKILL.md) | Translating MDX to ja/zh/ko, `pnpm translate`, glossary, changelog docs | +| **docs-i18n-translate** | [.cursor/skills/docs-i18n-translate/SKILL.md](.cursor/skills/docs-i18n-translate/SKILL.md) | Translating MDX to ja/zh/ko, `pnpm translate`, glossary | | **docs-i18n-review** | [.cursor/skills/docs-i18n-review/SKILL.md](.cursor/skills/docs-i18n-review/SKILL.md) | Reviewing translation quality, `pnpm translate:review` | -| **cms-changelog-sync** | [.cursor/skills/cms-changelog-sync/SKILL.md](.cursor/skills/cms-changelog-sync/SKILL.md) | Strapi release notes, popup simplify, `pnpm cms:prepare`, `pnpm cms:sync` | +| **cms-changelog-sync** | [.cursor/skills/cms-changelog-sync/SKILL.md](.cursor/skills/cms-changelog-sync/SKILL.md) | Strapi release notes, `pnpm cms:prepare`, `pnpm cms:sync` | -Always load the matching skill before changing that pipeline. +Load the matching skill and its README before changing that pipeline. -## Three pipelines +## Prose style (English MDX) -``` -┌─────────────────────────────────────────────────────────────────┐ -│ changelog/index.mdx (full English — edit here for releases) │ -└────────────┬───────────────────────────────┬────────────────────┘ - │ │ - ▼ ▼ - DOCS (Mintlify) CMS (Strapi popup) - pnpm translate Step 1: pnpm cms:prepare:en → staging/en/ - → zh/ ja/ ko/ Step 2: pnpm cms:prepare:locales → staging/{lang}/ - COMMIT to git Step 3: pnpm cms:sync → Strapi draft (after review) - │ │ - ▼ ▼ - Mintlify site In-app notification - (full changelog) (3–5 bullets, PR links) -``` +When writing or editing English documentation, follow [.cursor/rules/docs-prose.mdc](.cursor/rules/docs-prose.mdc): -| | Docs translation | CMS sync | -|--|------------------|----------| -| Command | `pnpm translate` | `pnpm cms:prepare:en` / `cms:prepare:locales` / `cms:sync` | -| Output | `{lang}/**/*.mdx` | `.github/scripts/cms/staging/` | -| English input | Full docs MDX | LLM-simplified popup copy | -| Commit? | **Yes** | **Yes** (staging is reviewed and committed) | -| Locales | ja, zh, ko | en, zh, ja, ko, fr, ru, es | +- **Avoid em dashes (—).** They read as generic AI copy. Use periods, commas, colons, parentheses, or a second sentence instead. +- Prefer short, direct sentences over stacked clauses joined by dashes. +- Match the tone of surrounding pages: technical reference, not marketing blog. -## Quick commands +**Instead of:** `Comfy Cloud MCP is in public beta — APIs may change.` +**Prefer:** `Comfy Cloud MCP is in public beta. APIs may change while we iterate.` -### Docs — after editing English MDX - -```bash -pnpm translate:dry-run -pnpm translate -- changelog/index.mdx # or specific paths -pnpm translate:check-truncation # long pages / changelog -pnpm translate:review # optional quality pass -``` - -### CMS — after editing `changelog/index.mdx` - -```bash -pnpm cms:prepare:en -- --force v0.25.1 # Step 1: simplify EN -pnpm cms:prepare:locales -- v0.25.1 # Step 2: translate (after EN approved) -pnpm cms:preview -- v0.25.1 # Step 3: dry-run -pnpm cms:sync -- v0.25.1 # Step 3: push drafts (after user confirms) -pnpm cms:publish -- v0.25.1 # publish + refresh published-versions.json -``` - -Prepare default: **comfyui + cloud** together so staging stays mirrored. -Sync/publish default for agents: **comfyui only** (`--project comfyui`). Run **cloud** sync or publish only after the user explicitly confirms cloud, using `--project cloud`. - -Local default for prepare/sync: **all unpublished EN versions** per `published-versions.json`. Use `CMS_SYNC_ALL=1` for full backfill. - -## Environment - -Copy `.env.local.example` → `.env.local` (never commit). - -| Variable | Docs translate | CMS prepare | CMS sync | -|----------|----------------|-------------|----------| -| `TRANSLATE_API_KEY` | ✓ | ✓ | | -| `TRANSLATE_API_BASE_URL` | ✓ | ✓ | | -| `TRANSLATE_API_MODEL` | ✓ | ✓ | | -| `CMS_BASE_URL` | | | ✓ | -| `CMS_API_TOKEN` | | | ✓ | - -## Agent rules - -1. **Do not shorten** `changelog/index.mdx` for CMS — use the staging + simplify pipeline. -2. **Do not use** `pnpm translate` to fill CMS staging — use `pnpm cms:prepare:en` then `cms:prepare:locales`. -3. **Do not commit** `.github/i18n-logs/`. -4. **Do commit** translated docs (`zh/`, `ja/`, `ko/`), `.github/scripts/cms/staging/`, and `published-versions.json` after Strapi publish. -5. Get user approval on **staging EN** before `cms:prepare:locales`; get approval on **all staging** before `cms:sync`. -6. Sync and publish **comfyui only** by default (`--project comfyui`). Cloud sync/publish requires separate explicit user confirmation (`--project cloud`). -7. Strapi publish is **manual by default** — use `bun run cms:publish` after review (not automatic on sync). - -## Prose style - -When editing English MDX, avoid em dashes (—). Use periods, commas, or colons instead. Stacked em-dash clauses tend to read as AI-generated; keep sentences direct. +**Instead of:** `**Discord** — #channel for questions.` +**Prefer:** `**Discord**: #channel for questions.` ## Reference docs -- i18n tooling: [.github/scripts/i18n/README.md](.github/scripts/i18n/README.md) -- CMS tooling: [.github/scripts/cms/README.md](.github/scripts/cms/README.md) +| Topic | Doc | +|-------|-----| +| Env / secrets | [`.env.local.example`](.env.local.example) | +| Docs i18n | [.github/scripts/i18n/README.md](.github/scripts/i18n/README.md) | +| CMS changelog | [.github/scripts/cms/README.md](.github/scripts/cms/README.md) | +| Mintlify analytics | [.github/scripts/analytics/README.md](.github/scripts/analytics/README.md) | diff --git a/package.json b/package.json index 8758ba020..b0320352c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,12 @@ "cms:preview": "bun .github/scripts/cms/sync-to-strapi.ts --preview", "cms:sync": "bun .github/scripts/cms/sync-to-strapi.ts", "cms:publish": "bun .github/scripts/cms/publish-cms-drafts.ts", - "cms:set-attention": "bun .github/scripts/cms/set-cms-attention.ts" + "cms:set-attention": "bun .github/scripts/cms/set-cms-attention.ts", + "analytics:fetch": "bun .github/scripts/analytics/fetch-assistant-insights.ts", + "analytics:fetch:all": "bun .github/scripts/analytics/fetch-assistant-insights.ts --all", + "analytics:fetch:assistant": "bun .github/scripts/analytics/fetch-assistant-insights.ts --assistant-only", + "analytics:fetch:resume": "bun .github/scripts/analytics/fetch-assistant-insights.ts --resume", + "analytics:fetch:dry-run": "bun .github/scripts/analytics/fetch-assistant-insights.ts --dry-run" }, "devDependencies": { "@executeautomation/playwright-mcp-server": "^1.0.5"