Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions website/docs/models-recipes/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@ description: Family-owned build, runtime, and test implementations in this check

import ModelSupportInventory from '@site/src/components/ModelSupportInventory';

Support is defined by files that live inside one self-contained
`families/<family>/` directory. Each row below points to that family&apos;s own
E2E manifest; there is no central model registry or duplicated support matrix.
Select a **checkpoint** to see its exact recipe: build settings, revision,
test inputs, and configuration checks. Select a **family** to browse its
recipes, command parameters, configuration fields, defaults, and constraints.
These pages are generated from the current `families/<family>/` sources.

<ModelSupportInventory variant="facts" />

Expand All @@ -17,10 +18,19 @@ An entry means the repository contains a builder, a native runtime DSO that
implements an abstract Task API, and a family-owned test recipe for that exact
checkpoint. It is not a promise that an unlisted fine-tune is compatible.

Configuration checks use the current command parser, build-request validation,
and family source declarations. A passing configuration check does not establish
a GPU build, inference result, or parity result. Each detail page reports the
checks that ran and any settings requiring further verification.

A manifest with `hf_id` names its Hugging Face source. A manifest without
`hf_id` intentionally represents a family-owned prepared local checkpoint; the
inventory labels it as such instead of inventing Hugging Face metadata.

<ModelSupportInventory variant="models" />

For task-oriented navigation, see [Model Recipes](model-recipes.md).

## All families

<ModelSupportInventory variant="families" />
4 changes: 2 additions & 2 deletions website/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
"version": "0.1.0",
"private": true,
"scripts": {
"build": "npm run diagrams:check && docusaurus build",
"build": "npm run diagrams:check && docusaurus build && node scripts/check-model-pages.js",
"clear": "docusaurus clear",
"diagrams:build": "node scripts/render-diagrams.mjs",
"diagrams:check": "node scripts/render-diagrams.mjs --check",
"serve": "docusaurus serve",
"start": "docusaurus start --host 0.0.0.0",
"test:model-support": "node --test plugins/model-support-inventory/index.test.js"
"test:model-support": "node --test plugins/model-support-inventory/index.test.js && python3 -m unittest discover -s scripts -p 'test_collect*py'"
},
"dependencies": {
"@docusaurus/core": "3.10.2",
Expand Down
80 changes: 72 additions & 8 deletions website/plugins/model-support-inventory/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,21 @@

const fs = require('fs');
const path = require('path');
const {execFileSync} = require('child_process');

const RECIPE_BASE = '/models-recipes/model-recipes';

function collectFamilyDetails(repoRoot) {
const output = execFileSync(process.env.PYTHON || 'python3', [
path.resolve(__dirname, '../../scripts/collect-family-details.py'),
'--repo-root', repoRoot,
], {encoding: 'utf8', maxBuffer: 64 * 1024 * 1024});
const details = JSON.parse(output);
if (details.schemaVersion !== 1 || !details.families) {
throw new Error('Unsupported family configuration inventory');
}
return details;
}

const TASKS = {
audio_generation: ['Text to speech', 'Audio', 'text-to-speech', 'generate-audio'],
Expand Down Expand Up @@ -250,12 +265,12 @@ function manifestFiles(family) {
.sort();
}

function profileFromManifest(repoRoot, family, filePath) {
function profileFromManifest(repoRoot, family, filePath, sourceBaseUrl) {
const manifest = readJson(filePath);
if (manifest.runtime_strategy !== undefined || manifest.task_strategy !== undefined) {
throw new Error(`${filePath} uses a removed strategy field`);
}
if (!manifest.name || !manifest.bundle
if (typeof manifest.name !== 'string' || !manifest.name || !manifest.bundle
|| manifest.family !== family.name || !manifest.task) {
throw new Error(`${filePath} must declare name, bundle, exact family, and task`);
}
Expand Down Expand Up @@ -283,8 +298,16 @@ function profileFromManifest(repoRoot, family, filePath) {
const parallelMode = contextParallelSize > 1
? 'context_parallel'
: tensorParallelSize > 1 ? 'tensor_parallel' : 'single_device';
const familySlug = family.name.replaceAll('_', '-');
const slug = encodeURIComponent(manifest.name);
const sourcePath = path.relative(repoRoot, filePath).replaceAll('\\', '/');
return {
profile: manifest.name,
slug,
url: `${RECIPE_BASE}/checkpoints/${familySlug}/${slug}`,
familyUrl: `${RECIPE_BASE}/families/${familySlug}`,
sourceUrl: `${sourceBaseUrl}/${sourcePath}`,
manifest,
hfId: manifest.hf_id || 'prepared local checkpoint',
revision: manifest.hf_id ? manifest.hf_revision || 'not pinned' : 'not applicable',
bundle: manifest.bundle,
Expand All @@ -301,7 +324,7 @@ function profileFromManifest(repoRoot, family, filePath) {
? manifest.testcases.map((testcase) => testcase.name).filter(Boolean)
: [],
fp32Layers: Array.isArray(manifest.fp32_layers) ? manifest.fp32_layers : [],
sourcePath: path.relative(repoRoot, filePath).replaceAll('\\', '/'),
sourcePath,
};
}

Expand Down Expand Up @@ -331,13 +354,14 @@ function taskRecipe(taskName, profiles) {
};
}

function collectModelSupportInventory(repoRoot) {
function collectModelSupportInventory(repoRoot, {familyDetails = null, sourceRevision = 'main'} = {}) {
const sourceBaseUrl = `https://github.com/NVIDIA/TensorRT-Model-Connect/blob/${sourceRevision}`;
const families = familyDirectories(repoRoot);
const profiles = [];
for (const family of families) {
requireFamilyShape(family);
for (const filePath of manifestFiles(family)) {
profiles.push(profileFromManifest(repoRoot, family, filePath));
profiles.push(profileFromManifest(repoRoot, family, filePath, sourceBaseUrl));
}
}
profiles.sort((left, right) =>
Expand All @@ -347,9 +371,20 @@ function collectModelSupportInventory(repoRoot) {

const familyRecipes = families.map((family) => {
const owned = profiles.filter((profile) => profile.family === family.name);
if (new Set(owned.map((profile) => profile.slug)).size !== owned.length) {
throw new Error(`${family.name} has duplicate recipe names and checkpoint routes`);
}
if (familyDetails && !Object.hasOwn(familyDetails.families, family.name)) {
throw new Error(`${family.name} is missing its configuration audit`);
}
return {
family: family.name,
slug: family.name.replaceAll('_', '-'),
url: `${RECIPE_BASE}/families/${family.name.replaceAll('_', '-')}`,
sourceUrl: `${sourceBaseUrl}/families/${family.name}`,
details: familyDetails?.families[family.name] || null,
declaredCliCommands: (familyDetails?.families[family.name]?.cli?.commands || [])
.filter((command) => command.executor === 'native').map((command) => command.name),
profiles: owned,
taskSlugs: [...new Set(owned.flatMap((profile) => profile.taskSlugs))].sort(),
cliCommands: [...new Set(owned.flatMap((profile) => profile.cliCommands))].sort(),
Expand All @@ -367,6 +402,10 @@ function collectModelSupportInventory(repoRoot) {
.sort((left, right) => left.label.localeCompare(right.label));

return {
sourceRevision,
sourceBaseUrl,
configurationSummary: familyDetails?.summary || null,
sharedBuildCommand: familyDetails?.sharedBuildCommand || null,
familyCount: families.length,
familyNames: families.map((family) => family.name),
manifestCount: profiles.length,
Expand All @@ -380,13 +419,24 @@ function modelSupportInventoryPlugin(context) {
return {
name: 'model-support-inventory',
loadContent() {
return collectModelSupportInventory(path.resolve(context.siteDir, '..'));
const repoRoot = path.resolve(context.siteDir, '..');
const sourceRevision = execFileSync('git', ['rev-parse', 'HEAD'], {
cwd: repoRoot, encoding: 'utf8',
}).trim();
return collectModelSupportInventory(repoRoot, {
sourceRevision,
familyDetails: collectFamilyDetails(repoRoot),
});
},
contentLoaded({content, actions}) {
actions.setGlobalData(content);
async contentLoaded({content, actions}) {
actions.setGlobalData({
...content,
familyRecipes: content.familyRecipes.map(({details, ...family}) => family),
});
const routeBase = context.baseUrl.replace(/\/$/, '');
const taskPage = path.join(context.siteDir, 'src/components/ModelRecipes/TaskPage.js');
const familyPage = path.join(context.siteDir, 'src/components/ModelRecipes/FamilyPage.js');
const checkpointPage = path.join(context.siteDir, 'src/components/ModelRecipes/CheckpointPage.js');
for (const task of content.taskRecipes) {
actions.addRoute({
path: `${routeBase}/models-recipes/model-recipes/tasks/${task.slug}`,
Expand All @@ -396,16 +446,30 @@ function modelSupportInventoryPlugin(context) {
});
}
for (const family of content.familyRecipes) {
const modules = family.details ? {
details: await actions.createData(`family-${family.slug}.json`, JSON.stringify(family.details)),
} : {};
actions.addRoute({
path: `${routeBase}/models-recipes/model-recipes/families/${family.slug}`,
component: familyPage,
exact: true,
props: {familySlug: family.slug},
modules,
});
for (const profile of family.profiles) {
actions.addRoute({
path: `${routeBase}${profile.url}`,
component: checkpointPage,
exact: true,
props: {familySlug: family.slug, profileSlug: profile.slug},
modules,
});
}
}
},
};
}

module.exports = modelSupportInventoryPlugin;
module.exports.collectModelSupportInventory = collectModelSupportInventory;
module.exports.collectFamilyDetails = collectFamilyDetails;
85 changes: 81 additions & 4 deletions website/plugins/model-support-inventory/index.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ function semanticTaskIds() {
}).sort();
}

test('every public semantic Task can generate a family recipe and route', (context) => {
test('every public semantic Task can generate a family recipe and route', async (context) => {
const tasks = semanticTaskIds();
assert.ok(tasks.includes('text_continuation'));
assert.ok(tasks.includes('batch_initial_image_text_to_audio_video'));
Expand All @@ -121,11 +121,11 @@ test('every public semantic Task can generate a family recipe and route', (conte
[...tasks, 'text_generation'].sort());
const routes = [];
const plugin = modelSupportInventoryPlugin({siteDir: path.join(root, 'website'), baseUrl: '/docs/'});
plugin.contentLoaded({content: inventory, actions: {
setGlobalData: (data) => assert.equal(data, inventory),
await plugin.contentLoaded({content: inventory, actions: {
setGlobalData: (data) => assert.equal(data.familyCount, inventory.familyCount),
addRoute: (route) => routes.push(route),
}});
assert.equal(routes.length, tasks.length + 1 + inventory.familyCount);
assert.equal(routes.length, tasks.length + 1 + inventory.familyCount + inventory.manifestCount);
for (const task of tasks) {
const recipe = inventory.taskRecipes.find((item) => item.task === task);
assert.ok(recipe.label.length > 0, task);
Expand Down Expand Up @@ -282,3 +282,80 @@ test('requires the complete family shape', () => {
fs.rmSync(path.join(root, 'families', 'beta', 'runtime', 'CMakeLists.txt'));
assert.throws(() => collectModelSupportInventory(root), /missing families\/beta\/runtime/);
});

test('every physical recipe has an exact detail route and keeps all manifest settings', async (context) => {
const root = repository();
context.after(() => fs.rmSync(root, {recursive: true, force: true}));
const sourcePath = 'families/alpha/tests/manifests/small.json';
const manifest = JSON.parse(fs.readFileSync(path.join(root, sourcePath)));
Object.assign(manifest, {max_sequence_length: 4096, dynamic_kv_cache: false,
quantization: 'fp8', fp32_layers: [0, 3], custom_owner_setting: {enabled: true}});
fs.writeFileSync(path.join(root, sourcePath), JSON.stringify(manifest));
const inventory = collectModelSupportInventory(root, {sourceRevision: 'abc123'});
const [profile] = inventory.modelProfiles;
assert.deepEqual(profile.manifest, manifest);
assert.equal(profile.url, '/models-recipes/model-recipes/checkpoints/alpha/alpha-small');
assert.equal(profile.familyUrl, '/models-recipes/model-recipes/families/alpha');
assert.equal(profile.sourceUrl,
`https://github.com/NVIDIA/TensorRT-Model-Connect/blob/abc123/${sourcePath}`);
const routes = [];
await modelSupportInventoryPlugin({siteDir: path.join(root, 'website'), baseUrl: '/docs/'})
.contentLoaded({content: inventory, actions: {
setGlobalData() {}, addRoute: (route) => routes.push(route),
}});
const route = routes.find((item) => item.path === `/docs${profile.url}`);
assert.deepEqual(route.props, {familySlug: 'alpha', profileSlug: 'alpha-small'});
assert.ok(route.component.endsWith('/CheckpointPage.js'));
assert.ok(routes.some((item) => item.path.endsWith('/families/beta')),
'families without recipes must still have a detail page');
});

test('configuration data is loaded per family and native commands come from that owner', async (context) => {
const root = repository();
context.after(() => fs.rmSync(root, {recursive: true, force: true}));
const alpha = {cli: {commands: [
{name: 'build', executor: 'python'}, {name: 'owner-command', executor: 'native'},
]}, configClasses: [{name: 'Config', fields: [{name: 'owner_field'}]}]};
const inventory = collectModelSupportInventory(root, {familyDetails: {families: {alpha, beta: {}}}});
const routes = [];
const dataFiles = new Map();
await modelSupportInventoryPlugin({siteDir: path.join(root, 'website'), baseUrl: '/docs/'})
.contentLoaded({content: inventory, actions: {
setGlobalData: (data) => {
assert.ok(data.familyRecipes.every((family) => !Object.hasOwn(family, 'details')));
assert.deepEqual(data.familyRecipes[0].declaredCliCommands, ['owner-command']);
},
createData: async (name, data) => { dataFiles.set(name, JSON.parse(data)); return name; },
addRoute: (route) => routes.push(route),
}});
assert.deepEqual(dataFiles.get('family-alpha.json'), alpha);
for (const route of routes.filter((route) => route.props.familySlug === 'alpha')) {
assert.deepEqual(route.modules, {details: 'family-alpha.json'});
}
});

test('rejects ambiguous checkpoint links and incomplete family audit coverage', (context) => {
const root = repository();
context.after(() => fs.rmSync(root, {recursive: true, force: true}));
assert.throws(() => collectModelSupportInventory(root, {familyDetails: {families: {alpha: {}}}}),
/beta is missing its configuration audit/);
fs.copyFileSync(path.join(root, 'families/alpha/tests/manifests/small.json'),
path.join(root, 'families/alpha/tests/manifests/duplicate.json'));
assert.throws(() => collectModelSupportInventory(root), /duplicate recipe names/);
});

test('real inventory audits every current family and every current recipe', () => {
const root = path.resolve(__dirname, '../../..');
const familyDetails = modelSupportInventoryPlugin.collectFamilyDetails(root);
const inventory = collectModelSupportInventory(root, {familyDetails});
assert.equal(Object.keys(familyDetails.families).length, inventory.familyCount);
for (const family of inventory.familyRecipes) {
assert.ok(family.details, family.family);
for (const profile of family.profiles) {
const check = family.details.profileChecks[profile.sourcePath];
assert.ok(check, profile.sourcePath);
assert.ok(['checked', 'issues', 'not-checked'].includes(check.status), profile.sourcePath);
assert.ok(Array.isArray(check.checks) && check.checks.length > 0, profile.sourcePath);
}
}
});
Loading
Loading