feat(bench): generate a shadow CLI from a spec, and compile mise's

.github/workflows/test.yml

@@ -41,10 +41,13 @@ jobs:
       - run: mise r build
       - run: mise r test
       - run: mise r render
-      - name: assert render produces no diff
+      # Same reasoning as `render`: the shadow is checked in, so a change to the derive's
+      # vocabulary that would alter it has to be committed rather than discovered later.
+      - run: mise r gen-shadow
+      - name: assert render and gen-shadow produce no diff
         run: |
           if [ -n "$(git status --porcelain)" ]; then
-            echo "::error::'mise run render' produced changes. Run it locally and commit."
+            echo "::error::'mise run render' or 'mise run gen-shadow' produced changes. Run them locally and commit."
             git status
             git diff
             exit 1

.gitignore

@@ -1,4 +1,6 @@
 /target
+# valgrind leaves these beside whatever directory it was run from
+cachegrind.out.*
 /schema/usage.json
 .aube/
 /node_modules/

Cargo.toml

@@ -7,8 +7,17 @@ members = [
     "cli",
     "conformance",
     "lib",
+    "xtask",
+    "benches/gate",
 ]
 
+# Generated, and not held to the same standard as code somebody wrote: a shadow of
+# mise's spec trips `large_enum_variant` on the commands with thirty flags, which the
+# real mise answers by boxing its variants — something the derive cannot express yet.
+# The gate depends on it by path, so it is still built, and its parsing is tested from
+# `benches/gate/tests`.
+exclude = ["benches/shadows"]
+
 [workspace.package]
 homepage = "https://usage.jdx.dev"
 documentation = "https://usage.jdx.dev"

PLAN.md

@@ -147,6 +147,10 @@ carrying them rather than being worked around.
       forty `conflicts_with` relationships mise declares in clap were being dropped
       by the bridge. Now a flag property, enforced by usage-lib, and a value from
       the environment counts on both sides of the check as clap does.
+- [x] **An argument after a variadic, when it needs a `--`** — the derive refused the
+      shape mise uses on `run`, `exec` and `git`: `[ARGS]…` for the words before the
+      separator and `[-- ARGS_LAST]…` for the ones after. Both parsers already bound it;
+      only the derive's validation disagreed. Found by compiling the mise shadow.
 - [ ] **A mount on the root command** — the spec accepts `mount` only inside a
       `cmd` block, so a CLI whose _top-level_ subcommands are discovered by running
       something cannot say so. Worth deciding whether that is a gap or a deliberate
@@ -173,8 +177,20 @@ Everything above is speculative until this passes. Baseline is mise's current
 clap parser at mise's real scale, using a shadow CLI generated from mise's
 checked-in `mise.usage.kdl` for both parsers.
 
-- [ ] **Bench harness** — `tak` instruction counts plus criterion, and an
-      `xtask gen-shadow` that turns any `.usage.kdl` into a shadow crate.
+- [x] **Shadow generation** — `xtask gen-shadow` turns any `.usage.kdl` into a crate
+      of derived types. mise's committed 5,592-line spec compiles: 211 commands, 711
+      flags, 128 arguments, four levels deep, in 2.6s. What it cannot express, it
+      counts: 91 command aliases (67 visible, 24 hidden), 13 secondary flag aliases, 3
+      `double_dash="automatic"`, 2 mounts, 2 restart tokens, 1 `default_subcommand`, 1
+      default on a collecting flag. Aliases are the largest gap and the next thing the
+      derive needs for mise; `default_subcommand` is the one that changes the _root's_
+      grammar, since `mise build` routes through `run` in mise and answers at the root
+      in the shadow.
+- [ ] **Bench harness** — the clap-equivalent shadow to measure against, `tak` gating in
+      CI, and criterion for wall clock. A first measurement of the usage side alone, at
+      mise's full scale, is 100k–106k instructions per parse for the parse itself
+      (process total minus a null binary that does everything but parse), near-constant
+      across invocation shapes — as static tables should be, with no tree to build.
 - [ ] **Differential fuzzing** — proptest over argv against usage-lib on the mise
       spec, to find disagreements the corpus did not think of.
 - [ ] **Perf report** — published honestly, whichever way it goes.

benches/gate/Cargo.toml

@@ -0,0 +1,14 @@
+# The gate: the same command line, parsed by the shadow and by clap, measured with
+# instruction counts. Two binaries rather than one benchmark harness, because the cost
+# being compared includes what happens before `main` gets a parser — clap builds its
+# command tree at runtime, and a harness that built it once outside the loop would
+# measure the half of clap that is already fast.
+[package]
+name = "gate"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+shadow-mise = { path = "../shadows/mise" }
+usage-argv = { path = "../../argv" }

benches/gate/src/bin/parse-none.rs

@@ -0,0 +1,13 @@
+//! Everything the other gate binaries do except parse.
+//!
+//! Subtracting this is what turns a process measurement into a parser measurement:
+//! roughly half of a small binary's instructions are the dynamic loader and libc
+//! starting up, and neither parser is responsible for those.
+
+use std::ffi::OsString;
+
+fn main() {
+    let args: Vec<OsString> = std::env::args_os().skip(1).collect();
+    let refs: Vec<&std::ffi::OsStr> = args.iter().map(|a| a.as_os_str()).collect();
+    println!("{}", !refs.is_empty());
+}

benches/gate/src/bin/parse-usage.rs

@@ -0,0 +1,22 @@
+//! Parse a mise command line with the generated shadow, once, and exit.
+//!
+//! One invocation is the unit under test: a CLI parses its arguments once per run, so
+//! what matters is the cost of a process reaching its first useful instruction — not a
+//! throughput loop with everything warm.
+
+use std::ffi::OsString;
+
+use shadow_mise::Cli;
+
+fn main() {
+    let args: Vec<OsString> = std::env::args_os().skip(1).collect();
+    let refs: Vec<&std::ffi::OsStr> = args.iter().map(|a| a.as_os_str()).collect();
+    match Cli::parse_from(&refs) {
+        // Printed, so the optimizer cannot decide the parse was unobservable.
+        Ok(cli) => println!("{}", cli.command.is_some()),
+        Err(e) => {
+            eprintln!("{e:?}");
+            std::process::exit(1)
+        }
+    }
+}

benches/gate/tests/parse.rs

@@ -0,0 +1,132 @@
+//! Real mise command lines, parsed by the generated shadow.
+//!
+//! Kept here rather than beside the generated crate: this is hand-written, and what
+//! the generator produces is only worth
+//! benchmarking if it parses the same words mise does. These are invocations out of
+//! mise's own docs.
+
+use std::ffi::OsStr;
+
+use shadow_mise::{Cli, Commands, TasksCommands};
+
+fn argv<const N: usize>(tokens: [&str; N]) -> [&OsStr; N] {
+    tokens.map(OsStr::new)
+}
+
+#[test]
+fn a_tool_is_installed_globally() {
+    let a = argv(["use", "-g", "node@20"]);
+    let cli = Cli::parse_from(&a).expect("should parse");
+    let Some(Commands::Use(use_args)) = cli.command else {
+        panic!("expected `use`")
+    };
+    assert!(use_args.global);
+    assert_eq!(use_args.tool_version, ["node@20"]);
+}
+
+#[test]
+fn a_task_runs_with_arguments_after_a_separator() {
+    // The shape that made the derive's validation wrong: `[ARGS]…` before the `--` and
+    // `[-- ARGS_LAST]…` after it. `tasks run` is where mise's spec declares it; the
+    // top-level `run` carries no positionals of its own — see the note in the PR.
+    let a = argv([
+        "tasks",
+        "run",
+        "build",
+        "extra",
+        "--dry-run",
+        "--",
+        "--verbose",
+    ]);
+    let cli = Cli::parse_from(&a).expect("should parse");
+    let Some(Commands::Tasks(tasks)) = cli.command else {
+        panic!("expected `tasks`")
+    };
+    let Some(TasksCommands::Run(run)) = tasks.command else {
+        panic!("expected `tasks run`")
+    };
+    // The words, not just that a command was selected: a regression that merged the two
+    // sides of the `--` or dropped either would otherwise leave this green.
+    assert_eq!(run.task.as_deref(), Some("build"));
+    assert_eq!(run.args, ["extra"]);
+    assert_eq!(run.args_last, ["--verbose"]);
+    assert!(run.dry_run);
+}
+
+#[test]
+fn a_bare_task_lands_on_the_root_positional() {
+    // `mise build -- --verbose` fills the root's own `[TASK]`, with the words after the
+    // separator kept apart.
+    //
+    // Real mise routes this through `run`, because its spec sets `default_subcommand
+    // run` — which the derive cannot declare, so the shadow answers at the root
+    // instead. One of the differences `gen-shadow` counts rather than one it hides.
+    let a = argv(["build", "--", "--verbose"]);
+    let cli = Cli::parse_from(&a).expect("should parse");
+    assert_eq!(cli.task.as_deref(), Some("build"));
+    assert_eq!(cli.task_args_last, ["--verbose"]);
+    assert!(cli.command.is_none(), "`build` is not a subcommand");
+}
+
+#[test]
+fn a_global_flag_is_accepted_before_or_after_the_command() {
+    let before = argv(["-C", "/tmp", "ls", "--installed"]);
+    let cli = Cli::parse_from(&before).expect("should parse");
+    assert_eq!(cli.cd.as_deref(), Some("/tmp"));
+
+    // Global means the subcommand takes it too, which is how `mise ls -C /tmp` works.
+    // Asserting the *value* matters here: unknown flag-like words are values by default
+    // and `ls` has a variadic positional, so a global that stopped being recognized
+    // after the command would still parse — `-C` and `/tmp` would land in the variadic
+    // and nothing would complain.
+    let after = argv(["ls", "-C", "/tmp"]);
+    let cli = Cli::parse_from(&after).expect("should parse");
+    assert_eq!(cli.cd.as_deref(), Some("/tmp"));
+    let Some(Commands::Ls(_)) = cli.command else {
+        panic!("expected `ls`")
+    };
+}
+
+#[test]
+fn a_nested_command_reaches_three_levels() {
+    let a = argv(["settings", "set", "experimental", "true"]);
+    let cli = Cli::parse_from(&a).expect("should parse");
+    let Some(Commands::Settings(settings)) = cli.command else {
+        panic!("expected `settings`")
+    };
+    assert!(
+        settings.command.is_some(),
+        "`set` should have been selected"
+    );
+}
+
+#[test]
+fn counted_verbosity_accumulates() {
+    let a = argv(["-vv", "ls"]);
+    let cli = Cli::parse_from(&a).expect("should parse");
+    assert_eq!(cli.verbose, 2);
+}
+
+#[test]
+fn a_command_that_requires_a_subcommand_refuses_to_stand_alone() {
+    // 27 of mise's commands set `subcommand_required`, and the shadow has to answer the
+    // same grammar: `mise bootstrap accounts` on its own is an error, not an empty
+    // invocation. (`bootstrap` itself does not require one, which is why the shadow has
+    // to read the spec rather than assume.)
+    // `Cli` is generated without `Debug` — 211 commands' worth of it would be dead
+    // weight — so the error is matched rather than unwrapped.
+    let a = argv(["bootstrap", "accounts"]);
+    match Cli::parse_from(&a) {
+        Err(usage_argv::Error::MissingSubcommand) => {}
+        Err(other) => panic!("wrong error: {other:?}"),
+        Ok(_) => panic!("`bootstrap accounts` needs a subcommand"),
+    }
+
+    // With one, it parses.
+    let a = argv(["bootstrap", "accounts", "status"]);
+    Cli::parse_from(&a).expect("`accounts status` should parse");
+
+    // And the root does not require one, because `mise <task>` is a whole invocation.
+    let a = argv(["build"]);
+    Cli::parse_from(&a).expect("a bare task should parse");
+}