From 2151c4d63f5cba134482e6698072370283618c45 Mon Sep 17 00:00:00 2001
From: Jeongkyu Shin <inureyes@gmail.com>
Date: Tue, 20 Oct 2026 14:15:04 +0900
Subject: [PATCH 1/3] bench: add M3 Ultra baselines for the three large MoE
 checkpoints

bench_mlxlm.py hardcoded its memory budget at 85% of 128 GB, so on the 512 GB M3 Ultra it skipped deepseek-v4-flash-4bit and both inkling-small checkpoints as SKIP:oom_estimate even though bench_decode.sh had just measured them. The budget now follows the host's own memory (hw.memsize on macOS, MemTotal on Linux), mirroring bench_decode.sh, and falls back to 128 GB when memory cannot be read. PYLM_BENCH_MAX_GB and the ROCm carve-out override still apply.

deepseek_v4 and inkling exist only in mlx-vlm, so the plain text path recorded FAIL:warmup for them. The new --text-via-vlm flag runs the same text protocol (synthesized 512-token prompt, no chat template, no image) but loads through mlx-vlm, records mlx-vlm as baseline_version, and writes to a separate pylm_*_textvlm_* file.

The ten CSVs are the 2026-09-08 M3 Ultra measurements: mlxcel text and VLM rows plus the mlx-vlm text and VLM baselines for the same three checkpoints.

Validated with py_compile, --help, and a module load that resolves the budget to 435 GB on this host.
---
 ...26-09-08_single_deepseek-v4-flash-4bit.csv |   2 +
 ...26-09-08_single_inkling-small-mlx-4bit.csv |   2 +
 ..._2026-09-08_single_inkling-small-nvfp4.csv |   2 +
 ...26-09-08_single_inkling-small-mlx-4bit.csv |   2 +
 ..._2026-09-08_single_inkling-small-nvfp4.csv |   2 +
 ...26-09-08_single_deepseek-v4-flash-4bit.csv |   2 +
 ...26-09-08_single_inkling-small-mlx-4bit.csv |   2 +
 ..._2026-09-08_single_inkling-small-nvfp4.csv |   2 +
 ...26-09-08_single_inkling-small-mlx-4bit.csv |   2 +
 ..._2026-09-08_single_inkling-small-nvfp4.csv |   2 +
 scripts/bench_mlxlm.py                        | 170 +++++++++++++++++-
 11 files changed, 181 insertions(+), 9 deletions(-)
 create mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv
 create mode 100644 benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv

diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv b/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
new file mode 100644
index 000000000..d76a3d9e0
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
+deepseek-v4-flash-4bit,models/deepseek-v4-flash-4bit,512,128,1885.97,271.48,14867.54,8.61,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..7ac108b7e
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
+inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,512,128,1016.72,503.58,16016.97,7.99,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..d86146a44
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
+inkling-small-nvfp4,models/inkling-small-nvfp4,512,128,1638.61,312.46,18697.77,6.85,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..ba7810edc
--- /dev/null
+++ b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
+inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,60,128,269.88,222.32,15730.40,8.14,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"What is in this image?",,b8fdddfa-dirty,9a795735
diff --git a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..0087cefc4
--- /dev/null
+++ b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
+inkling-small-nvfp4,models/inkling-small-nvfp4,60,128,339.63,176.66,18434.97,6.94,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"What is in this image?",,b8fdddfa-dirty,9a795735
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv
new file mode 100644
index 000000000..c7521729c
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+deepseek-v4-flash-4bit,./models/deepseek-v4-flash-4bit/,512,128,1133.85,451.56,3443.35,37.17,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..c0efd1073
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,512,128,835.61,612.72,2577.64,49.66,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..c7a57ebf5
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-nvfp4,./models/inkling-small-nvfp4/,512,128,908.64,563.48,3552.52,36.03,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..c00f46c56
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,60,128,199.92,300.13,2528.23,50.63,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"What is in this image?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..6af0eda1e
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-nvfp4,./models/inkling-small-nvfp4/,60,128,214.79,279.34,3462.46,36.97,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"What is in this image?"
diff --git a/scripts/bench_mlxlm.py b/scripts/bench_mlxlm.py
index f5fa737b1..e56b06a88 100755
--- a/scripts/bench_mlxlm.py
+++ b/scripts/bench_mlxlm.py
@@ -58,14 +58,46 @@
 # `models/mlx` works without symlinks, matching scripts/bench_decode.sh.
 MODELS_DIR = Path(os.environ.get("MODELS_DIR", "./models"))
 BENCHMARKS_DIR = Path("./benchmarks")
-# Memory budget: 85% of 128 GB. Override with PYLM_BENCH_MAX_GB env var to
-# enforce a tighter cap (e.g. PYLM_BENCH_MAX_GB=65 to skip very large MoE
-# models on a 128 GB host where 85% × 128 GB ≈ 108 GB still admits them).
+# Memory budget: 85% of the host's own memory, reserving 15% for OS and runtime
+# overhead. This mirrors detect_memory_bytes() and MEMORY_LIMIT_BYTES in
+# scripts/bench_decode.sh, and it has to: the two harnesses produce the paired
+# rows of one comparison, so a checkpoint either harness skips leaves the other
+# with a number and nothing to compare it against.
+#
+# The budget used to be a hardcoded 128 GB, which silently made every host look
+# like a 128 GB one. On the 2026-09-08 M3 Ultra run that skipped all three
+# large checkpoints as SKIP:oom_estimate (141.1 GB > 108.8 GB) on a 512 GB
+# machine where bench_decode.sh had just measured every one of them.
+#
+# PYLM_BENCH_MAX_GB still overrides, which is the way to enforce a *tighter* cap
+# than the host allows (e.g. PYLM_BENCH_MAX_GB=65 to skip very large MoE models
+# on a 128 GB host, where 85% would otherwise admit them).
+def _detect_memory_bytes() -> int:
+    try:
+        if sys.platform == "darwin":
+            out = subprocess.run(["sysctl", "-n", "hw.memsize"],
+                                 capture_output=True, text=True, timeout=10)
+            return int(out.stdout.strip()) if out.returncode == 0 else 0
+        with open("/proc/meminfo") as fh:
+            for line in fh:
+                if line.startswith("MemTotal:"):
+                    return int(line.split()[1]) * 1024
+    except Exception:
+        pass
+    return 0
+
+
 _env_max_gb = os.environ.get("PYLM_BENCH_MAX_GB")
 if _env_max_gb:
     MEMORY_LIMIT_BYTES = int(float(_env_max_gb) * 1024 * 1024 * 1024)
 else:
-    MEMORY_LIMIT_BYTES = int(128 * 1024 * 1024 * 1024 * 0.85)
+    _system_memory_bytes = _detect_memory_bytes()
+    # Undetectable memory falls back to the historical 128 GB assumption rather
+    # than to "unlimited", so a host this cannot read still refuses to load a
+    # checkpoint that would swap it to death.
+    if _system_memory_bytes <= 0:
+        _system_memory_bytes = 128 * 1024 * 1024 * 1024
+    MEMORY_LIMIT_BYTES = _system_memory_bytes * 85 // 100
 # Thermal cooldown thresholds
 BIG_MODEL_GB = 20
 
@@ -400,6 +432,107 @@ def _suppress_eos(tokens, logits):
 print('RESULT', json.dumps(result))
 """
 
+# Text workload measured through mlx-vlm instead of mlx-lm.
+#
+# Some architectures are implemented only in mlx-vlm. deepseek_v4 and inkling
+# are both in `mlx_vlm/models/` and in neither case does `mlx_lm.models` carry a
+# module, so the ordinary text path cannot load them at all and the sweep
+# records FAIL:warmup for what is really a missing implementation. Their text
+# decoders are still a real text workload, and mlxcel measures them in text
+# mode, so without this the comparison has no baseline column on exactly the
+# checkpoints that are too large to run anywhere else.
+#
+# This is the text protocol, not the VLM one: the same synthesized
+# LONG_PROMPT_CORPUS prompt, no chat template, no image. Only the package that
+# loads the model differs. Keeping the prompt construction identical to
+# SUBPROCESS_TEXT_CODE is the point, since a chat template or an image would
+# change the prefill shape and stop the row comparing against bench_decode.sh.
+SUBPROCESS_TEXT_VIA_VLM_CODE = """
+import json, sys, time
+from mlx_vlm import load, stream_generate
+
+model_path = sys.argv[1]
+prompt = sys.argv[2]
+warmup_tokens = int(sys.argv[3])
+max_tokens = int(sys.argv[4])
+prompt_tokens_target = int(sys.argv[5]) if len(sys.argv) > 5 else 0
+corpus = sys.argv[6] if len(sys.argv) > 6 else ''
+
+try:
+    model, processor = load(model_path, trust_remote_code=True)
+except Exception as e:
+    print('ERROR load:', repr(e), file=sys.stderr)
+    sys.exit(2)
+
+_tok = getattr(processor, 'tokenizer', processor)
+
+# Identical to the mlx-lm text path: repeat the shared corpus, tokenize with the
+# model's own tokenizer, truncate to exactly N tokens, decode back to text.
+if prompt_tokens_target > 0 and corpus:
+    per_copy = max(1, len(_tok.encode(corpus)))
+    repeats = prompt_tokens_target // per_copy + 4
+    ids = _tok.encode(corpus * repeats)[:prompt_tokens_target]
+    prompt = _tok.decode(ids)
+
+# EOS suppression. The stopping set can be wider than the tokenizer's own
+# eos_token_ids attribute, so read stopping_criteria first; see the VLM path.
+import mlx.core as mx
+_eos = set()
+_sc = getattr(_tok, 'stopping_criteria', None)
+if _sc is not None:
+    for _attr in ('eos_token_ids', '_eos_token_ids', 'stop_ids'):
+        _v = getattr(_sc, _attr, None)
+        if _v is not None:
+            _eos.update(_v if isinstance(_v, (list, tuple, set)) else [_v])
+for _attr in ('eos_token_ids', 'eos_token_id'):
+    _v = getattr(_tok, _attr, None)
+    if _v is None:
+        continue
+    _eos.update(_v if isinstance(_v, (list, tuple, set)) else [_v])
+_eos = sorted(int(i) for i in _eos if i is not None)
+if not _eos:
+    print('ERROR no_eos_ids', file=sys.stderr)
+    sys.exit(6)
+_eos_idx = mx.array(_eos)
+
+def _suppress_eos(tokens, logits):
+    return logits.at[..., _eos_idx].add(-float('inf'))
+
+_procs = [_suppress_eos]
+
+# Warmup
+try:
+    for _ in stream_generate(model, processor, prompt=prompt, image=[],
+                             max_tokens=warmup_tokens, logits_processors=_procs):
+        pass
+except Exception as e:
+    print('ERROR warmup:', repr(e), file=sys.stderr)
+    sys.exit(3)
+
+# Measured pass
+last = None
+try:
+    for r in stream_generate(model, processor, prompt=prompt, image=[],
+                             max_tokens=max_tokens, logits_processors=_procs):
+        last = r
+except Exception as e:
+    print('ERROR bench:', repr(e), file=sys.stderr)
+    sys.exit(4)
+
+if last is None:
+    print('ERROR no_output', file=sys.stderr)
+    sys.exit(5)
+
+result = {
+    'prompt_tokens': getattr(last, 'prompt_tokens', None),
+    'prefill_tps': getattr(last, 'prompt_tps', None),
+    'gen_tokens': getattr(last, 'generation_tokens', None),
+    'decode_tps': getattr(last, 'generation_tps', None),
+    'finish_reason': getattr(last, 'finish_reason', None),
+}
+print('RESULT', json.dumps(result))
+"""
+
 
 def resolve_python() -> list[str]:
     """Interpreter that has mlx-lm / mlx-vlm importable.
@@ -437,7 +570,7 @@ def resolve_python() -> list[str]:
 
 
 def bench_one(model_path: Path, vlm: bool, vlm_image: str, max_tokens: int,
-              warmup_tokens: int, timeout: int):
+              warmup_tokens: int, timeout: int, text_via_vlm: bool = False):
     """Returns (status, fields_dict_or_None)."""
     if vlm:
         prompt = VLM_PROMPT
@@ -446,6 +579,14 @@ def bench_one(model_path: Path, vlm: bool, vlm_image: str, max_tokens: int,
             *PYTHON_CMD, "-c", code,
             str(model_path), prompt, str(warmup_tokens), str(max_tokens), vlm_image,
         ]
+    elif text_via_vlm:
+        prompt = TEXT_PROMPT
+        code = SUBPROCESS_TEXT_VIA_VLM_CODE
+        args = [
+            *PYTHON_CMD, "-c", code,
+            str(model_path), prompt, str(warmup_tokens), str(max_tokens),
+            str(PROMPT_TOKENS), LONG_PROMPT_CORPUS,
+        ]
     else:
         prompt = TEXT_PROMPT
         code = SUBPROCESS_TEXT_CODE
@@ -503,6 +644,10 @@ def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("model", help='"all" for sweep, or models/<dir> for single')
     ap.add_argument("--vlm", action="store_true", help="VLM mode (use image prompt)")
+    ap.add_argument("--text-via-vlm", action="store_true",
+                    help="Text workload, but load through mlx-vlm. For architectures "
+                         "mlx-lm does not implement (deepseek_v4, inkling), whose text "
+                         "decoders otherwise get no baseline at all.")
     ap.add_argument("--image", default=VLM_IMAGE, help=f"VLM image (default: {VLM_IMAGE})")
     ap.add_argument("--cooldown", type=int, default=30, help="Seconds between models (default: 30)")
     ap.add_argument("--big-cooldown", type=int, default=30, help="Seconds after big model (default: 30)")
@@ -516,7 +661,7 @@ def main():
     rocm_gpu = detect_rocm_gpu() if sys.platform.startswith("linux") else None
     hw_short, hw_full = detect_hardware(rocm_gpu)
     today = date.today().isoformat()
-    # On a ROCm APU the GPU's budget is its carve-out, not the 128 GB Apple
+    # On a ROCm APU the GPU's budget is its carve-out, not the host-memory
     # default above; an explicit PYLM_BENCH_MAX_GB still wins.
     global MEMORY_LIMIT_BYTES
     if rocm_gpu and rocm_gpu["vram_bytes"] and not _env_max_gb:
@@ -526,8 +671,11 @@ def main():
     # this script. They differ whenever PYTHON_CMD resolves to the uv venv,
     # which is the normal case here: the system Python has no mlx wheels, so an
     # in-process import would always fail and record "unknown".
-    pkg = "mlx_vlm" if args.vlm else "mlx_lm"
-    label = "mlx-vlm" if args.vlm else "mlx-lm"
+    # --text-via-vlm measures a text workload but loads through mlx-vlm, so the
+    # recorded baseline_version must name mlx-vlm. Labelling it mlx-lm would put
+    # a version in the CSV that never ran.
+    pkg = "mlx_vlm" if (args.vlm or args.text_via_vlm) else "mlx_lm"
+    label = "mlx-vlm" if (args.vlm or args.text_via_vlm) else "mlx-lm"
     try:
         probe = subprocess.run(
             [*PYTHON_CMD, "-c", f"import {pkg}; print({pkg}.__version__)"],
@@ -553,6 +701,10 @@ def main():
     parts = ["pylm", hw_short, today]
     if args.vlm:
         parts.insert(2, "vlm")
+    elif args.text_via_vlm:
+        # Distinct from a plain text sweep: same workload, different package, so
+        # the two must not land in one file and look like a single baseline.
+        parts.insert(2, "textvlm")
     name = "_".join(parts)
     if args.suffix:
         name = f"{name}_{args.suffix}"
@@ -664,7 +816,7 @@ def append_row(line: str):
         status, fields = bench_one(
             mdir, vlm=args.vlm, vlm_image=args.image,
             max_tokens=args.max_tokens, warmup_tokens=args.warmup_tokens,
-            timeout=args.timeout,
+            timeout=args.timeout, text_via_vlm=args.text_via_vlm,
         )
         elapsed = time.perf_counter() - t0
         prompt = VLM_PROMPT if args.vlm else TEXT_PROMPT

From 058d7f78def5932e323b9ca01b1be1699776993f Mon Sep 17 00:00:00 2001
From: Jeongkyu Shin <inureyes@gmail.com>
Date: Tue, 20 Oct 2026 14:34:55 +0900
Subject: [PATCH 2/3] refactor(bench): reuse _host_memory_bytes for the pylm
 memory budget

The memory-budget fix added `_detect_memory_bytes()`, which shells out to `sysctl -n hw.memsize` on Darwin and parses `/proc/meminfo` on Linux. The file already had `_host_memory_bytes()`, a single portable `os.sysconf` call used by the ROCm hardware tagging, and both return the same value on the 512 GB M3 Ultra.

The budget now calls `_host_memory_bytes()` and the duplicate is gone. The helper moves above the module-level budget computation, since the budget runs at import time and the helper used to be defined later in the file. Behavior is unchanged: 435.2 GiB on the 512 GB host, `PYLM_BENCH_MAX_GB` still overrides (65 GiB checked), and the ROCm path still resolves the helper.
---
 scripts/bench_mlxlm.py | 25 +++++--------------------
 1 file changed, 5 insertions(+), 20 deletions(-)

diff --git a/scripts/bench_mlxlm.py b/scripts/bench_mlxlm.py
index e56b06a88..6a5e0b629 100755
--- a/scripts/bench_mlxlm.py
+++ b/scripts/bench_mlxlm.py
@@ -72,26 +72,18 @@
 # PYLM_BENCH_MAX_GB still overrides, which is the way to enforce a *tighter* cap
 # than the host allows (e.g. PYLM_BENCH_MAX_GB=65 to skip very large MoE models
 # on a 128 GB host, where 85% would otherwise admit them).
-def _detect_memory_bytes() -> int:
+def _host_memory_bytes():
     try:
-        if sys.platform == "darwin":
-            out = subprocess.run(["sysctl", "-n", "hw.memsize"],
-                                 capture_output=True, text=True, timeout=10)
-            return int(out.stdout.strip()) if out.returncode == 0 else 0
-        with open("/proc/meminfo") as fh:
-            for line in fh:
-                if line.startswith("MemTotal:"):
-                    return int(line.split()[1]) * 1024
-    except Exception:
-        pass
-    return 0
+        return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    except (ValueError, OSError, AttributeError):
+        return 0
 
 
 _env_max_gb = os.environ.get("PYLM_BENCH_MAX_GB")
 if _env_max_gb:
     MEMORY_LIMIT_BYTES = int(float(_env_max_gb) * 1024 * 1024 * 1024)
 else:
-    _system_memory_bytes = _detect_memory_bytes()
+    _system_memory_bytes = _host_memory_bytes()
     # Undetectable memory falls back to the historical 128 GB assumption rather
     # than to "unlimited", so a host this cannot read still refuses to load a
     # checkpoint that would swap it to death.
@@ -171,13 +163,6 @@ def detect_rocm_gpu():
     return {"gfx": gfx, "name": market or "AMD_GPU", "rocm_version": version, "vram_bytes": vram}
 
 
-def _host_memory_bytes():
-    try:
-        return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
-    except (ValueError, OSError, AttributeError):
-        return 0
-
-
 def rocm_hardware_names(gpu):
     """(short, full) tags for an AMD GPU, identical to bench_decode.sh's."""
     full = f"{gpu['name']}_{gpu['gfx']}"

From e70e5e6debfec5c440a2689d87bebde66e3fc065 Mon Sep 17 00:00:00 2001
From: Jeongkyu Shin <inureyes@gmail.com>
Date: Tue, 20 Oct 2026 15:32:16 +0900
Subject: [PATCH 3/3] bench: replace September CSVs with a post-#2310 rerun

The 2026-09-08 rows measured the pre-#2310 decode path, recorded two VLM rows on a dirty tree, and labeled the baseline with a stale 0.6.16 from a detached mlx-vlm pin. Swap all ten for a fresh 2026-10-20 four-way run at 058d7f78 on current main (MLX 243bdf09): mlxcel text and VLM rows plus mlx-vlm --text-via-vlm and VLM baselines for the same three checkpoints, baselines now recording mlx-vlm 0.7.9 from upstream main.

The decode gap to the mlx-vlm baseline persists on M3 Ultra after #2310 wired the fused MoE kernel: mlxcel decodes at 23.7% (deepseek-v4-flash-4bit), 15.9% (inkling-small-mlx-4bit) and 18.5% (inkling-small-nvfp4) of the baseline on the text protocol, and 15.8% / 18.5% on the VLM protocol, each within ~3% of the September numbers on both sides.

A same-build control with MLXCEL_FUSED_MOE=0 decodes at 7.80 tok/s against 7.78 with the fused kernel enabled (metal_m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv), so the kernel is not the lever on this host: the gap must come from elsewhere in the batch-1 expert path. The nvfp4 control arm stays on the gather path and moved by the same amount as the fused-path model, so the rerun is internally consistent.

Refs #1858, #1859.
---
 .../metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv  | 2 --
 .../metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv  | 2 --
 .../metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv     | 2 --
 ...m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv | 2 ++
 .../metal_m3ultra_2026-10-20_single_deepseek-v4-flash-4bit.csv  | 2 ++
 .../metal_m3ultra_2026-10-20_single_inkling-small-mlx-4bit.csv  | 2 ++
 .../metal_m3ultra_2026-10-20_single_inkling-small-nvfp4.csv     | 2 ++
 ...tal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv | 2 --
 .../metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv | 2 --
 ...tal_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv | 2 ++
 .../metal_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv | 2 ++
 ...lm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv | 2 --
 ...3ultra_textvlm_2026-10-20_single_deepseek-v4-flash-4bit.csv} | 2 +-
 ...3ultra_textvlm_2026-10-20_single_inkling-small-mlx-4bit.csv} | 2 +-
 ...lm_m3ultra_textvlm_2026-10-20_single_inkling-small-nvfp4.csv | 2 ++
 ...ylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv | 2 --
 .../pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv  | 2 --
 ...ylm_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv | 2 ++
 .../pylm_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv  | 2 ++
 19 files changed, 20 insertions(+), 18 deletions(-)
 delete mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
 delete mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
 delete mode 100644 benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-10-20_single_deepseek-v4-flash-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-nvfp4.csv
 delete mode 100644 benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
 delete mode 100644 benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv
 delete mode 100644 benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
 rename benchmarks/{pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv => pylm_m3ultra_textvlm_2026-10-20_single_deepseek-v4-flash-4bit.csv} (66%)
 rename benchmarks/{pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv => pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-mlx-4bit.csv} (67%)
 create mode 100644 benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-nvfp4.csv
 delete mode 100644 benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
 delete mode 100644 benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
 create mode 100644 benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
 create mode 100644 benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv

diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv b/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
deleted file mode 100644
index d76a3d9e0..000000000
--- a/benchmarks/metal_m3ultra_2026-09-08_single_deepseek-v4-flash-4bit.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
-deepseek-v4-flash-4bit,models/deepseek-v4-flash-4bit,512,128,1885.97,271.48,14867.54,8.61,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
deleted file mode 100644
index 7ac108b7e..000000000
--- a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-mlx-4bit.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
-inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,512,128,1016.72,503.58,16016.97,7.99,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
deleted file mode 100644
index d86146a44..000000000
--- a/benchmarks/metal_m3ultra_2026-09-08_single_inkling-small-nvfp4.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
-inkling-small-nvfp4,models/inkling-small-nvfp4,512,128,1638.61,312.46,18697.77,6.85,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"Hello, how are you today?",512,b8fdddfa,9a795735
diff --git a/benchmarks/metal_m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..d0bdc6dce
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-10-20_nofused_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,512,128,897.51,570.47,16410.81,7.80,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"Hello, how are you today?",512,058d7f78,243bdf09,engine
diff --git a/benchmarks/metal_m3ultra_2026-10-20_single_deepseek-v4-flash-4bit.csv b/benchmarks/metal_m3ultra_2026-10-20_single_deepseek-v4-flash-4bit.csv
new file mode 100644
index 000000000..4021a18bf
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-10-20_single_deepseek-v4-flash-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+deepseek-v4-flash-4bit,models/deepseek-v4-flash-4bit,512,128,1897.14,269.88,15278.99,8.38,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"Hello, how are you today?",512,058d7f78,243bdf09,engine
diff --git a/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..97606d0f8
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,512,128,897.97,570.17,16455.42,7.78,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"Hello, how are you today?",512,058d7f78,243bdf09,engine
diff --git a/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..428a9512e
--- /dev/null
+++ b/benchmarks/metal_m3ultra_2026-10-20_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+inkling-small-nvfp4,models/inkling-small-nvfp4,512,128,1638.73,312.44,19143.43,6.69,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"Hello, how are you today?",512,058d7f78,243bdf09,engine
diff --git a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
deleted file mode 100644
index ba7810edc..000000000
--- a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
-inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,60,128,269.88,222.32,15730.40,8.14,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"What is in this image?",,b8fdddfa-dirty,9a795735
diff --git a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
deleted file mode 100644
index 0087cefc4..000000000
--- a/benchmarks/metal_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit
-inkling-small-nvfp4,models/inkling-small-nvfp4,60,128,339.63,176.66,18434.97,6.94,2026-09-08,Apple_M3_Ultra_512GB,0.7.0-beta.1,release,128,"What is in this image?",,b8fdddfa-dirty,9a795735
diff --git a/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv b/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..d534f858e
--- /dev/null
+++ b/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+inkling-small-mlx-4bit,models/inkling-small-mlx-4bit,60,128,270.87,221.51,16148.45,7.93,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"What is in this image?",,058d7f78,243bdf09,engine
diff --git a/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv b/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..b52126152
--- /dev/null
+++ b/benchmarks/metal_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,mlxcel_version,build_type,max_tokens,prompt,prompt_target_len,mlxcel_commit,mlx_commit,decode_path
+inkling-small-nvfp4,models/inkling-small-nvfp4,60,128,337.17,177.95,18934.60,6.76,2026-10-20,Apple_M3_Ultra_512GB,0.7.0,release,128,"What is in this image?",,058d7f78,243bdf09,engine
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
deleted file mode 100644
index c7a57ebf5..000000000
--- a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-nvfp4.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
-inkling-small-nvfp4,./models/inkling-small-nvfp4/,512,128,908.64,563.48,3552.52,36.03,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_deepseek-v4-flash-4bit.csv
similarity index 66%
rename from benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv
rename to benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_deepseek-v4-flash-4bit.csv
index c7521729c..798e4926d 100644
--- a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_deepseek-v4-flash-4bit.csv
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_deepseek-v4-flash-4bit.csv
@@ -1,2 +1,2 @@
 model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
-deepseek-v4-flash-4bit,./models/deepseek-v4-flash-4bit/,512,128,1133.85,451.56,3443.35,37.17,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
+deepseek-v4-flash-4bit,./models/deepseek-v4-flash-4bit/,512,128,1041.80,491.46,3615.54,35.40,2026-10-20,Apple_M3_Ultra,mlx-vlm-0.7.9,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-mlx-4bit.csv
similarity index 67%
rename from benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv
rename to benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-mlx-4bit.csv
index c0efd1073..92cf7c972 100644
--- a/benchmarks/pylm_m3ultra_textvlm_2026-09-08_single_inkling-small-mlx-4bit.csv
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-mlx-4bit.csv
@@ -1,2 +1,2 @@
 model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
-inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,512,128,835.61,612.72,2577.64,49.66,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"Hello, how are you today?"
+inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,512,128,737.37,694.36,2615.58,48.94,2026-10-20,Apple_M3_Ultra,mlx-vlm-0.7.9,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..02cd7f90c
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_textvlm_2026-10-20_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-nvfp4,./models/inkling-small-nvfp4/,512,128,805.72,635.46,3543.38,36.12,2026-10-20,Apple_M3_Ultra,mlx-vlm-0.7.9,python,128,"Hello, how are you today?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
deleted file mode 100644
index c00f46c56..000000000
--- a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-mlx-4bit.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
-inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,60,128,199.92,300.13,2528.23,50.63,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"What is in this image?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
deleted file mode 100644
index 6af0eda1e..000000000
--- a/benchmarks/pylm_m3ultra_vlm_2026-09-08_single_inkling-small-nvfp4.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
-inkling-small-nvfp4,./models/inkling-small-nvfp4/,60,128,214.79,279.34,3462.46,36.97,2026-09-08,Apple_M3_Ultra,mlx-vlm-0.6.16,python,128,"What is in this image?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv b/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
new file mode 100644
index 000000000..3335add5e
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-mlx-4bit.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-mlx-4bit,./models/inkling-small-mlx-4bit/,60,128,221.39,271.02,2553.45,50.13,2026-10-20,Apple_M3_Ultra,mlx-vlm-0.7.9,python,128,"What is in this image?"
diff --git a/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv b/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv
new file mode 100644
index 000000000..060c783fe
--- /dev/null
+++ b/benchmarks/pylm_m3ultra_vlm_2026-10-20_single_inkling-small-nvfp4.csv
@@ -0,0 +1,2 @@
+model,model_path,prompt_tokens,generated_tokens,prefill_ms,prefill_tok_s,decode_ms,decode_tok_s,date,hardware,baseline_version,build_type,max_tokens,prompt
+inkling-small-nvfp4,./models/inkling-small-nvfp4/,60,128,245.24,244.65,3511.59,36.45,2026-10-20,Apple_M3_Ultra,mlx-vlm-0.7.9,python,128,"What is in this image?"