Fix L2 failures: arctic RoPE and Phi4-MM rope standardization (#230)

## Summary

Fixes 8 L2 test failures (arctic ×2, phi4_multimodal ×3, phi4mm ×3).

### Arctic (TypeError: rotary_emb is None)

Arctic's config JSON has `rope_theta=10000` (the default) and
`rope_scaling=null`. No signal triggers `_extract_rope_config()`, so
`rotary_emb` is never created — but the model uses RoPE.

**Fix:** Add `arctic` to `_IMPLICIT_ROPE_DEFAULTS` with
`rope_theta=10000.0`.

### Phi4-MM (AttributeError: max_position_embeddings)

Newer transformers (`convert_rope_params_to_dict` →
`standardize_rope_params`) accesses `self.max_position_embeddings`
before it's set as an attribute on `PretrainedConfig`. This crashes even
when `max_position_embeddings` is in the kwargs dict.

**Fix:** Two-layer defense in `_dict_to_pretrained_config`:
1. Proactively strip rope fields from **composite** configs (those with
nested text_config/thinker_config dicts) — the nested config carries its
own rope_scaling
2. For **flat** configs that still crash, catch the exception, strip
rope fields, reconstruct, then restore rope fields as attributes so
`_extract_rope_config` can still read them

---------

Signed-off-by: Justin Chu <justinchu@microsoft.com>
Signed-off-by: Justin Chu <justinchuby@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: justinchuby

src/mobius/_config_resolver.py

@@ -115,23 +115,14 @@ def _dict_to_pretrained_config(d: dict):
     """
     import transformers
 
-    # Some composite configs (e.g. Phi4-MM) have rope_scaling at the
-    # top level without max_position_embeddings, which crashes
-    # PretrainedConfig.__post_init__ rope standardization.  Strip rope
-    # fields proactively when the guard field is absent — the nested
-    # text_config will carry its own rope_scaling.
-    has_rope_fields = "rope_scaling" in d or "rope_parameters" in d
-    has_max_pos = "max_position_embeddings" in d
-    if has_rope_fields and not has_max_pos:
-        logger.debug(
-            "Stripping top-level rope fields from %s config: "
-            "rope_scaling present without max_position_embeddings",
-            d.get("model_type", "unknown"),
-        )
-        d = {k: v for k, v in d.items() if k not in ("rope_scaling", "rope_parameters")}
-    config = transformers.PretrainedConfig(**d)
-    # Recursively convert known nested config keys
-    nested_keys = (
+    # Composite configs (e.g. configs with text_config/thinker_config) may
+    # duplicate rope_scaling at the top level.  PretrainedConfig's rope
+    # standardization (__post_init__ → standardize_rope_params) can crash
+    # with AttributeError when self.max_position_embeddings is not yet set.
+    # Strip top-level rope fields ONLY for composite configs — the nested
+    # text_config will carry its own rope_scaling with correct context.
+    # Non-composite (flat) configs must keep rope fields intact.
+    nested_config_keys = (
         "thinker_config",
         "talker_config",
         "text_config",
@@ -141,7 +132,36 @@ def _dict_to_pretrained_config(d: dict):
         "code_predictor_config",
         "speaker_encoder_config",
     )
-    for key in nested_keys:
+    is_composite = any(isinstance(d.get(k), dict) for k in nested_config_keys)
+    rope_keys = ("rope_scaling", "rope_parameters")
+    if is_composite and any(k in d for k in rope_keys):
+        logger.debug(
+            "Stripping top-level rope fields from composite %s config",
+            d.get("model_type", "unknown"),
+        )
+        d = {k: v for k, v in d.items() if k not in rope_keys}
+
+    try:
+        config = transformers.PretrainedConfig(**d)
+    except (AttributeError, KeyError, TypeError) as e:
+        # Newer transformers may crash during rope standardization
+        # (e.g. Phi4-MM longrope format where PretrainedConfig doesn't
+        # set max_position_embeddings before accessing it).  Strip rope
+        # fields, construct the config, then restore them as attributes
+        # so _extract_rope_config can still read them.
+        logger.warning(
+            "Retrying %s config without rope fields after PretrainedConfig init failure: %s",
+            d.get("model_type", "unknown"),
+            e,
+        )
+        saved_rope = {k: d[k] for k in rope_keys if k in d}
+        d_clean = {k: v for k, v in d.items() if k not in rope_keys}
+        config = transformers.PretrainedConfig(**d_clean)
+        for k, v in saved_rope.items():
+            setattr(config, k, v)
+
+    # Recursively convert known nested config keys
+    for key in nested_config_keys:
         val = getattr(config, key, None)
         if isinstance(val, dict):
             setattr(config, key, _dict_to_pretrained_config(val))

src/mobius/_config_resolver_test.py

@@ -475,3 +475,49 @@ def test_thinker_config_nested(self):
         config = _dict_to_pretrained_config(d)
         assert config.thinker_config.model_type == "qwen3"
         assert config.thinker_config.text_config.model_type == "qwen3"
+
+    def test_composite_config_strips_rope_fields(self):
+        """Composite configs strip top-level rope_scaling/rope_parameters."""
+        d = {
+            "model_type": "composite",
+            "rope_scaling": {"type": "longrope"},
+            "rope_parameters": {"rope_type": "default"},
+            "text_config": {"model_type": "inner", "hidden_size": 256},
+        }
+        config = _dict_to_pretrained_config(d)
+        # Rope fields stripped from top level
+        assert not hasattr(config, "rope_scaling") or config.rope_scaling is None
+        # Nested config still works
+        assert config.text_config.model_type == "inner"
+
+    def test_flat_config_keeps_rope_fields(self):
+        """Non-composite (flat) configs keep rope_scaling as attribute."""
+        d = {
+            "model_type": "flat",
+            "hidden_size": 256,
+            "max_position_embeddings": 4096,
+            # Simple rope_scaling that won't crash PretrainedConfig
+            "rope_theta": 10000.0,
+        }
+        config = _dict_to_pretrained_config(d)
+        assert config.rope_theta == pytest.approx(10000.0)
+
+    def test_rope_retry_restores_fields(self):
+        """When PretrainedConfig init crashes on rope, fields are restored."""
+        # Simulate a config with rope_scaling that crashes standardization
+        # by including rope_scaling without the fields needed for
+        # standardization (matching Phi4-MM's failure pattern).
+        d = {
+            "model_type": "phi4mm",
+            "rope_scaling": {"type": "longrope", "long_factor": [1.0]},
+            "rope_theta": 10000.0,
+            "hidden_size": 256,
+            "max_position_embeddings": 4096,
+        }
+        # This should succeed (either directly or via retry)
+        config = _dict_to_pretrained_config(d)
+        assert config.model_type == "phi4mm"
+        # If the retry path ran, rope_scaling should be restored as attr
+        rope_scaling = getattr(config, "rope_scaling", None)
+        if rope_scaling is not None:
+            assert rope_scaling["type"] == "longrope"

src/mobius/_configs.py

@@ -323,6 +323,9 @@ def _first_not_none(*values, default=None):
 # TODO: migrate to a registry annotation (uses_rope: bool, rope_theta: float)
 # once the registry schema supports per-model capability flags.
 _IMPLICIT_ROPE_DEFAULTS: dict[str, float] = {
+    # arctic: config JSON has rope_theta=10000 (default) and rope_scaling=null;
+    # no signal triggers _extract_rope_config, but the model uses RoPE.
+    "arctic": 10_000.0,
     # chatglm: config JSON has no rope_theta/rope_scaling/rotary_* attrs;
     # uses default rope_theta=10000.0 hardcoded in modeling code.
     "chatglm": 10_000.0,

src/mobius/_configs_test.py

@@ -1050,3 +1050,29 @@ class FakeConfig:
         # With gelu_activation=False and no other activation attr,
         # hidden_act should be None (not "gelu")
         assert config.hidden_act is None
+
+
+class TestImplicitRopeDefaults:
+    """Tests for models in _IMPLICIT_ROPE_DEFAULTS."""
+
+    def test_arctic_gets_rope_config(self):
+        """Arctic (rope_theta=10000, rope_scaling=null) should get RoPE."""
+
+        class FakeConfig:
+            model_type = "arctic"
+            num_attention_heads = 8
+            num_key_value_heads = 8
+            num_hidden_layers = 2
+            vocab_size = 1000
+            hidden_size = 256
+            intermediate_size = 512
+            max_position_embeddings = 4096
+            head_dim = 32
+            hidden_act = "silu"
+            # Arctic has rope_theta=10000 (default) and no rope_scaling
+            rope_theta = 10_000.0
+
+        config = ArchitectureConfig.from_transformers(FakeConfig())
+        # Arctic must get RoPE via _IMPLICIT_ROPE_DEFAULTS
+        assert config.rope_type == "default"
+        assert config.rope_theta == pytest.approx(10_000.0)