Fix OOM regression for FSDP2 + cpu_ram_efficient_loading on large models

src/transformers/modeling_utils.py

@@ -4765,12 +4765,9 @@ def _move_missing_keys_from_meta_to_device(
         if is_deepspeed_zero3_enabled() and not is_quantized:
             return
 
-        # In this case we need to move everything back
+        # Non-persistent buffers need real placeholders as they won't be broadcasted by fsdp2
         if is_fsdp_enabled() and not is_local_dist_rank_0() and not is_quantized:
-            for key, param in self.named_parameters():
-                value = torch.zeros_like(param, device="cpu")
-                _load_parameter_into_model(self, key, value)
-            for key, buffer in self.named_buffers():
+            for key, buffer in self.named_non_persistent_buffers():
                 value = torch.zeros_like(buffer, device="cpu")
                 _load_parameter_into_model(self, key, value)
             return