diff --git a/torchao/quantization/pt2e/observer.py b/torchao/quantization/pt2e/observer.py
index 0294385324..0a5b42e31a 100644
--- a/torchao/quantization/pt2e/observer.py
+++ b/torchao/quantization/pt2e/observer.py
@@ -1157,7 +1157,8 @@ def _non_linear_param_search(self) -> tuple[torch.Tensor, torch.Tensor]:
         beta = 1.0  # upper bound
         start_bin = 0
         end_bin = self.bins - 1
-        norm_min = float("inf")
+        # Score the untrimmed range so a trim is only accepted if it lowers the error.
+        norm_min = self._compute_quantization_error(start_bin, end_bin)
 
         while alpha < beta:
             # Find the next step