fix(json): preserve Unicode boundaries in lexer errors

json/README.mbt.md

@@ -53,8 +53,8 @@ test "parse and validate jsons" {
 
 #### What may appear inside a string
 
-Every string in a parsed document is well-formed Unicode, so each `\uXXXX`
-escape must denote a Unicode scalar value on its own or be one half of a
+Each `\uXXXX` escape must denote a Unicode scalar value on its own or be one
+half of a
 correctly ordered surrogate pair. An escaped leading surrogate must be
 followed immediately by an escaped trailing surrogate, and the pair decodes
 to the single character it stands for; an escape that cannot pair up is a
@@ -71,10 +71,10 @@ test "surrogate escapes" {
 }
 ```
 
-RFC 8259 §9 leaves what a string may contain to the implementation, and this
-is where MoonBit draws that line: a `String` is required to be well-formed,
-so the alternatives would be to hand one back that is not, or to substitute
-U+FFFD and lose the difference between two distinct keys. Note that
+This restriction avoids manufacturing unpaired surrogates from escapes or
+substituting U+FFFD and losing the difference between distinct keys. It applies
+to escapes: raw UTF-16 code units are not validated, so callers using unchecked
+string construction must validate that input separately. Note that
 `JSON.stringify` in JavaScript does emit lone surrogates this way, so a
 document JavaScript and Python accept can be rejected here — as it is by
 Rust's serde_json when parsing into `String` or `Value`; Go's

json/lex_main.mbt

@@ -58,7 +58,7 @@ fn ParseContext::lex_value(
           )
           return Number(n, repr.map(repr => repr.to_owned()))
         }
-        Some(_) => ctx.invalid_char(shift=-1)
+        Some(c) => ctx.invalid_char(shift=-c.utf16_len())
         None => raise InvalidEof
       }
     Some('0') => {

json/lex_misc.mbt

@@ -34,11 +34,11 @@ fn ParseContext::read_char(ctx : ParseContext) -> Char? {
 }
 
 ///|
-/// low surrogate
+/// Lower bound of the UTF-16 surrogate range.
 const SURROGATE_LOW_CHAR = 0xD800
 
 ///|
-/// high surrogate
+/// Upper bound of the UTF-16 surrogate range.
 const SURROGATE_HIGH_CHAR = 0xDFFF
 
 ///|
@@ -53,9 +53,8 @@ fn ParseContext::expect_char(
   let c1 = ctx.input.unsafe_get(ctx.offset).to_int()
   ctx.offset += 1
   let c0 = c.to_int()
-  if c0 < 0xFFFF {
-    // c0 < SURROGATE_LOW_CHAR || c0 is (0xE000..=0XFFFF)  
-    // c0 is a valid char so only need check if c0<0xFFFF is BMP code point
+  if c0 <= 0xFFFF {
+    // c0 is a valid character in the BMP.
     if c0 != c1 {
       ctx.invalid_char(shift=-1)
     }
@@ -106,7 +105,6 @@ test "expect_char" {
 
 ///|
 test "expect_char with surrogate pair" {
-  // "\uD83D\uDE00" // todo: shall we allow this?
   let ctx = ParseContext::make("a\u{1F600}bc\u{1F600}c")
   ctx.expect_char('a')
   ctx.expect_char((0x1F600).unsafe_to_char())
@@ -179,7 +177,7 @@ fn ParseContext::lex_after_array_value(
   match ctx.read_char() {
     Some(']') => RBracket
     Some(',') => Comma
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
@@ -191,7 +189,7 @@ fn ParseContext::lex_after_property_name(
   ctx.lex_skip_whitespace()
   match ctx.read_char() {
     Some(':') => ()
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
@@ -204,7 +202,7 @@ fn ParseContext::lex_after_object_value(
   match ctx.read_char() {
     Some('}') => RBrace
     Some(',') => Comma
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
@@ -222,7 +220,7 @@ fn ParseContext::lex_property_name(
       let s = ctx.lex_string()
       String(s)
     }
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
@@ -240,7 +238,15 @@ fn ParseContext::lex_property_name2(
       let s = ctx.lex_string()
       String(s)
     }
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
+
+///|
+test "expect_char at the upper BMP boundary" {
+  let ctx = ParseContext::make("\u{FFFF}😀")
+  ctx.expect_char('\u{FFFF}')
+  ctx.expect_char('😀')
+  json_inspect(ctx.offset, content=3)
+}

json/lex_number.mbt

@@ -240,8 +240,8 @@ fn ParseContext::lex_integer_end(
       // exact source text in `repr` so `stringify` stays lossless. Only a
       // literal strconv itself rejects (beyond double range) keeps the
       // infinity sentinel.
-      // Malformed tokens may end inside a surrogate pair. Keep raw slicing
-      // here and in lex_number_end so parsing reports an error without aborting.
+      // The lexer leaves the first non-number character unconsumed,
+      // so the slice contains only the validated number token.
       let s = ctx.input.view(start_offset=start, end_offset=end)
       try {
         let value = @internal/strconv.parse_double(s)
@@ -269,8 +269,8 @@ fn ParseContext::lex_decimal_integer(
       Some('.') => return ctx.lex_decimal_point(start~)
       Some('e' | 'E') => return ctx.lex_decimal_exponent(start~)
       Some('0'..='9') => continue
-      Some(_) => {
-        ctx.offset -= 1
+      Some(c) => {
+        ctx.offset -= c.utf16_len()
         return ctx.lex_number_end(start, ctx.offset)
       }
       None => return ctx.lex_number_end(start, ctx.offset)
@@ -285,7 +285,7 @@ fn ParseContext::lex_decimal_point(
 ) -> LexedNumber raise ParseError {
   match ctx.read_char() {
     Some('0'..='9') => ctx.lex_decimal_fraction(start~)
-    Some(_) => ctx.invalid_char(shift=-1)
+    Some(c) => ctx.invalid_char(shift=-c.utf16_len())
     None => raise InvalidEof
   }
 }
@@ -299,8 +299,8 @@ fn ParseContext::lex_decimal_fraction(
     match ctx.read_char() {
       Some('e' | 'E') => return ctx.lex_decimal_exponent(start~)
       Some('0'..='9') => continue
-      Some(_) => {
-        ctx.offset -= 1
+      Some(c) => {
+        ctx.offset -= c.utf16_len()
         return ctx.lex_number_end(start, ctx.offset)
       }
       None => return ctx.lex_number_end(start, ctx.offset)
@@ -316,8 +316,8 @@ fn ParseContext::lex_decimal_exponent(
   match ctx.read_char() {
     Some('+' | '-') => return ctx.lex_decimal_exponent_sign(start~)
     Some('0'..='9') => return ctx.lex_decimal_exponent_integer(start~)
-    Some(_) => {
-      ctx.offset -= 1
+    Some(c) => {
+      ctx.offset -= c.utf16_len()
       ctx.invalid_char()
     }
     None => raise InvalidEof
@@ -331,8 +331,8 @@ fn ParseContext::lex_decimal_exponent_sign(
 ) -> LexedNumber raise ParseError {
   match ctx.read_char() {
     Some('0'..='9') => return ctx.lex_decimal_exponent_integer(start~)
-    Some(_) => {
-      ctx.offset -= 1
+    Some(c) => {
+      ctx.offset -= c.utf16_len()
       ctx.invalid_char()
     }
     None => raise InvalidEof
@@ -347,8 +347,8 @@ fn ParseContext::lex_decimal_exponent_integer(
   for ;; {
     match ctx.read_char() {
       Some('0'..='9') => continue
-      Some(_) => {
-        ctx.offset -= 1
+      Some(c) => {
+        ctx.offset -= c.utf16_len()
         return ctx.lex_number_end(start, ctx.offset)
       }
       None => return ctx.lex_number_end(start, ctx.offset)
@@ -368,8 +368,8 @@ fn ParseContext::lex_zero(
       ctx.offset -= 1
       ctx.invalid_char()
     }
-    Some(_) => {
-      ctx.offset -= 1
+    Some(c) => {
+      ctx.offset -= c.utf16_len()
       return ctx.lex_number_end(start, ctx.offset)
     }
     None => return ctx.lex_number_end(start, ctx.offset)

json/lex_string.mbt

@@ -77,7 +77,7 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError
               // immediately following trailing-surrogate escape into one
               // Unicode scalar value. Anything else would manufacture a
               // string containing an unpaired surrogate, which MoonBit
-              // strings disallow (RFC 8259 calls the behavior for such
+              // strings should avoid (RFC 8259 calls the behavior for such
               // escapes unpredictable; I-JSON forbids them).
               match ctx.read_char() {
                 Some('\\') => ()

json/parse.mbt

@@ -31,9 +31,8 @@ pub fn valid(input : StringView) -> Bool {
 ///
 /// ## What strings may contain
 ///
-/// Every string in the result is well-formed Unicode: each `\uXXXX` escape
-/// must denote a Unicode scalar value on its own, or be one half of a
-/// correctly ordered surrogate pair. An escaped leading surrogate
+/// Each `\uXXXX` escape must denote a Unicode scalar value on its own,
+/// or be one half of a correctly ordered surrogate pair. An escaped leading surrogate
 /// (`\uD800`–`\uDBFF`) must therefore be followed immediately by an escaped
 /// trailing surrogate (`\uDC00`–`\uDFFF`), and the pair is decoded as the
 /// one character it stands for.
@@ -48,11 +47,12 @@ pub fn valid(input : StringView) -> Bool {
 /// to the implementation — not a claim about which documents are
 /// grammatically well formed, since §8.2 admits unpaired surrogate escapes,
 /// nor a claim of I-JSON (RFC 7493) conformance, which restricts more than
-/// this. It is chosen because MoonBit's `String` is required to be
-/// well-formed, so the alternatives are to hand back a string that violates
-/// that invariant, or to substitute U+FFFD and silently lose the
-/// distinction between two different keys. A parse error is the only one of
-/// the three a caller can see and act on.
+/// this. It avoids manufacturing unpaired surrogates from escapes or
+/// substituting U+FFFD and silently losing the distinction between two
+/// different keys. A parse error lets the caller
+/// see and act on an invalid escape. This check applies to escapes; raw
+/// UTF-16 code units in the input are not validated. Callers constructing
+/// strings through unchecked APIs must validate that input separately.
 ///
 /// The cost is real: `JSON.stringify` in JavaScript emits lone surrogates as
 /// `\uXXXX`, so some JSON that JavaScript and Python accept is rejected

json/unicode_error_test.mbt

@@ -0,0 +1,42 @@
+// Copyright 2026 International Digital Economy Academy
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+///|
+test "syntax errors preserve supplementary characters and their positions" {
+  let cases : Array[(String, Int)] = [
+    ("[true😀]", 5),
+    ("{\"a\"😀0}", 4),
+    ("{\"a\":true😀}", 9),
+    ("{😀}", 1),
+    ("{\"a\":0,😀}", 7),
+    ("-😀", 1),
+    ("1.😀", 2),
+    ("1e😀", 2),
+    ("1e+😀", 3),
+    ("0😀", 1),
+    ("12😀", 2),
+    ("1.2😀", 3),
+    ("1e2😀", 3),
+  ]
+  for (input, column) in cases {
+    match expect_parse_error(input, "expected InvalidChar") {
+      InvalidChar(position, ch) => {
+        assert_eq(position.line, 1)
+        assert_eq(position.column, column)
+        assert_eq(ch, '😀')
+      }
+      _ => fail("expected InvalidChar")
+    }
+  }
+}