diff --git a/json/README.mbt.md b/json/README.mbt.md index b34fc8b30..2e66befc6 100644 --- a/json/README.mbt.md +++ b/json/README.mbt.md @@ -53,8 +53,8 @@ test "parse and validate jsons" { #### What may appear inside a string -Every string in a parsed document is well-formed Unicode, so each `\uXXXX` -escape must denote a Unicode scalar value on its own or be one half of a +Each `\uXXXX` escape must denote a Unicode scalar value on its own or be one +half of a correctly ordered surrogate pair. An escaped leading surrogate must be followed immediately by an escaped trailing surrogate, and the pair decodes to the single character it stands for; an escape that cannot pair up is a @@ -71,10 +71,10 @@ test "surrogate escapes" { } ``` -RFC 8259 ยง9 leaves what a string may contain to the implementation, and this -is where MoonBit draws that line: a `String` is required to be well-formed, -so the alternatives would be to hand one back that is not, or to substitute -U+FFFD and lose the difference between two distinct keys. Note that +This restriction avoids manufacturing unpaired surrogates from escapes or +substituting U+FFFD and losing the difference between distinct keys. It applies +to escapes: raw UTF-16 code units are not validated, so callers using unchecked +string construction must validate that input separately. Note that `JSON.stringify` in JavaScript does emit lone surrogates this way, so a document JavaScript and Python accept can be rejected here โ€” as it is by Rust's serde_json when parsing into `String` or `Value`; Go's diff --git a/json/lex_main.mbt b/json/lex_main.mbt index a6e101020..5b360f83b 100644 --- a/json/lex_main.mbt +++ b/json/lex_main.mbt @@ -58,7 +58,7 @@ fn ParseContext::lex_value( ) return Number(n, repr.map(repr => repr.to_owned())) } - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } Some('0') => { diff --git a/json/lex_misc.mbt b/json/lex_misc.mbt index 4e79e217e..861630e7e 100644 --- a/json/lex_misc.mbt +++ b/json/lex_misc.mbt @@ -34,11 +34,11 @@ fn ParseContext::read_char(ctx : ParseContext) -> Char? { } ///| -/// low surrogate +/// Lower bound of the UTF-16 surrogate range. const SURROGATE_LOW_CHAR = 0xD800 ///| -/// high surrogate +/// Upper bound of the UTF-16 surrogate range. const SURROGATE_HIGH_CHAR = 0xDFFF ///| @@ -53,9 +53,8 @@ fn ParseContext::expect_char( let c1 = ctx.input.unsafe_get(ctx.offset).to_int() ctx.offset += 1 let c0 = c.to_int() - if c0 < 0xFFFF { - // c0 < SURROGATE_LOW_CHAR || c0 is (0xE000..=0XFFFF) - // c0 is a valid char so only need check if c0<0xFFFF is BMP code point + if c0 <= 0xFFFF { + // c0 is a valid character in the BMP. if c0 != c1 { ctx.invalid_char(shift=-1) } @@ -106,7 +105,6 @@ test "expect_char" { ///| test "expect_char with surrogate pair" { - // "\uD83D\uDE00" // todo: shall we allow this? let ctx = ParseContext::make("a\u{1F600}bc\u{1F600}c") ctx.expect_char('a') ctx.expect_char((0x1F600).unsafe_to_char()) @@ -179,7 +177,7 @@ fn ParseContext::lex_after_array_value( match ctx.read_char() { Some(']') => RBracket Some(',') => Comma - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } @@ -191,7 +189,7 @@ fn ParseContext::lex_after_property_name( ctx.lex_skip_whitespace() match ctx.read_char() { Some(':') => () - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } @@ -204,7 +202,7 @@ fn ParseContext::lex_after_object_value( match ctx.read_char() { Some('}') => RBrace Some(',') => Comma - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } @@ -222,7 +220,7 @@ fn ParseContext::lex_property_name( let s = ctx.lex_string() String(s) } - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } @@ -240,7 +238,15 @@ fn ParseContext::lex_property_name2( let s = ctx.lex_string() String(s) } - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } + +///| +test "expect_char at the upper BMP boundary" { + let ctx = ParseContext::make("\u{FFFF}๐Ÿ˜€") + ctx.expect_char('\u{FFFF}') + ctx.expect_char('๐Ÿ˜€') + json_inspect(ctx.offset, content=3) +} diff --git a/json/lex_number.mbt b/json/lex_number.mbt index 23cd72ddb..f9eac61d5 100644 --- a/json/lex_number.mbt +++ b/json/lex_number.mbt @@ -240,8 +240,8 @@ fn ParseContext::lex_integer_end( // exact source text in `repr` so `stringify` stays lossless. Only a // literal strconv itself rejects (beyond double range) keeps the // infinity sentinel. - // Malformed tokens may end inside a surrogate pair. Keep raw slicing - // here and in lex_number_end so parsing reports an error without aborting. + // The lexer leaves the first non-number character unconsumed, + // so the slice contains only the validated number token. let s = ctx.input.view(start_offset=start, end_offset=end) try { let value = @internal/strconv.parse_double(s) @@ -269,8 +269,8 @@ fn ParseContext::lex_decimal_integer( Some('.') => return ctx.lex_decimal_point(start~) Some('e' | 'E') => return ctx.lex_decimal_exponent(start~) Some('0'..='9') => continue - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() return ctx.lex_number_end(start, ctx.offset) } None => return ctx.lex_number_end(start, ctx.offset) @@ -285,7 +285,7 @@ fn ParseContext::lex_decimal_point( ) -> LexedNumber raise ParseError { match ctx.read_char() { Some('0'..='9') => ctx.lex_decimal_fraction(start~) - Some(_) => ctx.invalid_char(shift=-1) + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) None => raise InvalidEof } } @@ -299,8 +299,8 @@ fn ParseContext::lex_decimal_fraction( match ctx.read_char() { Some('e' | 'E') => return ctx.lex_decimal_exponent(start~) Some('0'..='9') => continue - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() return ctx.lex_number_end(start, ctx.offset) } None => return ctx.lex_number_end(start, ctx.offset) @@ -316,8 +316,8 @@ fn ParseContext::lex_decimal_exponent( match ctx.read_char() { Some('+' | '-') => return ctx.lex_decimal_exponent_sign(start~) Some('0'..='9') => return ctx.lex_decimal_exponent_integer(start~) - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() ctx.invalid_char() } None => raise InvalidEof @@ -331,8 +331,8 @@ fn ParseContext::lex_decimal_exponent_sign( ) -> LexedNumber raise ParseError { match ctx.read_char() { Some('0'..='9') => return ctx.lex_decimal_exponent_integer(start~) - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() ctx.invalid_char() } None => raise InvalidEof @@ -347,8 +347,8 @@ fn ParseContext::lex_decimal_exponent_integer( for ;; { match ctx.read_char() { Some('0'..='9') => continue - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() return ctx.lex_number_end(start, ctx.offset) } None => return ctx.lex_number_end(start, ctx.offset) @@ -368,8 +368,8 @@ fn ParseContext::lex_zero( ctx.offset -= 1 ctx.invalid_char() } - Some(_) => { - ctx.offset -= 1 + Some(c) => { + ctx.offset -= c.utf16_len() return ctx.lex_number_end(start, ctx.offset) } None => return ctx.lex_number_end(start, ctx.offset) diff --git a/json/lex_string.mbt b/json/lex_string.mbt index 0852909bb..6ff6c29df 100644 --- a/json/lex_string.mbt +++ b/json/lex_string.mbt @@ -77,7 +77,7 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError // immediately following trailing-surrogate escape into one // Unicode scalar value. Anything else would manufacture a // string containing an unpaired surrogate, which MoonBit - // strings disallow (RFC 8259 calls the behavior for such + // strings should avoid (RFC 8259 calls the behavior for such // escapes unpredictable; I-JSON forbids them). match ctx.read_char() { Some('\\') => () diff --git a/json/parse.mbt b/json/parse.mbt index a6acc6d16..031f0b0ea 100644 --- a/json/parse.mbt +++ b/json/parse.mbt @@ -31,9 +31,8 @@ pub fn valid(input : StringView) -> Bool { /// /// ## What strings may contain /// -/// Every string in the result is well-formed Unicode: each `\uXXXX` escape -/// must denote a Unicode scalar value on its own, or be one half of a -/// correctly ordered surrogate pair. An escaped leading surrogate +/// Each `\uXXXX` escape must denote a Unicode scalar value on its own, +/// or be one half of a correctly ordered surrogate pair. An escaped leading surrogate /// (`\uD800`โ€“`\uDBFF`) must therefore be followed immediately by an escaped /// trailing surrogate (`\uDC00`โ€“`\uDFFF`), and the pair is decoded as the /// one character it stands for. @@ -48,11 +47,12 @@ pub fn valid(input : StringView) -> Bool { /// to the implementation โ€” not a claim about which documents are /// grammatically well formed, since ยง8.2 admits unpaired surrogate escapes, /// nor a claim of I-JSON (RFC 7493) conformance, which restricts more than -/// this. It is chosen because MoonBit's `String` is required to be -/// well-formed, so the alternatives are to hand back a string that violates -/// that invariant, or to substitute U+FFFD and silently lose the -/// distinction between two different keys. A parse error is the only one of -/// the three a caller can see and act on. +/// this. It avoids manufacturing unpaired surrogates from escapes or +/// substituting U+FFFD and silently losing the distinction between two +/// different keys. A parse error lets the caller +/// see and act on an invalid escape. This check applies to escapes; raw +/// UTF-16 code units in the input are not validated. Callers constructing +/// strings through unchecked APIs must validate that input separately. /// /// The cost is real: `JSON.stringify` in JavaScript emits lone surrogates as /// `\uXXXX`, so some JSON that JavaScript and Python accept is rejected diff --git a/json/unicode_error_test.mbt b/json/unicode_error_test.mbt new file mode 100644 index 000000000..665738828 --- /dev/null +++ b/json/unicode_error_test.mbt @@ -0,0 +1,42 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +test "syntax errors preserve supplementary characters and their positions" { + let cases : Array[(String, Int)] = [ + ("[true๐Ÿ˜€]", 5), + ("{\"a\"๐Ÿ˜€0}", 4), + ("{\"a\":true๐Ÿ˜€}", 9), + ("{๐Ÿ˜€}", 1), + ("{\"a\":0,๐Ÿ˜€}", 7), + ("-๐Ÿ˜€", 1), + ("1.๐Ÿ˜€", 2), + ("1e๐Ÿ˜€", 2), + ("1e+๐Ÿ˜€", 3), + ("0๐Ÿ˜€", 1), + ("12๐Ÿ˜€", 2), + ("1.2๐Ÿ˜€", 3), + ("1e2๐Ÿ˜€", 3), + ] + for (input, column) in cases { + match expect_parse_error(input, "expected InvalidChar") { + InvalidChar(position, ch) => { + assert_eq(position.line, 1) + assert_eq(position.column, column) + assert_eq(ch, '๐Ÿ˜€') + } + _ => fail("expected InvalidChar") + } + } +}