fix(lexer): interpret \xHH hex and \NNN octal escapes in interpolating strings

lexer/lexer.go

@@ -8,6 +8,7 @@ package lexer
 
 import (
 	"strings"
+	"unicode/utf8"
 
 	"github.com/go-ruby-parser/parser/token"
 )
@@ -1284,6 +1285,75 @@ func isHexDigit(c byte) bool {
 	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
 }
 
+// isOctalDigit reports whether c is an octal digit (0–7).
+func isOctalDigit(c byte) bool { return c >= '0' && c <= '7' }
+
+// hexVal returns the numeric value of a single hex digit. The caller must have
+// already established that c is a hex digit (isHexDigit).
+func hexVal(c byte) int {
+	switch {
+	case c >= '0' && c <= '9':
+		return int(c - '0')
+	case c >= 'a' && c <= 'f':
+		return int(c-'a') + 10
+	default: // 'A'..'F'
+		return int(c-'A') + 10
+	}
+}
+
+// appendUnicodeEscape resolves a `\u` escape at the cursor (the leading `\u`
+// already consumed) and appends the UTF-8 encoding of each codepoint to b. Two
+// forms are accepted, mirroring MRI: `\uHHHH` (exactly four hex digits) and
+// `\u{H… H… …}` (one or more whitespace-separated codepoints). Malformed input
+// (MRI raises a SyntaxError) is handled gracefully — only the well-formed
+// prefix contributes bytes — since this lexer entry point has no error channel.
+func (l *Lexer) appendUnicodeEscape(b []byte) []byte {
+	if l.peek() == '{' {
+		l.advance() // '{'
+		for {
+			for l.peek() == ' ' || l.peek() == '\t' {
+				l.advance()
+			}
+			if !isHexDigit(l.peek()) {
+				break
+			}
+			cp := 0
+			for isHexDigit(l.peek()) {
+				cp = cp*16 + hexVal(l.advance())
+			}
+			b = appendRune(b, cp)
+		}
+		for l.peek() != '}' && l.peek() != 0 {
+			l.advance()
+		}
+		if l.peek() == '}' {
+			l.advance()
+		}
+		return b
+	}
+	// `\uHHHH`: exactly four hex digits. With none present (MRI: SyntaxError)
+	// nothing is emitted, so a stray `\u` does not inject a NUL byte.
+	if !isHexDigit(l.peek()) {
+		return b
+	}
+	cp := 0
+	for i := 0; i < 4 && isHexDigit(l.peek()); i++ {
+		cp = cp*16 + hexVal(l.advance())
+	}
+	return appendRune(b, cp)
+}
+
+// appendRune appends the UTF-8 encoding of codepoint cp to b. An out-of-range
+// or surrogate value yields the U+FFFD replacement bytes, as Go's utf8 package
+// does for invalid runes.
+func appendRune(b []byte, cp int) []byte {
+	r := rune(cp)
+	if cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF) {
+		r = '�'
+	}
+	return utf8.AppendRune(b, r)
+}
+
 // percentDelimClose returns the closing delimiter for a %-literal opener: the
 // mate of a bracket pair, or the same character for a symmetric delimiter.
 func percentDelimClose(open byte) byte {
@@ -1993,8 +2063,34 @@ func (l *Lexer) scanStringSegment() (string, bool) {
 				b = append(b, '"')
 			case 'e':
 				b = append(b, 0x1b)
-			case '0':
-				b = append(b, 0)
+			case 'x':
+				// `\xHH`: 1–2 hex digits, greedy. MRI requires at least one hex
+				// digit (`\x`/`\xZ` is a SyntaxError); lacking an error channel
+				// here we fall back to emitting a literal 'x' so the byte stream
+				// stays well-defined.
+				if !isHexDigit(l.peek()) {
+					b = append(b, 'x')
+					continue
+				}
+				v := hexVal(l.advance())
+				if isHexDigit(l.peek()) {
+					v = v*16 + hexVal(l.advance())
+				}
+				b = append(b, byte(v))
+			case '0', '1', '2', '3', '4', '5', '6', '7':
+				// `\NNN` octal: 1–3 octal digits, greedy; the leading digit was
+				// consumed as `esc`. The value is masked to a single byte
+				// (MRI: `\400` → byte 0x00).
+				v := int(esc - '0')
+				for i := 0; i < 2 && isOctalDigit(l.peek()); i++ {
+					v = v*8 + int(l.advance()-'0')
+				}
+				b = append(b, byte(v))
+			case 'u':
+				// `\uHHHH` (exactly four hex digits) or `\u{H… H… …}` (one or
+				// more whitespace-separated codepoints). Each codepoint is
+				// emitted as UTF-8, matching MRI's `.bytes`.
+				b = l.appendUnicodeEscape(b)
 			default:
 				b = append(b, esc)
 			}

lexer/string_escape_test.go

@@ -0,0 +1,192 @@
+package lexer
+
+import (
+	"testing"
+
+	"github.com/go-ruby-parser/parser/token"
+)
+
+// firstStringLit tokenizes src and returns the literal of the first STRING,
+// STRBEG, STRMID, or STREND token (the interpreted body of a double-quoted-style
+// string), or ("", false) if none is found.
+func firstStringLit(src string) (string, bool) {
+	for _, t := range New(src).Tokenize() {
+		switch t.Type {
+		case token.STRING, token.STRBEG, token.STRMID, token.STREND:
+			return t.Lit, true
+		}
+	}
+	return "", false
+}
+
+// stringLits returns the literals of every STRING/STRBEG/STRMID/STREND token in
+// src, in order — used to inspect a string split across interpolations.
+func stringLits(src string) []string {
+	var out []string
+	for _, t := range New(src).Tokenize() {
+		switch t.Type {
+		case token.STRING, token.STRBEG, token.STRMID, token.STREND:
+			out = append(out, t.Lit)
+		}
+	}
+	return out
+}
+
+// TestStringHexEscape: `\xHH` consumes one or two hex digits (greedy) and yields
+// the corresponding byte, matching MRI 4.0.5.
+func TestStringHexEscape(t *testing.T) {
+	cases := map[string]string{
+		`"\x41"`:     "A",          // two digits
+		`"\x4"`:      "\x04",       // a single digit is enough
+		`"\xff"`:     "\xff",       // full byte, upper half
+		`"\xFF"`:     "\xff",       // uppercase hex digits
+		`"\xfff"`:    "\xff" + "f", // greedy stops after two digits
+		`"\x41\x42"`: "AB",         // adjacent escapes
+		`"a\x41b"`:   "aAb",        // surrounded by literal text
+	}
+	for src, want := range cases {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestStringHexEscapeNoDigit: `\x` with no following hex digit has no valid
+// interpretation (MRI raises a SyntaxError). Lacking an error channel here the
+// lexer degrades gracefully to a literal `x`, keeping the byte stream defined.
+func TestStringHexEscapeNoDigit(t *testing.T) {
+	for src, want := range map[string]string{
+		`"\xZ"`: "xZ", // non-hex follower
+		`"\x"`:  "x",  // end of string
+	} {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestStringOctalEscape: `\NNN` consumes one to three octal digits (greedy),
+// masking the value to a single byte, matching MRI 4.0.5.
+func TestStringOctalEscape(t *testing.T) {
+	cases := map[string]string{
+		`"\101"`:  "A",          // three digits -> 0o101 = 65
+		`"\12"`:   "\n",         // two digits -> 0o12 = 10 (newline)
+		`"\1"`:    "\x01",       // a single digit
+		`"\0"`:    "\x00",       // NUL (the previously-working single case)
+		`"\377"`:  "\xff",       // 0o377 = 255
+		`"\400"`:  "\x00",       // overflow: 0o400 & 0xFF = 0
+		`"\1010"`: "A0",         // greedy stops after three digits
+		`"\08"`:   "\x00" + "8", // non-octal digit ends the run
+		`"\779"`:  "?9",         // 0o77 = 63 ('?'), then literal 9
+		`"a\101"`: "aA",         // after literal text
+	}
+	for src, want := range cases {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestStringUnicodeEscape: `\uHHHH` (exactly four hex digits) and `\u{...}`
+// (whitespace-separated codepoints) both expand to UTF-8 bytes, matching MRI.
+func TestStringUnicodeEscape(t *testing.T) {
+	// bs is a single literal backslash; building the sources this way keeps the
+	// `\u` sequences intact (a bare `A` in Go source would be the compiler's
+	// own escape, not the two bytes the lexer must see).
+	bs := "\\"
+	cases := map[string]string{
+		`"` + bs + `u0041"`:       "A",          // four-digit form -> 'A'
+		`"` + bs + `u00e9"`:       "é",          // four-digit, two-byte UTF-8
+		`"a` + bs + `u0041b"`:     "aAb",        // four-digit among literal text
+		`"` + bs + `u{41}"`:       "A",          // brace form, single codepoint
+		`"` + bs + `u{41 42 43}"`: "ABC",        // multiple codepoints
+		`"` + bs + `u{1F600}"`:    "\U0001F600", // codepoint beyond the BMP
+		`"` + bs + `u{}"`:         "",           // empty braces -> nothing
+		`"` + bs + `u{ 41 }"`:     "A",          // surrounding whitespace tolerated
+	}
+	for src, want := range cases {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestStringUnicodeEscapeInvalid: out-of-range / surrogate codepoints fall back
+// to U+FFFD, and a `\u` with no hex digit contributes nothing.
+func TestStringUnicodeEscapeInvalid(t *testing.T) {
+	cases := map[string]string{
+		`"\u{110000}"`: "�", // beyond U+10FFFF
+		`"\u{D800}"`:   "�", // a surrogate half
+		`"\uZ"`:        "Z", // no hex digit after \u: the Z stays as literal text
+		`"\u{ZZ}"`:     "",  // no hex digit inside braces
+		`"\u{41ZZ}"`:   "A", // trailing junk before the close brace is skipped
+		`"\u{41`:       "A", // unterminated brace form runs into end of input
+	}
+	for src, want := range cases {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestStringBasicEscapesStillWork guards the pre-existing escapes the new code
+// sits beside, so the regression covers the whole switch.
+func TestStringBasicEscapesStillWork(t *testing.T) {
+	cases := map[string]string{
+		`"\n"`: "\n",
+		`"\t"`: "\t",
+		`"\r"`: "\r",
+		`"\a"`: "\a",
+		`"\b"`: "\b",
+		`"\v"`: "\v",
+		`"\f"`: "\f",
+		`"\e"`: "\x1b",
+		`"\s"`: " ",
+		`"\\"`: "\\",
+		`"\""`: "\"",
+		`"\q"`: "q", // unknown escape drops the backslash
+	}
+	for src, want := range cases {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}
+
+// TestNumericEscapesInPercentQAndInterp: the new escapes apply to every
+// interpolating string form (%Q, %{}, and the segment after an interpolation),
+// since they all funnel through the same scanner.
+func TestNumericEscapesInPercentQAndInterp(t *testing.T) {
+	if got, _ := firstStringLit(`%Q{\x41\101}`); got != "AA" {
+		t.Errorf(`%%Q{\x41\101}: got %q, want "AA"`, got)
+	}
+	if got, _ := firstStringLit(`%{A}`); got != "A" {
+		t.Errorf(`%%{A}: got %q, want "A"`, got)
+	}
+	// The trailing segment after an interpolation must interpret escapes too.
+	lits := stringLits("\"a#{x}\\x42\"")
+	if len(lits) < 2 || lits[len(lits)-1] != "B" {
+		t.Errorf("interpolation tail: lits=%q, want last segment %q", lits, "B")
+	}
+}
+
+// TestNumericEscapesNotInSingleQuote: single-quoted strings keep `\x`/`\NNN`
+// verbatim (only `\\` and `\'` are special), unchanged by this fix.
+func TestNumericEscapesNotInSingleQuote(t *testing.T) {
+	for src, want := range map[string]string{
+		`'\x41'`: `\x41`,
+		`'\101'`: `\101`,
+		`'A'`:    `A`,
+	} {
+		got, ok := firstStringLit(src)
+		if !ok || got != want {
+			t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want)
+		}
+	}
+}