diff --git a/ast/ast.go b/ast/ast.go index ca63f5e..7a5d4ba 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -21,6 +21,15 @@ type BignumLit struct{ Val *big.Int } // FloatLit is a floating-point literal. type FloatLit struct{ Value float64 } +// RationalLit is a rational literal (`2r`, `0.5r`): Value is the underlying +// numeric literal (an IntLit, BignumLit, or FloatLit) the `r` suffix promotes. +type RationalLit struct{ Value Node } + +// ImaginaryLit is an imaginary literal (`3i`, `2.5i`, and the combined `2ri` +// rational-imaginary): Value is the underlying numeric literal the `i` suffix +// promotes (itself possibly a RationalLit for the `ri` form). +type ImaginaryLit struct{ Value Node } + // StringLit is a (Phase 0: non-interpolated) string literal. type StringLit struct{ Value string } @@ -263,6 +272,7 @@ type SingletonClassDef struct { type Super struct { Args []Node Forward bool + Block *Block // a `do…end` / `{…}` block passed to super, or nil } // Break exits the innermost block (terminating its iterator) or loop. Value may @@ -431,14 +441,21 @@ type Begin struct { // rescue StandardError. type RescueClause struct { Classes []Node - Var string - Body []Node + Var string // plain-local capture name (`rescue => e`), or "" — see VarTarget + // VarTarget is the general capture target when it is not a plain local — + // an instance/class/global variable, a constant, or an attribute/index + // (`rescue => @error`, `rescue => $g`, `rescue => obj.err`). It is the LHS + // node the caught exception is assigned to; nil when Var carries a local. + VarTarget Node + Body []Node } func (*Program) node() {} func (*ScopedConst) node() {} func (*IntLit) node() {} func (*BignumLit) node() {} +func (*RationalLit) node() {} +func (*ImaginaryLit) node() {} func (*FloatLit) node() {} func (*StringLit) node() {} func (*SymbolLit) node() {} diff --git a/lexer/lexer.go b/lexer/lexer.go index 0d139eb..60905a8 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -44,6 +44,18 @@ type Lexer struct { // trailing-operator line continuation (a line ending in an infix operator // joins the next line, as MRI does). prevType token.Type + // prevBinary records whether the last token was an ambiguous operator + // (`|`/`&`/`^`/`<<`) lexed in binary (after-a-value) position. Only then does a + // trailing one continue the line; in operand position those open a block-param + // list, a block-pass, a heredoc, or a pattern pin instead. + prevBinary bool + // pendingBinary is set within lexToken when the token being emitted is an + // ambiguous operator in binary position; next() latches it into prevBinary. + pendingBinary bool + // inFitemList is set after `alias`/`undef` until the next newline: in that span + // a trailing operator names a method (`alias eql? ==`⏎`def …`) rather than + // continuing the line, so operator line-continuation is suppressed. + inFitemList bool } func New(src string) *Lexer { @@ -91,8 +103,10 @@ func (l *Lexer) Tokenize() []token.Token { // next returns the next token and records its type, so the lexer can recognise a // trailing-operator line continuation. The lexing itself is in lexToken. func (l *Lexer) next() token.Token { + l.pendingBinary = false t := l.lexToken() l.prevType = t.Type + l.prevBinary = l.pendingBinary return t } @@ -106,7 +120,7 @@ func (l *Lexer) next() token.Token { func isContinuationOp(t token.Type) bool { switch t { case token.PLUS, token.MINUS, token.STAR, token.POW, token.SLASH, token.PERCENT, - token.EQ, token.EQQ, token.MATCH, token.NEQ, token.LT, token.GT, token.LE, + token.EQ, token.EQQ, token.MATCH, token.NMATCH, token.NEQ, token.LT, token.GT, token.LE, token.GE, token.SPACESHIP, token.ANDAND, token.OROR, token.ASSIGN, token.OPASSIGN, token.COMMA, token.HASHROCKET, token.DOT, token.SAFEDOT, token.QUESTION, token.COLON, @@ -175,12 +189,28 @@ func (l *Lexer) lexToken() token.Token { if c == '\n' && l.nextLineStartsWithDot() { return l.next() } + // Inside an `alias`/`undef` fitem list, a trailing operator names a method + // (`alias eql? ==`) and must not continue the line; the newline closes it. + if l.inFitemList { + l.inFitemList = false + return mk(token.NEWLINE, "\\n") + } // Trailing-operator continuation: a line ending in an infix operator // (`a ||`, `x +`, a trailing comma, …) is incomplete and joins the next // line. `;` is an explicit terminator and is never suppressed this way. if c == '\n' && isContinuationOp(l.prevType) { return l.next() } + // The ambiguous bitwise/shift operators `|`/`&`/`^`/`<<` continue a line + // only when they were lexed in binary position (`new_args <<`⏎`x`, + // `… secure_compare(…) &`⏎`…`); in operand position they open a block param + // list, a block-pass, a heredoc, or a pattern pin and must not join. + if c == '\n' && l.prevBinary { + switch l.prevType { + case token.PIPE, token.AMPER, token.CARET, token.SHOVEL: + return l.next() + } + } return mk(token.NEWLINE, "\\n") case isDigit(c): return l.lexNumber(spaceBefore, line, col) @@ -190,6 +220,12 @@ func (l *Lexer) lexToken() token.Token { return l.lexString(spaceBefore, line, col) case c == '\'': return l.lexSingleQuote(spaceBefore, line, col) + case c == '`' && l.prevType == token.DEF: + // `def \`(cmd); end` — the backtick names the backtick method, not a + // command literal. Emit a SYMBOL-like XSTRING sentinel the parser maps to "`". + l.advance() + l.state = exprBegin + return mk(token.XSTRING, "`") case c == '`': return l.lexBacktick(spaceBefore, line, col) case c == '@': @@ -210,6 +246,12 @@ func (l *Lexer) lexToken() token.Token { } // Operators and delimiters. + // Record whether we are at expression-end (after a value) before consuming the + // operator. This disambiguates the ambiguous trailing operators `|`/`&`/`^`/`<<` + // at end-of-line: in binary position (after a value) a trailing one continues + // the line; in operand position it is a block param / block-pass / heredoc / + // pattern pin and does not. + binaryPos := l.state == exprEnd l.advance() switch c { case '+': @@ -307,6 +349,7 @@ func (l *Lexer) lexToken() token.Token { return mk(token.OPASSIGN, "|") } l.state = exprBegin + l.pendingBinary = binaryPos return mk(token.PIPE, "|") case '&': if l.peek() == '&' { @@ -330,6 +373,7 @@ func (l *Lexer) lexToken() token.Token { return mk(token.OPASSIGN, "&") } l.state = exprBegin + l.pendingBinary = binaryPos return mk(token.AMPER, "&") case ',': l.state = exprBegin @@ -376,6 +420,11 @@ func (l *Lexer) lexToken() token.Token { l.state = exprBegin return mk(token.NEQ, "!=") } + if l.peek() == '~' { // !~ does-not-match operator + l.advance() + l.state = exprBegin + return mk(token.NMATCH, "!~") + } l.state = exprBegin return mk(token.BANG, "!") case '<': @@ -400,6 +449,7 @@ func (l *Lexer) lexToken() token.Token { return l.lexHeredoc(spaceBefore, line, col) } l.state = exprBegin + l.pendingBinary = binaryPos return mk(token.SHOVEL, "<<") } l.state = exprBegin @@ -440,6 +490,7 @@ func (l *Lexer) lexToken() token.Token { return mk(token.OPASSIGN, "^") } l.state = exprBegin + l.pendingBinary = binaryPos return mk(token.CARET, "^") case '~': l.state = exprBegin @@ -599,12 +650,23 @@ func (l *Lexer) lexNumber(spaceBefore bool, line, col int) token.Token { } } lit := stripUnderscores(string(l.src[start:l.pos])) + // Trailing `r` (rational) and/or `i` (imaginary) suffixes: `2r`, `0.5r`, + // `3i`, `2.5ri`. Recorded in Flags so the parser wraps the literal accordingly. + suffix := "" + if l.peek() == 'r' { + l.advance() + suffix += "r" + } + if l.peek() == 'i' { + l.advance() + suffix += "i" + } l.state = exprEnd tt := token.INT if isFloat { tt = token.FLOAT } - return token.Token{Type: tt, Lit: lit, Line: line, Col: col, SpaceBefore: spaceBefore} + return token.Token{Type: tt, Lit: lit, Flags: suffix, Line: line, Col: col, SpaceBefore: spaceBefore} } // lexRadixInt lexes a prefixed integer literal (cursor on the leading '0'). The @@ -671,6 +733,14 @@ func (l *Lexer) lexIdent(spaceBefore bool, line, col int) token.Token { // Trailing ? or ! is part of a method name (e.g. empty?, save!). if c := l.peek(); c == '?' || c == '!' { l.advance() + // A predicate/bang method name immediately followed by a single ':' is a + // hash label too (`frozen?: …`, `has_key?: …`, `valid!: …`). + if l.peek() == ':' && l.peek2() != ':' { + lit := string(l.src[start:l.pos]) + l.advance() // ':' + l.state = exprBegin + return token.Token{Type: token.LABEL, Lit: lit, Line: line, Col: col, SpaceBefore: spaceBefore} + } } lit := string(l.src[start:l.pos]) tt := token.LookupIdent(lit) @@ -682,6 +752,9 @@ func (l *Lexer) lexIdent(spaceBefore bool, line, col int) token.Token { default: l.state = exprBegin } + if tt == token.ALIAS || tt == token.UNDEF { + l.inFitemList = true + } return token.Token{Type: tt, Lit: lit, Line: line, Col: col, SpaceBefore: spaceBefore} } @@ -690,8 +763,8 @@ func (l *Lexer) lexIdent(spaceBefore bool, line, col int) token.Token { // symbolOps are the operator method names that can appear as a symbol (`:+`, // `:[]=`, …), ordered so the first prefix match is the longest. var symbolOps = []string{ - "[]=", "<=>", "===", "[]", "==", "=~", "!=", "<<", ">>", "<=", ">=", "**", - "+@", "-@", "+", "-", "*", "/", "%", "<", ">", "&", "|", "^", "~", "!", + "[]=", "<=>", "===", "[]", "==", "=~", "!~", "!=", "<<", ">>", "<=", ">=", "**", + "+@", "-@", "+", "-", "*", "/", "%", "<", ">", "&", "|", "^", "~", "!", "`", } // symbolOpAt returns the operator-symbol name starting at src[i], or "". @@ -827,6 +900,12 @@ func (l *Lexer) lexGvar(spaceBefore bool, line, col int) token.Token { l.advance() // '$' start := l.pos switch c := l.peek(); { + case c == '-': + // Option globals: `$-I`, `$-d`, `$-w`, `$-0`, … — a `-` plus one name char. + l.advance() // '-' + if isIdentPart(l.peek()) { + l.advance() + } case isSpecialGvar(c): // Single-character special globals: $~ $& $` $' $! $@ $/ $\ $; $, $. // $< $> $? $* $$ $: $" $0 $+ (and the like). Each is exactly one byte. @@ -946,7 +1025,14 @@ func (l *Lexer) atCharLiteral() bool { if n == '\\' { // an escape always forms a char literal (?\n, ?\s, ?\\) return true } - // A plain single byte that begins an identifier must stand alone: the byte + // A multi-byte UTF-8 lead byte (>= 0x80) starts a single-rune payload (`?é`): + // it is taken whole, so the bytes that follow it are its own continuation + // bytes, not a longer word. (Checked before the identifier rule below, which + // would otherwise treat the rune's continuation bytes as more ident chars.) + if n >= 0x80 { + return true + } + // A plain ASCII byte that begins an identifier must stand alone: the byte // after it must not continue an identifier word (so `?a` is a char but `?ab` // is not). A non-identifier payload (`?|`, `?/`, `?.`) is always a char. if isIdentPart(n) { @@ -956,10 +1042,6 @@ func (l *Lexer) atCharLiteral() bool { } return !isIdentPart(third) } - // A multi-byte UTF-8 lead byte (>= 0x80) starts a single rune payload (`?é`). - if n >= 0x80 { - return true - } return true } @@ -1721,9 +1803,16 @@ func (l *Lexer) scanStringSegment() (string, bool) { } } -func isDigit(c byte) bool { return c >= '0' && c <= '9' } -func isIdentStart(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') } -func isIdentPart(c byte) bool { return isIdentStart(c) || isDigit(c) } +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +// isIdentStart reports whether c can begin an identifier. Besides ASCII letters +// and `_`, any byte >= 0x80 — a UTF-8 multi-byte lead/continuation byte — counts, +// so Unicode identifiers (`def なまえ`, `weird = Weird.create(なまえ: …)`) lex as one +// IDENT, matching Ruby's acceptance of non-ASCII characters in names. +func isIdentStart(c byte) bool { + return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 0x80 +} +func isIdentPart(c byte) bool { return isIdentStart(c) || isDigit(c) } // isSpecialGvar reports whether c is one of Ruby's single-character special // global-variable names that follow `$` (e.g. $! $@ $/ $\ $; $, $. $< $> $? diff --git a/parser.go b/parser.go index e165f19..8ba6b8b 100644 --- a/parser.go +++ b/parser.go @@ -107,6 +107,10 @@ type Parser struct { // default expression of an optional block parameter inside a `|...|` list, so // the `|` there closes the parameter list rather than continuing the default. noPipe bool + // noMasgn suppresses multiple-assignment detection while parsing a parameter + // default (`def f(a = c(1), b = nil)`), so the comma separating parameters is + // not mistaken for a masgn target separator (`c(1), b = nil`). + noMasgn bool } // parseHook, when non-nil, runs at the start of Parse. It exists only so a @@ -274,19 +278,14 @@ func (p *Parser) parseStatements(stop map[token.Type]bool) []ast.Node { func (p *Parser) parseStatement() ast.Node { switch p.cur().Type { case token.DEF: - return p.parseDef() + // A method definition may carry a trailing modifier (`def f; …; end if c`) + // and is occasionally chained (`def f; end.tap { … }`), so feed it through + // the postfix/modifier machinery rather than returning it raw. + return p.applyModifiers(p.parsePostfixTail(p.parseDef())) case token.CLASS: - return p.parseClass() + return p.applyModifiers(p.parsePostfixTail(p.parseClass())) case token.MODULE: - return p.parseModule() - case token.IF: - return p.parseIf() - case token.UNLESS: - return p.parseUnless() - case token.WHILE: - return p.parseWhile() - case token.UNTIL: - return p.parseUntil() + return p.applyModifiers(p.parsePostfixTail(p.parseModule())) case token.RETURN: return p.applyModifiers(p.parseReturn()) case token.BREAK: @@ -388,9 +387,20 @@ func (p *Parser) parseConstPath() (name string, path ast.Node) { if p.accept(token.SCOPE) { // leading `::Name` name = p.expect(token.CONST).Lit node = &ast.ScopedConst{Name: name, Global: true} - } else { - name = p.expect(token.CONST).Lit + } else if p.is(token.CONST) { + name = p.advance().Lit node = &ast.ConstRef{Name: name} + } else { + // An expression receiver before `::CONST` names a dynamically-scoped class + // or module: `class self.class::Foo`, `class obj::Bar`. parsePostfix reads + // the whole `recv::CONST` chain and yields a ScopedConst whose Name is the + // trailing segment, exactly the path node we want. + expr := p.parsePostfix() + sc, ok := expr.(*ast.ScopedConst) + if !ok { + p.fail("expected a class/module path") + } + return sc.Name, sc } scoped := node // becomes the *ScopedConst once a `::CONST` segment is seen for p.is(token.SCOPE) && p.peekTok().Type == token.CONST { @@ -552,6 +562,10 @@ func (p *Parser) parseDefName() (string, bool) { return "[]=", true } return "[]", true + case token.XSTRING: + // The backtick method name `def \`(cmd); end` (lexed as an empty XSTRING). + p.advance() + return "`", true } // A keyword used as a method name: def do / def then / def in / def class … // Ruby permits any reserved word in def-name position. @@ -625,7 +639,7 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a p.declareLocal(name) var def ast.Node if !p.is(token.COMMA) && !p.is(until) && !p.is(token.NEWLINE) { - def = p.parseExprOrAssign() + def = p.parseParamDefault() } kwParams = append(kwParams, ast.KwParam{Name: name, Default: def}) if !p.accept(token.COMMA) { @@ -650,12 +664,17 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a } name := p.expect(token.IDENT).Lit params = append(params, name) - p.declareLocal(name) + // A parameter is not yet in scope while its own default expression is + // parsed (MRI's rule), so a default may call a same-named method — + // `def f(secret = secret("x"))` is the secret() *method*, not a self + // reference. Declare the local only after the default is consumed; earlier + // parameters are already declared and so visible to this default. if p.accept(token.ASSIGN) { - defaults = append(defaults, p.parseExprOrAssign()) + defaults = append(defaults, p.parseParamDefault()) } else { defaults = append(defaults, nil) } + p.declareLocal(name) if !p.accept(token.COMMA) { break } @@ -666,6 +685,17 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a return params, defaults, splat, kwParams, kwRest, blockParam, forward } +// parseParamDefault parses a parameter's default-value expression with masgn +// detection suppressed, so the comma that separates parameters is not mistaken +// for a multiple-assignment target separator (`def f(a = c(1), b = nil)`). +func (p *Parser) parseParamDefault() ast.Node { + saved := p.noMasgn + p.noMasgn = true + def := p.parseExprOrAssign() + p.noMasgn = saved + return def +} + // parseCond parses an if/unless/while/until condition, where the low-precedence // keyword operators `and`/`or`/`not` are permitted (`if a and b`, `while x or // y`, `unless not done`). @@ -793,7 +823,7 @@ func (p *Parser) parseBreak() ast.Node { if p.atStatementEnd() { return &ast.Break{} } - return &ast.Break{Value: p.parseExprOrAssign()} + return &ast.Break{Value: p.parseJumpValue()} } func (p *Parser) parseNext() ast.Node { @@ -801,7 +831,22 @@ func (p *Parser) parseNext() ast.Node { if p.atStatementEnd() { return &ast.Next{} } - return &ast.Next{Value: p.parseExprOrAssign()} + return &ast.Next{Value: p.parseJumpValue()} +} + +// parseJumpValue parses the value of a `break`/`next` jump: a single expression, +// or a comma-separated list gathered into an array (`next table, true`, +// `break a, b`), matching `return a, b`. +func (p *Parser) parseJumpValue() ast.Node { + first := p.parseExprOrAssign() + if !p.is(token.COMMA) { + return first + } + elems := []ast.Node{first} + for p.accept(token.COMMA) { + elems = append(elems, p.parseExprOrAssign()) + } + return &ast.ArrayLit{Elems: elems} } // atStatementEnd reports whether the cursor is at a point where a value-less @@ -821,6 +866,14 @@ func (p *Parser) atStatementEnd() bool { // parseBegin parses `begin BODY (rescue [Classes] [=> var] BODY)* [else BODY] // [ensure BODY] end`. +// rescueVarTarget parses the non-local capture target of a `rescue => TARGET` +// clause (an ivar/cvar/gvar/const or an attribute/index target), reusing the +// masgn-target machinery so the resulting node is in the same assignable shape. +func (p *Parser) rescueVarTarget() ast.Node { + _, tgt, _ := p.parseMlhsTarget() + return tgt +} + func (p *Parser) parseBegin() ast.Node { p.expect(token.BEGIN) node := p.parseRescueTail(p.parseStatements(beginBodyEnd)) @@ -845,8 +898,15 @@ func (p *Parser) parseRescueTail(body []ast.Node) *ast.Begin { } } if p.accept(token.HASHROCKET) { - clause.Var = p.expect(token.IDENT).Lit - p.declareLocal(clause.Var) + // The capture target is usually a plain local (`rescue => e`), but may be + // any assignable: an ivar/cvar/gvar/const or an attribute (`rescue => + // @error`, `rescue => $g`, `rescue => obj.err`). + if p.is(token.IDENT) && !isPostfixStart(p.peekTok().Type) { + clause.Var = p.advance().Lit + p.declareLocal(clause.Var) + } else { + clause.VarTarget = p.rescueVarTarget() + } } p.accept(token.THEN) clause.Body = p.parseStatements(beginBodyEnd) @@ -1088,11 +1148,12 @@ func (p *Parser) parsePatternAtom() ast.Pattern { name := p.advance().Lit p.declareLocal(name) return &ast.BindPattern{Name: name} - case token.CONST: - c := &ast.ConstRef{Name: p.advance().Lit} - // Point[...] is a const array pattern (deconstruct); Point(x:, y:) is a - // const hash pattern (deconstruct_keys); otherwise the constant is a class - // match. + case token.CONST, token.SCOPE: + // A (possibly scope-resolved) constant pattern: `Point`, `Prism::CaseNode`, + // `::Foo`. Point[...] is a const array pattern (deconstruct); Point(x:, y:) + // is a const hash pattern (deconstruct_keys); otherwise the constant is a + // class match. + c := p.parsePatternConst() if p.is(token.LBRACKET) { return p.parseArrayPattern(c) } @@ -1109,15 +1170,39 @@ func (p *Parser) parsePatternAtom() ast.Pattern { } } +// parsePatternConst parses the constant reference at the head of a constant +// pattern, including a scope-resolution path (`Prism::CaseNode`) and a leading +// top-level `::Foo`. +func (p *Parser) parsePatternConst() ast.Node { + var node ast.Node + if p.accept(token.SCOPE) { + node = &ast.ScopedConst{Name: p.expect(token.CONST).Lit, Global: true} + } else { + node = &ast.ConstRef{Name: p.expect(token.CONST).Lit} + } + for p.is(token.SCOPE) && p.peekTok().Type == token.CONST { + p.advance() // :: + node = &ast.ScopedConst{Recv: node, Name: p.advance().Lit} + } + return node +} + // parsePatternValue parses the expression behind a value pattern: a literal or // a range of literals (`1..5`, `..10`, `1..`). func (p *Parser) parsePatternValue() ast.Node { return p.parseRange() } -// parseArrayPattern parses `[pat, …]`, with an optional leading constant. +// parseArrayPattern parses `[pat, …]`, with an optional leading constant. When +// the bracket body begins with a label or `**`, it is a hash pattern written +// with bracket delimiters (`Const[key:]`, deconstruct_keys), which MRI accepts. func (p *Parser) parseArrayPattern(constName ast.Node) ast.Pattern { p.expect(token.LBRACKET) + if p.is(token.LABEL) || p.is(token.POW) { + hp := p.parseHashPatternBody(constName, token.RBRACKET) + p.expect(token.RBRACKET) + return hp + } var elems []arrayElem if !p.accept(token.RBRACKET) { elems = append(elems, p.parseArrayPatternElem()) @@ -1260,6 +1345,7 @@ func (p *Parser) looksLikeMlhs() bool { i := p.pos sawComma := false sawSplat := false + sawGroup := false for { // Optional leading splat on this target. if p.toks[i].Type == token.STAR { @@ -1277,12 +1363,39 @@ func (p *Parser) looksLikeMlhs() bool { } } // A trailing comma before `=` ends the target list: `a, = x`. - if sawComma && p.toks[i].Type == token.ASSIGN { + if (sawComma || sawGroup) && p.toks[i].Type == token.ASSIGN { return true } - // A target must start with one of these kinds. + // A parenthesized nested target group: `(a, b) = …`, `((a, b), c) = …`, + // `(a, b), c = …`. A `(` here begins a target, not a postfix call (which + // only follows a name). Scan the balanced group, then a `,`/`=` must follow. + if p.toks[i].Type == token.LPAREN { + j := p.scanBalanced(i, token.LPAREN, token.RPAREN) + if j < 0 { + return false + } + sawGroup = true + i = j + switch p.toks[i].Type { + case token.COMMA: + sawComma = true + i++ + continue + case token.ASSIGN: + return true + default: + return false + } + } + // A target must start with one of these kinds. A leading `::` begins a + // top-level scoped target (`::Time.zone = …`). switch p.toks[i].Type { case token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, token.SELF: + case token.SCOPE: + i++ + if p.toks[i].Type != token.CONST { + return false + } default: return false } @@ -1411,6 +1524,39 @@ func (p *Parser) parseMlhs() ast.Node { return &ast.MultiAssign{Names: names, Targets: targets, SplatIndex: splat, Values: values} } +// parseMlhsGroup parses a parenthesized nested masgn target `(t1, t2, …)` and +// returns it as a *MultiAssign with Values left nil (it captures one destructured +// value rather than driving its own RHS). It reuses the same per-target grammar, +// so nesting (`((a, b), c)`) and an inner splat (`(a, *b)`) work. +func (p *Parser) parseMlhsGroup() ast.Node { + p.expect(token.LPAREN) + var names []string + var targets []ast.Node + splat := -1 + for { + isSplat := p.accept(token.STAR) + if isSplat { + splat = len(names) + } + if isSplat && (p.is(token.COMMA) || p.is(token.RPAREN)) { + names = append(names, "") + targets = append(targets, nil) + } else { + name, tgt, _ := p.parseMlhsTarget() + names = append(names, name) + targets = append(targets, tgt) + } + if !p.accept(token.COMMA) { + break + } + if p.is(token.RPAREN) { // trailing comma `(a, b,)` + break + } + } + p.expect(token.RPAREN) + return &ast.MultiAssign{Names: names, Targets: targets, SplatIndex: splat} +} + // parseSplatOrExpr parses an expression that may be prefixed by a `*splat`, // spreading an array in a position that accepts several values (a `when` // candidate list, a `rescue` class list). @@ -1440,6 +1586,12 @@ func (p *Parser) parseMasgnValue() ast.Node { // setter-call shape (Call with Name "x=" / "[]=") so consumers reuse their // single-assignment store logic. func (p *Parser) parseMlhsTarget() (string, ast.Node, bool) { + // Parenthesized nested target group: `(a, b) = …`, `((x, y), z) = …`. The + // group destructures one value into its own sub-targets; it is represented as + // a nested *MultiAssign with no Values (Values stays nil), stored as a target. + if p.is(token.LPAREN) { + return "", p.parseMlhsGroup(), false + } // Simple local: declare it and use a *VarRef (fast path). if p.is(token.IDENT) && !isPostfixStart(p.peekTok().Type) { name := p.advance().Lit @@ -1485,8 +1637,9 @@ func (p *Parser) parseExprOrAssign() ast.Node { if p.accept(token.NOT) { return not(p.parseExprOrAssign()) } - // Multiple assignment to local targets: a, b = … / a, *b = … . - if p.looksLikeMlhs() { + // Multiple assignment to local targets: a, b = … / a, *b = … . Disabled while + // parsing a parameter default, where a comma separates parameters. + if !p.noMasgn && p.looksLikeMlhs() { return p.parseMlhs() } // Simple local assignment: IDENT '=' expr (right-associative, chainable). @@ -1573,18 +1726,20 @@ func (p *Parser) parseExprOrAssign() ast.Node { } if p.is(token.ASSIGN) { if call, ok := left.(*ast.Call); ok && call.Recv != nil { - // Index assignment: recv[i] = v → recv.[]=(i, v). + // Index assignment: recv[i] = v → recv.[]=(i, v). A comma-separated + // right-hand side becomes an implicit array (`h[k] = 1, 2`). if call.Name == "[]" { p.advance() call.Name = "[]=" - call.Args = append(call.Args, p.parseExprOrAssign()) + call.Args = append(call.Args, p.parseAssignRhs()) return call } - // Attribute assignment: recv.attr = v → recv.attr=(v). + // Attribute assignment: recv.attr = v → recv.attr=(v). A comma list on + // the right is gathered into an array (`self.cache_store = :s, path`). if len(call.Args) == 0 && call.Block == nil { p.advance() call.Name += "=" - call.Args = []ast.Node{p.parseExprOrAssign()} + call.Args = []ast.Node{p.parseAssignRhs()} return call } } @@ -1654,9 +1809,12 @@ func (p *Parser) parseTernary() ast.Node { if !p.accept(token.QUESTION) { return cond } - then := p.parseTernary() + // Each branch may itself be an assignment (`c ? a = b : d`, + // `x ? ENV["k"] = v : super`), which MRI permits in this position even though + // `=` otherwise binds looser than `?:`. + then := p.maybeInlineAssign(p.parseTernary()) p.expect(token.COLON) - els := p.parseTernary() + els := p.maybeInlineAssign(p.parseTernary()) return &ast.If{Cond: cond, Then: []ast.Node{then}, Else: []ast.Node{els}} } @@ -1688,7 +1846,7 @@ func binBP(tt token.Type) int { return 4 case token.ANDAND: return 6 - case token.EQ, token.EQQ, token.NEQ, token.SPACESHIP, token.MATCH: + case token.EQ, token.EQQ, token.NEQ, token.SPACESHIP, token.MATCH, token.NMATCH: return 10 case token.LT, token.GT, token.LE, token.GE: return 20 @@ -1766,6 +1924,15 @@ func (p *Parser) valuelessJumpOperand() (ast.Node, bool) { // in parseExprOrAssign; this covers the nested operand case.) `==`/`=>`/`=~` are // their own tokens, so only a real assignment `=` is consumed here. func (p *Parser) maybeInlineAssign(node ast.Node) ast.Node { + // A compound assignment may also sit as a nested operand: `a && count += 1`, + // `x > 0 && h[k] -= 1`, `cond && precision ||= 1`. Desugar it the same way the + // statement-level OP= paths do. + if p.is(token.OPASSIGN) { + if assigned := p.inlineOpAssign(node); assigned != nil { + return assigned + } + return node + } if !p.is(token.ASSIGN) { return node } @@ -1813,6 +1980,58 @@ func (p *Parser) maybeInlineAssign(node ast.Node) ast.Node { return node } +// inlineOpAssign desugars a compound assignment `target OP= rhs` whose target is +// the already-parsed node, returning the assignment node, or nil if the node is +// not an assignable target (so the caller leaves it untouched). It mirrors the +// statement-level OP= handling for each target kind. +func (p *Parser) inlineOpAssign(node ast.Node) ast.Node { + switch n := node.(type) { + case *ast.VarRef: + op := p.advance().Lit + rhs := p.parseExprOrAssign() + p.declareLocal(n.Name) + return &ast.OpAssign{Name: n.Name, Op: op, Value: rhs} + case *ast.Call: + if n.Recv == nil && len(n.Args) == 0 && n.Block == nil { // a bare local + op := p.advance().Lit + rhs := p.parseExprOrAssign() + p.declareLocal(n.Name) + return &ast.OpAssign{Name: n.Name, Op: op, Value: rhs} + } + if n.Recv != nil && n.Block == nil { + // recv[i] OP= v → recv[i] = recv[i] OP v; recv.a OP= v → recv.a = recv.a OP v. + if n.Name == "[]" { + op := p.advance().Lit + rhs := p.parseExprOrAssign() + read := &ast.Call{Recv: n.Recv, Name: "[]", Args: n.Args} + newVal := &ast.BinaryExpr{Op: op, Left: read, Right: rhs} + args := append(append([]ast.Node{}, n.Args...), newVal) + return &ast.Call{Recv: n.Recv, Name: "[]=", Args: args} + } + if len(n.Args) == 0 { + op := p.advance().Lit + rhs := p.parseExprOrAssign() + read := &ast.Call{Recv: n.Recv, Name: n.Name} + newVal := &ast.BinaryExpr{Op: op, Left: read, Right: rhs} + return &ast.Call{Recv: n.Recv, Name: n.Name + "=", Args: []ast.Node{newVal}} + } + } + case *ast.IvarRef: + op := p.advance().Lit + rhs := p.parseExprOrAssign() + return &ast.IvarAssign{Name: n.Name, Value: &ast.BinaryExpr{Op: op, Left: &ast.IvarRef{Name: n.Name}, Right: rhs}} + case *ast.CVarRef: + op := p.advance().Lit + rhs := p.parseExprOrAssign() + return &ast.CVarAssign{Name: n.Name, Value: &ast.BinaryExpr{Op: op, Left: &ast.CVarRef{Name: n.Name}, Right: rhs}} + case *ast.GVarRef: + op := p.advance().Lit + rhs := p.parseExprOrAssign() + return &ast.GVarAssign{Name: n.Name, Value: &ast.BinaryExpr{Op: op, Left: &ast.GVarRef{Name: n.Name}, Right: rhs}} + } + return nil +} + // negateLiteral returns the negation of a numeric literal node. The MINUS path // in parseUnary reaches here only after parsePrimary consumed an INT or FLOAT // token, which yields exactly one of these three kinds: a FLOAT is always a @@ -1907,17 +2126,51 @@ func (p *Parser) parsePostfixTail(node ast.Node) ast.Node { node = &ast.Call{Recv: node, Name: name, Args: args, Safe: safe} case p.is(token.SCOPE): p.advance() - if p.is(token.CONST) { // Math::PI — a scoped constant - node = &ast.ScopedConst{Recv: node, Name: p.advance().Lit} + // `Const::Name(args)` — a capitalized scope-resolution method call: + // the `(` hugging the name (no space) means a send, not a constant + // (`Syslog::LOG_UPTO(Syslog::LOG_INFO)`). A bare `Const::Name` is a + // scoped constant (`Math::PI`). + if p.is(token.CONST) && p.peekTok().Type == token.LPAREN && !p.peekTok().SpaceBefore { + name := p.advance().Lit + p.advance() // consume '(' + args := p.parseCallArgs(token.RPAREN) + p.expect(token.RPAREN) + node = &ast.Call{Recv: node, Name: name, Args: args} break } - // Foo::bar(args) — a method call, like the dot form. + if p.is(token.CONST) { + name := p.advance().Lit + // A capitalized scope-resolution name with a space-separated command + // argument is a method call (`obj::Down x, y`); otherwise it is a + // scoped constant (`Math::PI`). + if p.canStartCommandArg() || p.atHuggingStringArg() { + call := &ast.Call{Recv: node, Name: name, Args: p.parseCommandArgs()} + if p.is(token.DO) && !p.noDo { + call.Block = p.parseDoBlock() + node = call + break + } + return call + } + node = &ast.ScopedConst{Recv: node, Name: name} + break + } + // Foo::bar(args) or `Mod::meth arg` — a method call, like the dot form. name := p.methodName() var args []ast.Node if p.is(token.LPAREN) && !p.cur().SpaceBefore { p.advance() args = p.parseCallArgs(token.RPAREN) p.expect(token.RPAREN) + } else if p.canStartCommandArg() || p.atHuggingStringArg() { + // Paren-less scope-resolution command call: `Mod::meth arg`. + call := &ast.Call{Recv: node, Name: name, Args: p.parseCommandArgs()} + if p.is(token.DO) && !p.noDo { + call.Block = p.parseDoBlock() + node = call + break + } + return call } node = &ast.Call{Recv: node, Name: name, Args: args} case p.is(token.LBRACKET): // index: recv[args] → recv.[](args) @@ -1927,15 +2180,26 @@ func (p *Parser) parsePostfixTail(node ast.Node) ast.Node { node = &ast.Call{Recv: node, Name: "[]", Args: args} case p.is(token.LBRACE) || (p.is(token.DO) && !p.noDo): // A block binds to the immediately preceding method call; chaining - // then continues (`recv.map { … }.join`). - call, ok := node.(*ast.Call) - if !ok || call.Block != nil { + // then continues (`recv.map { … }.join`). A `super` also takes a block + // (`super { … }`, `super(x) do … end`). + var blockSlot **ast.Block + switch n := node.(type) { + case *ast.Call: + if n.Block == nil { + blockSlot = &n.Block + } + case *ast.Super: + if n.Block == nil { + blockSlot = &n.Block + } + } + if blockSlot == nil { return node } if p.is(token.LBRACE) { - call.Block = p.parseBraceBlock() + *blockSlot = p.parseBraceBlock() } else { - call.Block = p.parseDoBlock() + *blockSlot = p.parseDoBlock() } default: return node @@ -2105,7 +2369,14 @@ func (p *Parser) parseBlockRest(stop map[token.Type]bool, end token.Type, withRe p.scope().explicitParams = true } bs := p.scope() + // A block body is a fresh statement context: a `do…end` inside it attaches + // normally even when this block is itself a paren-less command argument whose + // own trailing `do` was being held back (`include Module.new { … define_method + // (:x) do … end }`). Clear noDo for the duration of the body. + savedNoDo := p.noDo + p.noDo = false body := p.parseStatements(stop) + p.noDo = savedNoDo if withRescue && (p.is(token.RESCUE) || p.is(token.ELSE) || p.is(token.ENSURE)) { body = []ast.Node{p.parseRescueTail(body)} } @@ -2265,11 +2536,25 @@ func (p *Parser) methodName() string { return t.Lit } switch t.Type { - // Operator methods called explicitly: 1.+(2), a.<=>(b), … - case token.SPACESHIP, token.LT, token.GT, token.LE, token.GE, token.EQ, token.NEQ, - token.SHOVEL, token.PLUS, token.MINUS, token.STAR, token.SLASH, token.PERCENT, token.POW: + // Operator methods called explicitly: 1.+(2), a.<=>(b), obj.&(x), … + case token.SPACESHIP, token.LT, token.GT, token.LE, token.GE, token.EQ, token.EQQ, token.NEQ, + token.SHOVEL, token.RSHIFT, token.PLUS, token.MINUS, token.STAR, token.SLASH, token.PERCENT, token.POW, + token.AMPER, token.PIPE, token.CARET, token.TILDE, token.MATCH, token.NMATCH, token.BANG, + token.XSTRING: + // XSTRING here is an empty backtick literal `` produced when `` ` `` names the + // backtick method (`def \`(cmd); end`, `obj.\``). p.advance() - return t.Lit + return "`" + } + // The index methods `[]` / `[]=` named explicitly: `x&.[](i)`, `arr.[]=(i, v)`. + if t.Type == token.LBRACKET && p.peekTok().Type == token.RBRACKET { + p.advance() // [ + p.advance() // ] + if p.is(token.ASSIGN) && !p.cur().SpaceBefore { + p.advance() + return "[]=" + } + return "[]" } if _, isKeyword := token.Keywords[t.Lit]; isKeyword { p.advance() @@ -2279,6 +2564,20 @@ func (p *Parser) methodName() string { return "" } +// applyNumSuffix wraps a numeric literal in the rational/imaginary nodes named +// by a numeric suffix ("r", "i", or "ri"); an empty suffix returns base as-is. +func applyNumSuffix(base ast.Node, suffix string) ast.Node { + for _, c := range suffix { + switch c { + case 'r': + base = &ast.RationalLit{Value: base} + case 'i': + base = &ast.ImaginaryLit{Value: base} + } + } + return base +} + func (p *Parser) parsePrimary() ast.Node { t := p.cur() switch t.Type { @@ -2286,18 +2585,22 @@ func (p *Parser) parsePrimary() ast.Node { p.advance() // Base 0 decodes the radix prefix (0x/0o/0b) and treats a bare leading // zero as octal, matching Ruby. + var base ast.Node n, err := strconv.ParseInt(t.Lit, 0, 64) if err != nil { if z, ok := new(big.Int).SetString(t.Lit, 0); ok { - return &ast.BignumLit{Val: z} // valid digits, out of int64 range + base = &ast.BignumLit{Val: z} // valid digits, out of int64 range + } else { + p.fail("invalid integer literal: %s", t.Lit) // e.g. an invalid octal 08 } - p.fail("invalid integer literal: %s", t.Lit) // e.g. an invalid octal 08 + } else { + base = &ast.IntLit{Value: n} } - return &ast.IntLit{Value: n} + return applyNumSuffix(base, t.Flags) case token.FLOAT: p.advance() f, _ := strconv.ParseFloat(t.Lit, 64) - return &ast.FloatLit{Value: f} + return applyNumSuffix(&ast.FloatLit{Value: f}, t.Flags) case token.STRING, token.STRBEG: return p.parseStringConcat() case token.SYMBOL: @@ -2370,6 +2673,13 @@ func (p *Parser) parsePrimary() ast.Node { if p.atHuggingStringArg() { return &ast.Call{Name: t.Lit, Args: p.parseCommandArgs()} } + // A space-separated command argument makes the constant a method call: + // `BigDecimal "0.01"`, `Integer str`. Restricted to unambiguous starts (a + // string/number/symbol value) so a bare `Foo` followed by an unrelated + // token still reads as a constant reference. + if p.constCommandArgFollows() { + return &ast.Call{Name: t.Lit, Args: p.parseCommandArgs()} + } return &ast.ConstRef{Name: t.Lit} case token.SCOPE: // Leading `::Name` — a top-level constant lookup (`::Foo`, `defined?(::Foo)`). @@ -2388,6 +2698,13 @@ func (p *Parser) parsePrimary() ast.Node { // `def` in expression position evaluates to the defined method's name as a // symbol; it appears as a command argument (`private def foo; end`). return p.parseDef() + case token.CLASS: + // A class definition used as an rvalue (`c = class Foo < Bar; …; end`, + // `sc = class << obj; self; end`) evaluates to the body's last value. + return p.parseClass() + case token.MODULE: + // A module definition as an rvalue (`m = module M; …; end`). + return p.parseModule() case token.BEGIN: return p.parseBegin() case token.CASE: @@ -2497,6 +2814,24 @@ func isHuggingString(t token.Token) bool { return !t.SpaceBefore && (t.Type == token.STRING || t.Type == token.STRBEG) } +// constCommandArgFollows reports whether a constant is immediately followed by a +// space-separated literal value, making it a conversion-style command call +// (`BigDecimal "0.01"`, `Integer "42"`, `Float 1`). It is deliberately narrow — +// only a literal string/number/symbol argument — so an ordinary `Foo` reference +// next to other tokens (a binary operator, `.method`, `[`, a newline) is not +// mistaken for a call. +func (p *Parser) constCommandArgFollows() bool { + t := p.cur() + if !t.SpaceBefore { + return false + } + switch t.Type { + case token.STRING, token.STRBEG, token.INT, token.FLOAT, token.SYMBOL: + return true + } + return false +} + // canStartCommandArg decides whether the current token begins a paren-less // argument list. This is the `foo -1` (call) vs `foo - 1` (subtraction) // disambiguation, driven by SpaceBefore. @@ -2509,10 +2844,11 @@ func (p *Parser) canStartCommandArg() bool { case token.INT, token.FLOAT, token.STRING, token.STRBEG, token.SYMBOL, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, token.TRUE, token.FALSE, token.NIL, token.SELF, token.BANG, token.TILDE, token.LPAREN, token.LBRACKET, token.ARROW, token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING, - token.BEGIN, token.CASE, token.DEF: + token.BEGIN, token.CASE, token.DEF, token.SUPER: // Value-producing keywords: `p begin; 1; end`, `p case x; when 1; 2; end`, - // and `def` (which evaluates to the method's name symbol), as in - // `module_function def foo; end` / `private def bar; end`. + // `def` (which evaluates to the method's name symbol, as in + // `module_function def foo; end` / `private def bar; end`), and `super` + // (`number_to_currency super`, `Request.new super, url_helpers`). return true case token.LABEL: // Keyword/hash argument without parens: `render json: x`, `delegate to: :c`. @@ -2584,16 +2920,31 @@ func (p *Parser) parseCallArgs(until token.Type) []ast.Node { // parseOneCallArg parses a single call argument, routing `*splat` and positional // expressions into args, and `label: value` / `expr => value` pairs into kw. func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { - if p.is(token.DOTDOTDOT) { // `...` — forward the enclosing method's arguments + // `...` — forward the enclosing method's arguments, but only when it stands + // alone (closes the arg position). A `...` with an operand is a beginless + // exclusive range (`foo(...0)`, `[...11, 11]`), handled by the expression path. + if p.is(token.DOTDOTDOT) && (p.peekTok().Type == token.RPAREN || p.peekTok().Type == token.COMMA) { p.advance() *args = append(*args, &ast.ForwardArgs{}) return } if p.accept(token.AMPER) { // &expr — block-pass (coerced to a Proc) + // A bare `&` (no operand, at the end of the arg list) forwards the enclosing + // method's anonymous block parameter: `foo(&)`, `define_method(:x, &)`. + if p.atAnonForwardEnd() { + *args = append(*args, &ast.BlockPass{}) + return + } *args = append(*args, &ast.BlockPass{Value: p.parseExprOrAssign()}) return } if p.accept(token.POW) { // **expr — double-splat into the keyword hash + // A bare `**` forwards the enclosing method's anonymous keyword rest: + // `g(**)`, `view_context.render(inline: <<~ERB.strip, **)`. + if p.atAnonForwardEnd() { + p.addKwPair(kw, nil, nil) + return + } p.addKwPair(kw, nil, p.parseExprOrAssign()) return } @@ -2611,6 +2962,12 @@ func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { return } if p.accept(token.STAR) { + // A bare `*` forwards the enclosing method's anonymous rest parameter: + // `foo(*)`, `to_str[*]`. + if p.atAnonForwardEnd() { + *args = append(*args, &ast.SplatArg{}) + return + } *args = append(*args, &ast.SplatArg{Value: p.parseExprOrAssign()}) return } @@ -2640,6 +2997,17 @@ func (p *Parser) atKwShorthandEnd() bool { return false } +// atAnonForwardEnd reports whether a just-consumed `*`/`**`/`&` has no operand — +// i.e. it is an anonymous-argument forward at a call site (`foo(&)`, `bar(*)`, +// `g(**)`). That is the case when the next token closes the argument position. +func (p *Parser) atAnonForwardEnd() bool { + switch p.cur().Type { + case token.COMMA, token.RPAREN, token.RBRACKET, token.NEWLINE, token.EOF: + return true + } + return false +} + // addKwPair appends a key/value pair to the implicit trailing-hash argument, // allocating it on first use. func (p *Parser) addKwPair(kw **ast.HashLit, k, v ast.Node) { diff --git a/round4_coverage_test.go b/round4_coverage_test.go new file mode 100644 index 0000000..30e6d15 --- /dev/null +++ b/round4_coverage_test.go @@ -0,0 +1,76 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// Round-4 coverage fillers: exercise the remaining branches of the new code +// (paren-group masgn comma path, scoped masgn target rejection, nested OP= on a +// known local, scope-resolution command call with a do-block, and the lexer's +// unknown-character path). + +func TestParenGroupThenComma(t *testing.T) { + // `(a, b), c = …` — a leading paren group followed by another top-level target, + // hitting the group-then-COMMA branch of looksLikeMlhs and parseMlhs. + parsesOK(t, + "(a, b), c = [1, 2], 3\n", + "(a, b), (c, d) = x, y\n", + ) +} + +func TestScopeNotConstNotMasgn(t *testing.T) { + // A `::` target whose next token is not a CONST is not a masgn LHS, so the + // scan falls through and the line parses as an ordinary expression. + parsesOK(t, + "a::b\n", // method call via ::, not a target + "x = a::b, c\n", // RHS array, not an mlhs + ) + // A `::` at a (later) target-start position followed by a non-constant makes + // looksLikeMlhs reject the run as a masgn LHS (`a, ::b` — `::b` is not a valid + // scoped target), so the line is parsed by the ordinary expression path. + parseErrs(t, "a, ::b = 1, 2\n") +} + +func TestNestedOpAssignKnownLocal(t *testing.T) { + // `count` is a declared local before the nested `count += 1`, so the operand + // resolves to a VarRef and the VarRef branch of inlineOpAssign runs. + prog := mustParse(t, "count = 0\nok && count += 1\n") + if len(prog.Body) != 2 { + t.Fatalf("want 2 statements, got %d", len(prog.Body)) + } + bin, ok := prog.Body[1].(*ast.BinaryExpr) + if !ok { + t.Fatalf("second statement is %T, want *ast.BinaryExpr", prog.Body[1]) + } + if _, ok := bin.Right.(*ast.OpAssign); !ok { + t.Fatalf("RHS of && is %T, want *ast.OpAssign", bin.Right) + } +} + +func TestScopeCommandWithDoBlock(t *testing.T) { + parsesOK(t, + "Mod::run x do\n y\nend\n", // lowercase scope-resolution command + do + "obj::Down x do\n y\nend\n", // capitalized scope-resolution command + do + "obj::Down x, y\n", // capitalized scope-resolution command, no block + ) +} + +func TestInlineOpAssignNonAssignable(t *testing.T) { + // A compound-assignment operator following a non-assignable operand is left + // unconsumed by inlineOpAssign (it returns nil); the parser then reports the + // stray operator rather than crashing — exercising the nil-return path. + if _, err := parser.Parse("a && 1 += 1\n"); err == nil { + t.Fatalf("expected an error for `1 += 1`") + } +} + +func TestLexerUnknownChar(t *testing.T) { + // A stray control / unmapped byte yields a clean parse error, not a panic, + // covering lexToken's final ILLEGAL fallthrough. + if _, err := parser.Parse("a \x01 b\n"); err == nil { + t.Fatalf("expected an error for an unknown character") + } +} diff --git a/round4_features_test.go b/round4_features_test.go new file mode 100644 index 0000000..7b7ee3e --- /dev/null +++ b/round4_features_test.go @@ -0,0 +1,464 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// parsesOK asserts every source in srcs parses without error. Used for the +// many Round-4 features whose acceptance (matching MRI `ruby -c`) is what +// matters; AST-shape assertions follow in dedicated tests where the shape is +// load-bearing. +func parsesOK(t *testing.T, srcs ...string) { + t.Helper() + for _, src := range srcs { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +// parseErrs asserts every source fails to parse (a malformed-input guard). +func parseErrs(t *testing.T, srcs ...string) { + t.Helper() + for _, src := range srcs { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q): expected error, got none", src) + } + } +} + +// --- Feature 1: parenthesized masgn LHS --- + +func TestParenMasgn(t *testing.T) { + parsesOK(t, + "(a, b) = x, y\n", + "((a, b), c) = z\n", + "(a, *b) = list\n", + "(*a, b) = list\n", + "(a, *) = list\n", + "a, (b, c) = 1, [2, 3]\n", + "(last_wait, wait) = wait, last_wait + wait\n", + "(a, b,) = x\n", // trailing comma inside the group + "[1].each { |(a, b)| a }\n", + "[1].each { |(a, b), c| a }\n", + ) +} + +func TestParenMasgnShape(t *testing.T) { + ma, ok := mustParseSingle(t, "(a, b) = x, y\n").(*ast.MultiAssign) + if !ok { + t.Fatalf("top node is %T, want *ast.MultiAssign", mustParseSingle(t, "(a, b) = x, y\n")) + } + // The single paren group is one target whose node is a nested MultiAssign. + if len(ma.Targets) != 1 { + t.Fatalf("want 1 outer target, got %d", len(ma.Targets)) + } + inner, ok := ma.Targets[0].(*ast.MultiAssign) + if !ok { + t.Fatalf("nested target is %T, want *ast.MultiAssign", ma.Targets[0]) + } + if len(inner.Names) != 2 || inner.Names[0] != "a" || inner.Names[1] != "b" { + t.Fatalf("inner names = %v, want [a b]", inner.Names) + } + if inner.Values != nil { + t.Fatalf("nested group must have nil Values, got %v", inner.Values) + } +} + +// --- Feature 2: scope-resolution method call --- + +func TestScopeResolutionCall(t *testing.T) { + parsesOK(t, + "Syslog::LOG_UPTO(Syslog::LOG_INFO)\n", + "Mod::meth arg\n", + "Math::PI\n", + "Foo::bar(1, 2)\n", + "A::B::C\n", + "x = Math::PI + 1\n", + "Mod::run do\n x\nend\n", + "obj::Down x, y\n", + ) +} + +func TestScopeResolutionCallShape(t *testing.T) { + call, ok := mustParseSingle(t, "Syslog::LOG_UPTO(Syslog::LOG_INFO)\n").(*ast.Call) + if !ok { + t.Fatalf("want *ast.Call for scope-resolution call") + } + if call.Name != "LOG_UPTO" || call.Recv == nil { + t.Fatalf("call = %+v, want Name=LOG_UPTO with receiver", call) + } +} + +// --- Feature 3: trailing operator line continuation --- + +func TestTrailingOperatorContinuation(t *testing.T) { + parsesOK(t, + "batch <<\n x\n", + "new_args <<\n x\n", + "a = b &\n c\n", + "x = foo |\n bar\n", + "y = a ^\n b\n", + "batch << \"\\n\" <<\n x\n", + ) + // Operand-position |/&/<>(y)\n", + "x.===(y)\n", + "x.=~(y)\n", + ) +} + +// --- Feature 9: operator / backtick method symbols + !~ --- + +func TestOperatorSymbolsAndNMatch(t *testing.T) { + parsesOK(t, + "x = :`\n", + "receive(:`)\n", + "x = :!~\n", + "ignores = [:to_s, :=~, :!~, :===]\n", + "assert Mime[:js] !~ \"text/html\"\n", + "assert zone !~ /Nonexistent_Place/\n", + "a !~\n b\n", // !~ continuation + "def `(cmd); end\n", + ) + sym, ok := mustParseSingle(t, ":`\n").(*ast.SymbolLit) + if !ok || sym.Name != "`" { + t.Fatalf(":` should be a SymbolLit with Name=`") + } +} + +// --- Feature 10: ranges and patterns --- + +func TestBeginlessRangeAsArg(t *testing.T) { + parsesOK(t, + "x = [...11, 11]\n", + "assert_operator(...0, :overlap?, -1..0)\n", + "foo(..5)\n", + ) + // Bare `...` forwarding must still be recognised. + parsesOK(t, "def f(...); g(...); end\n", "foo(...)\n") +} + +func TestScopedConstPattern(t *testing.T) { + parsesOK(t, + "case x\nin Prism::CaseNode[a]\n 1\nend\n", + "case x\nin [Prism::SymbolNode[unescaped:]]\n 1\nend\n", + "case x\nin Point[a, b]\n 1\nend\n", + "case x\nin ::Foo\n 1\nend\n", + "case x\nin Foo(a:, b:)\n 1\nend\n", + "n in Prism::CaseNode[a]\n", + ) +} + +// --- Statement-level keyword postfix / modifiers --- + +func TestKeywordStatementPostfix(t *testing.T) { + parsesOK(t, + "if a\n 1\nelse\n 2\nend.html_safe\n", + "while a\n b\nend.foo\n", + "if a then 1 end if b\n", + "begin\n x\nend if Process.respond_to?(:fork)\n", + "def f; 1; end if cond\n", + "class A; end if cond\n", + "module M; end if cond\n", + "until a\n b\nend.to_s\n", + "unless a\n b\nend.to_s\n", + ) +} + +// --- super with a block --- + +func TestSuperBlock(t *testing.T) { + parsesOK(t, + "super do |record|\n x\nend\n", + "super { |t| yield t }\n", + "super() { |h, k| x }\n", + "super(operation, payload) do\n x\nend\n", + "number_to_currency super\n", + "Request.new super, url_helpers, @block\n", + ) + s, ok := mustParseSingle(t, "super { 1 }\n").(*ast.Super) + if !ok || s.Block == nil { + t.Fatalf("super { 1 } should be a Super with a Block") + } +} + +// --- alias / undef fitem list does not over-continue --- + +func TestAliasFitemList(t *testing.T) { + parsesOK(t, + "def ==(o); 1; end\nalias eql? ==\ndef hash; 1; end\n", + "alias eql? ==\n", + "alias foo bar\nbaz\n", + "undef ==\nx\n", + "alias x y; z\n", + "a ==\n b\n", // ordinary == continuation must still work + ) +} + +// --- attribute / index array-RHS assignment --- + +func TestAttributeArrayRHS(t *testing.T) { + parsesOK(t, + "self.cache_store = :file_store, X\n", + "@controller.cache_store = :file_store, @cache_path\n", + "Capybara.server = :puma, { Silent: true }\n", + "h[k] = 1, 2\n", + ) +} + +// --- jump values: break/next take a comma list --- + +func TestJumpCommaValues(t *testing.T) { + parsesOK(t, + "next table, true\n", + "break a, b\n", + "[1].each { next 1, 2 }\n", + "next\n", + "break\n", + ) + nx, ok := mustParseSingle(t, "next 1, 2\n").(*ast.Next) + if !ok { + t.Fatalf("want *ast.Next") + } + if _, ok := nx.Value.(*ast.ArrayLit); !ok { + t.Fatalf("next 1, 2 value = %T, want *ast.ArrayLit", nx.Value) + } +} + +// --- scoped masgn target (::Const) --- + +func TestScopedMasgnTarget(t *testing.T) { + parsesOK(t, + "old_zone, ::Time.zone = ::Time.zone, new_zone\n", + "::Foo, x = 1, 2\n", + ) +} + +// --- command-arg value forms (super / const conversion) --- + +func TestConstConversionCommand(t *testing.T) { + parsesOK(t, + "bd = BigDecimal \"0.01\"\n", + "Integer \"42\"\n", + "Float 1\n", + "Sym :x\n", + ) + // A constant next to a binary operator or a bare reference stays a const. + if _, ok := mustParseSingle(t, "Foo + 1\n").(*ast.BinaryExpr); !ok { + t.Fatalf("Foo + 1 should stay a BinaryExpr") + } + if _, ok := mustParseSingle(t, "Foo\n").(*ast.ConstRef); !ok { + t.Fatalf("Foo should be a ConstRef") + } +} + +// --- rescue into a non-local target --- + +func TestRescueIntoTarget(t *testing.T) { + parsesOK(t, + "begin\n x\nrescue => @error\n y\nend\n", + "begin\nrescue => @setup_exception; end\n", + "begin\nrescue => e\n y\nend\n", + "begin\nrescue Foo => $g\nend\n", + "begin\nrescue => @@c\nend\n", + "begin\nrescue => obj.err\nend\n", + ) +} + +// --- predicate / bang labels --- + +func TestPredicateLabels(t *testing.T) { + parsesOK(t, + "{ frozen?: frozen? }\n", + "h = { has_key?: true, include?: true }\n", + "deprecate auto_populated?: :x, deprecator: Y\n", + "foo valid!: 1\n", + ) + // Ternary with a predicate condition must still parse (no false label). + parsesOK(t, "x = empty? ? 1 : 2\n", "x = a ? b : c\n") +} + +// --- rational / imaginary literals --- + +func TestRationalImaginary(t *testing.T) { + parsesOK(t, + "x = 2r\n", "y = 0.5r\n", "z = 3i\n", "w = 2.5ri\n", + "Time.new(2002, 10, 31, 2, 2, 2.123456789r)\n", + "n = 1.upto(3)\n", + ) + if _, ok := mustParseSingle(t, "x = 2r\n").(*ast.Assign).Value.(*ast.RationalLit); !ok { + t.Fatalf("2r should be a RationalLit") + } + if _, ok := mustParseSingle(t, "x = 3i\n").(*ast.Assign).Value.(*ast.ImaginaryLit); !ok { + t.Fatalf("3i should be an ImaginaryLit") + } + // `ri` nests imaginary over rational. + im := mustParseSingle(t, "x = 2ri\n").(*ast.Assign).Value.(*ast.ImaginaryLit) + if _, ok := im.Value.(*ast.RationalLit); !ok { + t.Fatalf("2ri should be ImaginaryLit{RationalLit{...}}") + } + // A bignum literal with an `r` suffix exercises the bignum branch. + parsesOK(t, "x = 99999999999999999999999999999r\n") +} + +// --- nested compound assignment --- + +func TestNestedCompoundAssign(t *testing.T) { + parsesOK(t, + "distribution[record] > 0 && distribution[record] -= 1\n", + "record.car.save && count += 1\n", + "[1].each { |x| x.save && count += 1 }\n", + "@x.nil? && @x ||= 1\n", + "a += 1\n", + "cond && $g += 1\n", + "cond && @@c += 1\n", + "flag && obj.attr += 1\n", + ) +} + +// --- def parameter defaults (masgn suppression + self-name) --- + +func TestDefParamDefaults(t *testing.T) { + parsesOK(t, + "def make_codec(secret = secret(\"secret\"), v = nil, **options); end\n", + "def attach_to(ns, sub = new, notifier = AS::N.instance, inherit_all: false); end\n", + "def create_migration(p = default, c = {}, g = self, &block); end\n", + "def f(s = c(1), v = nil); end\n", + "def f(a = 1, b: a); end\n", + "def f(secret = secret(\"x\")); end\n", + ) +} + +// --- class name with an expression receiver path --- + +func TestClassPathExpression(t *testing.T) { + parsesOK(t, + "class self.class::TestRailtie < Rails::Railtie; end\n", + "class obj::Bar; end\n", + "class Foo::Bar; end\n", + "class ::Top; end\n", + "module A::B; end\n", + ) + parseErrs(t, "class 1.foo; end\n") // a non-path receiver is rejected +} + +// --- nested do-block inside a brace block that is a command argument --- + +func TestNestedBlockInCommandArg(t *testing.T) { + parsesOK(t, + "include Module.new {\n define_method(:x) do\n 1\n end\n}\n", + "while foo do\n bar\nend\n", // loop do must still bind to the loop + ) +} + +// --- option global variables ($-I) --- + +func TestOptionGlobals(t *testing.T) { + parsesOK(t, + "$-I.each { |p| p }\n", + "$stdout.puts 1\n", + "x = $$\n", + ) + parseErrs(t, "x = $\n") // a bare $ is still illegal +} diff --git a/token/token.go b/token/token.go index d16ee02..f7396f7 100644 --- a/token/token.go +++ b/token/token.go @@ -109,6 +109,7 @@ const ( CARET // ^ (pattern-matching pin operator) RSHIFT // >> TILDE // ~ (bitwise complement) + NMATCH // !~ (does-not-match operator) ) var typeNames = map[Type]string{ @@ -125,7 +126,7 @@ var typeNames = map[Type]string{ SPACESHIP: "<=>", SHOVEL: "<<", ANDAND: "&&", OROR: "||", OPASSIGN: "op=", QUESTION: "?", COLON: ":", SCOPE: "::", LPAREN: "(", RPAREN: ")", LBRACE: "{", RBRACE: "}", LBRACKET: "[", RBRACKET: "]", PIPE: "|", HASHROCKET: "=>", COMMA: ",", DOT: ".", DOTDOT: "..", DOTDOTDOT: "...", - AMPER: "&", SAFEDOT: "&.", ARROW: "->", CARET: "^", RSHIFT: ">>", TILDE: "~", + AMPER: "&", SAFEDOT: "&.", ARROW: "->", CARET: "^", RSHIFT: ">>", TILDE: "~", NMATCH: "!~", } func (t Type) String() string {