[Perf][SwiftParser] Record lookahead ranges only for a parse that hands them on

Sources/SwiftParser/IncrementalParseTransition.swift

@@ -35,6 +35,9 @@ extension Parser {
   /// been allocated by this parser, so an incremental parse can decide whether
   /// to reuse it.
   mutating func registerNodeForIncrementalParse(node: RawSyntax, startToken: Lexer.Lexeme) {
+    guard collectsLookaheadRanges else {
+      return
+    }
     arena.lookaheadLengths[node.id] =
       lexemes.lookaheadTracker.pointee.furthestOffset - self.lexemes.offsetToStart(startToken)
   }

Sources/SwiftParser/ParseSourceFile.swift

@@ -30,7 +30,10 @@ extension Parser {
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: nil,
       swiftVersion: swiftVersion,
-      languageFeatures: languageFeatures
+      languageFeatures: languageFeatures,
+      // This is not an entry point for incremental parsing, so it skips recording
+      // how far the parser looked ahead to parse each node.
+      collectsLookaheadRanges: false
     ) { SourceFileSyntax.parse(from: &$0) }
   }
 
@@ -47,7 +50,10 @@ extension Parser {
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: nil,
       swiftVersion: swiftVersion,
-      languageFeatures: languageFeatures
+      languageFeatures: languageFeatures,
+      // This is not an entry point for incremental parsing, so it skips recording
+      // how far the parser looked ahead to parse each node.
+      collectsLookaheadRanges: false
     ) { SourceFileSyntax.parse(from: &$0) }
   }
 
@@ -131,7 +137,8 @@ extension Parser {
       maximumNestingLevel: nil,
       parseTransition: parseTransition,
       swiftVersion: nil,
-      languageFeatures: []
+      languageFeatures: [],
+      collectsLookaheadRanges: true
     ) { IncrementalParseResult(tree: SourceFileSyntax.parse(from: &$0), lookaheadRanges: $0.lookaheadRanges) }
   }
 
@@ -150,7 +157,8 @@ extension Parser {
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: parseTransition,
       swiftVersion: nil,
-      languageFeatures: []
+      languageFeatures: [],
+      collectsLookaheadRanges: true
     ) { IncrementalParseResult(tree: SourceFileSyntax.parse(from: &$0), lookaheadRanges: $0.lookaheadRanges) }
   }
 }

Sources/SwiftParser/Parser.swift

@@ -118,6 +118,15 @@ public struct Parser {
   /// See comments in ``LookaheadRanges``
   public internal(set) var lookaheadRanges = LookaheadRanges()
 
+  /// Whether to record in the arena how far the parser looked ahead to parse each
+  /// node, for a later incremental reparse to consult.
+  ///
+  /// Recording costs a hash table insertion per node, and only an incremental
+  /// reparse reads it, so `Parser.parse(source:)` and its siblings opt out. A
+  /// caller holding a `Parser` of its own records by default, because that is how
+  /// the reparse after it decides what to reuse.
+  let collectsLookaheadRanges: Bool
+
   /// Parser should own a ``LookaheadTracker`` so that we can share one `furthestOffset` in a parse.
   private let lookaheadTrackerOwner: LookaheadTrackerOwner
 
@@ -247,7 +256,8 @@ public struct Parser {
     arena: ParsingRawSyntaxArena?,
     copySource: Bool,
     swiftVersion: SwiftVersion?,
-    languageFeatures: LanguageFeatures
+    languageFeatures: LanguageFeatures,
+    collectsLookaheadRanges: Bool
   ) {
     // A full parse allocates in proportion to the source, so the arena can size
     // its slabs for it. An incremental reparse allocates for what it re-lexes,
@@ -259,6 +269,10 @@ public struct Parser {
         sourceByteCount: parseTransition == nil ? input.count : nil
       )
 
+    // An incremental parse records whatever the caller asked for: the reparse after
+    // it decides what to reuse from what this one records.
+    self.collectsLookaheadRanges = collectsLookaheadRanges || parseTransition != nil
+
     // Each token's text is copied into the token's own node, so nothing in the
     // tree points into the buffer being lexed and the whole source does not need
     // copying into the arena first.
@@ -281,6 +295,12 @@ public struct Parser {
     self.lookaheadTrackerOwner = LookaheadTrackerOwner()
     self.lexerStateAllocator = Lexer.StateAllocator()
     self.nodeListAllocator = RawSyntaxNodeListAllocator()
+    // Only a full parse: an incremental one records only the nodes it re-parses.
+    // A node is registered roughly every 90 bytes of source; round that down so a
+    // file denser than average still does not have to grow the table.
+    if self.collectsLookaheadRanges, parseTransition == nil {
+      self.arena.lookaheadLengths.reserveCapacity(input.count / 80)
+    }
 
     self.lexemes = Lexer.tokenize(
       input,
@@ -313,7 +333,8 @@ public struct Parser {
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
     swiftVersion: SwiftVersion? = nil,
-    languageFeatures: LanguageFeatures = []
+    languageFeatures: LanguageFeatures = [],
+    collectsLookaheadRanges: Bool = true
   ) {
     var input = input
     input.makeContiguousUTF8()
@@ -328,7 +349,8 @@ public struct Parser {
         // copied into a parser-owned buffer.
         copySource: true,
         swiftVersion: swiftVersion,
-        languageFeatures: languageFeatures
+        languageFeatures: languageFeatures,
+        collectsLookaheadRanges: collectsLookaheadRanges
       )
     }
   }
@@ -353,7 +375,8 @@ public struct Parser {
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
     swiftVersion: SwiftVersion? = nil,
-    languageFeatures: LanguageFeatures = []
+    languageFeatures: LanguageFeatures = [],
+    collectsLookaheadRanges: Bool = true
   ) {
     // Copy the source so the caller may free `input` after this initializer
     // returns.
@@ -364,7 +387,8 @@ public struct Parser {
       arena: nil,
       copySource: true,
       swiftVersion: swiftVersion,
-      languageFeatures: languageFeatures
+      languageFeatures: languageFeatures,
+      collectsLookaheadRanges: collectsLookaheadRanges
     )
   }
 
@@ -377,7 +401,8 @@ public struct Parser {
     parseTransition: IncrementalParseTransition? = nil,
     arena: ParsingRawSyntaxArena,
     swiftVersion: SwiftVersion? = nil,
-    languageFeatures: LanguageFeatures = []
+    languageFeatures: LanguageFeatures = [],
+    collectsLookaheadRanges: Bool = true
   ) {
     // Copy the source so the caller may free `input` after this initializer
     // returns, and so the resulting tree does not depend on `input` (its tokens
@@ -389,7 +414,8 @@ public struct Parser {
       arena: arena,
       copySource: true,
       swiftVersion: swiftVersion,
-      languageFeatures: languageFeatures
+      languageFeatures: languageFeatures,
+      collectsLookaheadRanges: collectsLookaheadRanges
     )
   }
 
@@ -413,6 +439,7 @@ public struct Parser {
     parseTransition: IncrementalParseTransition? = nil,
     swiftVersion: SwiftVersion? = nil,
     languageFeatures: LanguageFeatures = [],
+    collectsLookaheadRanges: Bool = true,
     body: (inout Parser) -> T
   ) -> T {
     var parser = Parser(
@@ -422,7 +449,8 @@ public struct Parser {
       arena: nil,
       copySource: false,
       swiftVersion: swiftVersion,
-      languageFeatures: languageFeatures
+      languageFeatures: languageFeatures,
+      collectsLookaheadRanges: collectsLookaheadRanges
     )
     return body(&parser)
   }
@@ -441,6 +469,7 @@ public struct Parser {
     parseTransition: IncrementalParseTransition? = nil,
     swiftVersion: SwiftVersion? = nil,
     languageFeatures: LanguageFeatures = [],
+    collectsLookaheadRanges: Bool = true,
     body: (inout Parser) -> T
   ) -> T {
     var input = input
@@ -452,6 +481,7 @@ public struct Parser {
         parseTransition: parseTransition,
         swiftVersion: swiftVersion,
         languageFeatures: languageFeatures,
+        collectsLookaheadRanges: collectsLookaheadRanges,
         body: body
       )
     }

Tests/SwiftParserTest/IncrementalParsingTests.swift

@@ -186,6 +186,52 @@ class IncrementalParsingTests: ParserTestCase {
     }
   }
 
+  /// A client that drives `Parser` itself, rather than through `parseIncrementally`,
+  /// still gets lookahead ranges it can hand to the next parse. `SyntaxTreeManager` in
+  /// sourcekit-lsp does exactly this, because `parseIncrementally` cannot be given
+  /// language features: it constructs a `Parser`, parses, and builds an
+  /// `IncrementalParseResult` from `parser.lookaheadRanges`. A node with no recorded
+  /// range is never reused, so if the first parse in such a chain records none, the
+  /// parse after it reuses nothing — correct output, no diagnostic, and no incremental
+  /// parsing.
+  public func testLookaheadRangesAreRecordedForAParserDrivenDirectly() {
+    let originalSource = """
+      foo() {}
+      label: switch x {
+        default: break
+      }
+      """
+
+    // The first parse of the chain, driven the way a client with its own `Parser` does
+    // it. Whether the call's lookahead — past `label:`, to rule out a labeled trailing
+    // closure — is recorded is decided here.
+    var firstParser = Parser(originalSource, parseTransition: nil)
+    let originalResult = IncrementalParseResult(
+      tree: SourceFileSyntax.parse(from: &firstParser),
+      lookaheadRanges: firstParser.lookaheadRanges
+    )
+
+    // An edit after the call, so reuse is decided by the recorded range rather than by
+    // the node sitting past every edit.
+    let (editedSource, laterEdit) = replacing("default: break", with: "default: return", in: originalSource)
+    var reusedNodes: [Syntax] = []
+    var secondParser = Parser(
+      editedSource,
+      parseTransition: IncrementalParseTransition(
+        previousIncrementalParseResult: originalResult,
+        edits: ConcurrentEdits(laterEdit),
+        reusedNodeCallback: { reusedNodes.append($0) }
+      )
+    )
+    let editedTree = SourceFileSyntax.parse(from: &secondParser)
+
+    XCTAssertTrue(
+      reusedNodes.contains(where: { $0.trimmedDescription == "foo() {}" }),
+      "nothing the first parse produced was reusable: \(reusedNodes.map(\.trimmedDescription))"
+    )
+    XCTAssertEqual(editedTree.description, Parser.parse(source: editedSource).description)
+  }
+
   public func testAddElse() {
     assertIncrementalParse(
       """