#3814 (part 6): UAX #31 names for Ada, Tcl, Haskell, Scheme and PowerShell

gitgalaxy/core/detector.py

@@ -1978,6 +1978,10 @@ def _name_boundary_pattern_for(func_name: str, extra_chars: str) -> str:
 # scheme `set!`, C++ `operator==`) do NOT qualify and take the exact fallback.
 _WORD_NAME_RE = re.compile(r"\w+")
 
+# #3814: a character that continues a Haskell name (UAX #31, or a prime), compiled once -- a
+# per-name `name(?![...])` pattern would recompile the wide class for every function.
+_HASKELL_NAME_CHAR = re.compile(r"['" + ID_CONTINUE + r"]")
+
 # #3182: a name that is a single maximal word-or-hyphen run. The bisect fast
 # path in `_is_orphan` accepts these too, because coding_analysis indexes the
 # segment-aligned hyphenated sub-runs of every `[\w-]+` token (see the
@@ -7192,7 +7196,11 @@ def _slice_by_indentation(
                         # signature line -- caught and dropped by the `< 2`
                         # line-count floor below on every single-signature
                         # function, which is nearly all of them.
-                        if lang_id == "haskell" and re.match(re.escape(name) + r"(?!['\w])", stripped):
+                        if (
+                            lang_id == "haskell"
+                            and stripped.startswith(name)
+                            and not _HASKELL_NAME_CHAR.match(stripped, len(name))
+                        ):
                             if equation_start_idx is None:
                                 equation_start_idx = scan_pos
                             scan_pos = line_end
@@ -9177,7 +9185,10 @@ def _haskell_arrowless_signature_is_action(self, signature_text: str) -> bool:
         last_constraint = type_str.rfind("=>")
         if last_constraint != -1:
             type_str = type_str[last_constraint + 2 :].strip()
-        head_match = re.match(r"\(*\s*([A-Za-z_][A-Za-z0-9_']*(?:\.[A-Za-z_][A-Za-z0-9_']*)*)", type_str)
+        head_match = re.match(
+            r"\(*\s*([" + ID_START + r"][" + ID_CONTINUE + r"']*(?:\.[" + ID_START + r"][" + ID_CONTINUE + r"']*)*)",
+            type_str,
+        )
         if not head_match:
             return False
         head = head_match.group(1)

gitgalaxy/standards/language_standards/_shared_patterns.py

@@ -310,12 +310,18 @@
 
 # Lisp family (scheme): the callee is the first symbol after an open paren.
 # Charset includes lisp identifier punctuation (probe-branch, null?, set!).
-CALLS_OUT_LISP_FAMILY = re.compile(r"\(\s*([A-Za-z_][A-Za-z0-9_!?*<>=+-]*)")
+# #3814: the letters and digits are UAX #31's (`probe-नाम`), the punctuation stays.
+CALLS_OUT_LISP_FAMILY = re.compile(r"\(\s*([" + ID_START + r"][" + ID_CONTINUE + r"!?*<>=+-]*)")
 
 # Command-position family (tcl, powershell, livecode): the callee is the first
 # word on a statement line. Charset covers tcl `ns::proc` interior colons and
 # PS `Verb-Noun`; a leading-`::` fully-qualified tcl call is a known recall gap
 # (the first character must be a letter/underscore for precision).
 # Consumers MUST pair this with a `calls_out_ignore` set of the language's
 # statement keywords, since those also appear in command position.
-CALLS_OUT_COMMAND_POSITION = re.compile(r"(?m)^[ \t]*([A-Za-z_][\w:-]*)\b")
+# #3814: UAX #31 letters, digits and marks (`Get-नाम`). The trailing `\b` became what it meant
+# here -- the name ends on a name character and no name character follows -- in the same class,
+# so a name is never cut after a combining mark.
+CALLS_OUT_COMMAND_POSITION = re.compile(
+    r"(?m)^[ \t]*([" + ID_START + r"][" + ID_CONTINUE + r":-]*)(?<=[" + ID_CONTINUE + r"])(?![" + ID_CONTINUE + r"])"
+)

gitgalaxy/standards/language_standards/identifiers.py

@@ -47,8 +47,8 @@ def _class_contents(codepoints: list[int]) -> str:
     return "".join(out)
 
 
-def _build() -> tuple[str, str, str, str]:
-    start, cont, virama, capital = [], [], [], []
+def _build() -> tuple[str, str, str, str, str]:
+    start, cont, virama, capital, small = [], [], [], [], []
     category, combining = unicodedata.category, unicodedata.combining
     for plane in _PLANES:
         for cp in plane:
@@ -59,17 +59,23 @@ def _build() -> tuple[str, str, str, str]:
                     start.append(cp)
                 if cat in ("Lu", "Lt", "Lo"):
                     capital.append(cp)
+                if cat in ("Ll", "Lm", "Lo") or cp == 0x5F:
+                    small.append(cp)
                 if combining(chr(cp)) == 9:
                     virama.append(cp)
-    return _class_contents(start), _class_contents(cont), _class_contents(virama), _class_contents(capital)
+    c = _class_contents
+    return c(start), c(cont), c(virama), c(capital), c(small)
 
 
 # CAPITAL: what `[A-Z]` meant for a name that must start with a capital (a Java constructor):
 # upper- and title-case letters, and letters of scripts with no case at all (Devanagari, Han, ...).
 # VIRAMA: the marks (canonical combining class 9: Devanagari ्, Tamil ், ...) after which UAX #31
 # allows a ZWNJ / ZWJ inside a name (rules A1/A2) -- the one place those joiners are spelling,
 # not hidden text.
-ID_START, ID_CONTINUE, VIRAMA, CAPITAL = _build()
+# SMALL: what `[a-z_]` meant for a name that must NOT start with a capital (a Haskell variable, whose
+# first letter tells it from a constructor): lower-case letters, `_`, and letters of scripts with no
+# case (GHC reads an uncased letter as small).
+ID_START, ID_CONTINUE, VIRAMA, CAPITAL, SMALL = _build()
 NAME = f"[{ID_START}][{ID_CONTINUE}]*"
 
 # #3814 / #3810: MAINFRAME national characters. EBCDIC's national bytes 0x5B / 0x7B / 0x7C -- `$ # @`

gitgalaxy/standards/language_standards/languages/ada.py

@@ -11,6 +11,8 @@
 import re
 from typing import Any
 
+from gitgalaxy.standards.language_standards.identifiers import ID_CONTINUE, ID_START
+
 from .._shared_patterns import CALLS_OUT_C_STYLE, GLOBAL_FRAGILE_DEBT, GLOBAL_PLANNED_DEBT
 
 DEFINITION: dict[str, Any] = {
@@ -97,7 +99,7 @@
         # expressions routinely contain nested calls/aggregates, e.g.
         # `Y : Integer := Compute(A, (B + C))`.
         "args": re.compile(
-            r"\b(?:procedure|function)\b[ \t\n]+[A-Za-z_][A-Za-z0-9_]*[ \t\n]*"
+            r"\b(?:procedure|function)\b[ \t\n]+[" + ID_START + r"][" + ID_CONTINUE + r"]*[ \t\n]*"
             r"\(((?:[^()]|\((?:[^()]|\([^()]*\))*\))*)\)",
             re.I,
         ),
@@ -118,9 +120,9 @@
         # bounded (not unbounded `.*`) so it can't cross a statement
         # terminator or blow up on an adversarial payload.
         "func_start": re.compile(
-            r"\b(?:procedure|function)[ \t\n]+([A-Za-z_][A-Za-z0-9_]*)"
+            r"\b(?:procedure|function)[ \t\n]+([" + ID_START + r"][" + ID_CONTINUE + r"]*)"
             r"(?:[ \t\n]*\((?:[^()]|\((?:[^()]|\([^()]*\))*\))*\))?"
-            r"(?:[ \t\n]+return[ \t\n]+[A-Za-z_][A-Za-z0-9_.]*)?"
+            r"(?:[ \t\n]+return[ \t\n]+[" + ID_START + r"][" + ID_CONTINUE + r".]*)?"
             r"(?:[ \t\n]+with[ \t\n]+[^;{}]{0,300}?)?"
             r"[ \t\n]*is\b(?![ \t\n]*(?:abstract|null|new)\b)",
             re.I,
@@ -133,9 +135,9 @@
         # need their own alternative or the (more common, in an
         # OOP-heavy codebase) derived form is entirely invisible).
         "class_start": re.compile(
-            r"\btype[ \t]+([A-Za-z_][A-Za-z0-9_]*)[ \t\n]+is[ \t\n]+"
+            r"\btype[ \t]+([" + ID_START + r"][" + ID_CONTINUE + r"]*)[ \t\n]+is[ \t\n]+"
             r"(?:(?:abstract[ \t\n]+)?(?:limited[ \t\n]+)?tagged\b"
-            r"|(?:abstract[ \t\n]+)?new[ \t\n]+[A-Za-z_][A-Za-z0-9_.]*[ \t\n]+with\b)",
+            r"|(?:abstract[ \t\n]+)?new[ \t\n]+[" + ID_START + r"][" + ID_CONTINUE + r".]*[ \t\n]+with\b)",
             re.I,
         ),
         # --- PHASE 2: SAFETY & EXECUTION RISK ---
@@ -200,16 +202,20 @@
         # `^[ \t]*` here would count the private ones too. Needs re.M
         # (Rule 13).
         "api": re.compile(
-            r"\bpackage[ \t]+(?!body\b)(?:private[ \t]+)?([A-Za-z_][A-Za-z0-9_.]*)[ \t\n]+is\b(?![ \t\n]*new\b)|"
-            r"^(?:procedure|function)[ \t]+[A-Za-z_]",
+            r"\bpackage[ \t]+(?!body\b)(?:private[ \t]+)?(["
+            + ID_START
+            + r"]["
+            + ID_CONTINUE
+            + r".]*)[ \t\n]+is\b(?![ \t\n]*new\b)|"
+            r"^(?:procedure|function)[ \t]+[" + ID_START + r"]",
             re.M | re.I,
         ),
         # state_mutation: Ada's assignment operator `:=` is lexically
         # distinct from `=` (equality) -- no ambiguity to guard against.
         "state_mutation": re.compile(
             # #2765 contract: `X := v` at statement start writes; `X : T := v` declares
             # (the `:` before the type breaks the match), as does `X : constant T := v`.
-            r"^[ \t]*[A-Za-z_][\w.]*(?:[ \t]*\([^()\n]{0,80}\))?[ \t]*:=",
+            r"^[ \t]*[" + ID_START + r"][" + ID_CONTINUE + r".]*(?:[ \t]*\([^()\n]{0,80}\))?[ \t]*:=",
             re.M,
         ),
         # dead_code: commented-out structural code (line_exclusive_dash
@@ -274,7 +280,11 @@
         # with ...`), a genuinely different construct sharing the same
         # two words.
         "generics": re.compile(
-            r"\bgeneric\b|\b(?:package|procedure|function)\b[ \t]+[A-Za-z_][A-Za-z0-9_]*[ \t\n]+is[ \t\n]+new\b",
+            r"\bgeneric\b|\b(?:package|procedure|function)\b[ \t]+["
+            + ID_START
+            + r"]["
+            + ID_CONTINUE
+            + r"]*[ \t\n]+is[ \t\n]+new\b",
             re.I,
         ),
         # comprehensions: Ada 2012 quantified expressions (`for all` /
@@ -296,15 +306,23 @@
             r"|Convention|Import|Export|Inline|Volatile|Atomic|Pack|SPARK_Mode|No_Return"
             r"|Pure|Preelaborate|Elaborate_Body"
             r"|Abstract_State|Initializes|Refined_Global|Refined_Post|Refined_State|Refined_Depends)\b)"
-            r"[A-Za-z_][A-Za-z0-9_.]*(?:[ \t\n]*,[ \t\n]*[A-Za-z_][A-Za-z0-9_.]*){0,20}[ \t\n]*;",
+            r"["
+            + ID_START
+            + r"]["
+            + ID_CONTINUE
+            + r".]*(?:[ \t\n]*,[ \t\n]*["
+            + ID_START
+            + r"]["
+            + ID_CONTINUE
+            + r".]*){0,20}[ \t\n]*;",
             re.I | re.M,
         ),
         "_dependency_capture": re.compile(
             r"^[ \t]*with[ \t\n]+(?!(?:Pre|Post|Invariant|Static_Predicate|Dynamic_Predicate|Global|Depends"
             r"|Convention|Import|Export|Inline|Volatile|Atomic|Pack|SPARK_Mode|No_Return"
             r"|Pure|Preelaborate|Elaborate_Body"
             r"|Abstract_State|Initializes|Refined_Global|Refined_Post|Refined_State|Refined_Depends)\b)"
-            r"([A-Za-z_][A-Za-z0-9_.]*)",
+            r"([" + ID_START + r"][" + ID_CONTINUE + r".]*)",
             re.I | re.M,
         ),
         # ownership: header comment convention, same shape as the JCL/
@@ -321,7 +339,7 @@
         # for common secret-shaped identifiers.
         "hardcoded_secrets": re.compile(
             r"\b(?:password|secret|token|api[_-]?key|private[_-]?key|client[_-]?secret)\b"
-            r"[ \t]*:[ \t]*(?:constant[ \t]+)?[A-Za-z_][A-Za-z0-9_.]*[ \t]*:=[ \t]*\"[^\"]{8,}\"",
+            r"[ \t]*:[ \t]*(?:constant[ \t]+)?[" + ID_START + r"][" + ID_CONTINUE + r".]*[ \t]*:=[ \t]*\"[^\"]{8,}\"",
             re.I,
         ),
         "spec_exposure": re.compile(r"\[(?:SPEC[ \t]*-[ \t]*\d{1,6}|spec|audit)\]", re.I),
@@ -350,7 +368,7 @@
             re.I,
         ),
         # memory_alloc: the `new` allocator.
-        "memory_alloc": re.compile(r"\bnew[ \t]+[A-Za-z_]", re.I),
+        "memory_alloc": re.compile(r"\bnew[ \t]+[" + ID_START + r"]", re.I),
         # inline_asm: System.Machine_Code's Asm construct -- a real,
         # if niche, GNAT-standard extension used in kernel/embedded Ada.
         "inline_asm": re.compile(r"\bSystem\.Machine_Code\b|\bAsm[ \t]*\(", re.I),
@@ -367,7 +385,9 @@
         # explicit type conversion/qualification (unlike a bare type
         # conversion `T(Expr)`, which is syntactically identical to a
         # function call and can't be told apart without semantic info).
-        "explicit_casts": re.compile(r"\b[A-Za-z_][A-Za-z0-9_.]*'\(", re.I),
+        "explicit_casts": re.compile(
+            r"\b(?<![" + ID_CONTINUE + r"])[" + ID_START + r"][" + ID_CONTINUE + r".]*'\(", re.I
+        ),
         # panics_and_aborts: `raise` (exception propagation) and
         # `abort` (forceful task termination -- the schema's own
         # baseline definition lists "abort()" as a canonical example).