Foundation: host php-parser token map for PHP 8.2+ (#113)

composer.json

@@ -10,7 +10,8 @@
     ],
     "require": {
         "ext-ffi": "*",
-        "php": ">=7.4",
+        "php": ">=8.1",
+        "nikic/php-parser": "^4.19.1 || ^5.1.0",
         "ircmaxell/php-cfg": "^1.0",
         "ircmaxell/php-types": "^1.0",
         "ircmaxell/php-optimizer": "dev-master",

lib/Doctor.php

@@ -1256,6 +1256,7 @@ private static function collectChecks(string $repoRoot): array
         $checks[] = self::checkPhpVersion();
         $checks = array_merge($checks, self::checkExtensions());
         $checks[] = self::checkVendor($repoRoot);
+        $checks[] = self::checkPhpParserHostCompatibility($repoRoot);
         $checks[] = self::checkLlvm($repoRoot);
         $checks[] = self::checkJitCompliance($repoRoot);
         $checks[] = self::checkLoopback($repoRoot);
@@ -1323,6 +1324,114 @@ private static function checkVendor(string $repoRoot): array
         ];
     }
 
+    /**
+     * Host PHP tokenizer constants must match nikic/php-parser Lexer (#113).
+     *
+     * @return array{name: string, ok: bool, required: bool, detail: string, hint: string}
+     */
+    private static function checkPhpParserHostCompatibility(string $repoRoot): array
+    {
+        $autoload = $repoRoot.'/vendor/autoload.php';
+        if (!is_file($autoload)) {
+            return [
+                'name' => 'nikic/php-parser',
+                'ok' => false,
+                'required' => true,
+                'detail' => 'vendor/autoload.php missing',
+                'hint' => 'composer install --no-interaction && script/apply-patches.sh',
+            ];
+        }
+
+        $version = self::resolvePhpParserVersion($repoRoot);
+        try {
+            if (!class_exists(\PhpParser\Lexer::class, false)) {
+                require_once $autoload;
+            }
+            new \PhpParser\Lexer();
+        } catch (\Throwable $e) {
+            return [
+                'name' => 'nikic/php-parser',
+                'ok' => false,
+                'required' => true,
+                'detail' => 'Lexer load failed on PHP '.PHP_VERSION.($version !== '' ? " (parser {$version})" : ''),
+                'hint' => 'composer update nikic/php-parser --with-all-dependencies && script/apply-patches.sh (#113)',
+            ];
+        }
+
+        $missing = self::missingHostTokenizerConstantsForPhpParser();
+        if ([] !== $missing) {
+            return [
+                'name' => 'nikic/php-parser',
+                'ok' => false,
+                'required' => true,
+                'detail' => 'host PHP missing tokenizer constants: '.implode(', ', $missing),
+                'hint' => 'Use PHP 8.1+ or bump nikic/php-parser (composer update nikic/php-parser)',
+            ];
+        }
+
+        $detail = $version !== '' ? "Lexer OK ({$version})" : 'Lexer OK';
+
+        return [
+            'name' => 'nikic/php-parser',
+            'ok' => true,
+            'required' => true,
+            'detail' => $detail,
+            'hint' => '',
+        ];
+    }
+
+    /**
+     * Tokens referenced by modern php-parser Lexer maps on PHP 8.1+ hosts (#113).
+     *
+     * @return list<string>
+     */
+    private static function missingHostTokenizerConstantsForPhpParser(): array
+    {
+        if (version_compare(PHP_VERSION, '8.1.0', '<')) {
+            return [];
+        }
+
+        $required = [
+            'T_OPEN_TAG_WITH_ECHO',
+            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_READONLY',
+        ];
+        $missing = [];
+        foreach ($required as $token) {
+            if (!\defined($token)) {
+                $missing[] = $token;
+            }
+        }
+
+        return $missing;
+    }
+
+    private static function resolvePhpParserVersion(string $repoRoot): string
+    {
+        if (class_exists(\Composer\InstalledVersions::class)) {
+            if (\Composer\InstalledVersions::isInstalled('nikic/php-parser')) {
+                return \Composer\InstalledVersions::getPrettyVersion('nikic/php-parser') ?? '';
+            }
+        }
+
+        $lock = $repoRoot.'/composer.lock';
+        if (!is_readable($lock)) {
+            return '';
+        }
+        $json = json_decode((string) file_get_contents($lock), true);
+        if (!is_array($json) || !isset($json['packages']) || !is_array($json['packages'])) {
+            return '';
+        }
+        foreach ($json['packages'] as $package) {
+            if (is_array($package) && ($package['name'] ?? '') === 'nikic/php-parser') {
+                return (string) ($package['version'] ?? '');
+            }
+        }
+
+        return '';
+    }
+
     /**
      * @return array{name: string, ok: bool, required: bool, detail: string, hint: string}
      */

test/unit/PhpParserHostCompatibilityTest.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PHPCompiler;
+
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Host PHP + nikic/php-parser Lexer compatibility (#113).
+ */
+final class PhpParserHostCompatibilityTest extends TestCase
+{
+    private static string $root;
+
+    public static function setUpBeforeClass(): void
+    {
+        self::$root = dirname(__DIR__, 2);
+    }
+
+    public function testLexerInstantiatesOnHostPhp(): void
+    {
+        $autoload = self::$root.'/vendor/autoload.php';
+        if (!is_file($autoload)) {
+            $this->markTestSkipped('vendor/ not installed');
+        }
+
+        require_once $autoload;
+        $lexer = new \PhpParser\Lexer();
+        $this->assertInstanceOf(\PhpParser\Lexer::class, $lexer);
+    }
+
+    public function testPhp82TokenizerConstantsExistOnHost(): void
+    {
+        if (version_compare(PHP_VERSION, '8.1.0', '<')) {
+            $this->markTestSkipped('PHP 8.1+ required for tokenizer probe');
+        }
+
+        foreach ([
+            'T_OPEN_TAG_WITH_ECHO',
+            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_READONLY',
+        ] as $token) {
+            $this->assertTrue(\defined($token), $token.' must exist on host PHP '.PHP_VERSION);
+        }
+    }
+
+    public function testVmEntryRunsOnHost(): void
+    {
+        $autoload = self::$root.'/vendor/autoload.php';
+        if (!is_file($autoload)) {
+            $this->markTestSkipped('vendor/ not installed');
+        }
+
+        $example = self::$root.'/examples/000-HelloWorld/example.php';
+        if (!is_file($example)) {
+            $this->markTestSkipped('HelloWorld example missing');
+        }
+
+        $cmd = escapeshellarg(PHP_BINARY).' '.escapeshellarg(self::$root.'/bin/vm.php').' '
+            .escapeshellarg($example).' 2>&1';
+        $output = shell_exec($cmd);
+        $this->assertIsString($output);
+        $this->assertStringContainsString('Hello World', $output);
+        $this->assertStringNotContainsString('Undefined constant', $output);
+        $this->assertStringNotContainsString('Fatal error', $output);
+    }
+}

test/unit/PhpcDoctorTest.php

@@ -24,6 +24,7 @@ public function testDoctorPassesInHealthyRepo(): void
         $this->assertSame(0, $result['exit'], $result['stdout']."\n".$result['stderr']);
         $this->assertStringContainsString('[ok] PHP:', $result['stdout']);
         $this->assertStringContainsString('[ok] Composer deps:', $result['stdout']);
+        $this->assertStringContainsString('[ok] nikic/php-parser:', $result['stdout']);
         $this->assertStringContainsString('LLVM 9:', $result['stdout']);
         $this->assertStringContainsString('libLLVM-9.so.1:', $result['stdout']);
         $this->assertStringContainsString('JIT compliance:', $result['stdout']);