Stdlib: strtok() — VM + JIT + AOT (#3201)

docs/capabilities.md

@@ -304,6 +304,7 @@ Auto-generated by `script/capability-matrix.php`. Do not edit by hand.
 | `strrpos` | yes | yes | yes | standard | JIT PHPT; AOT PHPT |
 | `strspn` | yes | yes | yes | standard | AOT PHPT |
 | `strstr` | yes | yes | yes | standard | AOT PHPT |
+| `strtok` | yes | yes | yes | standard | JIT PHPT; AOT PHPT |
 | `strtolower` | yes | yes | yes | standard | AOT PHPT |
 | `strtoupper` | yes | yes | yes | standard | JIT PHPT; AOT PHPT |
 | `strtr` | yes | yes | yes | standard | JIT PHPT; AOT PHPT |

ext/standard/JitStrtok.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PHPCompiler\ext\standard;
+
+use PHPCompiler\JIT\BasicBlockHelper;
+use PHPCompiler\JIT\Context;
+use PHPCompiler\JIT\JitValueBox;
+use PHPLLVM\Builder;
+use PHPLLVM\Value;
+
+/**
+ * LLVM lowering for strtok() via phpc_strtok.c (issue #3201).
+ */
+final class JitStrtok
+{
+    private static int $blockSerial = 0;
+
+    public static function tokenize(Context $context, ?Value $str, Value $tok): Value
+    {
+        $id = (string) (++self::$blockSerial);
+        $i8 = $context->getTypeFromString('int8');
+        $strPtr = $context->getTypeFromString('__string__*')->constNull();
+        $init = $i8->constInt(0, true);
+        if (null !== $str) {
+            $strPtr = $str;
+            $init = $i8->constInt(1, true);
+        }
+        $fn = $context->lookupFunction('phpc_strtok');
+        $raw = $context->builder->call($fn, $strPtr, $tok, $init);
+        $null = $context->getTypeFromString('__string__*')->constNull();
+        $isNull = $context->builder->icmp(Builder::INT_EQ, $raw, $null);
+
+        $slot = JitValueBox::alloc($context);
+        $ptr = JitValueBox::pointer($context, $slot);
+        $failBlock = BasicBlockHelper::append($context, 'strtok_fail_'.$id);
+        $okBlock = BasicBlockHelper::append($context, 'strtok_ok_'.$id);
+        $doneBlock = BasicBlockHelper::append($context, 'strtok_done_'.$id);
+        $context->builder->branchIf($isNull, $failBlock, $okBlock);
+
+        $context->builder->positionAtEnd($failBlock);
+        $i1 = $context->getTypeFromString('int1');
+        JitValueBox::writeBool($context, $slot, $i1->constInt(0, false));
+        $context->builder->branch($doneBlock);
+
+        $context->builder->positionAtEnd($okBlock);
+        $context->builder->call(
+            $context->lookupFunction('__value__writeString'),
+            $ptr,
+            $raw
+        );
+        $context->builder->branch($doneBlock);
+
+        $context->builder->positionAtEnd($doneBlock);
+
+        return $ptr;
+    }
+}

ext/standard/Module.php

@@ -95,6 +95,7 @@ public function getFunctions(): array
             new str_shuffle(),
             new strpos(),
             new strstr(),
+            new strtok(),
             new strchr(),
             new stristr(),
             new strrchr(),
@@ -403,6 +404,15 @@ public function jitInit(JIT\Context $context): void
             $fn = $context->module->addFunction('substr_compare', $ft);
             $context->registerFunction('substr_compare', $fn);
         }
+        try {
+            $context->lookupFunction('phpc_strtok');
+        } catch (\Throwable $e) {
+            $strPtr = $context->getTypeFromString('__string__*');
+            $i8 = $context->getTypeFromString('int8');
+            $ft = $context->context->functionType($strPtr, false, $strPtr, $strPtr, $i8);
+            $fn = $context->module->addFunction('phpc_strtok', $ft);
+            $context->registerFunction('phpc_strtok', $fn);
+        }
         foreach (['strspn', 'strcspn'] as $name) {
             try {
                 $context->lookupFunction($name);

ext/standard/VmString.php

@@ -2538,6 +2538,89 @@ public static function pathFilename(string $path): string
         return self::byteSlice($base, 0, $baseLen - $extLen - 1);
     }
 
+    /** Source string for strtok() continuation (ext/standard/string.c; issue #3201). */
+    private static ?string $strtokString = null;
+
+    private static int $strtokLast = 0;
+
+    /**
+     * strtok() — tokenize with re-entrant static state (php-src ext/standard/string.c).
+     *
+     * @return string|false
+     */
+    public static function strtok(string $str, ?string $tok = null): string|false
+    {
+        if (null !== $tok) {
+            self::$strtokString = $str;
+            self::$strtokLast = 0;
+            $delimiter = $tok;
+        } else {
+            if (null === self::$strtokString) {
+                return false;
+            }
+            $delimiter = $str;
+        }
+
+        $len = self::byteLength(self::$strtokString);
+        $p = self::$strtokLast;
+        if ($p >= $len) {
+            self::strtokReset();
+
+            return false;
+        }
+
+        $table = array_fill(0, 256, false);
+        $delLen = self::byteLength($delimiter);
+        for ($i = 0; $i < $delLen; ++$i) {
+            $table[self::byteOrd($delimiter[$i])] = true;
+        }
+
+        $skipped = 0;
+        while ($p < $len && $table[self::byteOrd(self::$strtokString[$p])]) {
+            ++$p;
+            ++$skipped;
+            if ($p >= $len) {
+                self::strtokReset();
+
+                return false;
+            }
+        }
+
+        while (++$p < $len) {
+            if ($table[self::byteOrd(self::$strtokString[$p])]) {
+                $token = self::byteSlice(
+                    self::$strtokString,
+                    self::$strtokLast + $skipped,
+                    $p - self::$strtokLast - $skipped
+                );
+                self::$strtokLast = $p + 1;
+
+                return $token;
+            }
+        }
+
+        if ($p > self::$strtokLast) {
+            $token = self::byteSlice(
+                self::$strtokString,
+                self::$strtokLast + $skipped,
+                $p - self::$strtokLast - $skipped
+            );
+            self::strtokReset();
+
+            return $token;
+        }
+
+        self::strtokReset();
+
+        return false;
+    }
+
+    private static function strtokReset(): void
+    {
+        self::$strtokString = null;
+        self::$strtokLast = 0;
+    }
+
     private static function byteOrd(string $byte): int
     {
         return ord($byte);

ext/standard/strtok.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PHPCompiler\ext\standard;
+
+use PHPCompiler\Frame;
+use PHPCompiler\Func\Internal;
+use PHPCompiler\JIT\Builtin\StringStrtok;
+use PHPCompiler\JIT\Context;
+use PHPCompiler\JIT\Variable as JITVariable;
+use PHPCompiler\VM\Variable;
+use PHPLLVM\Value;
+
+/**
+ * strtok() — tokenize strings with static continuation state (php-src ext/standard/string.c; #3201).
+ */
+final class strtok extends Internal
+{
+    public function __construct()
+    {
+        parent::__construct('strtok');
+    }
+
+    public function execute(Frame $frame): void
+    {
+        $argc = \count($frame->calledArgs);
+        if ($argc < 1 || $argc > 2) {
+            throw new \LogicException('strtok() accepts one or two arguments in this compiler build');
+        }
+        if (null === $frame->returnVar) {
+            return;
+        }
+        $arg0 = $frame->calledArgs[0]->resolveIndirect();
+        if (Variable::TYPE_STRING !== $arg0->type) {
+            throw new \LogicException('strtok() argument #1 must be a string in this compiler build');
+        }
+        $tok = null;
+        if (2 === $argc) {
+            $arg1 = $frame->calledArgs[1]->resolveIndirect();
+            if (Variable::TYPE_STRING !== $arg1->type) {
+                throw new \LogicException('strtok() argument #2 must be a string in this compiler build');
+            }
+            $tok = $arg1->toString();
+        }
+        $result = VmString::strtok($arg0->toString(), $tok);
+        if (false === $result) {
+            $frame->returnVar->bool(false);
+        } else {
+            $frame->returnVar->string($result);
+        }
+    }
+
+    public function call(Context $context, JITVariable ...$args): Value
+    {
+        $argc = \count($args);
+        if ($argc < 1 || $argc > 2) {
+            throw new \LogicException('strtok() accepts one or two arguments in this compiler build');
+        }
+        StringStrtok::ensureLinked($context);
+        if (1 === $argc) {
+            return JitStrtok::tokenize(
+                $context,
+                null,
+                $this->jitString($context, $args[0], 'strtok() token')
+            );
+        }
+
+        return JitStrtok::tokenize(
+            $context,
+            $this->jitString($context, $args[0], 'strtok() string'),
+            $this->jitString($context, $args[1], 'strtok() token')
+        );
+    }
+}

lib/AOT/Linker.php

@@ -27,6 +27,7 @@ final class Linker
         __DIR__.'/runtime/phpc_metaphone.c',
         __DIR__.'/runtime/phpc_str_getcsv.c',
         __DIR__.'/runtime/phpc_uniqid.c',
+        __DIR__.'/runtime/phpc_strtok.c',
         __DIR__.'/runtime/password_crypto.c',
         __DIR__.'/runtime/crc32.c',
         __DIR__.'/runtime/strtr.c',

lib/AOT/runtime/builtin_function_names.inc

@@ -282,6 +282,7 @@ static const char *phpc_builtin_functions[] = {
     "strrpos",
     "strspn",
     "strstr",
+    "strtok",
     "strtolower",
     "str_word_count",
     "strtoupper",