diff --git a/docs/capabilities.md b/docs/capabilities.md index 07bdbc6740c..b6e4701d3c5 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -304,6 +304,7 @@ Auto-generated by `script/capability-matrix.php`. Do not edit by hand. | `strrpos` | yes | yes | yes | standard | JIT PHPT; AOT PHPT | | `strspn` | yes | yes | yes | standard | AOT PHPT | | `strstr` | yes | yes | yes | standard | AOT PHPT | +| `strtok` | yes | yes | yes | standard | JIT PHPT; AOT PHPT | | `strtolower` | yes | yes | yes | standard | AOT PHPT | | `strtoupper` | yes | yes | yes | standard | JIT PHPT; AOT PHPT | | `strtr` | yes | yes | yes | standard | JIT PHPT; AOT PHPT | diff --git a/ext/standard/JitStrtok.php b/ext/standard/JitStrtok.php new file mode 100644 index 00000000000..c5c7813f63c --- /dev/null +++ b/ext/standard/JitStrtok.php @@ -0,0 +1,59 @@ +getTypeFromString('int8'); + $strPtr = $context->getTypeFromString('__string__*')->constNull(); + $init = $i8->constInt(0, true); + if (null !== $str) { + $strPtr = $str; + $init = $i8->constInt(1, true); + } + $fn = $context->lookupFunction('phpc_strtok'); + $raw = $context->builder->call($fn, $strPtr, $tok, $init); + $null = $context->getTypeFromString('__string__*')->constNull(); + $isNull = $context->builder->icmp(Builder::INT_EQ, $raw, $null); + + $slot = JitValueBox::alloc($context); + $ptr = JitValueBox::pointer($context, $slot); + $failBlock = BasicBlockHelper::append($context, 'strtok_fail_'.$id); + $okBlock = BasicBlockHelper::append($context, 'strtok_ok_'.$id); + $doneBlock = BasicBlockHelper::append($context, 'strtok_done_'.$id); + $context->builder->branchIf($isNull, $failBlock, $okBlock); + + $context->builder->positionAtEnd($failBlock); + $i1 = $context->getTypeFromString('int1'); + JitValueBox::writeBool($context, $slot, $i1->constInt(0, false)); + $context->builder->branch($doneBlock); + + $context->builder->positionAtEnd($okBlock); + $context->builder->call( + $context->lookupFunction('__value__writeString'), + $ptr, + $raw + ); + $context->builder->branch($doneBlock); + + $context->builder->positionAtEnd($doneBlock); + + return $ptr; + } +} diff --git a/ext/standard/Module.php b/ext/standard/Module.php index 72f38403263..96944cc479b 100755 --- a/ext/standard/Module.php +++ b/ext/standard/Module.php @@ -95,6 +95,7 @@ public function getFunctions(): array new str_shuffle(), new strpos(), new strstr(), + new strtok(), new strchr(), new stristr(), new strrchr(), @@ -403,6 +404,15 @@ public function jitInit(JIT\Context $context): void $fn = $context->module->addFunction('substr_compare', $ft); $context->registerFunction('substr_compare', $fn); } + try { + $context->lookupFunction('phpc_strtok'); + } catch (\Throwable $e) { + $strPtr = $context->getTypeFromString('__string__*'); + $i8 = $context->getTypeFromString('int8'); + $ft = $context->context->functionType($strPtr, false, $strPtr, $strPtr, $i8); + $fn = $context->module->addFunction('phpc_strtok', $ft); + $context->registerFunction('phpc_strtok', $fn); + } foreach (['strspn', 'strcspn'] as $name) { try { $context->lookupFunction($name); diff --git a/ext/standard/VmString.php b/ext/standard/VmString.php index 19cab571140..2de5476ac33 100644 --- a/ext/standard/VmString.php +++ b/ext/standard/VmString.php @@ -2538,6 +2538,89 @@ public static function pathFilename(string $path): string return self::byteSlice($base, 0, $baseLen - $extLen - 1); } + /** Source string for strtok() continuation (ext/standard/string.c; issue #3201). */ + private static ?string $strtokString = null; + + private static int $strtokLast = 0; + + /** + * strtok() — tokenize with re-entrant static state (php-src ext/standard/string.c). + * + * @return string|false + */ + public static function strtok(string $str, ?string $tok = null): string|false + { + if (null !== $tok) { + self::$strtokString = $str; + self::$strtokLast = 0; + $delimiter = $tok; + } else { + if (null === self::$strtokString) { + return false; + } + $delimiter = $str; + } + + $len = self::byteLength(self::$strtokString); + $p = self::$strtokLast; + if ($p >= $len) { + self::strtokReset(); + + return false; + } + + $table = array_fill(0, 256, false); + $delLen = self::byteLength($delimiter); + for ($i = 0; $i < $delLen; ++$i) { + $table[self::byteOrd($delimiter[$i])] = true; + } + + $skipped = 0; + while ($p < $len && $table[self::byteOrd(self::$strtokString[$p])]) { + ++$p; + ++$skipped; + if ($p >= $len) { + self::strtokReset(); + + return false; + } + } + + while (++$p < $len) { + if ($table[self::byteOrd(self::$strtokString[$p])]) { + $token = self::byteSlice( + self::$strtokString, + self::$strtokLast + $skipped, + $p - self::$strtokLast - $skipped + ); + self::$strtokLast = $p + 1; + + return $token; + } + } + + if ($p > self::$strtokLast) { + $token = self::byteSlice( + self::$strtokString, + self::$strtokLast + $skipped, + $p - self::$strtokLast - $skipped + ); + self::strtokReset(); + + return $token; + } + + self::strtokReset(); + + return false; + } + + private static function strtokReset(): void + { + self::$strtokString = null; + self::$strtokLast = 0; + } + private static function byteOrd(string $byte): int { return ord($byte); diff --git a/ext/standard/strtok.php b/ext/standard/strtok.php new file mode 100644 index 00000000000..e3e5a7c5d28 --- /dev/null +++ b/ext/standard/strtok.php @@ -0,0 +1,75 @@ +calledArgs); + if ($argc < 1 || $argc > 2) { + throw new \LogicException('strtok() accepts one or two arguments in this compiler build'); + } + if (null === $frame->returnVar) { + return; + } + $arg0 = $frame->calledArgs[0]->resolveIndirect(); + if (Variable::TYPE_STRING !== $arg0->type) { + throw new \LogicException('strtok() argument #1 must be a string in this compiler build'); + } + $tok = null; + if (2 === $argc) { + $arg1 = $frame->calledArgs[1]->resolveIndirect(); + if (Variable::TYPE_STRING !== $arg1->type) { + throw new \LogicException('strtok() argument #2 must be a string in this compiler build'); + } + $tok = $arg1->toString(); + } + $result = VmString::strtok($arg0->toString(), $tok); + if (false === $result) { + $frame->returnVar->bool(false); + } else { + $frame->returnVar->string($result); + } + } + + public function call(Context $context, JITVariable ...$args): Value + { + $argc = \count($args); + if ($argc < 1 || $argc > 2) { + throw new \LogicException('strtok() accepts one or two arguments in this compiler build'); + } + StringStrtok::ensureLinked($context); + if (1 === $argc) { + return JitStrtok::tokenize( + $context, + null, + $this->jitString($context, $args[0], 'strtok() token') + ); + } + + return JitStrtok::tokenize( + $context, + $this->jitString($context, $args[0], 'strtok() string'), + $this->jitString($context, $args[1], 'strtok() token') + ); + } +} diff --git a/lib/AOT/Linker.php b/lib/AOT/Linker.php index fdde981671c..2a969f1f33c 100644 --- a/lib/AOT/Linker.php +++ b/lib/AOT/Linker.php @@ -27,6 +27,7 @@ final class Linker __DIR__.'/runtime/phpc_metaphone.c', __DIR__.'/runtime/phpc_str_getcsv.c', __DIR__.'/runtime/phpc_uniqid.c', + __DIR__.'/runtime/phpc_strtok.c', __DIR__.'/runtime/password_crypto.c', __DIR__.'/runtime/crc32.c', __DIR__.'/runtime/strtr.c', diff --git a/lib/AOT/runtime/builtin_function_names.inc b/lib/AOT/runtime/builtin_function_names.inc index e6dad85aa39..5b4ac0bc8f3 100644 --- a/lib/AOT/runtime/builtin_function_names.inc +++ b/lib/AOT/runtime/builtin_function_names.inc @@ -282,6 +282,7 @@ static const char *phpc_builtin_functions[] = { "strrpos", "strspn", "strstr", + "strtok", "strtolower", "str_word_count", "strtoupper", diff --git a/lib/AOT/runtime/phpc_strtok.c b/lib/AOT/runtime/phpc_strtok.c new file mode 100644 index 00000000000..be1e56b5b9c --- /dev/null +++ b/lib/AOT/runtime/phpc_strtok.c @@ -0,0 +1,134 @@ +/* + * strtok() runtime for VM/JIT/AOT (issue #3201). + * Mirrors php-src ext/standard/string.c PHP_FUNCTION(strtok) byte semantics. + */ + +#include +#include + +#define PHPC_STRTOK_MAX 65536 + +typedef struct __string__ __string__; + +extern __string__ *__string__init(long long size, const char *value); + +static char strtok_buf[PHPC_STRTOK_MAX]; +static size_t strtok_len = 0; +static const char *strtok_last = NULL; + +static size_t phpc_strlen(__string__ *s) +{ + if (NULL == s) { + return 0; + } + + return (size_t) *((long long *) ((char *) s + sizeof(void *))); +} + +static const char *phpc_strdata(__string__ *s) +{ + if (NULL == s) { + return ""; + } + + return (const char *) s + sizeof(void *) + sizeof(long long); +} + +static void phpc_strtok_reset(void) +{ + strtok_len = 0; + strtok_last = NULL; + strtok_buf[0] = '\0'; +} + +static void phpc_strtok_init(__string__ *str) +{ + const char *data; + size_t len; + + phpc_strtok_reset(); + if (NULL == str) { + return; + } + data = phpc_strdata(str); + len = phpc_strlen(str); + if (len >= PHPC_STRTOK_MAX) { + len = PHPC_STRTOK_MAX - 1; + } + if (len > 0) { + memcpy(strtok_buf, data, len); + } + strtok_buf[len] = '\0'; + strtok_len = len; + strtok_last = strtok_buf; +} + +__string__ *phpc_strtok(__string__ *str, __string__ *tok, int8_t init_string) +{ + unsigned char table[256]; + const char *p; + const char *pe; + const char *token; + const char *token_end; + size_t skipped; + size_t token_len; + + if (0 != init_string) { + phpc_strtok_init(str); + } else if (NULL == strtok_last) { + return NULL; + } + + if (NULL == tok) { + return NULL; + } + + p = strtok_last; + pe = strtok_buf + strtok_len; + if (p >= pe) { + phpc_strtok_reset(); + + return NULL; + } + + memset(table, 0, sizeof(table)); + token = phpc_strdata(tok); + token_end = token + phpc_strlen(tok); + while (token < token_end) { + table[(unsigned char) *token++] = 1; + } + + skipped = 0; + while (table[(unsigned char) *p]) { + if (++p >= pe) { + phpc_strtok_reset(); + + return NULL; + } + skipped++; + } + + while (++p < pe) { + if (table[(unsigned char) *p]) { + const char *start = strtok_last + skipped; + + token_len = (size_t) (p - strtok_last) - skipped; + strtok_last = p + 1; + + return __string__init((long long) token_len, start); + } + } + + if (p > strtok_last) { + const char *start = strtok_last + skipped; + + token_len = (size_t) (p - strtok_last) - skipped; + phpc_strtok_reset(); + + return __string__init((long long) token_len, start); + } + + phpc_strtok_reset(); + + return NULL; +} diff --git a/lib/JIT/Builtin/StringStrtok.php b/lib/JIT/Builtin/StringStrtok.php new file mode 100644 index 00000000000..81831779008 --- /dev/null +++ b/lib/JIT/Builtin/StringStrtok.php @@ -0,0 +1,184 @@ +loadType) { + $fn = $context->module->getNamedFunction('phpc_strtok'); + if (null !== $fn) { + $context->registerFunction('phpc_strtok', $fn); + } + + return; + } + + $probe = $context->module->getNamedFunction('phpc_strtok'); + if (null !== $probe && $probe->countBasicBlocks() > 0) { + self::registerLinkedRuntime($context); + + return; + } + + $bitcode = self::ensureBitcode(); + $data = file_get_contents($bitcode); + if (false === $data || '' === $data) { + throw new \LogicException('Failed to read strtok JIT bitcode: '.$bitcode); + } + $buffer = $context->llvm->createMemoryBufferWithString($data, 'phpc_strtok.bc'); + $runtimeModule = $buffer->parseBitcode($context->context); + if (!$context->module->link($runtimeModule)) { + throw new \LogicException('Failed to link strtok JIT runtime bitcode'); + } + self::registerLinkedRuntime($context); + } + + private static function registerLinkedRuntime(Context $context): void + { + $fn = $context->module->getNamedFunction('phpc_strtok'); + if (null === $fn) { + throw new \LogicException('phpc_strtok missing after strtok bitcode link'); + } + $context->registerFunction('phpc_strtok', $fn); + } + + private static function ensureBitcode(): string + { + $source = realpath(self::RUNTIME_SOURCE); + if (false === $source || !is_file($source)) { + throw new \LogicException('strtok runtime source not found: '.self::RUNTIME_SOURCE); + } + + $compiler = self::resolveCompiler(); + $cacheDir = sys_get_temp_dir().'/phpc-jit-runtime'; + if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) { + throw new \LogicException('Cannot create JIT runtime cache: '.$cacheDir); + } + + $cache = $cacheDir.'/'.basename($source, '.c').'-'.substr( + sha1($source.filemtime($source).$compiler.'host'), + 0, + 16 + ).'.bc'; + if (is_file($cache) && filemtime($cache) >= filemtime($source)) { + return $cache; + } + + $includes = self::hostLibcIncludeFlags(); + $cmd = escapeshellarg($compiler) + .' -emit-llvm -c -fPIC -O2'.$includes.' ' + .escapeshellarg($source).' -o '.escapeshellarg($cache).' 2>&1'; + $output = shell_exec($cmd); + if (!is_file($cache)) { + throw new \LogicException( + 'Failed to compile strtok JIT bitcode: '.trim((string) $output) + ); + } + + return $cache; + } + + private static function resolveCompiler(): string + { + $llvmDir = getenv('PHP_COMPILER_LLVM_PATH'); + if (false !== $llvmDir && '' !== $llvmDir) { + foreach (['clang-9', 'clang'] as $name) { + $candidate = $llvmDir.'/'.$name; + if (is_executable($candidate)) { + return $candidate; + } + } + } + + foreach (['clang-9', 'clang', 'gcc', 'cc'] as $name) { + $path = trim((string) shell_exec('command -v '.escapeshellarg($name).' 2>/dev/null')); + if ('' !== $path) { + return $path; + } + } + + throw new \LogicException('No C compiler found for strtok JIT runtime bitcode'); + } + + private static function hostLibcIncludeFlags(): string + { + $flags = ''; + foreach (self::discoverSystemIncludeDirs() as $dir) { + $flags .= ' -isystem '.escapeshellarg($dir); + } + if ('' === $flags && is_file('/usr/include/stdio.h')) { + $flags = ' -isystem /usr/include'; + } + + return $flags; + } + + /** + * @return list + */ + private static function discoverSystemIncludeDirs(): array + { + $dirs = []; + foreach (['gcc', 'cc', 'clang'] as $compiler) { + $path = trim((string) shell_exec('command -v '.escapeshellarg($compiler).' 2>/dev/null')); + if ('' === $path) { + continue; + } + $verbose = shell_exec( + escapeshellarg($path).' -E -Wp,-v -xc /dev/null 2>&1' + ); + if (!is_string($verbose)) { + continue; + } + $capture = false; + foreach (explode("\n", $verbose) as $line) { + if (str_contains($line, '#include <...> search starts here:')) { + $capture = true; + + continue; + } + if ($capture) { + if (str_contains($line, 'End of search list')) { + break; + } + $dir = trim($line); + if ('' !== $dir && is_dir($dir)) { + $dirs[$dir] = true; + } + } + } + if ([] !== $dirs) { + break; + } + } + + if ([] === $dirs) { + foreach (['/usr/include', '/usr/include/x86_64-linux-gnu'] as $fallback) { + if (is_dir($fallback)) { + $dirs[$fallback] = true; + } + } + } + + return array_keys($dirs); + } +} diff --git a/lib/JIT/SelfHostBuiltinPolicy.php b/lib/JIT/SelfHostBuiltinPolicy.php index 16542a7bdf9..41329dda746 100644 --- a/lib/JIT/SelfHostBuiltinPolicy.php +++ b/lib/JIT/SelfHostBuiltinPolicy.php @@ -101,7 +101,7 @@ final class SelfHostBuiltinPolicy 'chr' => 'string', 'chunk_split' => 'string', 'pack' => 'string', - 'strtolower' => 'string', 'strtoupper' => 'string', 'strcmp' => 'string', 'strncmp' => 'string', 'substr_compare' => 'string', + 'strtolower' => 'string', 'strtoupper' => 'string', 'strcmp' => 'string', 'strncmp' => 'string', 'substr_compare' => 'string', 'strtok' => 'string', 'strcasecmp' => 'string', 'strncasecmp' => 'string', 'strlen' => 'string', 'count' => 'string', 'sizeof' => 'string', 'gettype' => 'string', 'get_debug_type' => 'string', 'var_export' => 'string', 'str_replace' => 'string', 'str_ireplace' => 'string', 'strtr' => 'string', 'str_rot13' => 'string', diff --git a/test/compliance/cases/stdlib/strtok.phpt b/test/compliance/cases/stdlib/strtok.phpt new file mode 100644 index 00000000000..34106ffd311 --- /dev/null +++ b/test/compliance/cases/stdlib/strtok.phpt @@ -0,0 +1,12 @@ +--TEST-- +stdlib strtok() comma-separated tokens (#3201) +--FILE-- +