16 lines
428 B
Text
16 lines
428 B
Text
<?php
|
|
|
|
use Phpml\Tokenization\Tokenizer;
|
|
|
|
class WhitespaceLowerTokenizer implements Tokenizer
|
|
{
|
|
public function tokenize(string $text): array
|
|
{
|
|
$substrings = preg_split('/[\pZ\pC]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
|
|
if ($substrings === false) {
|
|
throw new InvalidArgumentException('preg_split failed on: ' . $text);
|
|
}
|
|
|
|
return array_map('mb_strtolower', $substrings);
|
|
}
|
|
}
|