26 lines
618 B
PHP
26 lines
618 B
PHP
|
<?php
|
||
|
|
|
||
|
|
namespace Tests\TextAnalysis\Tokenizers;
|
||
|
|
|
||
|
|
use TextAnalysis\Tokenizers\GeneralTokenizer;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* GeneralTokenizerTest
|
||
|
|
* @author yooper
|
||
|
|
*/
|
||
|
|
class GeneralTokenizerTest extends \PHPUnit\Framework\TestCase{
|
||
|
|
|
||
|
|
public function testSpaceTokenizer()
|
||
|
|
{
|
||
|
|
$tokenizer = new GeneralTokenizer(" ");
|
||
|
|
$this->assertCount(4, $tokenizer->tokenize("This has some words"));
|
||
|
|
}
|
||
|
|
|
||
|
|
public function testLineTokenizer(){
|
||
|
|
|
||
|
|
$tokenizer = new GeneralTokenizer(PHP_EOL);
|
||
|
|
$this->assertCount(4, $tokenizer->tokenize("This ".PHP_EOL." has".PHP_EOL." some".PHP_EOL." words"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|