58 lines
1.7 KiB
PHP
58 lines
1.7 KiB
PHP
|
<?php
|
||
|
|
|
||
|
|
namespace Tests\TextAnalysis\Tokenizers;
|
||
|
|
|
||
|
|
use TextAnalysis\Tokenizers\PennTreeBankTokenizer;
|
||
|
|
|
||
|
|
/**
|
||
|
|
*
|
||
|
|
* @author yooper
|
||
|
|
*/
|
||
|
|
class PennTreeBankTokenizerTest extends \PHPUnit\Framework\TestCase
|
||
|
|
{
|
||
|
|
|
||
|
|
public function testTokenizer()
|
||
|
|
{
|
||
|
|
$tokenizer = new PennTreeBankTokenizer();
|
||
|
|
$tokens = $tokenizer->tokenize("Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks.");
|
||
|
|
$this->assertCount(16, $tokens);
|
||
|
|
}
|
||
|
|
|
||
|
|
public function testTokenizer2()
|
||
|
|
{
|
||
|
|
$tokenizer = new PennTreeBankTokenizer();
|
||
|
|
$this->assertCount(7, $tokenizer->tokenize("They'll save and invest more."));
|
||
|
|
}
|
||
|
|
|
||
|
|
public function testTokenizer3()
|
||
|
|
{
|
||
|
|
$tokenizer = new PennTreeBankTokenizer();
|
||
|
|
$this->assertCount(4, $tokenizer->tokenize("I'm some text"));
|
||
|
|
}
|
||
|
|
|
||
|
|
public function testAgainstOriginalSedImplementation()
|
||
|
|
{
|
||
|
|
$tokenizer = new PennTreeBankTokenizer();
|
||
|
|
$tokenized = new \SplFileObject(TEST_DATA_DIR."/Tokenizers/PennTreeBankTokenizerTest/tokenized");
|
||
|
|
$tokenized->setFlags(\SplFileObject::DROP_NEW_LINE);
|
||
|
|
$sentences = new \SplFileObject(TEST_DATA_DIR."/Tokenizers/PennTreeBankTokenizerTest/test.txt");
|
||
|
|
$sentences->setFlags(\SplFileObject::DROP_NEW_LINE);
|
||
|
|
|
||
|
|
$tokenized->rewind();
|
||
|
|
foreach ($sentences as $sentence) {
|
||
|
|
if ($sentence) // skip empty lines
|
||
|
|
{
|
||
|
|
$this->assertEquals(
|
||
|
|
$tokenized->current(),
|
||
|
|
implode(" ",$tokenizer->tokenize($sentence)),
|
||
|
|
"Sentence: '$sentence' was not tokenized correctly"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
$tokenized->next();
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|