from domain.tokenizer import Tokenizer, WordPartOfSpeach


def test_tokenize_sentences():
    text = 'I hoped to see my friend. And shake his hand.'
    sentences = Tokenizer().sentences(text)

    assert list(sentences) == ['I hoped to see my friend.', 'And shake his hand.']


def test_tokenize_words():
    sentences = ['I hoped to see my friend.', 'And shake his hand.']
    words_by_sentence = list(Tokenizer().words(sentences))

    assert len(words_by_sentence) == 2
    assert list(words_by_sentence[0]) == [
        ('I', WordPartOfSpeach.OTHER),
        ('hoped', WordPartOfSpeach.VERB),
        ('to', WordPartOfSpeach.OTHER),
        ('see', WordPartOfSpeach.VERB),
        ('my',  WordPartOfSpeach.OTHER),
        ('friend', WordPartOfSpeach.NOUN),
        ('.', WordPartOfSpeach.OTHER)]
    assert list(words_by_sentence[1]) == [
        ('And', WordPartOfSpeach.OTHER),
        ('shake', WordPartOfSpeach.VERB),
        ('his', WordPartOfSpeach.OTHER),
        ('hand', WordPartOfSpeach.NOUN),
        ('.', WordPartOfSpeach.OTHER)]

@@ -327,32 +327,42 @@
 hoped to see my 
+wonderful 
 friend.', 'And s
@@ -370,24 +370,31 @@
 ake his hand
+ firmly
 .'%5D%0A    word
@@ -721,32 +721,78 @@
 fSpeach.OTHER),%0A
+        ('wonderful',  WordPartOfSpeach.ADJ),%0A
         ('friend
@@ -1016,32 +1016,32 @@
 fSpeach.OTHER),%0A
-
         ('hand',
@@ -1057,32 +1057,74 @@
 OfSpeach.NOUN),%0A
+        ('firmly', WordPartOfSpeach.ADV),%0A
         ('.', Wo

test all POS in Tokenizer