@@ -0,0 +1,344 @@
+import pytest%0A%0Afrom yargy.tokenizer import Tokenizer%0A%0A%0A@pytest.fixture%0Adef tokenizer():%0A    return Tokenizer()%0A%0A%0Adef test_that_tokenizer_doesnt_drop_symbols(tokenizer):%0A    tokens = list(tokenizer('%D0%A5%D0%B0%D0%BD%D1%82%D1%8B %E2%80%93 %D0%9C%D0%B0%D0%BD%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%BE%D0%B3%D0%BE'))%0A    assert %5Bt.value for t in tokens%5D == %5B'%D0%A5%D0%B0%D0%BD%D1%82%D1%8B', '%E2%80%93', '%D0%9C%D0%B0%D0%BD%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%BE%D0%B3%D0%BE'%5D%0A    assert tokens%5B1%5D.forms%5B0%5D.grammemes == %7B'OTHER'%7D%0A

Test that tokenizer doen't drop unknown symbols (closes #22)