@@ -0,0 +1,872 @@
+from unittest import TestCase%0A%0Afrom spicedham.split_tokenizer import SplitTokenizer%0A%0A%0Aclass TestSplitTokenizer(TestCase):%0A    def test_tokenize(self):%0A        tokenizer = SplitTokenizer(%7B%7D)%0A        # Text was chosen because it's a couple lines and includes ?!%5Cn,. etc.%0A        test_text = %22%22%22Not a whit, we defy augury; there's a special providence in%0A        the fall of a sparrow! If it be now, 'tis not to come; if it be%0A        not to come, will it be now?%22%22%22%0A        expected_text = %5B%22not%22, %22a%22, %22whit%22, %22we%22, %22defy%22, %22augury;%22,%0A        %22there's%22, %22a%22, %22special%22, %22providence%22, %22in%22, %22the%22, %22fall%22, %22of%22,%0A        %22a%22, %22sparrow%22, %22if%22, %22it%22, %22be%22, %22now%22, %22'tis%22, %22not%22, %22to%22, %22come;%22,%0A        %22if%22, %22it%22, %22be%22, %22not%22, %22to%22, %22come%22, %22will%22, %22it%22, %22be%22, %22now%22%5D%0A        received_text = tokenizer.tokenize(test_text)%0A        self.assertEqual(received_text, expected_text)%0A

Add tests for SplitTokenizer