@@ -0,0 +1,697 @@
+from nose.tools import eq_, assert_almost_equal%0Afrom wordfreq import tokenize, word_frequency%0A%0A%0Adef test_apostrophes():%0A    for lang in ('fr', 'ca', 'oc'):%0A        eq_(tokenize(%22langues d'o%C3%AFl%22, lang),%0A            %5B'langues', %22d%22, 'o%C3%AFl'%5D)%0A        eq_(tokenize(%22langues d'o%C3%AFl%22, lang, include_punctuation=True),%0A            %5B'langues', %22d'%22, 'o%C3%AFl'%5D)%0A        eq_(tokenize(%22l'heure%22, lang),%0A            %5B'l', 'heure'%5D)%0A        eq_(tokenize(%22l'heure%22, lang, include_punctuation=True),%0A            %5B%22l'%22, 'heure'%5D)%0A        eq_(tokenize(%22L'H%C3%B4pital%22, lang, include_punctuation=True),%0A            %5B%22l'%22, 'h%C3%B4pital'%5D)%0A        eq_(tokenize(%22This isn't French%22, lang),%0A            %5B'this', %22isn't%22, 'french'%5D)%0A%0A

add tests for French apostrophe tokenization