# coding: utf-8
from __future__ import unicode_literals

from ..util import make_tempdir

import pytest


@pytest.mark.parametrize('text', ["I can't do this"])
def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
    tokenizer_b = en_tokenizer.to_bytes()
    new_tokenizer = en_tokenizer.from_bytes(tokenizer_b)
    assert new_tokenizer.to_bytes() == tokenizer_b
    doc1 = en_tokenizer(text)
    doc2 = new_tokenizer(text)
    assert [token.text for token in doc1] == [token.text for token in doc2]


def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
    tokenizer = en_tokenizer
    with make_tempdir() as d:
        file_path = d / 'tokenizer'
        tokenizer.to_disk(file_path)
        tokenizer_d = en_tokenizer.from_disk(file_path)
        assert tokenizer.to_bytes() == tokenizer_d.to_bytes()

@@ -61,47 +61,228 @@
 m ..
-util import make_tempdir%0A%0Aimport pytest
+.util import get_lang_class%0Afrom ..util import make_tempdir, assert_packed_msg_equal%0A%0Aimport pytest%0A%0A%0Adef load_tokenizer(b):%0A    tok = get_lang_class('en').Defaults.create_tokenizer()%0A    tok.from_bytes(b)%0A    return tok
 %0A%0A%0A@
@@ -320,22 +320,34 @@
  %5B%22I
- can't do this
+%F0%9F%92%9Cyou%22, %22they%E2%80%99re%22, %22%E2%80%9Chello%E2%80%9D
 %22%5D)%0A
@@ -425,18 +425,16 @@
 okenizer
-_b
  = en_to
@@ -440,27 +440,16 @@
 okenizer
-.to_bytes()
 %0A    new
@@ -453,34 +453,100 @@
 new_tokenizer = 
-en
+load_tokenizer(tokenizer.to_bytes())%0A    assert_packed_msg_equal(new
 _tokenizer.from_
@@ -540,27 +540,28 @@
 kenizer.
-from
+to
 _bytes(
+), 
 tokenize
@@ -565,15 +565,26 @@
 izer
-_b
+.to_bytes()
 )%0A   
+ #
  ass
@@ -624,18 +624,27 @@
 okenizer
-_b
+.to_bytes()
 %0A    doc
@@ -647,19 +647,16 @@
  doc1 = 
-en_
 tokenize

Update serialization tests for tokenizer