






import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir


@dataclass
class      __SCREAMING_SNAKE_CASE       :
						_SCREAMING_SNAKE_CASE						:							str




class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    get_tests_dir('fixtures/test_sentencepiece.model' )
												lowerCAmelCase__         =    get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model' )

												lowerCAmelCase__         =    FakeOriginalTokenizer(vocab_file=_UpperCamelCase )

												with warnings.catch_warnings(record=_UpperCamelCase ) as w:
																		lowerCAmelCase__         =    SpmConverter(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    0 )

												lowerCAmelCase__         =    FakeOriginalTokenizer(vocab_file=_UpperCamelCase )

												with warnings.catch_warnings(record=_UpperCamelCase ) as w:
																		lowerCAmelCase__         =    SpmConverter(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    1 )

												self.assertIn(
												    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
												    ' which is not implemented in the fast tokenizers.'  ,    str(w[0].message )  ,    )









'''simple docstring'''



import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir



@dataclass
class       a							:


     """simple docstring"""






     SCREAMING_SNAKE_CASE			:					str



class       a							(						unittest.TestCase			):


     """simple docstring"""






     def 					lowerCamelCase__					(							self	:						str     )		->				Dict:
          __UpperCAmelCase      :					str    				=     get_tests_dir('''fixtures/test_sentencepiece.model'''     )
          __UpperCAmelCase      :					List[Any]    				=     get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model'''     )

          __UpperCAmelCase      :					Tuple    				=     FakeOriginalTokenizer(vocab_file=snake_case     )

          with warnings.catch_warnings(record=snake_case     ) as w:
               __UpperCAmelCase      :					Optional[int]    				=     SpmConverter(snake_case     )
          self.assertEqual(len(snake_case     )				,					0     )

          __UpperCAmelCase      :					str    				=     FakeOriginalTokenizer(vocab_file=snake_case     )

          with warnings.catch_warnings(record=snake_case     ) as w:
               __UpperCAmelCase      :					Optional[int]    				=     SpmConverter(snake_case     )
          self.assertEqual(len(snake_case     )				,					1     )

          self.assertIn(
              '''The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'''
              ''' which is not implemented in the fast tokenizers.'''				,					str(w[0].message     )				,					)