
'''simple docstring'''





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 			lowerCAmelCase__ (							UpperCamelCase       ,				unittest.TestCase							):



  '''simple docstring'''






  _UpperCamelCase							=			PreTrainedTokenizerFast
  _UpperCamelCase							=			False
  _UpperCamelCase							=			True
  _UpperCamelCase							=			'tokenizer_file'

  def 		_lowerCamelCase						(self	:			Any     ) ->							str:
        snake_case_    							=				False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        snake_case_    							=				True

        snake_case_    							=				["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
        snake_case_    							=				"SaulLu/dummy-tokenizer-bytelevel-bpe"

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        snake_case_    							=				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        snake_case_    							=				PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
        tokenizer.save_pretrained(self.tmpdirname     )

  def 		_lowerCamelCase						(self	:			Any     ) ->							List[Any]:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Dict:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Any     ) ->							Dict:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass

  def 		_lowerCamelCase						(self	:			Any     ) ->							Tuple:
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass

  def 		_lowerCamelCase						(self	:			List[str]     ) ->							List[str]:
        snake_case_    							=				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                    try:
                          snake_case_    							=				tempfile.mkdtemp()
                          snake_case_    							=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_						,			**lowerCAmelCase_     )

                          tokenizer.save_pretrained(self.tmpdirname     )
                          super().test_training_new_tokenizer()
                    finally:
                          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                          # is restored
                          shutil.rmtree(self.tmpdirname     )
                          snake_case_    							=				tmpdirname_orig

  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							Dict:
        snake_case_    							=				self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                    try:
                          snake_case_    							=				tempfile.mkdtemp()
                          snake_case_    							=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_						,			**lowerCAmelCase_     )

                          tokenizer.save_pretrained(self.tmpdirname     )
                          super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                          # is restored
                          shutil.rmtree(self.tmpdirname     )
                          snake_case_    							=				tmpdirname_orig


  def 		_lowerCamelCase						(self	:			Union[str, Any]     ) ->							Union[str, Any]:
        snake_case_    							=				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

        snake_case_    							=				("a" for _ in range(10_00     ))
        snake_case_    							=				tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase_						,			length=10_00						,			vocab_size=50     )

        snake_case_    							=				new_tokenizer.encode("a🤗"     )
        self.assertEqual(lowerCAmelCase_						,			[64, 1_72, 2_53, 97, 2_45]     )



@require_tokenizers
class 			lowerCAmelCase__ (							unittest.TestCase							):



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			Tuple     ) ->							Dict:
        snake_case_    							=				AutoTokenizer.from_pretrained("bert-base-cased"     )
        snake_case_    							=				json.loads(tokenizer._tokenizer.to_str()     )
        snake_case_    							=				len(lowerCAmelCase_     )

        with tempfile.TemporaryDirectory() as tmp_dir:
              # Hack to save this in the tokenizer_config.json
              snake_case_    							=				["tokenizer.4.0.0.json"]
              tokenizer.save_pretrained(lowerCAmelCase_     )
              json.dump(lowerCAmelCase_						,			open(os.path.join(lowerCAmelCase_						,			"tokenizer.4.0.0.json"     )						,			"w"     )     )

              # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
              snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
              self.assertEqual(len(lowerCAmelCase_     )						,			len(lowerCAmelCase_     ) + 1     )
              snake_case_    							=				json.loads(new_tokenizer._tokenizer.to_str()     )
              self.assertIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )

              # Will need to be adjusted if we reach v42 and this test is still here.
              # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
              shutil.move(os.path.join(lowerCAmelCase_						,			"tokenizer.4.0.0.json"     )						,			os.path.join(lowerCAmelCase_						,			"tokenizer.42.0.0.json"     )     )
              snake_case_    							=				["tokenizer.42.0.0.json"]
              tokenizer.save_pretrained(lowerCAmelCase_     )
              snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
              self.assertEqual(len(lowerCAmelCase_     )						,			len(lowerCAmelCase_     )     )
              snake_case_    							=				json.loads(new_tokenizer._tokenizer.to_str()     )
              self.assertNotIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )


  def 		_lowerCamelCase						(self	:			List[str]     ) ->							List[Any]:
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        snake_case_    							=				"hf-internal-testing/test-two-tokenizers"

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        snake_case_    							=				AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )						,			2_89_97     )
        snake_case_    							=				json.loads(tokenizer._tokenizer.to_str()     )
        self.assertIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        snake_case_    							=				"3.0.0"
        snake_case_    							=				old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertEqual(len(lowerCAmelCase_     )						,			2_89_96     )
        snake_case_    							=				json.loads(old_tokenizer._tokenizer.to_str()     )
        self.assertNotIn("huggingface"						,			json_tokenizer["model"]["vocab"]     )



@require_tokenizers
class 			lowerCAmelCase__ (							unittest.TestCase							):



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			List[Any]     ) ->							List[str]:
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        snake_case_    							=				PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"     )
        snake_case_    							=				"The Matrix is a 1999 science fiction action film."

        with concurrent.futures.ThreadPoolExecutor() as executor:
              snake_case_    							=				[executor.submit(self.fetch						,			lowerCAmelCase_						,			lowerCAmelCase_     ) for i in range(10     )]
              snake_case_    							=				[future.result() for future in futures]
              self.assertEqual(lowerCAmelCase_						,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10     )]     )


  def 		_lowerCamelCase						(self	:			Dict						,			lowerCAmelCase_	:			Optional[int]						,			lowerCAmelCase_	:			Optional[Any]     ) ->							Union[str, Any]:
        return tokenizer.encode(lowerCAmelCase_						,			truncation="longest_first"						,			padding="longest"     )






from __future__ import annotations

from typing import Generic, TypeVar

__snake_case		:str 						=	TypeVar('''T''')

class        _A			(      __UpperCAmelCase       ):
       pass


class        _A			(      __UpperCAmelCase       ):
       pass


class        _A			(      Generic[T]       ):



       def __init__(						self				:	List[Any]	,						__SCREAMING_SNAKE_CASE				:	int = 10):


           '''simple docstring'''

           __a					=							[]
           __a					=							limit



       def __bool__(						self				:	List[Any]):


           '''simple docstring'''

           return bool(self.stack)



       def __str__(						self				:	Any):


           '''simple docstring'''

           return str(self.stack)



       def 						_lowerCamelCase     (						self				:	List[Any]	,						__SCREAMING_SNAKE_CASE				:	T):


           '''simple docstring'''

           if len(self.stack) >= self.limit:
               raise StackOverflowError
           self.stack.append(__SCREAMING_SNAKE_CASE)



       def 						_lowerCamelCase     (						self				:	str):


           '''simple docstring'''

           if not self.stack:
               raise StackUnderflowError
           return self.stack.pop()



       def 						_lowerCamelCase     (						self				:	Optional[Any]):


           '''simple docstring'''

           if not self.stack:
               raise StackUnderflowError
           return self.stack[-1]



       def 						_lowerCamelCase     (						self				:	Union[str, Any]):


           '''simple docstring'''

           return not bool(self.stack)



       def 						_lowerCamelCase     (						self				:	Tuple):


           '''simple docstring'''

           return self.size() == self.limit



       def 						_lowerCamelCase     (						self				:	Union[str, Any]):


           '''simple docstring'''

           return len(self.stack)





       def __contains__(						self				:	int	,						__SCREAMING_SNAKE_CASE				:	T):


           '''simple docstring'''

           return item in self.stack





def 	__snake_case      (     ):
    __a					=							Stack(10							)
    assert bool(_UpperCAmelCase							) is False
    assert stack.is_empty() is True
    assert stack.is_full() is False
    assert str(_UpperCAmelCase							) == "[]"

    try:
        __a					=							stack.pop()
        raise AssertionError  # This should not happen
    except StackUnderflowError:
        assert True  # This should happen

    try:
        __a					=							stack.peek()
        raise AssertionError  # This should not happen
    except StackUnderflowError:
        assert True  # This should happen

    for i in range(10							):
        assert stack.size() == i
        stack.push(_UpperCAmelCase							)

    assert bool(_UpperCAmelCase							)
    assert not stack.is_empty()
    assert stack.is_full()
    assert str(_UpperCAmelCase							) == str(list(range(10							)							)							)
    assert stack.pop() == 9
    assert stack.peek() == 8

    stack.push(100							)
    assert str(_UpperCAmelCase							) == str([0, 1, 2, 3, 4, 5, 6, 7, 8, 100]							)

    try:
        stack.push(200							)
        raise AssertionError  # This should not happen
    except StackOverflowError:
        assert True  # This should happen

    assert not stack.is_empty()
    assert stack.size() == 10

    assert 5 in stack
    assert 55 not in stack


if __name__ == "__main__":
 test_stack()





