
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							A__	(			__SCREAMING_SNAKE_CASE						,     unittest.TestCase):
      _UpperCAmelCase       :					List[Any]					 =    PreTrainedTokenizerFast
      _UpperCAmelCase       :					Any					 =    False
      _UpperCAmelCase       :					List[Any]					 =    True
      _UpperCAmelCase       :					Optional[int]					 =    """tokenizer_file"""
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			List[str]           =   False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       lowerCamelCase	:			List[Any]           =   True

       lowerCamelCase	:			Tuple           =   ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
       lowerCamelCase	:			Optional[Any]           =   """SaulLu/dummy-tokenizer-bytelevel-bpe"""

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       lowerCamelCase	:			Union[str, Any]           =   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       lowerCamelCase	:			Dict           =   PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
       tokenizer.save_pretrained(self.tmpdirname   )
      def 		UpperCamelCase__							(   self   ):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass
      def 		UpperCamelCase__							(   self   ):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass
      def 		UpperCamelCase__							(   self   ):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass
      def 		UpperCamelCase__							(   self   ):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Dict           =   self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
         try:
          lowerCamelCase	:			Union[str, Any]           =   tempfile.mkdtemp()
          lowerCamelCase	:			Union[str, Any]           =   self.rust_tokenizer_class.from_pretrained(__magic_name__					,						**__magic_name__   )

          tokenizer.save_pretrained(self.tmpdirname   )
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname   )
          lowerCamelCase	:			Any           =   tmpdirname_orig
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			List[str]           =   self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
         try:
          lowerCamelCase	:			Optional[int]           =   tempfile.mkdtemp()
          lowerCamelCase	:			List[str]           =   self.rust_tokenizer_class.from_pretrained(__magic_name__					,						**__magic_name__   )

          tokenizer.save_pretrained(self.tmpdirname   )
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname   )
          lowerCamelCase	:			Optional[int]           =   tmpdirname_orig






      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Any           =   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

       lowerCamelCase	:			str           =   ("""a""" for _ in range(1_0_0_0   ))
       lowerCamelCase	:			str           =   tokenizer.train_new_from_iterator(text_iterator=__magic_name__					,						length=1_0_0_0					,						vocab_size=5_0   )

       lowerCamelCase	:			Tuple           =   new_tokenizer.encode("""a🤗"""   )
       self.assertEqual(__magic_name__					,						[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]   )





@require_tokenizers
class 							A__	(			unittest.TestCase):
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			Dict           =   AutoTokenizer.from_pretrained("""bert-base-cased"""   )
       lowerCamelCase	:			Optional[Any]           =   json.loads(tokenizer._tokenizer.to_str()   )
       lowerCamelCase	:			str           =   len(__magic_name__   )

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        lowerCamelCase	:			int           =   ["""tokenizer.4.0.0.json"""]
        tokenizer.save_pretrained(__magic_name__   )
        json.dump(__magic_name__					,						open(os.path.join(__magic_name__					,						"""tokenizer.4.0.0.json"""   )					,						"""w"""   )   )

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        lowerCamelCase	:			Optional[Any]           =   AutoTokenizer.from_pretrained(__magic_name__   )
        self.assertEqual(len(__magic_name__   )					,						len(__magic_name__   ) + 1   )
        lowerCamelCase	:			int           =   json.loads(new_tokenizer._tokenizer.to_str()   )
        self.assertIn("""huggingface"""					,						json_tokenizer["""model"""]["""vocab"""]   )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(__magic_name__					,						"""tokenizer.4.0.0.json"""   )					,						os.path.join(__magic_name__					,						"""tokenizer.42.0.0.json"""   )   )
        lowerCamelCase	:			Any           =   ["""tokenizer.42.0.0.json"""]
        tokenizer.save_pretrained(__magic_name__   )
        lowerCamelCase	:			Tuple           =   AutoTokenizer.from_pretrained(__magic_name__   )
        self.assertEqual(len(__magic_name__   )					,						len(__magic_name__   )   )
        lowerCamelCase	:			str           =   json.loads(new_tokenizer._tokenizer.to_str()   )
        self.assertNotIn("""huggingface"""					,						json_tokenizer["""model"""]["""vocab"""]   )






      def 		UpperCamelCase__							(   self   ):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       lowerCamelCase	:			int           =   """hf-internal-testing/test-two-tokenizers"""

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       lowerCamelCase	:			Any           =   AutoTokenizer.from_pretrained(__magic_name__   )
       self.assertEqual(len(__magic_name__   )					,						2_8_9_9_7   )
       lowerCamelCase	:			Dict           =   json.loads(tokenizer._tokenizer.to_str()   )
       self.assertIn("""huggingface"""					,						json_tokenizer["""model"""]["""vocab"""]   )

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       lowerCamelCase	:			Any           =   """3.0.0"""
       lowerCamelCase	:			Optional[Any]           =   old_transformers.models.auto.AutoTokenizer.from_pretrained(__magic_name__   )
       self.assertEqual(len(__magic_name__   )					,						2_8_9_9_6   )
       lowerCamelCase	:			Any           =   json.loads(old_tokenizer._tokenizer.to_str()   )
       self.assertNotIn("""huggingface"""					,						json_tokenizer["""model"""]["""vocab"""]   )





@require_tokenizers
class 							A__	(			unittest.TestCase):
      def 		UpperCamelCase__							(   self   ):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       lowerCamelCase	:			Optional[Any]           =   PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""   )
       lowerCamelCase	:			List[str]           =   """The Matrix is a 1999 science fiction action film."""

       with concurrent.futures.ThreadPoolExecutor() as executor:
        lowerCamelCase	:			Tuple           =   [executor.submit(self.fetch					,						__magic_name__					,						__magic_name__   ) for i in range(1_0   )]
        lowerCamelCase	:			Optional[int]           =   [future.result() for future in futures]
        self.assertEqual(__magic_name__					,						[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0   )]   )






      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__   ):
       return tokenizer.encode(__magic_name__					,						truncation="""longest_first"""					,						padding="""longest"""   )






from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase           =logging.get_logger(__name__)





class 							A__	(			__SCREAMING_SNAKE_CASE):
      _UpperCAmelCase       :					Tuple					 =    ["""input_ids""", """attention_mask"""]
      def __init__(   self					,						__magic_name__="[PAD]"					,						__magic_name__="[BOS]"					,						__magic_name__="[EOS]"					,						__magic_name__="[MASK]"					,						__magic_name__="[CLS]"					,						__magic_name__="[SEP]"					,						__magic_name__=2_0_4_8					,						**__magic_name__					,						):
       lowerCamelCase	:			List[str]           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else pad_token
       lowerCamelCase	:			Dict           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else bos_token
       lowerCamelCase	:			Any           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else eos_token
       lowerCamelCase	:			Any           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else mask_token
       lowerCamelCase	:			Any           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else cls_token
       lowerCamelCase	:			Optional[Any]           =   AddedToken(__magic_name__					,						lstrip=__magic_name__					,						rstrip=__magic_name__   ) if isinstance(__magic_name__					,						__magic_name__   ) else sep_token

       super().__init__(
           pad_token=__magic_name__					,						bos_token=__magic_name__					,						eos_token=__magic_name__					,						mask_token=__magic_name__					,						cls_token=__magic_name__					,						sep_token=__magic_name__					,						model_max_length=__magic_name__					,						**__magic_name__					,						)

       lowerCamelCase	:			List[Any]           =   2**8  # utf is 8 bits

       # define special tokens dict
       lowerCamelCase	:			Dict[str, int]           =   {
           self.pad_token: 0,
           self.bos_token: 1,
           self.eos_token: 2,
           self.mask_token: 3,
           self.cls_token: 4,
           self.sep_token: 5,
       }
       lowerCamelCase	:			Optional[int]           =   len(self.special_tokens_encoder   )
       lowerCamelCase	:			Dict[int, str]           =   {v: k for k, v in self.special_tokens_encoder.items()}
      def 		UpperCamelCase__							(   self   ):
       lowerCamelCase	:			List[str]           =   self.special_tokens_encoder.copy()
       vocab.update(self.added_tokens_encoder   )
       for i in range(self._utf_vocab_size   ):
        lowerCamelCase	:			List[Any]           =   chr(__magic_name__   )
        lowerCamelCase	:			Union[str, Any]           =   i + len(self.special_tokens_encoder   )
       return vocab
      @property
      def 		UpperCamelCase__							(   self   ):
       return self._utf_vocab_size + self._num_special_tokens
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None					,						__magic_name__ = False   ):
       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__magic_name__					,						token_ids_a=__magic_name__					,						already_has_special_tokens=__magic_name__   )

       # normal case: some special tokens
       if token_ids_a is None:
        return [1] + [0] * len(__magic_name__   ) + [1]
       return [1] + ([0] * len(__magic_name__   )) + [1] + ([0] * len(__magic_name__   )) + [1]
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):
       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       else:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       lowerCamelCase	:			Optional[int]           =   [chr(__magic_name__   ) for i in text.encode("""utf-8"""   )]
       return tokens
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       if token in self.special_tokens_encoder:
        lowerCamelCase	:			List[str]           =   self.special_tokens_encoder[token]
       elif token in self.added_tokens_encoder:
        lowerCamelCase	:			List[Any]           =   self.added_tokens_encoder[token]
       elif len(__magic_name__   ) != 1:
        lowerCamelCase	:			Union[str, Any]           =   self.unk_token_id
       else:
        lowerCamelCase	:			int           =   ord(__magic_name__   ) + self._num_special_tokens
       return token_id
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       if index in self.special_tokens_decoder:
        lowerCamelCase	:			Union[str, Any]           =   self.special_tokens_decoder[index]
       elif index in self.added_tokens_decoder:
        lowerCamelCase	:			List[str]           =   self.added_tokens_decoder[index]
       else:
        lowerCamelCase	:			Optional[int]           =   chr(index - self._num_special_tokens   )
       return token
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       lowerCamelCase	:			List[Any]           =   b""""""
       for token in tokens:
        if token in self.special_tokens_decoder:
         lowerCamelCase	:			List[str]           =   self.special_tokens_decoder[token].encode("""utf-8"""   )
        elif token in self.added_tokens_decoder:
         lowerCamelCase	:			Optional[int]           =   self.special_tokens_decoder[token].encode("""utf-8"""   )
        elif token in self.special_tokens_encoder:
         lowerCamelCase	:			Tuple           =   token.encode("""utf-8"""   )
        elif token in self.added_tokens_encoder:
         lowerCamelCase	:			Union[str, Any]           =   token.encode("""utf-8"""   )
        else:
         lowerCamelCase	:			Dict           =   bytes([ord(__magic_name__   )]   )
        bstring += tok_string
       lowerCamelCase	:			Any           =   bstring.decode("""utf-8"""					,						errors="""replace"""   )
       return string






      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):
       return ()




