






import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class       _UpperCAmelCase					(       A__		,unittest.TestCase	):

				"""simple docstring"""




				lowercase__											=       MgpstrTokenizer
				lowercase__											=       False
				lowercase__											=       {}
				lowercase__											=       False




				def 				lowercase__   (     self						:    Any  ):

								'''simple docstring'''
								super().setUp()

								# fmt: off
								lowercase__    =			['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
								# fmt: on
								lowercase__    =			dict(zip(lowerCamelCase,  range(len(lowerCamelCase  )  )  )  )

								lowercase__    =			os.path.join(self.tmpdirname,  VOCAB_FILES_NAMES['''vocab_file''']  )
								with open(self.vocab_file,  '''w''',  encoding='''utf-8'''  ) as fp:
												fp.write(json.dumps(lowerCamelCase  ) + '''\n'''  )




				def 				lowercase__   (     self						:    str,  **lowerCamelCase						:    Dict  ):

								'''simple docstring'''
								return MgpstrTokenizer.from_pretrained(self.tmpdirname,  **lowerCamelCase  )




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    Any  ):

								'''simple docstring'''
								lowercase__    =			'''tester'''
								lowercase__    =			'''tester'''
								return input_text, output_text




				@unittest.skip('''MGP-STR always lower cases letters.'''  )
				def 				lowercase__   (     self						:    Any  ):

								'''simple docstring'''
								pass




				def 				lowercase__   (     self						:    Optional[Any]  ):

								'''simple docstring'''
								lowercase__    =			self.get_tokenizers(do_lower_case=lowerCamelCase  )
								for tokenizer in tokenizers:
												with self.subTest(F"""{tokenizer.__class__.__name__}"""  ):
																lowercase__    =			'''[SPECIAL_TOKEN]'''

																tokenizer.add_special_tokens({'''cls_token''': special_token}  )
																lowercase__    =			tokenizer.encode([special_token],  add_special_tokens=lowerCamelCase  )
																self.assertEqual(len(lowerCamelCase  ),  1  )

																lowercase__    =			tokenizer.decode(lowerCamelCase,  skip_special_tokens=lowerCamelCase  )
																self.assertTrue(special_token not in decoded  )




				def 				lowercase__   (     self						:    Optional[Any]  ):

								'''simple docstring'''
								lowercase__    =			self.get_tokenizers()
								for tokenizer in tokenizers:
												with self.subTest(F"""{tokenizer.__class__.__name__}"""  ):
																lowercase__ ,						lowercase__    =			self.get_input_output_texts(lowerCamelCase  )

																lowercase__    =			tokenizer.tokenize(lowerCamelCase  )
																lowercase__    =			tokenizer.convert_tokens_to_ids(lowerCamelCase  )
																lowercase__    =			tokenizer.encode(lowerCamelCase,  add_special_tokens=lowerCamelCase  )
																self.assertListEqual(lowerCamelCase,  lowerCamelCase  )

																lowercase__    =			tokenizer.convert_ids_to_tokens(lowerCamelCase  )
																self.assertNotEqual(len(lowerCamelCase  ),  0  )
																lowercase__    =			tokenizer.decode(lowerCamelCase  )
																self.assertIsInstance(lowerCamelCase,  lowerCamelCase  )

																self.assertEqual(text_a.replace(''' ''',  ''''''  ),  lowerCamelCase  )




				@unittest.skip('''MGP-STR tokenizer only handles one sequence.'''  )
				def 				lowercase__   (     self						:    Union[str, Any]  ):

								'''simple docstring'''
								pass




				@unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''  )
				def 				lowercase__   (     self						:    Tuple  ):

								'''simple docstring'''
								pass












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A__	:			List[Any]			   = logging.get_logger(__name__)

A__	:			str			   = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:			int			   = {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__	:			Optional[Any]			   = {
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


A__	:			List[str]			   = {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       ["""input_ids""", """attention_mask"""]
				lowercase__											=       DistilBertTokenizer




				def __init__(     self						:    List[Any],  lowerCamelCase						:    List[Any]=None,  lowerCamelCase						:    Dict=None,  lowerCamelCase						:    str=True,  lowerCamelCase						:    Optional[int]="[UNK]",  lowerCamelCase						:    Optional[Any]="[SEP]",  lowerCamelCase						:    List[Any]="[PAD]",  lowerCamelCase						:    Any="[CLS]",  lowerCamelCase						:    Union[str, Any]="[MASK]",  lowerCamelCase						:    str=True,  lowerCamelCase						:    int=None,  **lowerCamelCase						:    Union[str, Any],  ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,  tokenizer_file=lowerCamelCase,  do_lower_case=lowerCamelCase,  unk_token=lowerCamelCase,  sep_token=lowerCamelCase,  pad_token=lowerCamelCase,  cls_token=lowerCamelCase,  mask_token=lowerCamelCase,  tokenize_chinese_chars=lowerCamelCase,  strip_accents=lowerCamelCase,  **lowerCamelCase,  )

								lowercase__    =			json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
								if (
								    normalizer_state.get('''lowercase''',  lowerCamelCase  ) != do_lower_case
								    or normalizer_state.get('''strip_accents''',  lowerCamelCase  ) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',  lowerCamelCase  ) != tokenize_chinese_chars
								):
												lowercase__    =			getattr(lowerCamelCase,  normalizer_state.pop('''type'''  )  )
												lowercase__    =			do_lower_case
												lowercase__    =			strip_accents
												lowercase__    =			tokenize_chinese_chars
												lowercase__    =			normalizer_class(**lowerCamelCase  )

								lowercase__    =			do_lower_case




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    List[Any]=None  ):

								'''simple docstring'''
								lowercase__    =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.sep_token_id]
								lowercase__    =			[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			self._tokenizer.model.save(lowerCamelCase,  name=lowerCamelCase  )
								return tuple(lowerCamelCase  )




