






'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

a_      = logging.get_logger(__name__)

a_      = {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a_      = {
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

a_      = {
    'gpt2': 1_024,
    'gpt2-medium': 1_024,
    'gpt2-large': 1_024,
    'gpt2-xl': 1_024,
    'distilgpt2': 1_024,
}



@lru_cache()
def 			__UpperCAmelCase				()		->				List[Any]:

						'''simple docstring'''


						a_				     =   (
						    list(range(ord("!"				)	,ord("~"				) + 1				)				) + list(range(ord("¡"				)	,ord("¬"				) + 1				)				) + list(range(ord("®"				)	,ord("ÿ"				) + 1				)				)
						)
						a_				     =   bs[:]
						a_				     =   0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(lowercase__				)
																		cs.append(2**8 + n				)
																		n += 1
						a_				     =   [chr(lowercase__				) for n in cs]
						return dict(zip(lowercase__	,lowercase__				)				)



def 			__UpperCAmelCase				(lowercase__				)		->				Optional[Any]:

						'''simple docstring'''


						a_				     =   set()
						a_				     =   word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												a_				     =   char
						return pairs





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):
		_UpperCAmelCase         =VOCAB_FILES_NAMES
		_UpperCAmelCase         =PRETRAINED_VOCAB_FILES_MAP
		_UpperCAmelCase         =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_UpperCAmelCase         =['''input_ids''', '''attention_mask''']



		def __init__(    self:    Any				,   a:    Optional[Any]				,   a:    Optional[int]				,   a:    Union[str, Any]="replace"				,   a:    str="<|endoftext|>"				,   a:    Optional[int]="<|endoftext|>"				,   a:    str="<|endoftext|>"				,   a:    str=None				,   a:    Union[str, Any]=False				,   a:    str=False				,   **a:    List[Any]				,   )  ->Union[str, Any]:






								'''simple docstring'''


								a_				     =   AddedToken(a				,   lstrip=a				,   rstrip=a) if isinstance(a				,   a) else bos_token
								a_				     =   AddedToken(a				,   lstrip=a				,   rstrip=a) if isinstance(a				,   a) else eos_token
								a_				     =   AddedToken(a				,   lstrip=a				,   rstrip=a) if isinstance(a				,   a) else unk_token
								a_				     =   AddedToken(a				,   lstrip=a				,   rstrip=a) if isinstance(a				,   a) else pad_token
								super().__init__(
								    errors=a				,   unk_token=a				,   bos_token=a				,   eos_token=a				,   pad_token=a				,   add_prefix_space=a				,   add_bos_token=a				,   **a				,   )
								a_				     =   add_bos_token

								with open(a				,   encoding="utf-8") as vocab_handle:
														a_				     =   json.load(a)
								a_				     =   {v: k for k, v in self.encoder.items()}
								a_				     =   errors  # how to handle errors in decoding
								a_				     =   bytes_to_unicode()
								a_				     =   {v: k for k, v in self.byte_encoder.items()}
								with open(a				,   encoding="utf-8") as merges_handle:
														a_				     =   merges_handle.read().split("\n")[1:-1]
								a_				     =   [tuple(merge.split()) for merge in bpe_merges]
								a_				     =   dict(zip(a				,   range(len(a))))
								a_				     =   {}
								a_				     =   add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								a_				     =   re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")



		@property
		def _lowerCAmelCase     (    self:    Optional[int])  ->Optional[Any]:






								'''simple docstring'''


								return len(self.encoder)



		def _lowerCAmelCase     (    self:    Union[str, Any])  ->Union[str, Any]:






								'''simple docstring'''


								return dict(self.encoder				,   **self.added_tokens_encoder)



		def _lowerCAmelCase     (    self:    Optional[Any]				,   a:    str)  ->Dict:






								'''simple docstring'''


								if token in self.cache:
														return self.cache[token]
								a_				     =   tuple(a)
								a_				     =   get_pairs(a)

								if not pairs:
														return token

								while True:
														a_				     =   min(a				,   key=lambda a: self.bpe_ranks.get(a				,   float("inf")))
														if bigram not in self.bpe_ranks:
																				break
														a_     ,       a_				     =   bigram
														a_				     =   []
														a_				     =   0
														while i < len(a):
																				try:
																										a_				     =   word.index(a				,   a)
																				except ValueError:
																										new_word.extend(word[i:])
																										break
																				else:
																										new_word.extend(word[i:j])
																										a_				     =   j

																				if word[i] == first and i < len(a) - 1 and word[i + 1] == second:
																										new_word.append(first + second)
																										i += 2
																				else:
																										new_word.append(word[i])
																										i += 1
														a_				     =   tuple(a)
														a_				     =   new_word
														if len(a) == 1:
																				break
														else:
																				a_				     =   get_pairs(a)
								a_				     =   " ".join(a)
								a_				     =   word
								return word



		def _lowerCAmelCase     (    self:    Any				,   a:    Optional[Any]				,   a:    int=None)  ->Any:






								'''simple docstring'''


								if self.add_bos_token:
														a_				     =   [self.bos_token_id]
								else:
														a_				     =   []

								a_				     =   bos_token_ids + token_ids_a

								if token_ids_a is None:
														return output

								return output + bos_token_ids + token_ids_a



		def _lowerCAmelCase     (    self:    Tuple				,   a:    List[int]				,   a:    Optional[List[int]] = None				,   a:    bool = False)  ->List[int]:






								'''simple docstring'''


								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=a				,   token_ids_a=a				,   already_has_special_tokens=a)

								if not self.add_bos_token:
														return super().get_special_tokens_mask(
														    token_ids_a=a				,   token_ids_a=a				,   already_has_special_tokens=a)

								if token_ids_a is None:
														return [1] + ([0] * len(a))
								return [1] + ([0] * len(a)) + [1] + ([0] * len(a))



		def _lowerCAmelCase     (    self:    int				,   a:    Optional[Any])  ->Any:






								'''simple docstring'''


								a_				     =   []
								for token in re.findall(self.pat				,   a):
														a_				     =   "".join(
														    self.byte_encoder[b] for b in token.encode("utf-8"))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
														bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a).split(" "))
								return bpe_tokens



		def _lowerCAmelCase     (    self:    List[Any]				,   a:    Dict)  ->List[str]:






								'''simple docstring'''


								return self.encoder.get(a				,   self.encoder.get(self.unk_token))



		def _lowerCAmelCase     (    self:    Optional[int]				,   a:    List[Any])  ->int:






								'''simple docstring'''


								return self.decoder.get(a)



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   a:    Tuple)  ->str:






								'''simple docstring'''


								a_				     =   "".join(a)
								a_				     =   bytearray([self.byte_decoder[c] for c in text]).decode("utf-8"				,   errors=self.errors)
								return text



		def _lowerCAmelCase     (    self:    int				,   a:    str				,   a:    Optional[str] = None)  ->Tuple[str]:






								'''simple docstring'''


								if not os.path.isdir(a):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
														return
								a_				     =   os.path.join(
								    a				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
								a_				     =   os.path.join(
								    a				,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

								with open(a				,   "w"				,   encoding="utf-8") as f:
														f.write(json.dumps(self.encoder				,   indent=2				,   sort_keys=a				,   ensure_ascii=a) + "\n")

								a_				     =   0
								with open(a				,   "w"				,   encoding="utf-8") as writer:
														writer.write("#version: 0.2\n")
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,   key=lambda a: kv[1]):
																				if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																										    " Please check that the tokenizer is not corrupted!")
																										a_				     =   token_index
																				writer.write(" ".join(a) + "\n")
																				index += 1

								return vocab_file, merge_file



		def _lowerCAmelCase     (    self:    Union[str, Any]				,   a:    Optional[Any]				,   a:    str=False				,   **a:    Optional[int])  ->List[Any]:






								'''simple docstring'''


								a_				     =   kwargs.pop("add_prefix_space"				,   self.add_prefix_space)
								if is_split_into_words or add_prefix_space:
														a_				     =   " " + text
								return (text, kwargs)




		def _lowerCAmelCase     (    self:    Dict				,   a:    "Conversation")  ->List[int]:






								'''simple docstring'''


								a_				     =   []
								for is_user, text in conversation.iter_texts():
														input_ids.extend(self.encode(a				,   add_special_tokens=a) + [self.eos_token_id])
								if len(a) > self.model_max_length:
														a_				     =   input_ids[-self.model_max_length :]
								return input_ids











'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a_      = logging.get_logger(__name__)

a_      = {
    'google/mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/config.json'
}





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):
		_UpperCAmelCase         =MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
		_UpperCAmelCase         ='''mobilebert'''



		def __init__(    self:    Union[str, Any]				,   a:    str=3_05_22				,   a:    List[str]=5_12				,   a:    List[str]=24				,   a:    Dict=4				,   a:    List[str]=5_12				,   a:    Optional[int]="relu"				,   a:    Dict=0.0				,   a:    Union[str, Any]=0.1				,   a:    int=5_12				,   a:    Optional[Any]=2				,   a:    List[Any]=0.02				,   a:    Optional[Any]=1e-12				,   a:    List[Any]=0				,   a:    Optional[int]=1_28				,   a:    Union[str, Any]=True				,   a:    Dict=True				,   a:    List[Any]=1_28				,   a:    Any=False				,   a:    Any=True				,   a:    Dict=4				,   a:    List[Any]="no_norm"				,   a:    Any=True				,   a:    Tuple=None				,   **a:    Any				,   )  ->Any:






								'''simple docstring'''


								super().__init__(pad_token_id=a				,   **a)

								a_				     =   vocab_size
								a_				     =   hidden_size
								a_				     =   num_hidden_layers
								a_				     =   num_attention_heads
								a_				     =   hidden_act
								a_				     =   intermediate_size
								a_				     =   hidden_dropout_prob
								a_				     =   attention_probs_dropout_prob
								a_				     =   max_position_embeddings
								a_				     =   type_vocab_size
								a_				     =   initializer_range
								a_				     =   layer_norm_eps
								a_				     =   embedding_size
								a_				     =   trigram_input
								a_				     =   use_bottleneck
								a_				     =   intra_bottleneck_size
								a_				     =   use_bottleneck_attention
								a_				     =   key_query_shared_bottleneck
								a_				     =   num_feedforward_networks
								a_				     =   normalization_type
								a_				     =   classifier_activation

								if self.use_bottleneck:
														a_				     =   intra_bottleneck_size
								else:
														a_				     =   hidden_size

								a_				     =   classifier_dropout





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):



		@property
		def _lowerCAmelCase     (    self:    Optional[int])  ->Mapping[str, Mapping[int, str]]:






								'''simple docstring'''


								if self.task == "multiple-choice":
														a_				     =   {0: "batch", 1: "choice", 2: "sequence"}
								else:
														a_				     =   {0: "batch", 1: "sequence"}
								return OrderedDict(
								    [
								        ("input_ids", dynamic_axis),
								        ("attention_mask", dynamic_axis),
								        ("token_type_ids", dynamic_axis),
								    ])



