code: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  lowercase__    :       Optional[Any]						=			logging.get_logger(__name__)
  
  lowercase__    :       Optional[int]						=			{
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  lowercase__    :       List[str]						=			{
      "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
      "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
  }
  
  lowercase__    :       Optional[int]						=			{
      "openai-gpt": 5_1_2,
  }
  
  def      SCREAMING_SNAKE_CASE_  (		snake_case__							)  ->			str:
  							lowerCAmelCase				=	text.strip()
  							if not text:
  														return []
  							lowerCAmelCase				=	text.split()
  							return tokens
  
  
  
  
  class      lowercase_						(						SCREAMING_SNAKE_CASE__				):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=None	,    __SCREAMING_SNAKE_CASE=True	,    __SCREAMING_SNAKE_CASE=None	,    __SCREAMING_SNAKE_CASE=True	,    )		->Any:
  
  
  
  								if never_split is None:
  															lowerCAmelCase				=	[]
  								lowerCAmelCase				=	do_lower_case
  								lowerCAmelCase				=	set(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	tokenize_chinese_chars
  								lowerCAmelCase				=	strip_accents
  								lowerCAmelCase				=	do_split_on_punc
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=None       )		->Dict:
  								lowerCAmelCase				=	self.never_split.union(set(__SCREAMING_SNAKE_CASE       )       ) if never_split else self.never_split
  								lowerCAmelCase				=	self._clean_text(__SCREAMING_SNAKE_CASE       )
  
  								# This was added on November 1st, 2018 for the multilingual and Chinese
  								# models. This is also applied to the English models now, but it doesn't
  								# matter since the English models were not trained on any Chinese data
  								# and generally don't have any Chinese data in them (there are Chinese
  								# characters in the vocabulary because Wikipedia does have some Chinese
  								# words in the English Wikipedia.).
  								if self.tokenize_chinese_chars:
  															lowerCAmelCase				=	self._tokenize_chinese_chars(__SCREAMING_SNAKE_CASE       )
  								# prevents treating the same character with different unicode codepoints as different characters
  								lowerCAmelCase				=	unicodedata.normalize('''NFC'''	,    __SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	whitespace_tokenize(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	[]
  								for token in orig_tokens:
  															if token not in never_split:
  																						if self.do_lower_case:
  																													lowerCAmelCase				=	token.lower()
  																													if self.strip_accents is not False:
  																																				lowerCAmelCase				=	self._run_strip_accents(__SCREAMING_SNAKE_CASE       )
  																						elif self.strip_accents:
  																													lowerCAmelCase				=	self._run_strip_accents(__SCREAMING_SNAKE_CASE       )
  															split_tokens.extend(self._run_split_on_punc(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )
  
  								lowerCAmelCase				=	whitespace_tokenize(''' '''.join(__SCREAMING_SNAKE_CASE       )       )
  								return output_tokens
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->int:
  								lowerCAmelCase				=	unicodedata.normalize('''NFD'''	,    __SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	[]
  								for char in text:
  															lowerCAmelCase				=	unicodedata.category(__SCREAMING_SNAKE_CASE       )
  															if cat == "Mn":
  																						continue
  															output.append(__SCREAMING_SNAKE_CASE       )
  								return "".join(__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=None       )		->List[Any]:
  
  
  
  								if not self.do_split_on_punc or (never_split is not None and text in never_split):
  															return [text]
  								lowerCAmelCase				=	list(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	0
  								lowerCAmelCase				=	True
  								lowerCAmelCase				=	[]
  								while i < len(__SCREAMING_SNAKE_CASE       ):
  															lowerCAmelCase				=	chars[i]
  															if _is_punctuation(__SCREAMING_SNAKE_CASE       ):
  																						output.append([char]       )
  																						lowerCAmelCase				=	True
  															else:
  																						if start_new_word:
  																													output.append([]       )
  																						lowerCAmelCase				=	False
  																						output[-1].append(__SCREAMING_SNAKE_CASE       )
  															i += 1
  
  								return ["".join(__SCREAMING_SNAKE_CASE       ) for x in output]
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->List[str]:
  								lowerCAmelCase				=	[]
  								for char in text:
  															lowerCAmelCase				=	ord(__SCREAMING_SNAKE_CASE       )
  															if self._is_chinese_char(__SCREAMING_SNAKE_CASE       ):
  																						output.append(''' '''       )
  																						output.append(__SCREAMING_SNAKE_CASE       )
  																						output.append(''' '''       )
  															else:
  																						output.append(__SCREAMING_SNAKE_CASE       )
  								return "".join(__SCREAMING_SNAKE_CASE       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
  
  
  
  								if (
  								    (cp >= 0X4_E00 and cp <= 0X9_FFF)
  								    or (cp >= 0X3_400 and cp <= 0X4_DBF)  #
  								    or (cp >= 0X20_000 and cp <= 0X2A_6DF)  #
  								    or (cp >= 0X2A_700 and cp <= 0X2B_73F)  #
  								    or (cp >= 0X2B_740 and cp <= 0X2B_81F)  #
  								    or (cp >= 0X2B_820 and cp <= 0X2C_EAF)  #
  								    or (cp >= 0XF_900 and cp <= 0XF_AFF)
  								    or (cp >= 0X2F_800 and cp <= 0X2F_A1F)  #
  								):  #
  															return True
  
  								return False
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								lowerCAmelCase				=	[]
  								for char in text:
  															lowerCAmelCase				=	ord(__SCREAMING_SNAKE_CASE       )
  															if cp == 0 or cp == 0XF_FFD or _is_control(__SCREAMING_SNAKE_CASE       ):
  																						continue
  															if _is_whitespace(__SCREAMING_SNAKE_CASE       ):
  																						output.append(''' '''       )
  															else:
  																						output.append(__SCREAMING_SNAKE_CASE       )
  								return "".join(__SCREAMING_SNAKE_CASE       )
  
  def      SCREAMING_SNAKE_CASE_  (		snake_case__							)  ->			Tuple:
  							lowerCAmelCase				=	set()
  							lowerCAmelCase				=	word[0]
  							for char in word[1:]:
  														pairs.add((prev_char, char)							)
  														lowerCAmelCase				=	char
  							return pairs
  
  def      SCREAMING_SNAKE_CASE_  (		snake_case__							)  ->			Optional[Any]:
  							lowerCAmelCase				=	text.replace('''—'''     ,						'''-'''							)
  							lowerCAmelCase				=	text.replace('''–'''     ,						'''-'''							)
  							lowerCAmelCase				=	text.replace('''―'''     ,						'''-'''							)
  							lowerCAmelCase				=	text.replace('''…'''     ,						'''...'''							)
  							lowerCAmelCase				=	text.replace('''´'''     ,						'''\''''							)
  							lowerCAmelCase				=	re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''     ,						R''' \1 '''     ,						snake_case__							)
  							lowerCAmelCase				=	re.sub(R'''\s*\n\s*'''     ,						''' \n '''     ,						snake_case__							)
  							lowerCAmelCase				=	re.sub(R'''[^\S\n]+'''     ,						''' '''     ,						snake_case__							)
  							return text.strip()
  
  
  
  
  
  
  class      lowercase_						(						SCREAMING_SNAKE_CASE__				):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	UpperCAmelCase_						:      Any     =       VOCAB_FILES_NAMES
  	UpperCAmelCase_						:      List[str]     =       PRETRAINED_VOCAB_FILES_MAP
  	UpperCAmelCase_						:      Tuple     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	UpperCAmelCase_						:      Any     =       ["""input_ids""", """attention_mask"""]
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE="<unk>"	,    **__SCREAMING_SNAKE_CASE       )		->Any:
  								super().__init__(unk_token=__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE       )
  
  								try:
  															import ftfy
  															from spacy.lang.en import English
  
  															lowerCAmelCase				=	English()
  															lowerCAmelCase				=	_nlp.tokenizer
  															lowerCAmelCase				=	ftfy.fix_text
  								except ImportError:
  															logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''       )
  															lowerCAmelCase				=	BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )
  															lowerCAmelCase				=	None
  
  								with open(__SCREAMING_SNAKE_CASE	,    encoding='''utf-8'''       ) as vocab_handle:
  															lowerCAmelCase				=	json.load(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	{v: k for k, v in self.encoder.items()}
  								with open(__SCREAMING_SNAKE_CASE	,    encoding='''utf-8'''       ) as merges_handle:
  															lowerCAmelCase				=	merges_handle.read().split('''\n'''       )[1:-1]
  								lowerCAmelCase				=	[tuple(merge.split()       ) for merge in merges]
  								lowerCAmelCase				=	dict(zip(__SCREAMING_SNAKE_CASE	,    range(len(__SCREAMING_SNAKE_CASE       )       )       )       )
  								lowerCAmelCase				=	{}
  
  
  	@property
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Union[str, Any]:
  								return True
  
  
  	@property
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								return len(self.encoder       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[Any]:
  								return dict(self.encoder	,    **self.added_tokens_encoder       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->List[Any]:
  								lowerCAmelCase				=	tuple(token[:-1]       ) + (token[-1] + '''</w>''',)
  								if token in self.cache:
  															return self.cache[token]
  								lowerCAmelCase				=	get_pairs(__SCREAMING_SNAKE_CASE       )
  
  								if not pairs:
  															return token + "</w>"
  
  								while True:
  															lowerCAmelCase				=	min(__SCREAMING_SNAKE_CASE	,    key=lambda __SCREAMING_SNAKE_CASE       : self.bpe_ranks.get(__SCREAMING_SNAKE_CASE	,    float('''inf'''       )       )       )
  															if bigram not in self.bpe_ranks:
  																						break
  															lowerCAmelCase	,   lowerCAmelCase				=	bigram
  															lowerCAmelCase				=	[]
  															lowerCAmelCase				=	0
  															while i < len(__SCREAMING_SNAKE_CASE       ):
  																						try:
  																													lowerCAmelCase				=	word.index(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
  																						except ValueError:
  																													new_word.extend(word[i:]       )
  																													break
  																						else:
  																													new_word.extend(word[i:j]       )
  																													lowerCAmelCase				=	j
  
  																						if word[i] == first and i < len(__SCREAMING_SNAKE_CASE       ) - 1 and word[i + 1] == second:
  																													new_word.append(first + second       )
  																													i += 2
  																						else:
  																													new_word.append(word[i]       )
  																													i += 1
  															lowerCAmelCase				=	tuple(__SCREAMING_SNAKE_CASE       )
  															lowerCAmelCase				=	new_word
  															if len(__SCREAMING_SNAKE_CASE       ) == 1:
  																						break
  															else:
  																						lowerCAmelCase				=	get_pairs(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	''' '''.join(__SCREAMING_SNAKE_CASE       )
  								if word == "\n  </w>":
  															lowerCAmelCase				=	'''\n</w>'''
  								lowerCAmelCase				=	word
  								return word
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Optional[int]:
  								lowerCAmelCase				=	[]
  								if self.fix_text is None:
  															# Using BERT's BasicTokenizer
  															lowerCAmelCase				=	self.nlp.tokenize(__SCREAMING_SNAKE_CASE       )
  															for token in text:
  																						split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE       ).split(''' '''       )       )       )
  								else:
  															# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  															lowerCAmelCase				=	self.nlp(text_standardize(self.fix_text(__SCREAMING_SNAKE_CASE       )       )       )
  															for token in text:
  																						split_tokens.extend(list(self.bpe(token.text.lower()       ).split(''' '''       )       )       )
  								return split_tokens
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Optional[int]:
  								return self.encoder.get(__SCREAMING_SNAKE_CASE	,    self.encoder.get(self.unk_token       )       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								return self.decoder.get(__SCREAMING_SNAKE_CASE	,    self.unk_token       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Optional[Any]:
  								lowerCAmelCase				=	''''''.join(__SCREAMING_SNAKE_CASE       ).replace('''</w>'''	,    ''' '''       ).strip()
  								return out_string
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = None       )		->Tuple[str]:
  
  
  
  								if not os.path.isdir(__SCREAMING_SNAKE_CASE       ):
  															logger.error(F"Vocabulary path ({save_directory}) should be a directory"       )
  															return
  								lowerCAmelCase				=	os.path.join(
  								    __SCREAMING_SNAKE_CASE	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
  								lowerCAmelCase				=	os.path.join(
  								    __SCREAMING_SNAKE_CASE	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )
  
  								with open(__SCREAMING_SNAKE_CASE	,    '''w'''	,    encoding='''utf-8'''       ) as f:
  															f.write(json.dumps(self.encoder	,    indent=2	,    sort_keys=__SCREAMING_SNAKE_CASE	,    ensure_ascii=__SCREAMING_SNAKE_CASE       ) + '''\n'''       )
  
  								lowerCAmelCase				=	0
  								with open(__SCREAMING_SNAKE_CASE	,    '''w'''	,    encoding='''utf-8'''       ) as writer:
  															writer.write('''#version: 0.2\n'''       )
  															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,    key=lambda __SCREAMING_SNAKE_CASE       : kv[1]       ):
  																						if index != token_index:
  																													logger.warning(
  																													    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
  																													    ''' Please check that the tokenizer is not corrupted!'''       )
  																													lowerCAmelCase				=	token_index
  																						writer.write(''' '''.join(__SCREAMING_SNAKE_CASE       ) + '''\n'''       )
  																						index += 1
  
  								return vocab_file, merge_file
  
  
  
  
style_context: from typing import Callable, Optional, Union
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  lowercase__    :       Optional[Any]						=			logging.get_logger(__name__)
  
  lowercase__    :       Union[str, Any]						=			{
      '''microsoft/prophetnet-large-uncased''': (
          '''https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'''
      ),
  }
  
  
  
  
  class      lowercase_						(						UpperCamelCase_		):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	UpperCAmelCase_						:      Optional[int]     =       """prophetnet"""
  	UpperCAmelCase_						:      Optional[int]     =       ["""past_key_values"""]
  	UpperCAmelCase_						:      Any     =       {
  	    """num_attention_heads""": """num_encoder_attention_heads""",
  	}
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = "gelu"	,    __SCREAMING_SNAKE_CASE = 30522	,    __SCREAMING_SNAKE_CASE = 1024	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 512	,    __SCREAMING_SNAKE_CASE = 0.0_2	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 2	,    __SCREAMING_SNAKE_CASE = 32	,    __SCREAMING_SNAKE_CASE = 128	,    __SCREAMING_SNAKE_CASE = False	,    __SCREAMING_SNAKE_CASE = 0.0	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 1	,    __SCREAMING_SNAKE_CASE = 2	,    **__SCREAMING_SNAKE_CASE	,    )		->int:
  								lowerCAmelCase				=	vocab_size
  								lowerCAmelCase				=	hidden_size
  								lowerCAmelCase				=	encoder_ffn_dim
  								lowerCAmelCase				=	num_encoder_layers
  								lowerCAmelCase				=	num_encoder_attention_heads
  								lowerCAmelCase				=	decoder_ffn_dim
  								lowerCAmelCase				=	num_decoder_layers
  								lowerCAmelCase				=	num_decoder_attention_heads
  								lowerCAmelCase				=	max_position_embeddings
  								lowerCAmelCase				=	init_std  # Normal(0, this parameter)
  								lowerCAmelCase				=	activation_function
  
  								# parameters for prophetnet
  								lowerCAmelCase				=	ngram
  								lowerCAmelCase				=	num_buckets
  								lowerCAmelCase				=	relative_max_distance
  								lowerCAmelCase				=	disable_ngram_loss
  								lowerCAmelCase				=	eps
  
  								# 3 Types of Dropout
  								lowerCAmelCase				=	attention_dropout
  								lowerCAmelCase				=	activation_dropout
  								lowerCAmelCase				=	dropout
  
  								lowerCAmelCase				=	use_cache
  
  								super().__init__(
  								    pad_token_id=__SCREAMING_SNAKE_CASE	,    bos_token_id=__SCREAMING_SNAKE_CASE	,    eos_token_id=__SCREAMING_SNAKE_CASE	,    is_encoder_decoder=__SCREAMING_SNAKE_CASE	,    add_cross_attention=__SCREAMING_SNAKE_CASE	,    decoder_start_token_id=__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE	,    )
  
  
  	@property
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								return self.num_encoder_layers + self.num_decoder_layers
  
  
  
  
  	@num_hidden_layers.setter
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
  								raise NotImplementedError(
  								    '''This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'''
  								    ''' `num_decoder_layers`.'''       )
  
  
  
  