code: import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple, Union
  
  import numpy as np
  import regex as re
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  if TYPE_CHECKING:
  	if is_torch_available():
  		import torch
  	if is_tf_available():
  		import tensorflow as tf
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  
  
  lowercase__    :       Dict						=			logging.get_logger(__name__)
  
  lowercase__    :       Union[str, Any]						=			{
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  lowercase__    :       int						=			{
      '''vocab_file''': {
          '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
      },
      '''merges_file''': {
          '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
      },
  }
  
  lowercase__    :       List[Any]						=			{
      '''Salesforce/codegen-350M-mono''': 2_0_4_8,
  }
  
  @lru_cache()
  def      SCREAMING_SNAKE_CASE_  (		)  ->			Dict:
  							lowerCAmelCase				=	(
  							    list(range(ord('''!'''							)     ,						ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)     ,						ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)     ,						ord('''ÿ'''							) + 1							)							)
  							)
  							lowerCAmelCase				=	bs[:]
  							lowerCAmelCase				=	0
  							for b in range(2**8							):
  														if b not in bs:
  																					bs.append(__a							)
  																					cs.append(2**8 + n							)
  																					n += 1
  							lowerCAmelCase				=	[chr(__a							) for n in cs]
  							return dict(zip(__a     ,						__a							)							)
  
  def      SCREAMING_SNAKE_CASE_  (		snake_case__							)  ->			Tuple:
  							lowerCAmelCase				=	set()
  							lowerCAmelCase				=	word[0]
  							for char in word[1:]:
  														pairs.add((prev_char, char)							)
  														lowerCAmelCase				=	char
  							return pairs
  
  
  
  
  
  
  class      lowercase_						(						UpperCamelCase__				):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	UpperCAmelCase_						:      Optional[int]     =       VOCAB_FILES_NAMES
  	UpperCAmelCase_						:      List[str]     =       PRETRAINED_VOCAB_FILES_MAP
  	UpperCAmelCase_						:      Dict     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	UpperCAmelCase_						:      Optional[Any]     =       ["""input_ids""", """attention_mask"""]
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE="replace"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE=None	,    __SCREAMING_SNAKE_CASE=False	,    __SCREAMING_SNAKE_CASE=False	,    **__SCREAMING_SNAKE_CASE	,    )		->Optional[Any]:
  								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else bos_token
  								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else eos_token
  								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else unk_token
  								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else pad_token
  								super().__init__(
  								    errors=snake_case_	,    unk_token=snake_case_	,    bos_token=snake_case_	,    eos_token=snake_case_	,    pad_token=snake_case_	,    add_prefix_space=snake_case_	,    add_bos_token=snake_case_	,    **snake_case_	,    )
  								lowerCAmelCase				=	add_bos_token
  
  								with open(snake_case_	,    encoding='''utf-8'''       ) as vocab_handle:
  															lowerCAmelCase				=	json.load(snake_case_       )
  								lowerCAmelCase				=	{v: k for k, v in self.encoder.items()}
  								lowerCAmelCase				=	errors  # how to handle errors in decoding
  								lowerCAmelCase				=	bytes_to_unicode()
  								lowerCAmelCase				=	{v: k for k, v in self.byte_encoder.items()}
  								with open(snake_case_	,    encoding='''utf-8'''       ) as merges_handle:
  															lowerCAmelCase				=	merges_handle.read().split('''\n'''       )[1:-1]
  								lowerCAmelCase				=	[tuple(merge.split()       ) for merge in bpe_merges]
  								lowerCAmelCase				=	dict(zip(snake_case_	,    range(len(snake_case_       )       )       )       )
  								lowerCAmelCase				=	{}
  								lowerCAmelCase				=	add_prefix_space
  
  								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  								lowerCAmelCase				=	re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )
  
  
  	@property
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
  								return len(self.encoder       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Tuple:
  								return dict(self.encoder	,    **self.added_tokens_encoder       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Optional[int]:
  								if token in self.cache:
  															return self.cache[token]
  								lowerCAmelCase				=	tuple(snake_case_       )
  								lowerCAmelCase				=	get_pairs(snake_case_       )
  
  								if not pairs:
  															return token
  
  								while True:
  															lowerCAmelCase				=	min(snake_case_	,    key=lambda __SCREAMING_SNAKE_CASE       : self.bpe_ranks.get(snake_case_	,    float('''inf'''       )       )       )
  															if bigram not in self.bpe_ranks:
  																						break
  															lowerCAmelCase				=	bigram
  															lowerCAmelCase				=	[]
  															lowerCAmelCase				=	0
  															while i < len(snake_case_       ):
  																						try:
  																													lowerCAmelCase				=	word.index(snake_case_	,    snake_case_       )
  																						except ValueError:
  																													new_word.extend(word[i:]       )
  																													break
  																						else:
  																													new_word.extend(word[i:j]       )
  																													lowerCAmelCase				=	j
  
  																						if word[i] == first and i < len(snake_case_       ) - 1 and word[i + 1] == second:
  																													new_word.append(first + second       )
  																													i += 2
  																						else:
  																													new_word.append(word[i]       )
  																													i += 1
  															lowerCAmelCase				=	tuple(snake_case_       )
  															lowerCAmelCase				=	new_word
  															if len(snake_case_       ) == 1:
  																						break
  															else:
  																						lowerCAmelCase				=	get_pairs(snake_case_       )
  								lowerCAmelCase				=	''' '''.join(snake_case_       )
  								lowerCAmelCase				=	word
  								return word
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=None       )		->Any:
  								if self.add_bos_token:
  															lowerCAmelCase				=	[self.bos_token_id]
  								else:
  															lowerCAmelCase				=	[]
  
  								lowerCAmelCase				=	bos_token_ids + token_ids_a
  
  								if token_ids_a is None:
  															return output
  
  								return output + bos_token_ids + token_ids_a
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->List[Any]:
  								lowerCAmelCase				=	[]
  								for token in re.findall(self.pat	,    snake_case_       ):
  															lowerCAmelCase				=	''''''.join(
  															    self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_       ).split(''' '''       )       )
  								return bpe_tokens
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Dict:
  								return self.encoder.get(snake_case_	,    self.encoder.get(self.unk_token       )       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								return self.decoder.get(snake_case_       )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								lowerCAmelCase				=	''''''.join(snake_case_       )
  								lowerCAmelCase				=	bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''	,    errors=self.errors       )
  								return text
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = None       )		->Tuple[str]:
  								if not os.path.isdir(snake_case_       ):
  															logger.error(F"Vocabulary path ({save_directory}) should be a directory"       )
  															return
  								lowerCAmelCase				=	os.path.join(
  								    snake_case_	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
  								lowerCAmelCase				=	os.path.join(
  								    snake_case_	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )
  
  								with open(snake_case_	,    '''w'''	,    encoding='''utf-8'''       ) as f:
  															f.write(json.dumps(self.encoder	,    indent=2	,    sort_keys=snake_case_	,    ensure_ascii=snake_case_       ) + '''\n'''       )
  
  								lowerCAmelCase				=	0
  								with open(snake_case_	,    '''w'''	,    encoding='''utf-8'''       ) as writer:
  															writer.write('''#version: 0.2\n'''       )
  															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,    key=lambda __SCREAMING_SNAKE_CASE       : kv[1]       ):
  																						if index != token_index:
  																													logger.warning(
  																													    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
  																													    ''' Please check that the tokenizer is not corrupted!'''       )
  																													lowerCAmelCase				=	token_index
  																						writer.write(''' '''.join(snake_case_       ) + '''\n'''       )
  																						index += 1
  
  								return vocab_file, merge_file
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=False	,    **__SCREAMING_SNAKE_CASE       )		->Dict:
  								lowerCAmelCase				=	kwargs.pop('''add_prefix_space'''	,    self.add_prefix_space       )
  								if is_split_into_words or add_prefix_space:
  															lowerCAmelCase				=	''' ''' + text
  								return (text, kwargs)
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = False	,    __SCREAMING_SNAKE_CASE = None	,    __SCREAMING_SNAKE_CASE = None	,    **__SCREAMING_SNAKE_CASE	,    )		->str:
  								lowerCAmelCase				=	super()._decode(
  								    token_ids=snake_case_	,    skip_special_tokens=snake_case_	,    clean_up_tokenization_spaces=snake_case_	,    **snake_case_	,    )
  
  								if truncate_before_pattern is not None and len(snake_case_       ) > 0:
  															lowerCAmelCase				=	self.truncate(snake_case_	,    snake_case_       )
  
  								return decoded_text
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
  								def find_re(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
  															lowerCAmelCase				=	pattern.search(snake_case_	,    snake_case_       )
  															return m.start() if m else -1
  
  								lowerCAmelCase				=	[re.compile(snake_case_	,    re.MULTILINE       ) for pattern in truncate_before_pattern]
  
  								lowerCAmelCase				=	list(re.finditer('''^print'''	,    snake_case_	,    re.MULTILINE       )       )
  
  								if len(snake_case_       ) > 1:
  															lowerCAmelCase				=	completion[: prints[1].start()]
  
  								lowerCAmelCase				=	list(re.finditer('''^def'''	,    snake_case_	,    re.MULTILINE       )       )
  
  								if len(snake_case_       ) > 1:
  															lowerCAmelCase				=	completion[: defs[1].start()]
  
  								lowerCAmelCase				=	0
  
  								lowerCAmelCase				=	[
  								    pos for pos in [find_re(snake_case_	,    snake_case_	,    snake_case_       ) for terminal in terminals] if pos != -1
  								]
  
  								if len(snake_case_       ) > 0:
  															return completion[: min(snake_case_       )]
  								else:
  															return completion
  
  
  
  
style_context: from typing import Callable, Optional, Union
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  lowercase__    :       Optional[Any]						=			logging.get_logger(__name__)
  
  lowercase__    :       Union[str, Any]						=			{
      '''microsoft/prophetnet-large-uncased''': (
          '''https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'''
      ),
  }
  
  
  
  
  class      lowercase_						(						UpperCamelCase_		):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	UpperCAmelCase_						:      Optional[int]     =       """prophetnet"""
  	UpperCAmelCase_						:      Optional[int]     =       ["""past_key_values"""]
  	UpperCAmelCase_						:      Any     =       {
  	    """num_attention_heads""": """num_encoder_attention_heads""",
  	}
  
  
  	def __init__(							self	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = "gelu"	,    __SCREAMING_SNAKE_CASE = 30522	,    __SCREAMING_SNAKE_CASE = 1024	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 512	,    __SCREAMING_SNAKE_CASE = 0.0_2	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 2	,    __SCREAMING_SNAKE_CASE = 32	,    __SCREAMING_SNAKE_CASE = 128	,    __SCREAMING_SNAKE_CASE = False	,    __SCREAMING_SNAKE_CASE = 0.0	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 1	,    __SCREAMING_SNAKE_CASE = 2	,    **__SCREAMING_SNAKE_CASE	,    )		->int:
  								lowerCAmelCase				=	vocab_size
  								lowerCAmelCase				=	hidden_size
  								lowerCAmelCase				=	encoder_ffn_dim
  								lowerCAmelCase				=	num_encoder_layers
  								lowerCAmelCase				=	num_encoder_attention_heads
  								lowerCAmelCase				=	decoder_ffn_dim
  								lowerCAmelCase				=	num_decoder_layers
  								lowerCAmelCase				=	num_decoder_attention_heads
  								lowerCAmelCase				=	max_position_embeddings
  								lowerCAmelCase				=	init_std  # Normal(0, this parameter)
  								lowerCAmelCase				=	activation_function
  
  								# parameters for prophetnet
  								lowerCAmelCase				=	ngram
  								lowerCAmelCase				=	num_buckets
  								lowerCAmelCase				=	relative_max_distance
  								lowerCAmelCase				=	disable_ngram_loss
  								lowerCAmelCase				=	eps
  
  								# 3 Types of Dropout
  								lowerCAmelCase				=	attention_dropout
  								lowerCAmelCase				=	activation_dropout
  								lowerCAmelCase				=	dropout
  
  								lowerCAmelCase				=	use_cache
  
  								super().__init__(
  								    pad_token_id=__SCREAMING_SNAKE_CASE	,    bos_token_id=__SCREAMING_SNAKE_CASE	,    eos_token_id=__SCREAMING_SNAKE_CASE	,    is_encoder_decoder=__SCREAMING_SNAKE_CASE	,    add_cross_attention=__SCREAMING_SNAKE_CASE	,    decoder_start_token_id=__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE	,    )
  
  
  	@property
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
  								return self.num_encoder_layers + self.num_decoder_layers
  
  
  
  
  	@num_hidden_layers.setter
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
  								raise NotImplementedError(
  								    '''This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'''
  								    ''' `num_decoder_layers`.'''       )
  
  
  
  