code: 
  
  
  
  import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple, Union
  
  import numpy as np
  import regex as re
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  if TYPE_CHECKING:
  				if is_torch_available():
  								import torch
  				if is_tf_available():
  								import tensorflow as tf
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  
  
  __lowerCamelCase  				=					logging.get_logger(__name__)
  
  __lowerCamelCase  				=					{
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  __lowerCamelCase  				=					{
      """vocab_file""": {
          """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
      },
      """merges_file""": {
          """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
      },
  }
  
  __lowerCamelCase  				=					{
      """Salesforce/codegen-350M-mono""": 2_0_4_8,
  }
  
  
  
  
  @lru_cache()
  def     UpperCAmelCase__ (							)   -> Dict:
  		_A           =    (
  		    list(range(ord('''!'''							)						,				ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)						,				ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)						,				ord('''ÿ'''							) + 1							)							)
  		)
  		_A           =    bs[:]
  		_A           =    0
  		for b in range(2**8							):
  				if b not in bs:
  						bs.append(__snake_case							)
  						cs.append(2**8 + n							)
  						n += 1
  		_A           =    [chr(__snake_case							) for n in cs]
  		return dict(zip(__snake_case						,				__snake_case							)							)
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case							)   -> List[str]:
  		_A           =    set()
  		_A           =    word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)							)
  				_A           =    char
  		return pairs
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
  	"""simple docstring"""
  
  
  
  	lowerCamelCase_          =			VOCAB_FILES_NAMES
  	lowerCamelCase_          =			PRETRAINED_VOCAB_FILES_MAP
  	lowerCamelCase_          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	lowerCamelCase_          =			['''input_ids''', '''attention_mask''']
  	def __init__(						self	,    a	,    a	,    a="replace"	,    a="<|endoftext|>"	,    a="<|endoftext|>"	,    a="<|endoftext|>"	,    a=None	,    a=False	,    a=False	,    **a	,    )	->							Tuple:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else bos_token
  			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else eos_token
  			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else unk_token
  			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else pad_token
  			super().__init__(
  			    errors=a	,    unk_token=a	,    bos_token=a	,    eos_token=a	,    pad_token=a	,    add_prefix_space=a	,    add_bos_token=a	,    **a	,    )
  			_A           =    add_bos_token
  
  			with open(a	,    encoding='''utf-8'''					) as vocab_handle:
  					_A           =    json.load(a					)
  			_A           =    {v: k for k, v in self.encoder.items()}
  			_A           =    errors  # how to handle errors in decoding
  			_A           =    bytes_to_unicode()
  			_A           =    {v: k for k, v in self.byte_encoder.items()}
  			with open(a	,    encoding='''utf-8'''					) as merges_handle:
  					_A           =    merges_handle.read().split('''\n'''					)[1:-1]
  			_A           =    [tuple(merge.split()					) for merge in bpe_merges]
  			_A           =    dict(zip(a	,    range(len(a					)					)					)					)
  			_A           =    {}
  			_A           =    add_prefix_space
  
  			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  			_A           =    re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)
  	@property
  	def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			return len(self.encoder					)
  	def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			return dict(self.encoder	,    **self.added_tokens_encoder					)
  	def lowercase_		(						self	,    a					)	->							int:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			if token in self.cache:
  					return self.cache[token]
  			_A           =    tuple(a					)
  			_A           =    get_pairs(a					)
  
  			if not pairs:
  					return token
  
  			while True:
  					_A           =    min(a	,    key=lambda a					: self.bpe_ranks.get(a	,    float('''inf'''					)					)					)
  					if bigram not in self.bpe_ranks:
  							break
  					_A    ,   _A           =    bigram
  					_A           =    []
  					_A           =    0
  					while i < len(a					):
  							try:
  									_A           =    word.index(a	,    a					)
  							except ValueError:
  									new_word.extend(word[i:]					)
  									break
  							else:
  									new_word.extend(word[i:j]					)
  									_A           =    j
  
  							if word[i] == first and i < len(a					) - 1 and word[i + 1] == second:
  									new_word.append(first + second					)
  									i += 2
  							else:
  									new_word.append(word[i]					)
  									i += 1
  					_A           =    tuple(a					)
  					_A           =    new_word
  					if len(a					) == 1:
  							break
  					else:
  							_A           =    get_pairs(a					)
  			_A           =    ''' '''.join(a					)
  			_A           =    word
  			return word
  	def lowercase_		(						self	,    a	,    a=None					)	->							List[Any]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			if self.add_bos_token:
  					_A           =    [self.bos_token_id]
  			else:
  					_A           =    []
  
  			_A           =    bos_token_ids + token_ids_a
  
  			if token_ids_a is None:
  					return output
  
  			return output + bos_token_ids + token_ids_a
  	def lowercase_		(						self	,    a					)	->							List[Any]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			_A           =    []
  			for token in re.findall(self.pat	,    a					):
  					_A           =    ''''''.join(
  					    self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a					).split(''' '''					)					)
  			return bpe_tokens
  	def lowercase_		(						self	,    a					)	->							Union[str, Any]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			return self.encoder.get(a	,    self.encoder.get(self.unk_token					)					)
  	def lowercase_		(						self	,    a					)	->							Any:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			return self.decoder.get(a					)
  	def lowercase_		(						self	,    a					)	->							Optional[int]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			_A           =    ''''''.join(a					)
  			_A           =    bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''	,    errors=self.errors					)
  			return text
  	def lowercase_		(						self	,    a	,    a = None					)	->							Tuple[str]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			if not os.path.isdir(a					):
  					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
  					return
  			_A           =    os.path.join(
  			    a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
  			_A           =    os.path.join(
  			    a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)
  
  			with open(a	,    '''w'''	,    encoding='''utf-8'''					) as f:
  					f.write(json.dumps(self.encoder	,    indent=2	,    sort_keys=a	,    ensure_ascii=a					) + '''\n'''					)
  
  			_A           =    0
  			with open(a	,    '''w'''	,    encoding='''utf-8'''					) as writer:
  					writer.write('''#version: 0.2\n'''					)
  					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,    key=lambda a					: kv[1]					):
  							if index != token_index:
  									logger.warning(
  									    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
  									    ''' Please check that the tokenizer is not corrupted!'''					)
  									_A           =    token_index
  							writer.write(''' '''.join(a					) + '''\n'''					)
  							index += 1
  
  			return vocab_file, merge_file
  	def lowercase_		(						self	,    a	,    a=False	,    **a					)	->							Tuple:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			_A           =    kwargs.pop('''add_prefix_space'''	,    self.add_prefix_space					)
  			if is_split_into_words or add_prefix_space:
  					_A           =    ''' ''' + text
  			return (text, kwargs)
  	def lowercase_		(						self	,    a	,    a = False	,    a = None	,    a = None	,    **a	,    )	->							str:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			_A           =    super()._decode(
  			    token_ids=a	,    skip_special_tokens=a	,    clean_up_tokenization_spaces=a	,    **a	,    )
  
  			if truncate_before_pattern is not None and len(a					) > 0:
  					_A           =    self.truncate(a	,    a					)
  
  			return decoded_text
  
  
  
  	def lowercase_		(						self	,    a	,    a					)	->							Optional[Any]:
  
  
  
  
  			"""simple docstring"""
  
  
  
  
  
  
  
  			def find_re(a	,    a	,    a					):
  					_A           =    pattern.search(a	,    a					)
  					return m.start() if m else -1
  
  			_A           =    [re.compile(a	,    re.MULTILINE					) for pattern in truncate_before_pattern]
  
  			_A           =    list(re.finditer('''^print'''	,    a	,    re.MULTILINE					)					)
  
  			if len(a					) > 1:
  					_A           =    completion[: prints[1].start()]
  
  			_A           =    list(re.finditer('''^def'''	,    a	,    re.MULTILINE					)					)
  
  			if len(a					) > 1:
  					_A           =    completion[: defs[1].start()]
  
  			_A           =    0
  
  			_A           =    [
  			    pos for pos in [find_re(a	,    a	,    a					) for terminal in terminals] if pos != -1
  			]
  
  			if len(a					) > 0:
  					return completion[: min(a					)]
  			else:
  					return completion
style_context: 
  
  
  
  import copy
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  __lowerCamelCase  				=					logging.get_logger(__name__)
  
  __lowerCamelCase  				=					{}
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			'''fsmt_decoder'''
   def __init__(						self	,    a=0	,    a=0					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     super().__init__()
     _A           =    vocab_size
     _A           =    bos_token_id
  
  
  
  
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			'''fsmt'''
   lowerCamelCase_          =			{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}
   def __init__(						self	,    a=["en", "de"]	,    a=4_2_0_2_4	,    a=4_2_0_2_4	,    a="relu"	,    a=1_0_2_4	,    a=2_0_0	,    a=1_0_2_4	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=0.0	,    a=0.1	,    a=0.0	,    a=0.02	,    a=2	,    a=True	,    a=True	,    a=False	,    a=5	,    a=1.0	,    a=False	,    a=True	,    a=1	,    a=0	,    a=2	,    a=2	,    **a	,    )	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    langs
     _A           =    src_vocab_size
     _A           =    tgt_vocab_size
     _A           =    d_model  # encoder_embed_dim and decoder_embed_dim
  
     _A           =    encoder_ffn_dim
     _A           =    _A           =    encoder_layers
     _A           =    encoder_attention_heads
     _A           =    encoder_layerdrop
     _A           =    decoder_layerdrop
     _A           =    decoder_ffn_dim
     _A           =    decoder_layers
     _A           =    decoder_attention_heads
     _A           =    max_position_embeddings
     _A           =    init_std  # Normal(0, this parameter)
     _A           =    activation_function
  
     _A           =    DecoderConfig(vocab_size=a	,    bos_token_id=a					)
     if "decoder" in common_kwargs:
       del common_kwargs["decoder"]
  
     _A           =    scale_embedding  # scale factor will be sqrt(d_model) if True
  
     # 3 Types of Dropout
     _A           =    attention_dropout
     _A           =    activation_dropout
     _A           =    dropout
  
     _A           =    use_cache
     super().__init__(
         pad_token_id=a	,    bos_token_id=a	,    eos_token_id=a	,    decoder_start_token_id=a	,    is_encoder_decoder=a	,    tie_word_embeddings=a	,    forced_eos_token_id=a	,    max_length=a	,    num_beams=a	,    length_penalty=a	,    early_stopping=a	,    **a	,    )
  
  
  
   def lowercase_		(						self					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    copy.deepcopy(self.__dict__					)
     _A           =    self.decoder.to_dict()
     _A           =    self.__class__.model_type
     return output