code: 
  
  
  import json
  import os
  from functools import lru_cache
  from typing import List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  lowerCAmelCase					 =	logging.get_logger(__name__)
  
  lowerCAmelCase					 =	{
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  lowerCAmelCase					 =	{
      '''vocab_file''': {
          '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
          '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
          '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
          '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
          '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
          '''roberta-large-openai-detector''': (
              '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
          ),
      },
      '''merges_file''': {
          '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
          '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
          '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
          '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
          '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
          '''roberta-large-openai-detector''': (
              '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
          ),
      },
  }
  
  lowerCAmelCase					 =	{
      '''roberta-base''': 5_1_2,
      '''roberta-large''': 5_1_2,
      '''roberta-large-mnli''': 5_1_2,
      '''distilroberta-base''': 5_1_2,
      '''roberta-base-openai-detector''': 5_1_2,
      '''roberta-large-openai-detector''': 5_1_2,
  }
  
  
  
  
  
  
  
  @lru_cache()
  def  _lowerCamelCase(			)		->			Optional[int]:
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowercase=    (
  		    list(range(ord('!'			)     ,				ord('~'			) + 1			)			) + list(range(ord('¡'			)     ,				ord('¬'			) + 1			)			) + list(range(ord('®'			)     ,				ord('ÿ'			) + 1			)			)
  		)
  		__lowercase=    bs[:]
  		__lowercase=    0
  		for b in range(2**8			):
  				if b not in bs:
  						bs.append(lowercase__			)
  						cs.append(2**8 + n			)
  						n += 1
  		__lowercase=    [chr(lowercase__			) for n in cs]
  		return dict(zip(lowercase__     ,				lowercase__			)			)
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__			)		->			List[str]:
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowercase=    set()
  		__lowercase=    word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)			)
  				__lowercase=    char
  		return pairs
  
  
  
  
  
  class    A					(     A_				):
  							UpperCamelCase_     :							Optional[Any] =VOCAB_FILES_NAMES
  							UpperCamelCase_     :							int =PRETRAINED_VOCAB_FILES_MAP
  							UpperCamelCase_     :							str =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							UpperCamelCase_     :							List[Any] =['''input_ids''', '''attention_mask''']
  
  
  							def __init__(self   ,       lowerCAmelCase   ,       lowerCAmelCase   ,       lowerCAmelCase="replace"   ,       lowerCAmelCase="<s>"   ,       lowerCAmelCase="</s>"   ,       lowerCAmelCase="</s>"   ,       lowerCAmelCase="<s>"   ,       lowerCAmelCase="<unk>"   ,       lowerCAmelCase="<pad>"   ,       lowerCAmelCase="<mask>"   ,       lowerCAmelCase=False   ,       **lowerCAmelCase   ,       ):
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else bos_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else eos_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else sep_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else cls_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else unk_token
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else pad_token
  
  									# Mask token behave like a normal word, i.e. include the space before it
  									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else mask_token
  
  									super().__init__(
  									    errors=lowerCAmelCase   ,       bos_token=lowerCAmelCase   ,       eos_token=lowerCAmelCase   ,       unk_token=lowerCAmelCase   ,       sep_token=lowerCAmelCase   ,       cls_token=lowerCAmelCase   ,       pad_token=lowerCAmelCase   ,       mask_token=lowerCAmelCase   ,       add_prefix_space=lowerCAmelCase   ,       **lowerCAmelCase   ,       )
  
  									with open(lowerCAmelCase   ,       encoding='utf-8'      ) as vocab_handle:
  											__lowercase=    json.load(lowerCAmelCase      )
  									__lowercase=    {v: k for k, v in self.encoder.items()}
  									__lowercase=    errors  # how to handle errors in decoding
  									__lowercase=    bytes_to_unicode()
  									__lowercase=    {v: k for k, v in self.byte_encoder.items()}
  									with open(lowerCAmelCase   ,       encoding='utf-8'      ) as merges_handle:
  											__lowercase=    merges_handle.read().split('\n'      )[1:-1]
  									__lowercase=    [tuple(merge.split()      ) for merge in bpe_merges]
  									__lowercase=    dict(zip(lowerCAmelCase   ,       range(len(lowerCAmelCase      )      )      )      )
  									__lowercase=    {}
  									__lowercase=    add_prefix_space
  
  									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  									__lowercase=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )
  
  
  							@property
  							def     _A (self      ):
  									return len(self.encoder      )
  
  
  							def     _A (self      ):
  									return dict(self.encoder   ,       **self.added_tokens_encoder      )
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									if token in self.cache:
  											return self.cache[token]
  									__lowercase=    tuple(lowerCAmelCase      )
  									__lowercase=    get_pairs(lowerCAmelCase      )
  
  									if not pairs:
  											return token
  
  									while True:
  											__lowercase=    min(lowerCAmelCase   ,       key=lambda lowerCAmelCase      : self.bpe_ranks.get(lowerCAmelCase   ,       float('inf'      )      )      )
  											if bigram not in self.bpe_ranks:
  													break
  											__lowercase,							__lowercase=    bigram
  											__lowercase=    []
  											__lowercase=    0
  											while i < len(lowerCAmelCase      ):
  													try:
  															__lowercase=    word.index(lowerCAmelCase   ,       lowerCAmelCase      )
  													except ValueError:
  															new_word.extend(word[i:]      )
  															break
  													else:
  															new_word.extend(word[i:j]      )
  															__lowercase=    j
  
  													if word[i] == first and i < len(lowerCAmelCase      ) - 1 and word[i + 1] == second:
  															new_word.append(first + second      )
  															i += 2
  													else:
  															new_word.append(word[i]      )
  															i += 1
  											__lowercase=    tuple(lowerCAmelCase      )
  											__lowercase=    new_word
  											if len(lowerCAmelCase      ) == 1:
  													break
  											else:
  													__lowercase=    get_pairs(lowerCAmelCase      )
  									__lowercase=    ' '.join(lowerCAmelCase      )
  									__lowercase=    word
  									return word
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									__lowercase=    []
  									for token in re.findall(self.pat   ,       lowerCAmelCase      ):
  											__lowercase=    ''.join(
  											    self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase      ).split(' '      )      )
  									return bpe_tokens
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									return self.encoder.get(lowerCAmelCase   ,       self.encoder.get(self.unk_token      )      )
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									return self.decoder.get(lowerCAmelCase      )
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  									__lowercase=    ''.join(lowerCAmelCase      )
  									__lowercase=    bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'   ,       errors=self.errors      )
  									return text
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
  									if not os.path.isdir(lowerCAmelCase      ):
  											logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
  											return
  									__lowercase=    os.path.join(
  									    lowerCAmelCase   ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
  									__lowercase=    os.path.join(
  									    lowerCAmelCase   ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )
  
  									with open(lowerCAmelCase   ,       'w'   ,       encoding='utf-8'      ) as f:
  											f.write(json.dumps(self.encoder   ,       indent=2   ,       sort_keys=lowerCAmelCase   ,       ensure_ascii=lowerCAmelCase      ) + '\n'      )
  
  									__lowercase=    0
  									with open(lowerCAmelCase   ,       'w'   ,       encoding='utf-8'      ) as writer:
  											writer.write('#version: 0.2\n'      )
  											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,       key=lambda lowerCAmelCase      : kv[1]      ):
  													if index != token_index:
  															logger.warning(
  															    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
  															    ' Please check that the tokenizer is not corrupted!'      )
  															__lowercase=    token_index
  													writer.write(' '.join(lowerCAmelCase      ) + '\n'      )
  													index += 1
  
  									return vocab_file, merge_file
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
  									if token_ids_a is None:
  											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  									__lowercase=    [self.cls_token_id]
  									__lowercase=    [self.sep_token_id]
  									return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None   ,       lowerCAmelCase = False      ):
  									if already_has_special_tokens:
  											return super().get_special_tokens_mask(
  											    token_ids_a=lowerCAmelCase   ,       token_ids_a=lowerCAmelCase   ,       already_has_special_tokens=lowerCAmelCase      )
  
  									if token_ids_a is None:
  											return [1] + ([0] * len(lowerCAmelCase      )) + [1]
  									return [1] + ([0] * len(lowerCAmelCase      )) + [1, 1] + ([0] * len(lowerCAmelCase      )) + [1]
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
  									__lowercase=    [self.sep_token_id]
  									__lowercase=    [self.cls_token_id]
  
  									if token_ids_a is None:
  											return len(cls + token_ids_a + sep      ) * [0]
  									return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]
  
  
  
  
  
  
  
  							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase=False   ,       **lowerCAmelCase      ):
  									__lowercase=    kwargs.pop('add_prefix_space'   ,       self.add_prefix_space      )
  									if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase      ) > 0 and not text[0].isspace()):
  											__lowercase=    ' ' + text
  									return (text, kwargs)
  
  
  
  
  
  
  
style_context: 
  
  
  from __future__ import annotations
  
  from enum import Enum
  
  
  
  
  
  class    A					(     A_				):
  							UpperCamelCase_     :							List[Any] =24
  							UpperCamelCase_     :							List[Any] =21
  							UpperCamelCase_     :							int =18
  							UpperCamelCase_     :							Optional[int] =15
  							UpperCamelCase_     :							Tuple =12
  							UpperCamelCase_     :							List[str] =9
  							UpperCamelCase_     :							Any =6
  							UpperCamelCase_     :							List[str] =3
  							UpperCamelCase_     :							str =2
  							UpperCamelCase_     :							str =1
  							UpperCamelCase_     :							Optional[int] =-1
  							UpperCamelCase_     :							List[Any] =-2
  							UpperCamelCase_     :							str =-3
  							UpperCamelCase_     :							Tuple =-6
  							UpperCamelCase_     :							Tuple =-9
  							UpperCamelCase_     :							Optional[int] =-12
  							UpperCamelCase_     :							str =-15
  							UpperCamelCase_     :							Optional[Any] =-18
  							UpperCamelCase_     :							Any =-21
  							UpperCamelCase_     :							List[str] =-24
  
  
  
  
  
  class    A					(     A_				):
  							UpperCamelCase_     :							str =8
  							UpperCamelCase_     :							Optional[Any] =7
  							UpperCamelCase_     :							Tuple =6
  							UpperCamelCase_     :							List[Any] =5
  							UpperCamelCase_     :							Optional[Any] =4
  							UpperCamelCase_     :							Union[str, Any] =3
  							UpperCamelCase_     :							Union[str, Any] =2
  							UpperCamelCase_     :							int =1
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__     ,				lowercase__     ,				)		->			float:
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if isinstance(lowercase__     ,				lowercase__			):
  				__lowercase=    SIUnit[known_prefix.lower()]
  		if isinstance(lowercase__     ,				lowercase__			):
  				__lowercase=    SIUnit[unknown_prefix.lower()]
  		__lowercase=    known_amount * (
  		    1_0 ** (known_prefix.value - unknown_prefix.value)
  		)
  		return unknown_amount
  
  
  
  
  
  
  
  def  _lowerCamelCase(			lowercase__     ,				lowercase__     ,				lowercase__     ,				)		->			float:
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		if isinstance(lowercase__     ,				lowercase__			):
  				__lowercase=    BinaryUnit[known_prefix.lower()]
  		if isinstance(lowercase__     ,				lowercase__			):
  				__lowercase=    BinaryUnit[unknown_prefix.lower()]
  		__lowercase=    known_amount * (
  		    2 ** ((known_prefix.value - unknown_prefix.value) * 1_0)
  		)
  		return unknown_amount
  
  
  if __name__ == "__main__":
  	import doctest
  
  	doctest.testmod()
  
  
  
  
  
  
  