code: 
  import json
  import os
  from functools import lru_cache
  from typing import List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  A_														=      logging.get_logger(__name__)
  
  A_														=      {
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  A_														=      {
      "vocab_file": {
          "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
          "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
          "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
          "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
          "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
          "roberta-large-openai-detector": (
              "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
          ),
      },
      "merges_file": {
          "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
          "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
          "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
          "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
          "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
          "roberta-large-openai-detector": (
              "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
          ),
      },
  }
  
  A_														=      {
      "roberta-base": 5_1_2,
      "roberta-large": 5_1_2,
      "roberta-large-mnli": 5_1_2,
      "distilroberta-base": 5_1_2,
      "roberta-base-openai-detector": 5_1_2,
      "roberta-large-openai-detector": 5_1_2,
  }
  
  
  
  
  
  
  
  @lru_cache()
  def 							UpperCAmelCase				(     )->		Dict:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					SCREAMING_SNAKE_CASE_	      =       (
  					    list(range(ord('''!'''      )				,ord('''~'''      ) + 1      )      ) + list(range(ord('''¡'''      )				,ord('''¬'''      ) + 1      )      ) + list(range(ord('''®'''      )				,ord('''ÿ'''      ) + 1      )      )
  					)
  					SCREAMING_SNAKE_CASE_	      =       bs[:]
  					SCREAMING_SNAKE_CASE_	      =       0
  					for b in range(2**8      ):
  										if b not in bs:
  															bs.append(UpperCAmelCase      )
  															cs.append(2**8 + n      )
  															n += 1
  					SCREAMING_SNAKE_CASE_	      =       [chr(UpperCAmelCase      ) for n in cs]
  					return dict(zip(UpperCAmelCase				,UpperCAmelCase      )      )
  
  
  
  
  
  
  
  def 							UpperCAmelCase				(     UpperCAmelCase      )->		Dict:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					SCREAMING_SNAKE_CASE_	      =       set()
  					SCREAMING_SNAKE_CASE_	      =       word[0]
  					for char in word[1:]:
  										pairs.add((prev_char, char)      )
  										SCREAMING_SNAKE_CASE_	      =       char
  					return pairs
  
  
  
  
  
  
  class 						snake_case			(       lowerCAmelCase__	):
  			'''simple docstring'''
  
  			UpperCAmelCase       :       Any       = VOCAB_FILES_NAMES
  			UpperCAmelCase       :       List[str]       = PRETRAINED_VOCAB_FILES_MAP
  			UpperCAmelCase       :       int       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  			UpperCAmelCase       :       Dict       = ["input_ids", "attention_mask"]
  
  			def __init__(	self     :     Dict							,							lowerCAmelCase_     :     List[Any]							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     List[Any]="replace"							,							lowerCAmelCase_     :     Union[str, Any]="<s>"							,							lowerCAmelCase_     :     int="</s>"							,							lowerCAmelCase_     :     Optional[int]="</s>"							,							lowerCAmelCase_     :     List[str]="<s>"							,							lowerCAmelCase_     :     Tuple="<unk>"							,							lowerCAmelCase_     :     Tuple="<pad>"							,							lowerCAmelCase_     :     Optional[Any]="<mask>"							,							lowerCAmelCase_     :     Tuple=False							,							**lowerCAmelCase_     :     int							,							)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else bos_token
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else eos_token
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else sep_token
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else cls_token
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else unk_token
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else pad_token
  
  								# Mask token behave like a normal word, i.e. include the space before it
  								SCREAMING_SNAKE_CASE_	      =       AddedToken(UpperCamelCase_							,							lstrip=UpperCamelCase_							,							rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_							,							UpperCamelCase_					) else mask_token
  
  								super().__init__(
  								    errors=UpperCamelCase_							,							bos_token=UpperCamelCase_							,							eos_token=UpperCamelCase_							,							unk_token=UpperCamelCase_							,							sep_token=UpperCamelCase_							,							cls_token=UpperCamelCase_							,							pad_token=UpperCamelCase_							,							mask_token=UpperCamelCase_							,							add_prefix_space=UpperCamelCase_							,							**UpperCamelCase_							,							)
  
  								with open(UpperCamelCase_							,							encoding='''utf-8'''					) as vocab_handle:
  													SCREAMING_SNAKE_CASE_	      =       json.load(UpperCamelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       {v: k for k, v in self.encoder.items()}
  								SCREAMING_SNAKE_CASE_	      =       errors  # how to handle errors in decoding
  								SCREAMING_SNAKE_CASE_	      =       bytes_to_unicode()
  								SCREAMING_SNAKE_CASE_	      =       {v: k for k, v in self.byte_encoder.items()}
  								with open(UpperCamelCase_							,							encoding='''utf-8'''					) as merges_handle:
  													SCREAMING_SNAKE_CASE_	      =       merges_handle.read().split('''\n'''					)[1:-1]
  								SCREAMING_SNAKE_CASE_	      =       [tuple(merge.split()					) for merge in bpe_merges]
  								SCREAMING_SNAKE_CASE_	      =       dict(zip(UpperCamelCase_							,							range(len(UpperCamelCase_					)					)					)					)
  								SCREAMING_SNAKE_CASE_	      =       {}
  								SCREAMING_SNAKE_CASE_	      =       add_prefix_space
  
  								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  								SCREAMING_SNAKE_CASE_	      =       re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)
  
  			@property
  			def _lowercase (	self     :     str					)					->					List[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return len(self.encoder					)
  
  			def _lowercase (	self     :     List[str]					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return dict(self.encoder							,							**self.added_tokens_encoder					)
  
  			def _lowercase (	self     :     Tuple							,							lowerCAmelCase_     :     Optional[Any]					)					->					Optional[int]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if token in self.cache:
  													return self.cache[token]
  								SCREAMING_SNAKE_CASE_	      =       tuple(UpperCamelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       get_pairs(UpperCamelCase_					)
  
  								if not pairs:
  													return token
  
  								while True:
  													SCREAMING_SNAKE_CASE_	      =       min(UpperCamelCase_							,							key=lambda lowerCAmelCase_					: self.bpe_ranks.get(UpperCamelCase_							,							float('''inf'''					)					)					)
  													if bigram not in self.bpe_ranks:
  																		break
  													SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_	      =       bigram
  													SCREAMING_SNAKE_CASE_	      =       []
  													SCREAMING_SNAKE_CASE_	      =       0
  													while i < len(UpperCamelCase_					):
  																		try:
  																							SCREAMING_SNAKE_CASE_	      =       word.index(UpperCamelCase_							,							UpperCamelCase_					)
  																		except ValueError:
  																							new_word.extend(word[i:]					)
  																							break
  																		else:
  																							new_word.extend(word[i:j]					)
  																							SCREAMING_SNAKE_CASE_	      =       j
  
  																		if word[i] == first and i < len(UpperCamelCase_					) - 1 and word[i + 1] == second:
  																							new_word.append(first + second					)
  																							i += 2
  																		else:
  																							new_word.append(word[i]					)
  																							i += 1
  													SCREAMING_SNAKE_CASE_	      =       tuple(UpperCamelCase_					)
  													SCREAMING_SNAKE_CASE_	      =       new_word
  													if len(UpperCamelCase_					) == 1:
  																		break
  													else:
  																		SCREAMING_SNAKE_CASE_	      =       get_pairs(UpperCamelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       ''' '''.join(UpperCamelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       word
  								return word
  
  			def _lowercase (	self     :     Tuple							,							lowerCAmelCase_     :     Union[str, Any]					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       []
  								for token in re.findall(self.pat							,							UpperCamelCase_					):
  													SCREAMING_SNAKE_CASE_	      =       ''''''.join(
  													    self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  													bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_					).split(''' '''					)					)
  								return bpe_tokens
  
  			def _lowercase (	self     :     Optional[Any]							,							lowerCAmelCase_     :     Optional[int]					)					->					Optional[int]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.encoder.get(UpperCamelCase_							,							self.encoder.get(self.unk_token					)					)
  
  			def _lowercase (	self     :     Optional[Any]							,							lowerCAmelCase_     :     List[str]					)					->					Optional[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.decoder.get(UpperCamelCase_					)
  
  			def _lowercase (	self     :     str							,							lowerCAmelCase_     :     str					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       ''''''.join(UpperCamelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''							,							errors=self.errors					)
  								return text
  
  			def _lowercase (	self     :     Optional[int]							,							lowerCAmelCase_     :     str							,							lowerCAmelCase_     :     Optional[str] = None					)					->					Any:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if not os.path.isdir(UpperCamelCase_					):
  													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
  													return
  								SCREAMING_SNAKE_CASE_	      =       os.path.join(
  								    UpperCamelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
  								SCREAMING_SNAKE_CASE_	      =       os.path.join(
  								    UpperCamelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)
  
  								with open(UpperCamelCase_							,							'''w'''							,							encoding='''utf-8'''					) as f:
  													f.write(json.dumps(self.encoder							,							indent=2							,							sort_keys=UpperCamelCase_							,							ensure_ascii=UpperCamelCase_					) + '''\n'''					)
  
  								SCREAMING_SNAKE_CASE_	      =       0
  								with open(UpperCamelCase_							,							'''w'''							,							encoding='''utf-8'''					) as writer:
  													writer.write('''#version: 0.2\n'''					)
  													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,							key=lambda lowerCAmelCase_					: kv[1]					):
  																		if index != token_index:
  																							logger.warning(
  																							    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
  																							    ''' Please check that the tokenizer is not corrupted!'''					)
  																							SCREAMING_SNAKE_CASE_	      =       token_index
  																		writer.write(''' '''.join(UpperCamelCase_					) + '''\n'''					)
  																		index += 1
  
  								return vocab_file, merge_file
  
  			def _lowercase (	self     :     int							,							lowerCAmelCase_     :     List[int]							,							lowerCAmelCase_     :     Optional[List[int]] = None					)					->					Union[str, Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if token_ids_a is None:
  													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  								SCREAMING_SNAKE_CASE_	      =       [self.cls_token_id]
  								SCREAMING_SNAKE_CASE_	      =       [self.sep_token_id]
  								return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  			def _lowercase (	self     :     Union[str, Any]							,							lowerCAmelCase_     :     List[int]							,							lowerCAmelCase_     :     Optional[List[int]] = None							,							lowerCAmelCase_     :     bool = False					)					->					Optional[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if already_has_special_tokens:
  													return super().get_special_tokens_mask(
  													    token_ids_a=UpperCamelCase_							,							token_ids_a=UpperCamelCase_							,							already_has_special_tokens=UpperCamelCase_					)
  
  								if token_ids_a is None:
  													return [1] + ([0] * len(UpperCamelCase_					)) + [1]
  								return [1] + ([0] * len(UpperCamelCase_					)) + [1, 1] + ([0] * len(UpperCamelCase_					)) + [1]
  
  			def _lowercase (	self     :     Union[str, Any]							,							lowerCAmelCase_     :     List[int]							,							lowerCAmelCase_     :     Optional[List[int]] = None					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       [self.sep_token_id]
  								SCREAMING_SNAKE_CASE_	      =       [self.cls_token_id]
  
  								if token_ids_a is None:
  													return len(cls + token_ids_a + sep					) * [0]
  								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]
  
  
  
  
  			def _lowercase (	self     :     Union[str, Any]							,							lowerCAmelCase_     :     Tuple							,							lowerCAmelCase_     :     Any=False							,							**lowerCAmelCase_     :     Dict					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       kwargs.pop('''add_prefix_space'''							,							self.add_prefix_space					)
  								if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_					) > 0 and not text[0].isspace()):
  													SCREAMING_SNAKE_CASE_	      =       ''' ''' + text
  								return (text, kwargs)
  
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_sentencepiece_available,
      is_tf_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  A_														=      {
      "configuration_switch_transformers": [
          "SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP",
          "SwitchTransformersConfig",
          "SwitchTransformersOnnxConfig",
      ]
  }
  
  try:
  		if not is_torch_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		A_														=      [
  		    "SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST",
  		    "SwitchTransformersEncoderModel",
  		    "SwitchTransformersForConditionalGeneration",
  		    "SwitchTransformersModel",
  		    "SwitchTransformersPreTrainedModel",
  		    "SwitchTransformersTop1Router",
  		    "SwitchTransformersSparseMLP",
  		]
  
  
  if TYPE_CHECKING:
  		from .configuration_switch_transformers import (
  		    SWITCH_TRANSFORMERS_PRETRAINED_CONFIG_ARCHIVE_MAP,
  		    SwitchTransformersConfig,
  		    SwitchTransformersOnnxConfig,
  		)
  
  		try:
  				if not is_torch_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .modeling_switch_transformers import (
  				    SWITCH_TRANSFORMERS_PRETRAINED_MODEL_ARCHIVE_LIST,
  				    SwitchTransformersEncoderModel,
  				    SwitchTransformersForConditionalGeneration,
  				    SwitchTransformersModel,
  				    SwitchTransformersPreTrainedModel,
  				    SwitchTransformersSparseMLP,
  				    SwitchTransformersTopaRouter,
  				)
  
  
  else:
  		import sys
  
  		A_														=      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
  
  
  