code: 
  
  
  import json
  import os
  from functools import lru_cache
  from typing import List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  UpperCAmelCase      :						List[str]    =		logging.get_logger(__name__)
  
  UpperCAmelCase      :						str    =		{
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  UpperCAmelCase      :						Union[str, Any]    =		{
      "vocab_file": {
          "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
          "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
          "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
          "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
          "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
          "roberta-large-openai-detector": (
              "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
          ),
      },
      "merges_file": {
          "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
          "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
          "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
          "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
          "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
          "roberta-large-openai-detector": (
              "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
          ),
      },
  }
  
  UpperCAmelCase      :						Optional[Any]    =		{
      "roberta-base": 5_12,
      "roberta-large": 5_12,
      "roberta-large-mnli": 5_12,
      "distilroberta-base": 5_12,
      "roberta-base-openai-detector": 5_12,
      "roberta-large-openai-detector": 5_12,
  }
  
  
  
  
  
  
  @lru_cache()
  def   __lowerCamelCase	(       ):
  
  
  
  
  
  			'''simple docstring'''
  
  
  
  
  			lowerCamelCase								=     (
  			    list(range(ord("""!"""			)     ,       ord("""~"""			) + 1			)			) + list(range(ord("""¡"""			)     ,       ord("""¬"""			) + 1			)			) + list(range(ord("""®"""			)     ,       ord("""ÿ"""			) + 1			)			)
  			)
  			lowerCamelCase								=     bs[:]
  			lowerCamelCase								=     0
  			for b in range(2**8			):
  						if b not in bs:
  									bs.append(_A			)
  									cs.append(2**8 + n			)
  									n += 1
  			lowerCamelCase								=     [chr(_A			) for n in cs]
  			return dict(zip(_A     ,       _A			)			)
  
  
  
  
  
  
  def   __lowerCamelCase	(       lowerCamelCase__		:	str			):
  
  
  
  
  
  			'''simple docstring'''
  
  
  
  
  			lowerCamelCase								=     set()
  			lowerCamelCase								=     word[0]
  			for char in word[1:]:
  						pairs.add((prev_char, char)			)
  						lowerCamelCase								=     char
  			return pairs
  
  
  
  
  
  
  
  class 					__lowercase					(       lowerCamelCase__     ):
  
  
  
  		"""simple docstring"""
  
  		UpperCamelCase    :				List[Any] 					=		VOCAB_FILES_NAMES
  		UpperCamelCase    :				List[Any] 					=		PRETRAINED_VOCAB_FILES_MAP
  		UpperCamelCase    :				Tuple 					=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  		UpperCamelCase    :				Any 					=		['input_ids', 'attention_mask']
  
  
  
  
  
  
  
  		def __init__(   self		,			A		,			A		,			A="replace"		,			A="<s>"		,			A="</s>"		,			A="</s>"		,			A="<s>"		,			A="<unk>"		,			A="<pad>"		,			A="<mask>"		,			A=False		,			**A		,			)     ->   Optional[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else bos_token
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else eos_token
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else sep_token
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else cls_token
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else unk_token
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else pad_token
  
  					# Mask token behave like a normal word, i.e. include the space before it
  					lowerCamelCase								=     AddedToken(lowercase_		,			lstrip=lowercase_		,			rstrip=lowercase_  ) if isinstance(lowercase_		,			lowercase_  ) else mask_token
  
  					super().__init__(
  					    errors=lowercase_		,			bos_token=lowercase_		,			eos_token=lowercase_		,			unk_token=lowercase_		,			sep_token=lowercase_		,			cls_token=lowercase_		,			pad_token=lowercase_		,			mask_token=lowercase_		,			add_prefix_space=lowercase_		,			**lowercase_		,			)
  
  					with open(lowercase_		,			encoding="""utf-8"""  ) as vocab_handle:
  								lowerCamelCase								=     json.load(lowercase_  )
  					lowerCamelCase								=     {v: k for k, v in self.encoder.items()}
  					lowerCamelCase								=     errors  # how to handle errors in decoding
  					lowerCamelCase								=     bytes_to_unicode()
  					lowerCamelCase								=     {v: k for k, v in self.byte_encoder.items()}
  					with open(lowercase_		,			encoding="""utf-8"""  ) as merges_handle:
  								lowerCamelCase								=     merges_handle.read().split("""\n"""  )[1:-1]
  					lowerCamelCase								=     [tuple(merge.split()  ) for merge in bpe_merges]
  					lowerCamelCase								=     dict(zip(lowercase_		,			range(len(lowercase_  )  )  )  )
  					lowerCamelCase								=     {}
  					lowerCamelCase								=     add_prefix_space
  
  					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  					lowerCamelCase								=     re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )
  
  
  
  
  
  
  
  		@property
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return len(self.encoder  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Tuple:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return dict(self.encoder		,			**self.added_tokens_encoder  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if token in self.cache:
  								return self.cache[token]
  					lowerCamelCase								=     tuple(lowercase_  )
  					lowerCamelCase								=     get_pairs(lowercase_  )
  
  					if not pairs:
  								return token
  
  					while True:
  								lowerCamelCase								=     min(lowercase_		,			key=lambda A  : self.bpe_ranks.get(lowercase_		,			float("""inf"""  )  )  )
  								if bigram not in self.bpe_ranks:
  											break
  								lowerCamelCase		,		lowerCamelCase								=     bigram
  								lowerCamelCase								=     []
  								lowerCamelCase								=     0
  								while i < len(lowercase_  ):
  											try:
  														lowerCamelCase								=     word.index(lowercase_		,			lowercase_  )
  											except ValueError:
  														new_word.extend(word[i:]  )
  														break
  											else:
  														new_word.extend(word[i:j]  )
  														lowerCamelCase								=     j
  
  											if word[i] == first and i < len(lowercase_  ) - 1 and word[i + 1] == second:
  														new_word.append(first + second  )
  														i += 2
  											else:
  														new_word.append(word[i]  )
  														i += 1
  								lowerCamelCase								=     tuple(lowercase_  )
  								lowerCamelCase								=     new_word
  								if len(lowercase_  ) == 1:
  											break
  								else:
  											lowerCamelCase								=     get_pairs(lowercase_  )
  					lowerCamelCase								=     """ """.join(lowercase_  )
  					lowerCamelCase								=     word
  					return word
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     []
  					for token in re.findall(self.pat		,			lowercase_  ):
  								lowerCamelCase								=     """""".join(
  								    self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_  ).split(""" """  )  )
  					return bpe_tokens
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   Optional[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return self.encoder.get(lowercase_		,			self.encoder.get(self.unk_token  )  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return self.decoder.get(lowercase_  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     """""".join(lowercase_  )
  					lowerCamelCase								=     bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8"""		,			errors=self.errors  )
  					return text
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A = None  )     ->   Tuple[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if not os.path.isdir(lowercase_  ):
  								logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
  								return
  					lowerCamelCase								=     os.path.join(
  					    lowercase_		,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
  					lowerCamelCase								=     os.path.join(
  					    lowercase_		,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )
  
  					with open(lowercase_		,			"""w"""		,			encoding="""utf-8"""  ) as f:
  								f.write(json.dumps(self.encoder		,			indent=2		,			sort_keys=lowercase_		,			ensure_ascii=lowercase_  ) + """\n"""  )
  
  					lowerCamelCase								=     0
  					with open(lowercase_		,			"""w"""		,			encoding="""utf-8"""  ) as writer:
  								writer.write("""#version: 0.2\n"""  )
  								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,			key=lambda A  : kv[1]  ):
  											if index != token_index:
  														logger.warning(
  														    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
  														    """ Please check that the tokenizer is not corrupted!"""  )
  														lowerCamelCase								=     token_index
  											writer.write(""" """.join(lowercase_  ) + """\n"""  )
  											index += 1
  
  					return vocab_file, merge_file
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A = None  )     ->   List[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if token_ids_a is None:
  								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  					lowerCamelCase								=     [self.cls_token_id]
  					lowerCamelCase								=     [self.sep_token_id]
  					return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A = None		,			A = False  )     ->   List[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if already_has_special_tokens:
  								return super().get_special_tokens_mask(
  								    token_ids_a=lowercase_		,			token_ids_a=lowercase_		,			already_has_special_tokens=lowercase_  )
  
  					if token_ids_a is None:
  								return [1] + ([0] * len(lowercase_  )) + [1]
  					return [1] + ([0] * len(lowercase_  )) + [1, 1] + ([0] * len(lowercase_  )) + [1]
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A = None  )     ->   List[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     [self.sep_token_id]
  					lowerCamelCase								=     [self.cls_token_id]
  
  					if token_ids_a is None:
  								return len(cls + token_ids_a + sep  ) * [0]
  					return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A=False		,			**A  )     ->   List[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     kwargs.pop("""add_prefix_space"""		,			self.add_prefix_space  )
  					if (is_split_into_words or add_prefix_space) and (len(lowercase_  ) > 0 and not text[0].isspace()):
  								lowerCamelCase								=     """ """ + text
  					return (text, kwargs)
style_context: 
  
  
  from __future__ import annotations
  
  from collections.abc import Callable
  from typing import Generic, TypeVar
  
  UpperCAmelCase      :						List[str]    =		TypeVar("T")
  UpperCAmelCase      :						Dict    =		TypeVar("U")
  
  
  
  
  
  
  
  class 					__lowercase					(       Generic[T, U]     ):
  
  
  
  		"""simple docstring"""
  
  		def __init__(   self		,			A		,			A  )     ->   Any:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     key
  					lowerCamelCase								=     val
  					lowerCamelCase								=     0
  					lowerCamelCase								=     None
  					lowerCamelCase								=     None
  
  
  
  
  
  
  
  		def __repr__(   self  )     ->   str:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return "Node: key: {}, val: {}, freq: {}, has next: {}, has prev: {}".format(
  					    self.key		,			self.val		,			self.freq		,			self.next is not None		,			self.prev is not None  )
  
  
  
  
  
  
  
  class 					__lowercase					(       Generic[T, U]     ):
  
  
  
  		"""simple docstring"""
  
  		def __init__(   self  )     ->   None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     DoubleLinkedListNode(A		,			A  )
  					lowerCamelCase								=     DoubleLinkedListNode(A		,			A  )
  					lowerCamelCase		,		lowerCamelCase								=     self.rear, self.head
  
  
  
  
  
  
  
  		def __repr__(   self  )     ->   str:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""DoubleLinkedList"""]
  					lowerCamelCase								=     self.head
  					while node.next is not None:
  								rep.append(str(A  )  )
  								lowerCamelCase								=     node.next
  					rep.append(str(self.rear  )  )
  					return ",\n    ".join(A  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     self.rear.prev
  
  					# All nodes other than self.head are guaranteed to have non-None previous
  					assert previous is not None
  
  					lowerCamelCase								=     node
  					lowerCamelCase								=     previous
  					lowerCamelCase								=     node
  					lowerCamelCase								=     self.rear
  					node.freq += 1
  					self._position_node(A  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					while node.prev is not None and node.prev.freq > node.freq:
  								# swap node with previous node
  								lowerCamelCase								=     node.prev
  
  								lowerCamelCase								=     previous_node.prev
  								lowerCamelCase								=     node.prev
  								lowerCamelCase								=     previous_node
  								lowerCamelCase								=     node
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   DoubleLinkedListNode[T, U] | None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if node.prev is None or node.next is None:
  								return None
  
  					lowerCamelCase								=     node.next
  					lowerCamelCase								=     node.prev
  					lowerCamelCase								=     None
  					lowerCamelCase								=     None
  					return node
  
  
  
  
  
  
  
  class 					__lowercase					(       Generic[T, U]     ):
  
  
  
  		"""simple docstring"""
  
  		UpperCamelCase    :				dict[Callable[[T], U], LFUCache[T, U]] 					=		{}
  
  
  
  
  
  
  
  		def __init__(   self		,			A  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     DoubleLinkedList()
  					lowerCamelCase								=     capacity
  					lowerCamelCase								=     0
  					lowerCamelCase								=     0
  					lowerCamelCase								=     0
  					lowerCamelCase								=     {}
  
  
  
  
  
  
  
  		def __repr__(   self  )     ->   str:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return (
  					    F'CacheInfo(hits={self.hits}, misses={self.miss}, '
  					    F'capacity={self.capacity}, current_size={self.num_keys})'
  					)
  
  
  
  
  
  
  
  		def __contains__(   self		,			A  )     ->   bool:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return key in self.cache
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A  )     ->   U | None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if key in self.cache:
  								self.hits += 1
  								lowerCamelCase								=     self.cache[key]
  								lowerCamelCase								=     self.list.remove(self.cache[key]  )
  								assert node == value_node
  
  								# node is guaranteed not None because it is in self.cache
  								assert node is not None
  								self.list.add(A  )
  								return node.val
  					self.miss += 1
  					return None
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A  )     ->   None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					if key not in self.cache:
  								if self.num_keys >= self.capacity:
  											# delete first node when over capacity
  											lowerCamelCase								=     self.list.head.next
  
  											# guaranteed to have a non-None first node when num_keys > 0
  											# explain to type checker via assertions
  											assert first_node is not None
  											assert first_node.key is not None
  											assert self.list.remove(A  ) is not None
  											# first_node guaranteed to be in list
  
  											del self.cache[first_node.key]
  											self.num_keys -= 1
  								lowerCamelCase								=     DoubleLinkedListNode(A		,			A  )
  								self.list.add(self.cache[key]  )
  								self.num_keys += 1
  
  					else:
  								lowerCamelCase								=     self.list.remove(self.cache[key]  )
  								assert node is not None  # node guaranteed to be in list
  								lowerCamelCase								=     value
  								self.list.add(A  )
  
  
  
  
  
  
  
  		@classmethod
  		def 				__A      (   cls		,			A = 1_28  )     ->   Callable[[Callable[[T], U]], Callable[..., U]]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					def cache_decorator_inner(A  ) -> Callable[..., U]:
  								def cache_decorator_wrapper(*A  ) -> U:
  											if func not in cls.decorator_function_to_instance_map:
  														lowerCamelCase								=     LFUCache(A  )
  
  											lowerCamelCase								=     cls.decorator_function_to_instance_map[func].get(args[0]  )
  											if result is None:
  														lowerCamelCase								=     func(*A  )
  														cls.decorator_function_to_instance_map[func].put(args[0]		,			A  )
  											return result
  
  								def cache_info() -> LFUCache[T, U]:
  											return cls.decorator_function_to_instance_map[func]
  
  								setattr(A		,			"""cache_info"""		,			A  )  # noqa: B010
  
  								return cache_decorator_wrapper
  
  					return cache_decorator_inner
  
  
  if __name__ == "__main__":
  		import doctest
  
  		doctest.testmod()
  
  