code: 
  
  import json
  import os
  from functools import lru_cache
  from typing import List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  UpperCAmelCase_       :				List[str]		   =      logging.get_logger(__name__)
  
  UpperCAmelCase_       :				Optional[int]		   =      {
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  UpperCAmelCase_       :				Optional[Any]		   =      {
      '''vocab_file''': {
          '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
          '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
          '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
          '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
          '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
          '''roberta-large-openai-detector''': (
              '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
          ),
      },
      '''merges_file''': {
          '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
          '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
          '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
          '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
          '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
          '''roberta-large-openai-detector''': (
              '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
          ),
      },
  }
  
  UpperCAmelCase_       :				Tuple		   =      {
      '''roberta-base''': 512,
      '''roberta-large''': 512,
      '''roberta-large-mnli''': 512,
      '''distilroberta-base''': 512,
      '''roberta-base-openai-detector''': 512,
      '''roberta-large-openai-detector''': 512,
  }
  @lru_cache()
  def        __SCREAMING_SNAKE_CASE    (      )				->							Dict:
  	__A   : Optional[Any]								= (
  	    list(range(ord("""!"""					) ,ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					) ,ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					) ,ord("""ÿ"""					) + 1					)					)
  	)
  	__A   : str								= bs[:]
  	__A   : Any								= 0
  	for b in range(2**8					):
  		if b not in bs:
  			bs.append(a__					)
  			cs.append(2**8 + n					)
  			n += 1
  	__A   : Dict								= [chr(a__					) for n in cs]
  	return dict(zip(a__ ,a__					)					)
  def        __SCREAMING_SNAKE_CASE    (      a__						:      int					)				->							int:
  	__A   : List[Any]								= set()
  	__A   : Tuple								= word[0]
  	for char in word[1:]:
  		pairs.add((prev_char, char)					)
  		__A   : int								= char
  	return pairs
  
  
  
  
  
  
  
  class     lowerCamelCase_			(  _lowercase  ):
  	_lowercase		:      List[str]    				=							VOCAB_FILES_NAMES
  	_lowercase		:      List[str]    				=							PRETRAINED_VOCAB_FILES_MAP
  	_lowercase		:      List[Any]    				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	_lowercase		:      Union[str, Any]    				=							['''input_ids''', '''attention_mask''']
  
  
  
  
  	def __init__(    self	:    Optional[Any]				,			__A	:    Optional[int]				,			__A	:    Optional[Any]				,			__A	:    List[str]="replace"				,			__A	:    Any="<s>"				,			__A	:    int="</s>"				,			__A	:    Optional[Any]="</s>"				,			__A	:    int="<s>"				,			__A	:    Dict="<unk>"				,			__A	:    Optional[Any]="<pad>"				,			__A	:    Optional[Any]="<mask>"				,			__A	:    int=False				,			**__A	:    Dict				,			):
  		__A   : Dict								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else bos_token
  		__A   : Any								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else eos_token
  		__A   : List[Any]								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else sep_token
  		__A   : Union[str, Any]								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else cls_token
  		__A   : Any								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else unk_token
  		__A   : List[Any]								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else pad_token
  
  		# Mask token behave like a normal word, i.e. include the space before it
  		__A   : int								= AddedToken(__A				,			lstrip=__A				,			rstrip=__A       ) if isinstance(__A				,			__A       ) else mask_token
  
  		super().__init__(
  		    errors=__A				,			bos_token=__A				,			eos_token=__A				,			unk_token=__A				,			sep_token=__A				,			cls_token=__A				,			pad_token=__A				,			mask_token=__A				,			add_prefix_space=__A				,			**__A				,			)
  
  		with open(__A				,			encoding="""utf-8"""       ) as vocab_handle:
  			__A   : Optional[Any]								= json.load(__A       )
  		__A   : List[str]								= {v: k for k, v in self.encoder.items()}
  		__A   : List[str]								= errors  # how to handle errors in decoding
  		__A   : Tuple								= bytes_to_unicode()
  		__A   : List[str]								= {v: k for k, v in self.byte_encoder.items()}
  		with open(__A				,			encoding="""utf-8"""       ) as merges_handle:
  			__A   : List[Any]								= merges_handle.read().split("""\n"""       )[1:-1]
  		__A   : Any								= [tuple(merge.split()       ) for merge in bpe_merges]
  		__A   : List[str]								= dict(zip(__A				,			range(len(__A       )       )       )       )
  		__A   : Tuple								= {}
  		__A   : List[str]								= add_prefix_space
  
  		# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  		__A   : int								= re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )
  
  
  
  
  	@property
  	def 					lowerCAmelCase_					(    self	:    Any       ):
  		return len(self.encoder       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		return dict(self.encoder				,			**self.added_tokens_encoder       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]				,			__A	:    Union[str, Any]       ):
  		if token in self.cache:
  			return self.cache[token]
  		__A   : Tuple								= tuple(__A       )
  		__A   : str								= get_pairs(__A       )
  
  		if not pairs:
  			return token
  
  		while True:
  			__A   : Tuple								= min(__A				,			key=lambda __A       : self.bpe_ranks.get(__A				,			float("""inf"""       )       )       )
  			if bigram not in self.bpe_ranks:
  				break
  			__A   : int								= bigram
  			__A   : Any								= []
  			__A   : Optional[Any]								= 0
  			while i < len(__A       ):
  				try:
  					__A   : Optional[int]								= word.index(__A				,			__A       )
  				except ValueError:
  					new_word.extend(word[i:]       )
  					break
  				else:
  					new_word.extend(word[i:j]       )
  					__A   : Tuple								= j
  
  				if word[i] == first and i < len(__A       ) - 1 and word[i + 1] == second:
  					new_word.append(first + second       )
  					i += 2
  				else:
  					new_word.append(word[i]       )
  					i += 1
  			__A   : Tuple								= tuple(__A       )
  			__A   : Union[str, Any]								= new_word
  			if len(__A       ) == 1:
  				break
  			else:
  				__A   : List[Any]								= get_pairs(__A       )
  		__A   : Optional[Any]								= """ """.join(__A       )
  		__A   : Optional[int]								= word
  		return word
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]				,			__A	:    Any       ):
  		__A   : str								= []
  		for token in re.findall(self.pat				,			__A       ):
  			__A   : Union[str, Any]								= """""".join(
  			    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A       ).split(""" """       )       )
  		return bpe_tokens
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    List[str]       ):
  		return self.encoder.get(__A				,			self.encoder.get(self.unk_token       )       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[Any]				,			__A	:    int       ):
  		return self.decoder.get(__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    List[str]       ):
  		__A   : Tuple								= """""".join(__A       )
  		__A   : Any								= bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""				,			errors=self.errors       )
  		return text
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    str				,			__A	:    Optional[str] = None       ):
  		if not os.path.isdir(__A       ):
  			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
  			return
  		__A   : int								= os.path.join(
  		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
  		__A   : Optional[int]								= os.path.join(
  		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )
  
  		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as f:
  			f.write(json.dumps(self.encoder				,			indent=2				,			sort_keys=__A				,			ensure_ascii=__A       ) + """\n"""       )
  
  		__A   : List[str]								= 0
  		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as writer:
  			writer.write("""#version: 0.2\n"""       )
  			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,			key=lambda __A       : kv[1]       ):
  				if index != token_index:
  					logger.warning(
  					    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  					    """ Please check that the tokenizer is not corrupted!"""       )
  					__A   : Tuple								= token_index
  				writer.write(""" """.join(__A       ) + """\n"""       )
  				index += 1
  
  		return vocab_file, merge_file
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Tuple				,			__A	:    List[int]				,			__A	:    Optional[List[int]] = None       ):
  		if token_ids_a is None:
  			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  		__A   : List[str]								= [self.cls_token_id]
  		__A   : Optional[int]								= [self.sep_token_id]
  		return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    List[int]				,			__A	:    Optional[List[int]] = None				,			__A	:    bool = False       ):
  		if already_has_special_tokens:
  			return super().get_special_tokens_mask(
  			    token_ids_a=__A				,			token_ids_a=__A				,			already_has_special_tokens=__A       )
  
  		if token_ids_a is None:
  			return [1] + ([0] * len(__A       )) + [1]
  		return [1] + ([0] * len(__A       )) + [1, 1] + ([0] * len(__A       )) + [1]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    List[int]				,			__A	:    Optional[List[int]] = None       ):
  		__A   : Any								= [self.sep_token_id]
  		__A   : Any								= [self.cls_token_id]
  
  		if token_ids_a is None:
  			return len(cls + token_ids_a + sep       ) * [0]
  		return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    str				,			__A	:    List[Any]=False				,			**__A	:    List[Any]       ):
  		__A   : Tuple								= kwargs.pop("""add_prefix_space"""				,			self.add_prefix_space       )
  		if (is_split_into_words or add_prefix_space) and (len(__A       ) > 0 and not text[0].isspace()):
  			__A   : Dict								= """ """ + text
  		return (text, kwargs)
  
  
  
  
  
  
style_context: 
  
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  
  
  
  
  
  
  class     lowerCamelCase_			(  tf.keras.layers.Layer  ):
  
  
  
  
  	def __init__(    self	:    int				,			__A	:    List				,			__A	:    bool				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    str = "longest"				,			__A	:    bool = True				,			__A	:    int = 512				,			__A	:    int = None				,			__A	:    bool = True				,			__A	:    bool = True				,			__A	:    bool = True				,			**__A	:    Any				,			):
  		super().__init__()
  		if use_fast_bert_tokenizer:
  			__A   : Optional[int]								= FastBertTokenizer(
  			    __A				,			token_out_type=tf.intaa				,			lower_case_nfd_strip_accents=__A				,			**__A       )
  		else:
  			__A   : Optional[Any]								= tf.lookup.StaticVocabularyTable(
  			    tf.lookup.KeyValueTensorInitializer(
  			        keys=__A				,			key_dtype=tf.string				,			values=tf.range(tf.size(__A				,			out_type=tf.intaa       )				,			dtype=tf.intaa       )				,			value_dtype=tf.intaa				,			)				,			num_oov_buckets=1				,			)
  			__A   : Any								= BertTokenizerLayer(
  			    __A				,			token_out_type=tf.intaa				,			lower_case=__A				,			**__A       )
  
  		__A   : Union[str, Any]								= vocab_list
  		__A   : List[str]								= do_lower_case
  		__A   : List[Any]								= cls_token_id or vocab_list.index("""[CLS]"""       )
  		__A   : Tuple								= sep_token_id or vocab_list.index("""[SEP]"""       )
  		__A   : str								= pad_token_id or vocab_list.index("""[PAD]"""       )
  		__A   : Optional[int]								= ShrinkLongestTrimmer(max_length - 3				,			axis=1       )  # Allow room for special tokens
  		__A   : Union[str, Any]								= max_length
  		__A   : List[Any]								= padding
  		__A   : Optional[int]								= truncation
  		__A   : Tuple								= pad_to_multiple_of
  		__A   : Optional[Any]								= return_token_type_ids
  		__A   : int								= return_attention_mask
  
  
  
  
  	@classmethod
  	def 					lowerCAmelCase_					(    cls	:    List[Any]				,			__A	:    "PreTrainedTokenizerBase"				,			**__A	:    List[str]       ):  # noqa: F821
  		__A   : Any								= kwargs.pop("""do_lower_case"""				,			__A       )
  		__A   : Dict								= tokenizer.do_lower_case if do_lower_case is None else do_lower_case
  		__A   : List[str]								= kwargs.pop("""cls_token_id"""				,			__A       )
  		__A   : str								= tokenizer.cls_token_id if cls_token_id is None else cls_token_id
  		__A   : str								= kwargs.pop("""sep_token_id"""				,			__A       )
  		__A   : Dict								= tokenizer.sep_token_id if sep_token_id is None else sep_token_id
  		__A   : Tuple								= kwargs.pop("""pad_token_id"""				,			__A       )
  		__A   : List[Any]								= tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
  		__A   : Union[str, Any]								= tokenizer.get_vocab()
  		__A   : Any								= sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]				,			key=lambda __A       : x[1]       )
  		__A   : int								= [entry[0] for entry in vocab]
  		return cls(
  		    vocab_list=__A				,			do_lower_case=__A				,			cls_token_id=__A				,			sep_token_id=__A				,			pad_token_id=__A				,			**__A				,			)
  
  
  
  
  	@classmethod
  	def 					lowerCAmelCase_					(    cls	:    List[str]				,			__A	:    Union[str, os.PathLike]				,			*__A	:    Tuple				,			**__A	:    Optional[int]       ):
  		try:
  			__A   : int								= BertTokenizer.from_pretrained(__A				,			*__A				,			**__A       )
  		except:  # noqa: E722
  			from .tokenization_bert_fast import BertTokenizerFast
  
  			__A   : Optional[int]								= BertTokenizerFast.from_pretrained(__A				,			*__A				,			**__A       )
  		return cls.from_tokenizer(__A				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    Optional[Any]       ):
  		if self.do_lower_case:
  			__A   : Tuple								= case_fold_utfa(__A       )
  		__A   : Optional[int]								= self.tf_tokenizer.tokenize(__A       )
  		return tokens.merge_dims(1				,			-1       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Optional[Any]				,			__A	:    Dict=None				,			__A	:    List[Any]=None				,			__A	:    str=None				,			__A	:    Union[str, Any]=None				,			__A	:    Union[str, Any]=None				,			__A	:    int=None				,			__A	:    Any=None				,			):
  		if padding is None:
  			__A   : List[Any]								= self.padding
  		if padding not in ("longest", "max_length"):
  			raise ValueError("""Padding must be either 'longest' or 'max_length'!"""       )
  		if max_length is not None and text_pair is not None:
  			# Because we have to instantiate a Trimmer to do it properly
  			raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""       )
  		if max_length is None:
  			__A   : Any								= self.max_length
  		if truncation is None:
  			__A   : int								= self.truncation
  		if pad_to_multiple_of is None:
  			__A   : List[str]								= self.pad_to_multiple_of
  		if return_token_type_ids is None:
  			__A   : int								= self.return_token_type_ids
  		if return_attention_mask is None:
  			__A   : Any								= self.return_attention_mask
  		if not isinstance(__A				,			tf.Tensor       ):
  			__A   : str								= tf.convert_to_tensor(__A       )
  		if text_pair is not None and not isinstance(__A				,			tf.Tensor       ):
  			__A   : Optional[Any]								= tf.convert_to_tensor(__A       )
  		if text_pair is not None:
  			if text.shape.rank > 1:
  				raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""       )
  			if text_pair.shape.rank > 1:
  				raise ValueError("""text_pair should not be multidimensional!"""       )
  		if text.shape.rank == 2:
  			__A						,							__A   : Optional[int]								= text[:, 0], text[:, 1]
  		__A   : List[Any]								= self.unpaired_tokenize(__A       )
  		if text_pair is None:  # Unpaired text
  			if truncation:
  				__A   : Union[str, Any]								= text[:, : max_length - 2]  # Allow room for special tokens
  			__A						,							__A   : List[Any]								= combine_segments(
  			    (text,)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
  		else:  # Paired text
  			__A   : Tuple								= self.unpaired_tokenize(__A       )
  			if truncation:
  				__A						,							__A   : Optional[int]								= self.paired_trimmer.trim([text, text_pair]       )
  			__A						,							__A   : int								= combine_segments(
  			    (text, text_pair)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
  		if padding == "longest":
  			__A   : str								= input_ids.bounding_shape(axis=1       )
  			if pad_to_multiple_of is not None:
  				# No ceiling division in tensorflow, so we negate floordiv instead
  				__A   : Optional[Any]								= pad_to_multiple_of * (-tf.math.floordiv(-pad_length				,			__A       ))
  		else:
  			__A   : Dict								= max_length
  
  		__A						,							__A   : str								= pad_model_inputs(__A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
  		__A   : List[str]								= {"""input_ids""": input_ids}
  		if return_attention_mask:
  			__A   : Any								= attention_mask
  		if return_token_type_ids:
  			__A						,							__A   : Tuple								= pad_model_inputs(
  			    __A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
  			__A   : List[str]								= token_type_ids
  		return output
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
  		return {
  		    "vocab_list": self.vocab_list,
  		    "do_lower_case": self.do_lower_case,
  		    "cls_token_id": self.cls_token_id,
  		    "sep_token_id": self.sep_token_id,
  		    "pad_token_id": self.pad_token_id,
  		}
  
  
  
  
  
  