code: 
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  UpperCAmelCase_       :				List[Any]		   =      logging.get_logger(__name__)
  
  UpperCAmelCase_       :				Union[str, Any]		   =      {
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  UpperCAmelCase_       :				Union[str, Any]		   =      {
      '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
      '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
  }
  
  UpperCAmelCase_       :				Tuple		   =      {
      '''openai-gpt''': 512,
  }
  def        __SCREAMING_SNAKE_CASE    (      a__						:      Dict					)				->							Any:
  	__A   : int								= text.strip()
  	if not text:
  		return []
  	__A   : int								= text.split()
  	return tokens
  
  
  
  
  
  
  class     lowerCamelCase_			(  _lowercase  ):
  
  
  
  
  	def __init__(    self	:    List[str]				,			__A	:    Optional[int]=True				,			__A	:    int=None				,			__A	:    int=True				,			__A	:    int=None				,			__A	:    str=True				,			):
  		if never_split is None:
  			__A   : Tuple								= []
  		__A   : Any								= do_lower_case
  		__A   : Any								= set(__A       )
  		__A   : Optional[int]								= tokenize_chinese_chars
  		__A   : Union[str, Any]								= strip_accents
  		__A   : str								= do_split_on_punc
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Optional[Any]				,			__A	:    Optional[int]=None       ):
  		__A   : Optional[Any]								= self.never_split.union(set(__A       )       ) if never_split else self.never_split
  		__A   : List[Any]								= self._clean_text(__A       )
  
  		# This was added on November 1st, 2018 for the multilingual and Chinese
  		# models. This is also applied to the English models now, but it doesn't
  		# matter since the English models were not trained on any Chinese data
  		# and generally don't have any Chinese data in them (there are Chinese
  		# characters in the vocabulary because Wikipedia does have some Chinese
  		# words in the English Wikipedia.).
  		if self.tokenize_chinese_chars:
  			__A   : List[str]								= self._tokenize_chinese_chars(__A       )
  		# prevents treating the same character with different unicode codepoints as different characters
  		__A   : List[Any]								= unicodedata.normalize("""NFC"""				,			__A       )
  		__A   : Tuple								= whitespace_tokenize(__A       )
  		__A   : Any								= []
  		for token in orig_tokens:
  			if token not in never_split:
  				if self.do_lower_case:
  					__A   : Optional[int]								= token.lower()
  					if self.strip_accents is not False:
  						__A   : Dict								= self._run_strip_accents(__A       )
  				elif self.strip_accents:
  					__A   : List[Any]								= self._run_strip_accents(__A       )
  			split_tokens.extend(self._run_split_on_punc(__A				,			__A       )       )
  
  		__A   : str								= whitespace_tokenize(""" """.join(__A       )       )
  		return output_tokens
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Dict       ):
  		__A   : Any								= unicodedata.normalize("""NFD"""				,			__A       )
  		__A   : Dict								= []
  		for char in text:
  			__A   : Union[str, Any]								= unicodedata.category(__A       )
  			if cat == "Mn":
  				continue
  			output.append(__A       )
  		return "".join(__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    int				,			__A	:    List[str]=None       ):
  		if not self.do_split_on_punc or (never_split is not None and text in never_split):
  			return [text]
  		__A   : Union[str, Any]								= list(__A       )
  		__A   : str								= 0
  		__A   : int								= True
  		__A   : Dict								= []
  		while i < len(__A       ):
  			__A   : Tuple								= chars[i]
  			if _is_punctuation(__A       ):
  				output.append([char]       )
  				__A   : Tuple								= True
  			else:
  				if start_new_word:
  					output.append([]       )
  				__A   : str								= False
  				output[-1].append(__A       )
  			i += 1
  
  		return ["".join(__A       ) for x in output]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Tuple				,			__A	:    Any       ):
  		__A   : Optional[Any]								= []
  		for char in text:
  			__A   : Optional[int]								= ord(__A       )
  			if self._is_chinese_char(__A       ):
  				output.append(""" """       )
  				output.append(__A       )
  				output.append(""" """       )
  			else:
  				output.append(__A       )
  		return "".join(__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict				,			__A	:    Optional[int]       ):
  		# This defines a "chinese character" as anything in the CJK Unicode block:
  		#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
  		#
  		# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
  		# despite its name. The modern Korean Hangul alphabet is a different block,
  		# as is Japanese Hiragana and Katakana. Those alphabets are used to write
  		# space-separated words, so they are not treated specially and handled
  		# like the all of the other languages.
  		if (
  		    (cp >= 0X4E00 and cp <= 0X9FFF)
  		    or (cp >= 0X3400 and cp <= 0X4DBF)  #
  		    or (cp >= 0X20000 and cp <= 0X2A6DF)  #
  		    or (cp >= 0X2A700 and cp <= 0X2B73F)  #
  		    or (cp >= 0X2B740 and cp <= 0X2B81F)  #
  		    or (cp >= 0X2B820 and cp <= 0X2CEAF)  #
  		    or (cp >= 0XF900 and cp <= 0XFAFF)
  		    or (cp >= 0X2F800 and cp <= 0X2FA1F)  #
  		):  #
  			return True
  
  		return False
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Union[str, Any]				,			__A	:    Any       ):
  		__A   : Tuple								= []
  		for char in text:
  			__A   : int								= ord(__A       )
  			if cp == 0 or cp == 0XFFFD or _is_control(__A       ):
  				continue
  			if _is_whitespace(__A       ):
  				output.append(""" """       )
  			else:
  				output.append(__A       )
  		return "".join(__A       )
  
  
  
  
  
  
  def        __SCREAMING_SNAKE_CASE    (      a__						:      Tuple					)				->							Any:
  	__A   : Dict								= set()
  	__A   : Dict								= word[0]
  	for char in word[1:]:
  		pairs.add((prev_char, char)					)
  		__A   : Optional[int]								= char
  	return pairs
  def        __SCREAMING_SNAKE_CASE    (      a__						:      Any					)				->							Dict:
  	__A   : List[Any]								= text.replace("""—""" ,"""-"""					)
  	__A   : Union[str, Any]								= text.replace("""–""" ,"""-"""					)
  	__A   : Optional[Any]								= text.replace("""―""" ,"""-"""					)
  	__A   : str								= text.replace("""…""" ,"""..."""					)
  	__A   : Any								= text.replace("""´""" ,"""'"""					)
  	__A   : List[str]								= re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""" ,r""" \1 """ ,a__					)
  	__A   : str								= re.sub(r"""\s*\n\s*""" ,""" \n """ ,a__					)
  	__A   : Optional[Any]								= re.sub(r"""[^\S\n]+""" ,""" """ ,a__					)
  	return text.strip()
  
  
  
  
  
  
  
  class     lowerCamelCase_			(  _lowercase  ):
  	_lowercase		:      str    				=							VOCAB_FILES_NAMES
  	_lowercase		:      Dict    				=							PRETRAINED_VOCAB_FILES_MAP
  	_lowercase		:      Any    				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	_lowercase		:      Union[str, Any]    				=							['''input_ids''', '''attention_mask''']
  
  
  
  
  	def __init__(    self	:    Union[str, Any]				,			__A	:    Optional[Any]				,			__A	:    Dict				,			__A	:    str="<unk>"				,			**__A	:    Dict       ):
  		super().__init__(unk_token=__A				,			**__A       )
  
  		try:
  			import ftfy
  			from spacy.lang.en import English
  
  			__A   : int								= English()
  			__A   : Union[str, Any]								= _nlp.tokenizer
  			__A   : Tuple								= ftfy.fix_text
  		except ImportError:
  			logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""       )
  			__A   : int								= BasicTokenizer(do_lower_case=__A       )
  			__A   : Union[str, Any]								= None
  
  		with open(__A				,			encoding="""utf-8"""       ) as vocab_handle:
  			__A   : Union[str, Any]								= json.load(__A       )
  		__A   : Any								= {v: k for k, v in self.encoder.items()}
  		with open(__A				,			encoding="""utf-8"""       ) as merges_handle:
  			__A   : Union[str, Any]								= merges_handle.read().split("""\n"""       )[1:-1]
  		__A   : List[Any]								= [tuple(merge.split()       ) for merge in merges]
  		__A   : Union[str, Any]								= dict(zip(__A				,			range(len(__A       )       )       )       )
  		__A   : Union[str, Any]								= {}
  
  
  
  
  	@property
  	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
  		return True
  
  
  
  
  	@property
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		return len(self.encoder       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		return dict(self.encoder				,			**self.added_tokens_encoder       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict				,			__A	:    Any       ):
  		__A   : Any								= tuple(token[:-1]       ) + (token[-1] + """</w>""",)
  		if token in self.cache:
  			return self.cache[token]
  		__A   : Dict								= get_pairs(__A       )
  
  		if not pairs:
  			return token + "</w>"
  
  		while True:
  			__A   : Dict								= min(__A				,			key=lambda __A       : self.bpe_ranks.get(__A				,			float("""inf"""       )       )       )
  			if bigram not in self.bpe_ranks:
  				break
  			__A						,							__A   : List[str]								= bigram
  			__A   : Union[str, Any]								= []
  			__A   : List[str]								= 0
  			while i < len(__A       ):
  				try:
  					__A   : Optional[int]								= word.index(__A				,			__A       )
  				except ValueError:
  					new_word.extend(word[i:]       )
  					break
  				else:
  					new_word.extend(word[i:j]       )
  					__A   : List[str]								= j
  
  				if word[i] == first and i < len(__A       ) - 1 and word[i + 1] == second:
  					new_word.append(first + second       )
  					i += 2
  				else:
  					new_word.append(word[i]       )
  					i += 1
  			__A   : List[str]								= tuple(__A       )
  			__A   : Dict								= new_word
  			if len(__A       ) == 1:
  				break
  			else:
  				__A   : List[Any]								= get_pairs(__A       )
  		__A   : Any								= """ """.join(__A       )
  		if word == "\n  </w>":
  			__A   : Optional[Any]								= """\n</w>"""
  		__A   : Dict								= word
  		return word
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Tuple       ):
  		__A   : Union[str, Any]								= []
  		if self.fix_text is None:
  			# Using BERT's BasicTokenizer
  			__A   : Union[str, Any]								= self.nlp.tokenize(__A       )
  			for token in text:
  				split_tokens.extend(list(self.bpe(__A       ).split(""" """       )       )       )
  		else:
  			# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  			__A   : Union[str, Any]								= self.nlp(text_standardize(self.fix_text(__A       )       )       )
  			for token in text:
  				split_tokens.extend(list(self.bpe(token.text.lower()       ).split(""" """       )       )       )
  		return split_tokens
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Union[str, Any]       ):
  		return self.encoder.get(__A				,			self.encoder.get(self.unk_token       )       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    Union[str, Any]       ):
  		return self.decoder.get(__A				,			self.unk_token       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    Optional[int]       ):
  		__A   : Union[str, Any]								= """""".join(__A       ).replace("""</w>"""				,			""" """       ).strip()
  		return out_string
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[Any]				,			__A	:    str				,			__A	:    Optional[str] = None       ):
  		if not os.path.isdir(__A       ):
  			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
  			return
  		__A   : List[str]								= os.path.join(
  		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
  		__A   : Dict								= os.path.join(
  		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )
  
  		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as f:
  			f.write(json.dumps(self.encoder				,			indent=2				,			sort_keys=__A				,			ensure_ascii=__A       ) + """\n"""       )
  
  		__A   : Dict								= 0
  		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as writer:
  			writer.write("""#version: 0.2\n"""       )
  			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,			key=lambda __A       : kv[1]       ):
  				if index != token_index:
  					logger.warning(
  					    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  					    """ Please check that the tokenizer is not corrupted!"""       )
  					__A   : Any								= token_index
  				writer.write(""" """.join(__A       ) + """\n"""       )
  				index += 1
  
  		return vocab_file, merge_file
  
  
  
  
  
  
style_context: 
  
  import os
  import tempfile
  import unittest
  from typing import List
  
  from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
  from transformers.testing_utils import require_tokenizers
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers.tokenization_utils_base import PreTrainedTokenizerBase
  
  
  
  
  
  
  @require_tokenizers
  class     lowerCamelCase_			(  unittest.TestCase  ):
  	_lowercase		:      List[str]    				=							EsmTokenizer
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		super().setUp()
  		__A   : Dict								= tempfile.mkdtemp()
  		# fmt: off
  		__A   : List[str]								= ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
  		# fmt: on
  		__A   : int								= os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]       )
  		with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""       ) as vocab_writer:
  			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict				,			**__A	:    int       ):
  		return [self.get_tokenizer(**__A       )]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			**__A	:    List[Any]       ):
  		return self.tokenizer_class.from_pretrained(self.tmpdirname				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= tokenizer.tokenize("""LAGVS"""       )
  		self.assertListEqual(__A				,			["""L""", """A""", """G""", """V""", """S"""]       )
  		self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )				,			[4, 5, 6, 7, 8]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= """LAGVS"""
  		self.assertListEqual(tokenizer.encode(__A       )				,			[0, 4, 5, 6, 7, 8, 2]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict       ):
  		__A   : str								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Tuple								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : Dict								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Any								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Optional[int]								= self.get_tokenizers(fast=__A       )
  		for tokenizer in tokenizers:
  			with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
  				__A   : Any								= """<unk>"""
  				__A   : str								= """<mask>"""
  
  				__A   : Optional[Any]								= tokenizer.tokenize(__A       )
  				__A   : str								= tokenizer.tokenize(__A       )
  
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(token_a[0]				,			__A       )
  				self.assertEqual(token_a[0]				,			__A       )
  
  
  
  
  
  