code: 
  
  
  
  
  
  import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  if TYPE_CHECKING:
  		from transformers.pipelines.conversational import Conversation
  
  _lowercase		:     Tuple 					=logging.get_logger(__name__)
  
  _lowercase		:     List[Any] 					={
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  _lowercase		:     int 					={
      """vocab_file""": {
          """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
          """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
          """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
          """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
          """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
      },
      """merges_file""": {
          """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
          """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
          """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
          """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
          """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
      },
  }
  
  _lowercase		:     List[Any] 					={
      """gpt2""": 1024,
      """gpt2-medium""": 1024,
      """gpt2-large""": 1024,
      """gpt2-xl""": 1024,
      """distilgpt2""": 1024,
  }
  
  
  @lru_cache()
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		lowerCamelCase_  :    str   	=			(
  		    list(range(ord('!'	)       ,ord('~'	) + 1	)	) + list(range(ord('¡'	)       ,ord('¬'	) + 1	)	) + list(range(ord('®'	)       ,ord('ÿ'	) + 1	)	)
  		)
  		lowerCamelCase_  :    Optional[int]   	=			bs[:]
  		lowerCamelCase_  :    List[Any]   	=			0
  		for b in range(2**8	):
  				if b not in bs:
  						bs.append(lowerCAmelCase__	)
  						cs.append(2**8 + n	)
  						n += 1
  		lowerCamelCase_  :    Union[str, Any]   	=			[chr(lowerCAmelCase__	) for n in cs]
  		return dict(zip(lowerCAmelCase__       ,lowerCAmelCase__	)	)
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		lowerCamelCase_  :    Tuple   	=			set()
  		lowerCamelCase_  :    Any   	=			word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)	)
  				lowerCamelCase_  :    List[str]   	=			char
  		return pairs
  
  
  
  
  
  
  class 	UpperCamelCase_     (     __lowerCamelCase       ):
  					_a			:	Optional[Any]							=				VOCAB_FILES_NAMES
  					_a			:	str							=				PRETRAINED_VOCAB_FILES_MAP
  					_a			:	str							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					_a			:	Dict							=				['input_ids', 'attention_mask']
  
  
  
  					def __init__(		self    :    List[str]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]="replace"					,			lowerCamelCase    :    int="<|endoftext|>"					,			lowerCamelCase    :    Optional[Any]="<|endoftext|>"					,			lowerCamelCase    :    List[Any]="<|endoftext|>"					,			lowerCamelCase    :    Optional[Any]=None					,			lowerCamelCase    :    int=False					,			lowerCamelCase    :    Any=False					,			**lowerCamelCase    :    Union[str, Any]					,			):
  							lowerCamelCase_  :    Any   	=			AddedToken(lowerCamelCase					,			lstrip=lowerCamelCase					,			rstrip=lowerCamelCase	) if isinstance(lowerCamelCase					,			lowerCamelCase	) else bos_token
  							lowerCamelCase_  :    Dict   	=			AddedToken(lowerCamelCase					,			lstrip=lowerCamelCase					,			rstrip=lowerCamelCase	) if isinstance(lowerCamelCase					,			lowerCamelCase	) else eos_token
  							lowerCamelCase_  :    Tuple   	=			AddedToken(lowerCamelCase					,			lstrip=lowerCamelCase					,			rstrip=lowerCamelCase	) if isinstance(lowerCamelCase					,			lowerCamelCase	) else unk_token
  							lowerCamelCase_  :    Any   	=			AddedToken(lowerCamelCase					,			lstrip=lowerCamelCase					,			rstrip=lowerCamelCase	) if isinstance(lowerCamelCase					,			lowerCamelCase	) else pad_token
  							super().__init__(
  							    errors=lowerCamelCase					,			unk_token=lowerCamelCase					,			bos_token=lowerCamelCase					,			eos_token=lowerCamelCase					,			pad_token=lowerCamelCase					,			add_prefix_space=lowerCamelCase					,			add_bos_token=lowerCamelCase					,			**lowerCamelCase					,			)
  							lowerCamelCase_  :    Dict   	=			add_bos_token
  
  							with open(lowerCamelCase					,			encoding='utf-8'	) as vocab_handle:
  									lowerCamelCase_  :    Tuple   	=			json.load(lowerCamelCase	)
  							lowerCamelCase_  :    Tuple   	=			{v: k for k, v in self.encoder.items()}
  							lowerCamelCase_  :    List[Any]   	=			errors  # how to handle errors in decoding
  							lowerCamelCase_  :    Optional[int]   	=			bytes_to_unicode()
  							lowerCamelCase_  :    List[Any]   	=			{v: k for k, v in self.byte_encoder.items()}
  							with open(lowerCamelCase					,			encoding='utf-8'	) as merges_handle:
  									lowerCamelCase_  :    int   	=			merges_handle.read().split('\n'	)[1:-1]
  							lowerCamelCase_  :    Tuple   	=			[tuple(merge.split()	) for merge in bpe_merges]
  							lowerCamelCase_  :    List[str]   	=			dict(zip(lowerCamelCase					,			range(len(lowerCamelCase	)	)	)	)
  							lowerCamelCase_  :    int   	=			{}
  							lowerCamelCase_  :    Optional[int]   	=			add_prefix_space
  
  							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  							lowerCamelCase_  :    Any   	=			re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)
  
  
  
  					@property
  					def      __a    (		self    :    Optional[int]	):
  							return len(self.encoder	)
  
  
  
  					def      __a    (		self    :    Tuple	):
  							return dict(self.encoder					,			**self.added_tokens_encoder	)
  
  
  
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    Optional[Any]	):
  
  
  
  
  
  
  							if token in self.cache:
  									return self.cache[token]
  							lowerCamelCase_  :    str   	=			tuple(lowerCamelCase	)
  							lowerCamelCase_  :    Union[str, Any]   	=			get_pairs(lowerCamelCase	)
  
  							if not pairs:
  									return token
  
  							while True:
  									lowerCamelCase_  :    str   	=			min(lowerCamelCase					,			key=lambda lowerCamelCase	: self.bpe_ranks.get(lowerCamelCase					,			float('inf'	)	)	)
  									if bigram not in self.bpe_ranks:
  											break
  									lowerCamelCase_	,			lowerCamelCase_  :    List[Any]   	=			bigram
  									lowerCamelCase_  :    Union[str, Any]   	=			[]
  									lowerCamelCase_  :    Tuple   	=			0
  									while i < len(lowerCamelCase	):
  											try:
  													lowerCamelCase_  :    Any   	=			word.index(lowerCamelCase					,			lowerCamelCase	)
  											except ValueError:
  													new_word.extend(word[i:]	)
  													break
  											else:
  													new_word.extend(word[i:j]	)
  													lowerCamelCase_  :    List[Any]   	=			j
  
  											if word[i] == first and i < len(lowerCamelCase	) - 1 and word[i + 1] == second:
  													new_word.append(first + second	)
  													i += 2
  											else:
  													new_word.append(word[i]	)
  													i += 1
  									lowerCamelCase_  :    Any   	=			tuple(lowerCamelCase	)
  									lowerCamelCase_  :    List[Any]   	=			new_word
  									if len(lowerCamelCase	) == 1:
  											break
  									else:
  											lowerCamelCase_  :    Optional[Any]   	=			get_pairs(lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			' '.join(lowerCamelCase	)
  							lowerCamelCase_  :    List[Any]   	=			word
  							return word
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    str=None	):
  
  
  
  
  
  
  							if self.add_bos_token:
  									lowerCamelCase_  :    List[str]   	=			[self.bos_token_id]
  							else:
  									lowerCamelCase_  :    Optional[int]   	=			[]
  
  							lowerCamelCase_  :    Any   	=			bos_token_ids + token_ids_a
  
  							if token_ids_a is None:
  									return output
  
  							return output + bos_token_ids + token_ids_a
  
  
  
  					def      __a    (		self    :    Tuple					,			lowerCamelCase    :    List[int]					,			lowerCamelCase    :    Optional[List[int]] = None					,			lowerCamelCase    :    bool = False	):
  
  
  
  
  
  
  							if already_has_special_tokens:
  									return super().get_special_tokens_mask(
  									    token_ids_a=lowerCamelCase					,			token_ids_a=lowerCamelCase					,			already_has_special_tokens=lowerCamelCase	)
  
  							if not self.add_bos_token:
  									return super().get_special_tokens_mask(
  									    token_ids_a=lowerCamelCase					,			token_ids_a=lowerCamelCase					,			already_has_special_tokens=lowerCamelCase	)
  
  							if token_ids_a is None:
  									return [1] + ([0] * len(lowerCamelCase	))
  							return [1] + ([0] * len(lowerCamelCase	)) + [1] + ([0] * len(lowerCamelCase	))
  
  
  
  					def      __a    (		self    :    int					,			lowerCamelCase    :    Union[str, Any]	):
  							lowerCamelCase_  :    List[str]   	=			[]
  							for token in re.findall(self.pat					,			lowerCamelCase	):
  									lowerCamelCase_  :    List[str]   	=			''.join(
  									    self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase	).split(' '	)	)
  							return bpe_tokens
  
  
  
  					def      __a    (		self    :    str					,			lowerCamelCase    :    List[Any]	):
  							return self.encoder.get(lowerCamelCase					,			self.encoder.get(self.unk_token	)	)
  
  
  
  					def      __a    (		self    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]	):
  							return self.decoder.get(lowerCamelCase	)
  
  
  
  					def      __a    (		self    :    str					,			lowerCamelCase    :    Union[str, Any]	):
  							lowerCamelCase_  :    Tuple   	=			''.join(lowerCamelCase	)
  							lowerCamelCase_  :    str   	=			bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'					,			errors=self.errors	)
  							return text
  
  
  
  					def      __a    (		self    :    Any					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[str] = None	):
  
  
  
  
  
  
  							if not os.path.isdir(lowerCamelCase	):
  									logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
  									return
  							lowerCamelCase_  :    Dict   	=			os.path.join(
  							    lowerCamelCase					,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
  							lowerCamelCase_  :    str   	=			os.path.join(
  							    lowerCamelCase					,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)
  
  							with open(lowerCamelCase					,			'w'					,			encoding='utf-8'	) as f:
  									f.write(json.dumps(self.encoder					,			indent=2					,			sort_keys=lowerCamelCase					,			ensure_ascii=lowerCamelCase	) + '\n'	)
  
  							lowerCamelCase_  :    List[Any]   	=			0
  							with open(lowerCamelCase					,			'w'					,			encoding='utf-8'	) as writer:
  									writer.write('#version: 0.2\n'	)
  									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,			key=lambda lowerCamelCase	: kv[1]	):
  											if index != token_index:
  													logger.warning(
  													    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
  													    ' Please check that the tokenizer is not corrupted!'	)
  													lowerCamelCase_  :    List[Any]   	=			token_index
  											writer.write(' '.join(lowerCamelCase	) + '\n'	)
  											index += 1
  
  							return vocab_file, merge_file
  
  
  
  					def      __a    (		self    :    Dict					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    List[str]=False					,			**lowerCamelCase    :    Optional[int]	):
  							lowerCamelCase_  :    Union[str, Any]   	=			kwargs.pop('add_prefix_space'					,			self.add_prefix_space	)
  							if is_split_into_words or add_prefix_space:
  									lowerCamelCase_  :    Dict   	=			' ' + text
  							return (text, kwargs)
  
  
  
  
  
  					def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    "Conversation"	):
  							lowerCamelCase_  :    Any   	=			[]
  							for is_user, text in conversation.iter_texts():
  									input_ids.extend(self.encode(lowerCamelCase					,			add_special_tokens=lowerCamelCase	) + [self.eos_token_id]	)
  							if len(lowerCamelCase	) > self.model_max_length:
  									lowerCamelCase_  :    List[Any]   	=			input_ids[-self.model_max_length :]
  							return input_ids
  
style_context: 
  
  
  
  
  
  from __future__ import annotations
  
  
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		lowerCamelCase_  :    int
  		lowerCamelCase_  :    list[str]   	=			list(lowerCAmelCase__	)
  		lowerCamelCase_	,			lowerCamelCase_  :    List[str]   	=			_vector[:-1], int(_vector[-1]	)
  		lowerCamelCase_  :    list[int]   	=			[int(lowerCAmelCase__	) for digit in __vector]
  
  		vector.reverse()
  		for i, digit in enumerate(lowerCAmelCase__	):
  				if i & 1 == 0:
  						lowerCamelCase_  :    int   	=			digit * 2
  						if doubled > 9:
  								doubled -= 9
  						check_digit += doubled
  				else:
  						check_digit += digit
  
  		return check_digit % 10 == 0
  
  
  if __name__ == "__main__":
  		import doctest
  
  		doctest.testmod()
  		assert is_luhn("""79927398713""")
  		assert not is_luhn("""79927398714""")
  