code: 
  
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  lowercase_								=    logging.get_logger(__name__)
  
  lowercase_								=    {
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  lowercase_								=    {
      'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
      'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
  }
  
  lowercase_								=    {
      'openai-gpt': 5_12,
  }
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  Union[str, Any]:
  						_a     =			text.strip()
  						if not text:
  												return []
  						_a     =			text.split()
  						return tokens
  
  
  
  
  class 					_UpperCamelCase (      lowerCamelCase__	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  
  
  
  
  
  
  
  							def __init__(		self  :				List[Any]		,				SCREAMING_SNAKE_CASE_  :				List[str]=True		,				SCREAMING_SNAKE_CASE_  :				Tuple=None		,				SCREAMING_SNAKE_CASE_  :				Any=True		,				SCREAMING_SNAKE_CASE_  :				int=None		,				SCREAMING_SNAKE_CASE_  :				Optional[int]=True		,				):
  													if never_split is None:
  																			_a     =			[]
  													_a     =			do_lower_case
  													_a     =			set(SCREAMING_SNAKE_CASE_		)
  													_a     =			tokenize_chinese_chars
  													_a     =			strip_accents
  													_a     =			do_split_on_punc
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				Optional[int]=None		):
  													_a     =			self.never_split.union(set(SCREAMING_SNAKE_CASE_		)		) if never_split else self.never_split
  													_a     =			self._clean_text(SCREAMING_SNAKE_CASE_		)
  
  													# This was added on November 1st, 2018 for the multilingual and Chinese
  													# models. This is also applied to the English models now, but it doesn't
  													# matter since the English models were not trained on any Chinese data
  													# and generally don't have any Chinese data in them (there are Chinese
  													# characters in the vocabulary because Wikipedia does have some Chinese
  													# words in the English Wikipedia.).
  													if self.tokenize_chinese_chars:
  																			_a     =			self._tokenize_chinese_chars(SCREAMING_SNAKE_CASE_		)
  													# prevents treating the same character with different unicode codepoints as different characters
  													_a     =			unicodedata.normalize('NFC'		,				SCREAMING_SNAKE_CASE_		)
  													_a     =			whitespace_tokenize(SCREAMING_SNAKE_CASE_		)
  													_a     =			[]
  													for token in orig_tokens:
  																			if token not in never_split:
  																									if self.do_lower_case:
  																															_a     =			token.lower()
  																															if self.strip_accents is not False:
  																																					_a     =			self._run_strip_accents(SCREAMING_SNAKE_CASE_		)
  																									elif self.strip_accents:
  																															_a     =			self._run_strip_accents(SCREAMING_SNAKE_CASE_		)
  																			split_tokens.extend(self._run_split_on_punc(SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		)		)
  
  													_a     =			whitespace_tokenize(' '.join(SCREAMING_SNAKE_CASE_		)		)
  													return output_tokens
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Union[str, Any]		,				SCREAMING_SNAKE_CASE_  :				Dict		):
  													_a     =			unicodedata.normalize('NFD'		,				SCREAMING_SNAKE_CASE_		)
  													_a     =			[]
  													for char in text:
  																			_a     =			unicodedata.category(SCREAMING_SNAKE_CASE_		)
  																			if cat == "Mn":
  																									continue
  																			output.append(SCREAMING_SNAKE_CASE_		)
  													return "".join(SCREAMING_SNAKE_CASE_		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[Any]		,				SCREAMING_SNAKE_CASE_  :				Dict		,				SCREAMING_SNAKE_CASE_  :				Optional[Any]=None		):
  													if not self.do_split_on_punc or (never_split is not None and text in never_split):
  																			return [text]
  													_a     =			list(SCREAMING_SNAKE_CASE_		)
  													_a     =			0
  													_a     =			True
  													_a     =			[]
  													while i < len(SCREAMING_SNAKE_CASE_		):
  																			_a     =			chars[i]
  																			if _is_punctuation(SCREAMING_SNAKE_CASE_		):
  																									output.append([char]		)
  																									_a     =			True
  																			else:
  																									if start_new_word:
  																															output.append([]		)
  																									_a     =			False
  																									output[-1].append(SCREAMING_SNAKE_CASE_		)
  																			i += 1
  
  													return ["".join(SCREAMING_SNAKE_CASE_		) for x in output]
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Optional[int]		,				SCREAMING_SNAKE_CASE_  :				Union[str, Any]		):
  													_a     =			[]
  													for char in text:
  																			_a     =			ord(SCREAMING_SNAKE_CASE_		)
  																			if self._is_chinese_char(SCREAMING_SNAKE_CASE_		):
  																									output.append(' '		)
  																									output.append(SCREAMING_SNAKE_CASE_		)
  																									output.append(' '		)
  																			else:
  																									output.append(SCREAMING_SNAKE_CASE_		)
  													return "".join(SCREAMING_SNAKE_CASE_		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Optional[int]		,				SCREAMING_SNAKE_CASE_  :				Union[str, Any]		):
  													# This defines a "chinese character" as anything in the CJK Unicode block:
  													#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
  													#
  													# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
  													# despite its name. The modern Korean Hangul alphabet is a different block,
  													# as is Japanese Hiragana and Katakana. Those alphabets are used to write
  													# space-separated words, so they are not treated specially and handled
  													# like the all of the other languages.
  													if (
  													    (cp >= 0X4e00 and cp <= 0X9fff)
  													    or (cp >= 0X3400 and cp <= 0X4dbf)  #
  													    or (cp >= 0X20000 and cp <= 0X2a6df)  #
  													    or (cp >= 0X2a700 and cp <= 0X2b73f)  #
  													    or (cp >= 0X2b740 and cp <= 0X2b81f)  #
  													    or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
  													    or (cp >= 0Xf900 and cp <= 0Xfaff)
  													    or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
  													):  #
  																			return True
  
  													return False
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Dict		,				SCREAMING_SNAKE_CASE_  :				Tuple		):
  													_a     =			[]
  													for char in text:
  																			_a     =			ord(SCREAMING_SNAKE_CASE_		)
  																			if cp == 0 or cp == 0Xfffd or _is_control(SCREAMING_SNAKE_CASE_		):
  																									continue
  																			if _is_whitespace(SCREAMING_SNAKE_CASE_		):
  																									output.append(' '		)
  																			else:
  																									output.append(SCREAMING_SNAKE_CASE_		)
  													return "".join(SCREAMING_SNAKE_CASE_		)
  
  
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  Optional[Any]:
  						_a     =			set()
  						_a     =			word[0]
  						for char in word[1:]:
  												pairs.add((prev_char, char)				)
  												_a     =			char
  						return pairs
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  List[str]:
  						_a     =			text.replace('—'							,     '-'				)
  						_a     =			text.replace('–'							,     '-'				)
  						_a     =			text.replace('―'							,     '-'				)
  						_a     =			text.replace('…'							,     '...'				)
  						_a     =			text.replace('´'							,     '\''				)
  						_a     =			re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'							,     R' \1 '							,     _UpperCAmelCase				)
  						_a     =			re.sub(R'\s*\n\s*'							,     ' \n '							,     _UpperCAmelCase				)
  						_a     =			re.sub(R'[^\S\n]+'							,     ' '							,     _UpperCAmelCase				)
  						return text.strip()
  
  
  
  
  
  
  class 					_UpperCamelCase (      lowerCamelCase__	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  							_A  				=  VOCAB_FILES_NAMES
  							_A  				=  PRETRAINED_VOCAB_FILES_MAP
  							_A  				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							_A  				=  ["input_ids", "attention_mask"]
  
  
  
  
  
  
  
  							def __init__(		self  :				Tuple		,				SCREAMING_SNAKE_CASE_  :				Tuple		,				SCREAMING_SNAKE_CASE_  :				Dict		,				SCREAMING_SNAKE_CASE_  :				Tuple="<unk>"		,				**SCREAMING_SNAKE_CASE_  :				int		):
  													super().__init__(unk_token=SCREAMING_SNAKE_CASE_		,				**SCREAMING_SNAKE_CASE_		)
  
  													try:
  																			import ftfy
  																			from spacy.lang.en import English
  
  																			_a     =			English()
  																			_a     =			_nlp.tokenizer
  																			_a     =			ftfy.fix_text
  													except ImportError:
  																			logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'		)
  																			_a     =			BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_		)
  																			_a     =			None
  
  													with open(SCREAMING_SNAKE_CASE_		,				encoding='utf-8'		) as vocab_handle:
  																			_a     =			json.load(SCREAMING_SNAKE_CASE_		)
  													_a     =			{v: k for k, v in self.encoder.items()}
  													with open(SCREAMING_SNAKE_CASE_		,				encoding='utf-8'		) as merges_handle:
  																			_a     =			merges_handle.read().split('\n'		)[1:-1]
  													_a     =			[tuple(merge.split()		) for merge in merges]
  													_a     =			dict(zip(SCREAMING_SNAKE_CASE_		,				range(len(SCREAMING_SNAKE_CASE_		)		)		)		)
  													_a     =			{}
  
  
  
  
  
  
  
  							@property
  							def 		_UpperCAmelCase	(		self  :				int		):
  													return True
  
  
  
  
  
  
  
  							@property
  							def 		_UpperCAmelCase	(		self  :				Union[str, Any]		):
  													return len(self.encoder		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Union[str, Any]		):
  													return dict(self.encoder		,				**self.added_tokens_encoder		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				List[Any]		):
  													_a     =			tuple(token[:-1]		) + (token[-1] + '</w>',)
  													if token in self.cache:
  																			return self.cache[token]
  													_a     =			get_pairs(SCREAMING_SNAKE_CASE_		)
  
  													if not pairs:
  																			return token + "</w>"
  
  													while True:
  																			_a     =			min(SCREAMING_SNAKE_CASE_		,				key=lambda SCREAMING_SNAKE_CASE_		: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_		,				float('inf'		)		)		)
  																			if bigram not in self.bpe_ranks:
  																									break
  																			_a ,	_a     =			bigram
  																			_a     =			[]
  																			_a     =			0
  																			while i < len(SCREAMING_SNAKE_CASE_		):
  																									try:
  																															_a     =			word.index(SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		)
  																									except ValueError:
  																															new_word.extend(word[i:]		)
  																															break
  																									else:
  																															new_word.extend(word[i:j]		)
  																															_a     =			j
  
  																									if word[i] == first and i < len(SCREAMING_SNAKE_CASE_		) - 1 and word[i + 1] == second:
  																															new_word.append(first + second		)
  																															i += 2
  																									else:
  																															new_word.append(word[i]		)
  																															i += 1
  																			_a     =			tuple(SCREAMING_SNAKE_CASE_		)
  																			_a     =			new_word
  																			if len(SCREAMING_SNAKE_CASE_		) == 1:
  																									break
  																			else:
  																									_a     =			get_pairs(SCREAMING_SNAKE_CASE_		)
  													_a     =			' '.join(SCREAMING_SNAKE_CASE_		)
  													if word == "\n  </w>":
  																			_a     =			'\n</w>'
  													_a     =			word
  													return word
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				Optional[int]		):
  													_a     =			[]
  													if self.fix_text is None:
  																			# Using BERT's BasicTokenizer
  																			_a     =			self.nlp.tokenize(SCREAMING_SNAKE_CASE_		)
  																			for token in text:
  																									split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_		).split(' '		)		)		)
  													else:
  																			# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  																			_a     =			self.nlp(text_standardize(self.fix_text(SCREAMING_SNAKE_CASE_		)		)		)
  																			for token in text:
  																									split_tokens.extend(list(self.bpe(token.text.lower()		).split(' '		)		)		)
  													return split_tokens
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		,				SCREAMING_SNAKE_CASE_  :				List[str]		):
  													return self.encoder.get(SCREAMING_SNAKE_CASE_		,				self.encoder.get(self.unk_token		)		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Any		,				SCREAMING_SNAKE_CASE_  :				List[Any]		):
  													return self.decoder.get(SCREAMING_SNAKE_CASE_		,				self.unk_token		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Tuple		,				SCREAMING_SNAKE_CASE_  :				int		):
  													_a     =			''.join(SCREAMING_SNAKE_CASE_		).replace('</w>'		,				' '		).strip()
  													return out_string
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Tuple		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				Optional[str] = None		):
  													if not os.path.isdir(SCREAMING_SNAKE_CASE_		):
  																			logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
  																			return
  													_a     =			os.path.join(
  													    SCREAMING_SNAKE_CASE_		,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
  													_a     =			os.path.join(
  													    SCREAMING_SNAKE_CASE_		,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']		)
  
  													with open(SCREAMING_SNAKE_CASE_		,				'w'		,				encoding='utf-8'		) as f:
  																			f.write(json.dumps(self.encoder		,				indent=2		,				sort_keys=SCREAMING_SNAKE_CASE_		,				ensure_ascii=SCREAMING_SNAKE_CASE_		) + '\n'		)
  
  													_a     =			0
  													with open(SCREAMING_SNAKE_CASE_		,				'w'		,				encoding='utf-8'		) as writer:
  																			writer.write('#version: 0.2\n'		)
  																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,				key=lambda SCREAMING_SNAKE_CASE_		: kv[1]		):
  																									if index != token_index:
  																															logger.warning(
  																															    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  																															    ' Please check that the tokenizer is not corrupted!'		)
  																															_a     =			token_index
  																									writer.write(' '.join(SCREAMING_SNAKE_CASE_		) + '\n'		)
  																									index += 1
  
  													return vocab_file, merge_file
  
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available
  
  
  lowercase_								=    {
      'configuration_codegen': ['CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CodeGenConfig', 'CodeGenOnnxConfig'],
      'tokenization_codegen': ['CodeGenTokenizer'],
  }
  
  try:
  		if not is_tokenizers_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		lowercase_								=    ['CodeGenTokenizerFast']
  
  try:
  		if not is_torch_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		lowercase_								=    [
  		    'CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST',
  		    'CodeGenForCausalLM',
  		    'CodeGenModel',
  		    'CodeGenPreTrainedModel',
  		]
  
  if TYPE_CHECKING:
  		from .configuration_codegen import CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP, CodeGenConfig, CodeGenOnnxConfig
  		from .tokenization_codegen import CodeGenTokenizer
  
  		try:
  				if not is_tokenizers_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .tokenization_codegen_fast import CodeGenTokenizerFast
  
  		try:
  				if not is_torch_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .modeling_codegen import (
  				    CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST,
  				    CodeGenForCausalLM,
  				    CodeGenModel,
  				    CodeGenPreTrainedModel,
  				)
  
  else:
  		import sys
  
  		lowercase_								=    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
  
  
  