from ctypes import c_float, sizeof
from enum import Enum
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
 from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore




class    __lowerCAmelCase  (       a_						):



 '''simple docstring'''
 _lowerCamelCase						:	int				    = c_float

 @property
 def 					__snake_case					(			self       :		Tuple  ) ->							int:
        return sizeof(self.value  )







def SCREAMING_SNAKE_CASE				(     lowercase_:	int		,						lowercase_:	int		,						lowercase_:	int = 0				):
       # < 0 is possible if using a dynamic axis
       if dimension <= 0:
              _UpperCamelCase     :       List[str]  						=  fixed_dimension

       dimension -= num_token_to_add
       return dimension


def SCREAMING_SNAKE_CASE				(     lowercase_:	int		,						lowercase_:	ParameterFormat				):
       return num_parameters * dtype.size





def SCREAMING_SNAKE_CASE				(     lowercase_:	str				):
       from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore

       try:
              return AutoProcessor.from_pretrained(lowercase_				)
       except (ValueError, OSError, KeyError):
              _UpperCamelCase						,      _UpperCamelCase     :       Dict  						=  None, None
              try:
                     _UpperCamelCase     :       List[Any]  						=  AutoTokenizer.from_pretrained(lowercase_				)
              except (OSError, KeyError):
                     pass
              try:
                     _UpperCamelCase     :       List[str]  						=  AutoFeatureExtractor.from_pretrained(lowercase_				)
              except (OSError, KeyError):
                     pass

              if tokenizer is not None and feature_extractor is not None:
                     raise ValueError(
                         F"Couldn't auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor."				)
              elif tokenizer is None and feature_extractor is None:
                     return None
              elif tokenizer is not None:
                     return tokenizer
              else:
                     return feature_extractor




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase					=				logging.get_logger(__name__)

UpperCAmelCase					=				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase					=				{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

UpperCAmelCase					=				{
    '''openai-gpt''': 5_12,
}


def SCREAMING_SNAKE_CASE				(     lowercase_:	int				):
       _UpperCamelCase     :       Optional[Any]  						=  text.strip()
       if not text:
              return []
       _UpperCamelCase     :       Optional[Any]  						=  text.split()
       return tokens




class    __lowerCAmelCase  (       a_						):



 '''simple docstring'''

 def __init__(			self       :		str       ,			lowerCamelCase       :		Union[str, Any]=True       ,			lowerCamelCase       :		Optional[Any]=None       ,			lowerCamelCase       :		Dict=True       ,			lowerCamelCase       :		List[Any]=None       ,			lowerCamelCase       :		Optional[Any]=True       ,			) ->							Optional[Any]:
        if never_split is None:
               _UpperCamelCase     :       Union[str, Any]  						=  []
        _UpperCamelCase     :       List[Any]  						=  do_lower_case
        _UpperCamelCase     :       List[Any]  						=  set(lowerCamelCase  )
        _UpperCamelCase     :       int  						=  tokenize_chinese_chars
        _UpperCamelCase     :       Dict  						=  strip_accents
        _UpperCamelCase     :       List[Any]  						=  do_split_on_punc

 def 					__snake_case					(			self       :		int       ,			lowerCamelCase       :		List[str]       ,			lowerCamelCase       :		List[Any]=None  ) ->							List[str]:
        _UpperCamelCase     :       Any  						=  self.never_split.union(set(lowerCamelCase  )  ) if never_split else self.never_split
        _UpperCamelCase     :       int  						=  self._clean_text(lowerCamelCase  )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
               _UpperCamelCase     :       Optional[int]  						=  self._tokenize_chinese_chars(lowerCamelCase  )
        # prevents treating the same character with different unicode codepoints as different characters
        _UpperCamelCase     :       int  						=  unicodedata.normalize('NFC'       ,			lowerCamelCase  )
        _UpperCamelCase     :       Tuple  						=  whitespace_tokenize(lowerCamelCase  )
        _UpperCamelCase     :       Dict  						=  []
        for token in orig_tokens:
               if token not in never_split:
                      if self.do_lower_case:
                             _UpperCamelCase     :       Tuple  						=  token.lower()
                             if self.strip_accents is not False:
                                    _UpperCamelCase     :       List[Any]  						=  self._run_strip_accents(lowerCamelCase  )
                      elif self.strip_accents:
                             _UpperCamelCase     :       List[str]  						=  self._run_strip_accents(lowerCamelCase  )
               split_tokens.extend(self._run_split_on_punc(lowerCamelCase       ,			lowerCamelCase  )  )

        _UpperCamelCase     :       str  						=  whitespace_tokenize(' '.join(lowerCamelCase  )  )
        return output_tokens

 def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		Union[str, Any]  ) ->							str:
        _UpperCamelCase     :       Optional[Any]  						=  unicodedata.normalize('NFD'       ,			lowerCamelCase  )
        _UpperCamelCase     :       Dict  						=  []
        for char in text:
               _UpperCamelCase     :       Tuple  						=  unicodedata.category(lowerCamelCase  )
               if cat == "Mn":
                      continue
               output.append(lowerCamelCase  )
        return "".join(lowerCamelCase  )

 def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		List[Any]       ,			lowerCamelCase       :		int=None  ) ->							List[Any]:
        if not self.do_split_on_punc or (never_split is not None and text in never_split):
               return [text]
        _UpperCamelCase     :       Optional[int]  						=  list(lowerCamelCase  )
        _UpperCamelCase     :       Dict  						=  0
        _UpperCamelCase     :       Union[str, Any]  						=  True
        _UpperCamelCase     :       int  						=  []
        while i < len(lowerCamelCase  ):
               _UpperCamelCase     :       List[str]  						=  chars[i]
               if _is_punctuation(lowerCamelCase  ):
                      output.append([char]  )
                      _UpperCamelCase     :       str  						=  True
               else:
                      if start_new_word:
                             output.append([]  )
                      _UpperCamelCase     :       List[str]  						=  False
                      output[-1].append(lowerCamelCase  )
               i += 1

        return ["".join(lowerCamelCase  ) for x in output]

 def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		int  ) ->							Optional[Any]:
        _UpperCamelCase     :       Dict  						=  []
        for char in text:
               _UpperCamelCase     :       Optional[int]  						=  ord(lowerCamelCase  )
               if self._is_chinese_char(lowerCamelCase  ):
                      output.append(' '  )
                      output.append(lowerCamelCase  )
                      output.append(' '  )
               else:
                      output.append(lowerCamelCase  )
        return "".join(lowerCamelCase  )

 def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		int  ) ->							Optional[Any]:
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0x4_E00 and cp <= 0x9_FFF)
            or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
            or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
            or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
            or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
            or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
            or (cp >= 0xF_900 and cp <= 0xF_AFF)
            or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
        ):  #
               return True

        return False

 def 					__snake_case					(			self       :		Any       ,			lowerCamelCase       :		int  ) ->							Dict:
        _UpperCamelCase     :       int  						=  []
        for char in text:
               _UpperCamelCase     :       str  						=  ord(lowerCamelCase  )
               if cp == 0 or cp == 0xF_FFD or _is_control(lowerCamelCase  ):
                      continue
               if _is_whitespace(lowerCamelCase  ):
                      output.append(' '  )
               else:
                      output.append(lowerCamelCase  )
        return "".join(lowerCamelCase  )







def SCREAMING_SNAKE_CASE				(     lowercase_:	Optional[Any]				):
       _UpperCamelCase     :       List[Any]  						=  set()
       _UpperCamelCase     :       Tuple  						=  word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)				)
              _UpperCamelCase     :       Tuple  						=  char
       return pairs


def SCREAMING_SNAKE_CASE				(     lowercase_:	Dict				):
       _UpperCamelCase     :       Union[str, Any]  						=  text.replace('—'		,						'-'				)
       _UpperCamelCase     :       int  						=  text.replace('–'		,						'-'				)
       _UpperCamelCase     :       Dict  						=  text.replace('―'		,						'-'				)
       _UpperCamelCase     :       Any  						=  text.replace('…'		,						'...'				)
       _UpperCamelCase     :       Dict  						=  text.replace('´'		,						'\''				)
       _UpperCamelCase     :       Union[str, Any]  						=  re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'		,						R' \1 '		,						lowercase_				)
       _UpperCamelCase     :       Dict  						=  re.sub(R'\s*\n\s*'		,						' \n '		,						lowercase_				)
       _UpperCamelCase     :       Union[str, Any]  						=  re.sub(R'[^\S\n]+'		,						' '		,						lowercase_				)
       return text.strip()





class    __lowerCAmelCase  (       a_						):



 '''simple docstring'''
 _lowerCamelCase						:	int				    = VOCAB_FILES_NAMES
 _lowerCamelCase						:	Any				    = PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase						:	Optional[int]				    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase						:	List[str]				    = ['input_ids', 'attention_mask']

 def __init__(			self       :		str       ,			lowerCamelCase       :		Optional[Any]       ,			lowerCamelCase       :		Tuple       ,			lowerCamelCase       :		Union[str, Any]="<unk>"       ,			**lowerCamelCase       :		Any  ) ->							Dict:
        super().__init__(unk_token=lowerCamelCase       ,			**lowerCamelCase  )

        try:
               import ftfy
               from spacy.lang.en import English

               _UpperCamelCase     :       Any  						=  English()
               _UpperCamelCase     :       Any  						=  _nlp.tokenizer
               _UpperCamelCase     :       Dict  						=  ftfy.fix_text
        except ImportError:
               logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'  )
               _UpperCamelCase     :       int  						=  BasicTokenizer(do_lower_case=lowerCamelCase  )
               _UpperCamelCase     :       List[Any]  						=  None

        with open(lowerCamelCase       ,			encoding='utf-8'  ) as vocab_handle:
               _UpperCamelCase     :       Tuple  						=  json.load(lowerCamelCase  )
        _UpperCamelCase     :       List[Any]  						=  {v: k for k, v in self.encoder.items()}
        with open(lowerCamelCase       ,			encoding='utf-8'  ) as merges_handle:
               _UpperCamelCase     :       List[str]  						=  merges_handle.read().split('\n'  )[1:-1]
        _UpperCamelCase     :       Optional[int]  						=  [tuple(merge.split()  ) for merge in merges]
        _UpperCamelCase     :       Dict  						=  dict(zip(lowerCamelCase       ,			range(len(lowerCamelCase  )  )  )  )
        _UpperCamelCase     :       Optional[Any]  						=  {}

 @property
 def 					__snake_case					(			self       :		str  ) ->							int:
        return True

 @property
 def 					__snake_case					(			self       :		int  ) ->							Dict:
        return len(self.encoder  )

 def 					__snake_case					(			self       :		Optional[Any]  ) ->							Union[str, Any]:
        return dict(self.encoder       ,			**self.added_tokens_encoder  )

 def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		Optional[int]  ) ->							Union[str, Any]:
        _UpperCamelCase     :       int  						=  tuple(token[:-1]  ) + (token[-1] + '</w>',)
        if token in self.cache:
               return self.cache[token]
        _UpperCamelCase     :       Dict  						=  get_pairs(lowerCamelCase  )

        if not pairs:
               return token + "</w>"

        while True:
               _UpperCamelCase     :       List[str]  						=  min(lowerCamelCase       ,			key=lambda lowerCamelCase  : self.bpe_ranks.get(lowerCamelCase       ,			float('inf'  )  )  )
               if bigram not in self.bpe_ranks:
                      break
               _UpperCamelCase						,      _UpperCamelCase     :       Any  						=  bigram
               _UpperCamelCase     :       Optional[Any]  						=  []
               _UpperCamelCase     :       Optional[int]  						=  0
               while i < len(lowerCamelCase  ):
                      try:
                             _UpperCamelCase     :       int  						=  word.index(lowerCamelCase       ,			lowerCamelCase  )
                      except ValueError:
                             new_word.extend(word[i:]  )
                             break
                      else:
                             new_word.extend(word[i:j]  )
                             _UpperCamelCase     :       Union[str, Any]  						=  j

                      if word[i] == first and i < len(lowerCamelCase  ) - 1 and word[i + 1] == second:
                             new_word.append(first + second  )
                             i += 2
                      else:
                             new_word.append(word[i]  )
                             i += 1
               _UpperCamelCase     :       str  						=  tuple(lowerCamelCase  )
               _UpperCamelCase     :       Optional[int]  						=  new_word
               if len(lowerCamelCase  ) == 1:
                      break
               else:
                      _UpperCamelCase     :       List[Any]  						=  get_pairs(lowerCamelCase  )
        _UpperCamelCase     :       Any  						=  ' '.join(lowerCamelCase  )
        if word == "\n  </w>":
               _UpperCamelCase     :       Union[str, Any]  						=  '\n</w>'
        _UpperCamelCase     :       Tuple  						=  word
        return word

 def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		Optional[Any]  ) ->							List[Any]:
        _UpperCamelCase     :       Optional[int]  						=  []
        if self.fix_text is None:
               # Using BERT's BasicTokenizer
               _UpperCamelCase     :       int  						=  self.nlp.tokenize(lowerCamelCase  )
               for token in text:
                      split_tokens.extend(list(self.bpe(lowerCamelCase  ).split(' '  )  )  )
        else:
               # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
               _UpperCamelCase     :       Optional[Any]  						=  self.nlp(text_standardize(self.fix_text(lowerCamelCase  )  )  )
               for token in text:
                      split_tokens.extend(list(self.bpe(token.text.lower()  ).split(' '  )  )  )
        return split_tokens

 def 					__snake_case					(			self       :		int       ,			lowerCamelCase       :		Optional[int]  ) ->							int:
        return self.encoder.get(lowerCamelCase       ,			self.encoder.get(self.unk_token  )  )

 def 					__snake_case					(			self       :		Dict       ,			lowerCamelCase       :		List[str]  ) ->							str:
        return self.decoder.get(lowerCamelCase       ,			self.unk_token  )

 def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		Dict  ) ->							Tuple:
        _UpperCamelCase     :       str  						=  ''.join(lowerCamelCase  ).replace('</w>'       ,			' '  ).strip()
        return out_string

 def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		str       ,			lowerCamelCase       :		Optional[str] = None  ) ->							Tuple[str]:
        if not os.path.isdir(lowerCamelCase  ):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"  )
               return
        _UpperCamelCase     :       str  						=  os.path.join(
            lowerCamelCase       ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
        _UpperCamelCase     :       List[str]  						=  os.path.join(
            lowerCamelCase       ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

        with open(lowerCamelCase       ,			'w'       ,			encoding='utf-8'  ) as f:
               f.write(json.dumps(self.encoder       ,			indent=2       ,			sort_keys=lowerCamelCase       ,			ensure_ascii=lowerCamelCase  ) + '\n'  )

        _UpperCamelCase     :       List[Any]  						=  0
        with open(lowerCamelCase       ,			'w'       ,			encoding='utf-8'  ) as writer:
               writer.write('#version: 0.2\n'  )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,			key=lambda lowerCamelCase  : kv[1]  ):
                      if index != token_index:
                             logger.warning(
                                 f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                 ' Please check that the tokenizer is not corrupted!'  )
                             _UpperCamelCase     :       str  						=  token_index
                      writer.write(' '.join(lowerCamelCase  ) + '\n'  )
                      index += 1

        return vocab_file, merge_file



