



"""simple docstring"""







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase__						:							Union[str, Any]        =      logging.get_logger(__name__)

lowercase__						:							Dict        =      {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowercase__						:							Tuple        =      {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

lowercase__						:							Dict        =      {
    """openai-gpt""": 5_1_2,
}

def 							UpperCamelCase_       (       lowerCAmelCase__ :							Optional[Any]		)		->  Tuple:

  """simple docstring"""




  lowerCAmelCase_  :							List[Any]								=					text.strip()
  if not text:
    return []
  lowerCAmelCase_  :							Optional[int]								=					text.split()
  return tokens







class  UpperCamelCase__  (      __lowerCAmelCase    ):




      """simple docstring"""




      def __init__( self							: Any       ,     SCREAMING_SNAKE_CASE_							: str=True       ,     SCREAMING_SNAKE_CASE_							: Dict=None       ,     SCREAMING_SNAKE_CASE_							: Union[str, Any]=True       ,     SCREAMING_SNAKE_CASE_							: List[str]=None       ,     SCREAMING_SNAKE_CASE_							: Optional[int]=True       ,     ):






        if never_split is None:
          lowerCAmelCase_  :							Optional[Any]								=					[]
        lowerCAmelCase_  :							Tuple								=					do_lower_case
        lowerCAmelCase_  :							Optional[Any]								=					set(__UpperCamelCase			)
        lowerCAmelCase_  :							Any								=					tokenize_chinese_chars
        lowerCAmelCase_  :							str								=					strip_accents
        lowerCAmelCase_  :							Union[str, Any]								=					do_split_on_punc



      def 	SCREAMING_SNAKE_CASE__					( self							: str       ,     SCREAMING_SNAKE_CASE_							: Optional[int]       ,     SCREAMING_SNAKE_CASE_							: Any=None			):
        lowerCAmelCase_  :							Tuple								=					self.never_split.union(set(__UpperCamelCase			)			) if never_split else self.never_split
        lowerCAmelCase_  :							List[Any]								=					self._clean_text(__UpperCamelCase			)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
          lowerCAmelCase_  :							Any								=					self._tokenize_chinese_chars(__UpperCamelCase			)
        # prevents treating the same character with different unicode codepoints as different characters
        lowerCAmelCase_  :							Dict								=					unicodedata.normalize('NFC'       ,     __UpperCamelCase			)
        lowerCAmelCase_  :							Optional[Any]								=					whitespace_tokenize(__UpperCamelCase			)
        lowerCAmelCase_  :							Optional[int]								=					[]
        for token in orig_tokens:
          if token not in never_split:
            if self.do_lower_case:
              lowerCAmelCase_  :							Any								=					token.lower()
              if self.strip_accents is not False:
                lowerCAmelCase_  :							Any								=					self._run_strip_accents(__UpperCamelCase			)
            elif self.strip_accents:
              lowerCAmelCase_  :							Tuple								=					self._run_strip_accents(__UpperCamelCase			)
          split_tokens.extend(self._run_split_on_punc(__UpperCamelCase       ,     __UpperCamelCase			)			)

        lowerCAmelCase_  :							str								=					whitespace_tokenize(' '.join(__UpperCamelCase			)			)
        return output_tokens



      def 	SCREAMING_SNAKE_CASE__					( self							: Optional[Any]       ,     SCREAMING_SNAKE_CASE_							: Tuple			):
        lowerCAmelCase_  :							Optional[Any]								=					unicodedata.normalize('NFD'       ,     __UpperCamelCase			)
        lowerCAmelCase_  :							List[Any]								=					[]
        for char in text:
          lowerCAmelCase_  :							Dict								=					unicodedata.category(__UpperCamelCase			)
          if cat == "Mn":
            continue
          output.append(__UpperCamelCase			)
        return "".join(__UpperCamelCase			)



      def 	SCREAMING_SNAKE_CASE__					( self							: Optional[Any]       ,     SCREAMING_SNAKE_CASE_							: Any       ,     SCREAMING_SNAKE_CASE_							: int=None			):






        if not self.do_split_on_punc or (never_split is not None and text in never_split):
          return [text]
        lowerCAmelCase_  :							Dict								=					list(__UpperCamelCase			)
        lowerCAmelCase_  :							List[str]								=					0
        lowerCAmelCase_  :							Optional[Any]								=					True
        lowerCAmelCase_  :							List[str]								=					[]
        while i < len(__UpperCamelCase			):
          lowerCAmelCase_  :							Tuple								=					chars[i]
          if _is_punctuation(__UpperCamelCase			):
            output.append([char]			)
            lowerCAmelCase_  :							int								=					True
          else:
            if start_new_word:
              output.append([]			)
            lowerCAmelCase_  :							List[Any]								=					False
            output[-1].append(__UpperCamelCase			)
          i += 1

        return ["".join(__UpperCamelCase			) for x in output]



      def 	SCREAMING_SNAKE_CASE__					( self							: Tuple       ,     SCREAMING_SNAKE_CASE_							: Optional[Any]			):
        lowerCAmelCase_  :							Optional[Any]								=					[]
        for char in text:
          lowerCAmelCase_  :							str								=					ord(__UpperCamelCase			)
          if self._is_chinese_char(__UpperCamelCase			):
            output.append(' '			)
            output.append(__UpperCamelCase			)
            output.append(' '			)
          else:
            output.append(__UpperCamelCase			)
        return "".join(__UpperCamelCase			)



      def 	SCREAMING_SNAKE_CASE__					( self							: Any       ,     SCREAMING_SNAKE_CASE_							: str			):






        if (
            (cp >= 0X4E_00 and cp <= 0X9F_FF)
            or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
            or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
            or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
            or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
            or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
            or (cp >= 0XF9_00 and cp <= 0XFA_FF)
            or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
        ):  #
          return True

        return False





      def 	SCREAMING_SNAKE_CASE__					( self							: str       ,     SCREAMING_SNAKE_CASE_							: Any			):
        lowerCAmelCase_  :							Dict								=					[]
        for char in text:
          lowerCAmelCase_  :							str								=					ord(__UpperCamelCase			)
          if cp == 0 or cp == 0XFF_FD or _is_control(__UpperCamelCase			):
            continue
          if _is_whitespace(__UpperCamelCase			):
            output.append(' '			)
          else:
            output.append(__UpperCamelCase			)
        return "".join(__UpperCamelCase			)




def 							UpperCamelCase_       (       lowerCAmelCase__ :							Optional[Any]		)		->  int:

  """simple docstring"""




  lowerCAmelCase_  :							str								=					set()
  lowerCAmelCase_  :							Union[str, Any]								=					word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    lowerCAmelCase_  :							Union[str, Any]								=					char
  return pairs

def 							UpperCamelCase_       (       lowerCAmelCase__ :							Union[str, Any]		)		->  Union[str, Any]:

  """simple docstring"""




  lowerCAmelCase_  :							Any								=					text.replace('—'       ,  '-'		)
  lowerCAmelCase_  :							List[str]								=					text.replace('–'       ,  '-'		)
  lowerCAmelCase_  :							Any								=					text.replace('―'       ,  '-'		)
  lowerCAmelCase_  :							Union[str, Any]								=					text.replace('…'       ,  '...'		)
  lowerCAmelCase_  :							Dict								=					text.replace('´'       ,  '\''		)
  lowerCAmelCase_  :							str								=					re.sub(R'(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'       ,  R' \1 '       ,  lowerCAmelCase__		)
  lowerCAmelCase_  :							List[str]								=					re.sub(R'\s*\n\s*'       ,  ' \n '       ,  lowerCAmelCase__		)
  lowerCAmelCase_  :							int								=					re.sub(R'[^\S\n]+'       ,  ' '       ,  lowerCAmelCase__		)
  return text.strip()







class  UpperCamelCase__  (      __lowerCAmelCase    ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE    							=	VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE    							=	PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE    							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE    							=	["""input_ids""", """attention_mask"""]



      def __init__( self							: int       ,     SCREAMING_SNAKE_CASE_							: Optional[int]       ,     SCREAMING_SNAKE_CASE_							: str       ,     SCREAMING_SNAKE_CASE_							: Dict="<unk>"       ,     **SCREAMING_SNAKE_CASE_							: Optional[Any]			):
        super().__init__(unk_token=__UpperCamelCase       ,     **__UpperCamelCase			)

        try:
          import ftfy
          from spacy.lang.en import English

          lowerCAmelCase_  :							Optional[int]								=					English()
          lowerCAmelCase_  :							Optional[int]								=					_nlp.tokenizer
          lowerCAmelCase_  :							str								=					ftfy.fix_text
        except ImportError:
          logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'			)
          lowerCAmelCase_  :							Dict								=					BasicTokenizer(do_lower_case=__UpperCamelCase			)
          lowerCAmelCase_  :							Optional[Any]								=					None

        with open(__UpperCamelCase       ,     encoding='utf-8'			) as vocab_handle:
          lowerCAmelCase_  :							List[str]								=					json.load(__UpperCamelCase			)
        lowerCAmelCase_  :							List[Any]								=					{v: k for k, v in self.encoder.items()}
        with open(__UpperCamelCase       ,     encoding='utf-8'			) as merges_handle:
          lowerCAmelCase_  :							Any								=					merges_handle.read().split('\n'			)[1:-1]
        lowerCAmelCase_  :							Union[str, Any]								=					[tuple(merge.split()			) for merge in merges]
        lowerCAmelCase_  :							int								=					dict(zip(__UpperCamelCase       ,     range(len(__UpperCamelCase			)			)			)			)
        lowerCAmelCase_  :							Dict								=					{}



      @property
      def 	SCREAMING_SNAKE_CASE__					( self							: str			):
        return True



      @property
      def 	SCREAMING_SNAKE_CASE__					( self							: str			):
        return len(self.encoder			)



      def 	SCREAMING_SNAKE_CASE__					( self							: Union[str, Any]			):
        return dict(self.encoder       ,     **self.added_tokens_encoder			)



      def 	SCREAMING_SNAKE_CASE__					( self							: Optional[Any]       ,     SCREAMING_SNAKE_CASE_							: Optional[int]			):
        lowerCAmelCase_  :							int								=					tuple(token[:-1]			) + (token[-1] + '</w>',)
        if token in self.cache:
          return self.cache[token]
        lowerCAmelCase_  :							List[Any]								=					get_pairs(__UpperCamelCase			)

        if not pairs:
          return token + "</w>"

        while True:
          lowerCAmelCase_  :							Optional[int]								=					min(__UpperCamelCase       ,     key=lambda SCREAMING_SNAKE_CASE_			: self.bpe_ranks.get(__UpperCamelCase       ,     float('inf'			)			)			)
          if bigram not in self.bpe_ranks:
            break
          lowerCAmelCase_		,lowerCAmelCase_  :							int								=					bigram
          lowerCAmelCase_  :							Tuple								=					[]
          lowerCAmelCase_  :							Tuple								=					0
          while i < len(__UpperCamelCase			):
            try:
              lowerCAmelCase_  :							Optional[Any]								=					word.index(__UpperCamelCase       ,     __UpperCamelCase			)
            except ValueError:
              new_word.extend(word[i:]			)
              break
            else:
              new_word.extend(word[i:j]			)
              lowerCAmelCase_  :							int								=					j

            if word[i] == first and i < len(__UpperCamelCase			) - 1 and word[i + 1] == second:
              new_word.append(first + second			)
              i += 2
            else:
              new_word.append(word[i]			)
              i += 1
          lowerCAmelCase_  :							Union[str, Any]								=					tuple(__UpperCamelCase			)
          lowerCAmelCase_  :							Optional[int]								=					new_word
          if len(__UpperCamelCase			) == 1:
            break
          else:
            lowerCAmelCase_  :							str								=					get_pairs(__UpperCamelCase			)
        lowerCAmelCase_  :							Any								=					' '.join(__UpperCamelCase			)
        if word == "\n  </w>":
          lowerCAmelCase_  :							List[str]								=					'\n</w>'
        lowerCAmelCase_  :							str								=					word
        return word



      def 	SCREAMING_SNAKE_CASE__					( self							: str       ,     SCREAMING_SNAKE_CASE_							: Optional[int]			):
        lowerCAmelCase_  :							Optional[Any]								=					[]
        if self.fix_text is None:
          # Using BERT's BasicTokenizer
          lowerCAmelCase_  :							List[str]								=					self.nlp.tokenize(__UpperCamelCase			)
          for token in text:
            split_tokens.extend(list(self.bpe(__UpperCamelCase			).split(' '			)			)			)
        else:
          # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
          lowerCAmelCase_  :							Any								=					self.nlp(text_standardize(self.fix_text(__UpperCamelCase			)			)			)
          for token in text:
            split_tokens.extend(list(self.bpe(token.text.lower()			).split(' '			)			)			)
        return split_tokens



      def 	SCREAMING_SNAKE_CASE__					( self							: List[Any]       ,     SCREAMING_SNAKE_CASE_							: str			):
        return self.encoder.get(__UpperCamelCase       ,     self.encoder.get(self.unk_token			)			)



      def 	SCREAMING_SNAKE_CASE__					( self							: str       ,     SCREAMING_SNAKE_CASE_							: str			):
        return self.decoder.get(__UpperCamelCase       ,     self.unk_token			)



      def 	SCREAMING_SNAKE_CASE__					( self							: Union[str, Any]       ,     SCREAMING_SNAKE_CASE_							: int			):
        lowerCAmelCase_  :							List[Any]								=					''.join(__UpperCamelCase			).replace('</w>'       ,     ' '			).strip()
        return out_string





      def 	SCREAMING_SNAKE_CASE__					( self							: Any       ,     SCREAMING_SNAKE_CASE_							: str       ,     SCREAMING_SNAKE_CASE_							: Optional[str] = None			):






        if not os.path.isdir(__UpperCamelCase			):
          logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
          return
        lowerCAmelCase_  :							str								=					os.path.join(
            __UpperCamelCase       ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
        lowerCAmelCase_  :							Optional[int]								=					os.path.join(
            __UpperCamelCase       ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

        with open(__UpperCamelCase       ,     'w'       ,     encoding='utf-8'			) as f:
          f.write(json.dumps(self.encoder       ,     indent=2       ,     sort_keys=__UpperCamelCase       ,     ensure_ascii=__UpperCamelCase			) + '\n'			)

        lowerCAmelCase_  :							Optional[int]								=					0
        with open(__UpperCamelCase       ,     'w'       ,     encoding='utf-8'			) as writer:
          writer.write('#version: 0.2\n'			)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,     key=lambda SCREAMING_SNAKE_CASE_			: kv[1]			):
            if index != token_index:
              logger.warning(
                  F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                  ' Please check that the tokenizer is not corrupted!'			)
              lowerCAmelCase_  :							Union[str, Any]								=					token_index
            writer.write(' '.join(__UpperCamelCase			) + '\n'			)
            index += 1

        return vocab_file, merge_file











'''simple docstring'''

import warnings

from ...utils import logging
from .image_processing_vit import ViTImageProcessor


a__         =     logging.get_logger(__name__)





class __magic_name__(						__lowerCAmelCase      ):






						def __init__(   self						:						List[str]					,   *__UpperCamelCase						:						Tuple					,   **__UpperCamelCase						:						Tuple       ):



									'''simple docstring'''






									warnings.warn(
									    """The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"""
									    """ use ViTImageProcessor instead."""					,   __UpperCamelCase					,   )
									super().__init__(*__UpperCamelCase					,   **__UpperCamelCase       )