





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCamelCase       = logging.get_logger(__name__)

_UpperCamelCase       = {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCamelCase       = {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_UpperCamelCase       = {
    '''openai-gpt''': 512,
}







def       lowerCAmelCase__(	lowercase       :    Union[str, Any]				) ->    List[str]:
  __snake_case   :				List[str]        =       text.strip()
  if not text:
    return []
  __snake_case   :				List[Any]        =       text.split()
  return tokens




class 				_lowerCamelCase (    a    ):



      """simple docstring"""






      def __init__( self				,						UpperCAmelCase=True				,						UpperCAmelCase=None				,						UpperCAmelCase=True				,						UpperCAmelCase=None				,						UpperCAmelCase=True				,						)       ->   Optional[Any]:






        '''simple docstring'''






        if never_split is None:
          __snake_case   :				Tuple        =       []
        __snake_case   :				str        =       do_lower_case
        __snake_case   :				Optional[Any]        =       set(UpperCAmelCase   )
        __snake_case   :				Dict        =       tokenize_chinese_chars
        __snake_case   :				str        =       strip_accents
        __snake_case   :				int        =       do_split_on_punc







      def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase=None   )       ->   Optional[int]:






        '''simple docstring'''






        __snake_case   :				int        =       self.never_split.union(set(UpperCAmelCase   )   ) if never_split else self.never_split
        __snake_case   :				Tuple        =       self._clean_text(UpperCAmelCase   )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
          __snake_case   :				int        =       self._tokenize_chinese_chars(UpperCAmelCase   )
        # prevents treating the same character with different unicode codepoints as different characters
        __snake_case   :				List[Any]        =       unicodedata.normalize("NFC"				,						UpperCAmelCase   )
        __snake_case   :				Tuple        =       whitespace_tokenize(UpperCAmelCase   )
        __snake_case   :				Union[str, Any]        =       []
        for token in orig_tokens:
          if token not in never_split:
            if self.do_lower_case:
              __snake_case   :				Optional[int]        =       token.lower()
              if self.strip_accents is not False:
                __snake_case   :				Optional[int]        =       self._run_strip_accents(UpperCAmelCase   )
            elif self.strip_accents:
              __snake_case   :				List[str]        =       self._run_strip_accents(UpperCAmelCase   )
          split_tokens.extend(self._run_split_on_punc(UpperCAmelCase				,						UpperCAmelCase   )   )

        __snake_case   :				Optional[Any]        =       whitespace_tokenize(" ".join(UpperCAmelCase   )   )
        return output_tokens







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[Any]:






        '''simple docstring'''






        __snake_case   :				Tuple        =       unicodedata.normalize("NFD"				,						UpperCAmelCase   )
        __snake_case   :				List[Any]        =       []
        for char in text:
          __snake_case   :				Tuple        =       unicodedata.category(UpperCAmelCase   )
          if cat == "Mn":
            continue
          output.append(UpperCAmelCase   )
        return "".join(UpperCAmelCase   )







      def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase=None   )       ->   Union[str, Any]:






        '''simple docstring'''






        if not self.do_split_on_punc or (never_split is not None and text in never_split):
          return [text]
        __snake_case   :				Union[str, Any]        =       list(UpperCAmelCase   )
        __snake_case   :				Optional[Any]        =       0
        __snake_case   :				Optional[int]        =       True
        __snake_case   :				Any        =       []
        while i < len(UpperCAmelCase   ):
          __snake_case   :				str        =       chars[i]
          if _is_punctuation(UpperCAmelCase   ):
            output.append([char]   )
            __snake_case   :				Any        =       True
          else:
            if start_new_word:
              output.append([]   )
            __snake_case   :				Dict        =       False
            output[-1].append(UpperCAmelCase   )
          i += 1

        return ["".join(UpperCAmelCase   ) for x in output]







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Any:






        '''simple docstring'''






        __snake_case   :				Optional[int]        =       []
        for char in text:
          __snake_case   :				Tuple        =       ord(UpperCAmelCase   )
          if self._is_chinese_char(UpperCAmelCase   ):
            output.append(" "   )
            output.append(UpperCAmelCase   )
            output.append(" "   )
          else:
            output.append(UpperCAmelCase   )
        return "".join(UpperCAmelCase   )







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Tuple:






        '''simple docstring'''






        if (
            (cp >= 0x4E00 and cp <= 0x9FFF)
            or (cp >= 0x3400 and cp <= 0x4DBF)  #
            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
            or (cp >= 0xF900 and cp <= 0xFAFF)
            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
        ):  #
          return True

        return False







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Dict:






        '''simple docstring'''






        __snake_case   :				Tuple        =       []
        for char in text:
          __snake_case   :				Union[str, Any]        =       ord(UpperCAmelCase   )
          if cp == 0 or cp == 0xFFFD or _is_control(UpperCAmelCase   ):
            continue
          if _is_whitespace(UpperCAmelCase   ):
            output.append(" "   )
          else:
            output.append(UpperCAmelCase   )
        return "".join(UpperCAmelCase   )







def       lowerCAmelCase__(	lowercase       :    int				) ->    Dict:
  __snake_case   :				int        =       set()
  __snake_case   :				Dict        =       word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)				)
    __snake_case   :				List[str]        =       char
  return pairs







def       lowerCAmelCase__(	lowercase       :    Union[str, Any]				) ->    Union[str, Any]:
  __snake_case   :				List[Any]        =       text.replace("—"   ,     "-"				)
  __snake_case   :				Optional[Any]        =       text.replace("–"   ,     "-"				)
  __snake_case   :				List[Any]        =       text.replace("―"   ,     "-"				)
  __snake_case   :				Any        =       text.replace("…"   ,     "..."				)
  __snake_case   :				str        =       text.replace("´"   ,     "'"				)
  __snake_case   :				str        =       re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"   ,     R" \1 "   ,     lowercase				)
  __snake_case   :				Union[str, Any]        =       re.sub(R"\s*\n\s*"   ,     " \n "   ,     lowercase				)
  __snake_case   :				Any        =       re.sub(R"[^\S\n]+"   ,     " "   ,     lowercase				)
  return text.strip()




class 				_lowerCamelCase (    a    ):



      """simple docstring"""






      UpperCAmelCase_						:      List[str]       =VOCAB_FILES_NAMES
      UpperCAmelCase_						:      int       =PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_						:      int       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase_						:      Union[str, Any]       =["input_ids", "attention_mask"]







      def __init__( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase="<unk>"				,						**UpperCAmelCase   )       ->   Dict:






        '''simple docstring'''






        super().__init__(unk_token=UpperCAmelCase				,						**UpperCAmelCase   )

        try:
          import ftfy
          from spacy.lang.en import English

          __snake_case   :				List[Any]        =       English()
          __snake_case   :				Optional[int]        =       _nlp.tokenizer
          __snake_case   :				Tuple        =       ftfy.fix_text
        except ImportError:
          logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
          __snake_case   :				Dict        =       BasicTokenizer(do_lower_case=UpperCAmelCase   )
          __snake_case   :				str        =       None

        with open(UpperCAmelCase				,						encoding="utf-8"   ) as vocab_handle:
          __snake_case   :				str        =       json.load(UpperCAmelCase   )
        __snake_case   :				Union[str, Any]        =       {v: k for k, v in self.encoder.items()}
        with open(UpperCAmelCase				,						encoding="utf-8"   ) as merges_handle:
          __snake_case   :				Dict        =       merges_handle.read().split("\n"   )[1:-1]
        __snake_case   :				Dict        =       [tuple(merge.split()   ) for merge in merges]
        __snake_case   :				str        =       dict(zip(UpperCAmelCase				,						range(len(UpperCAmelCase   )   )   )   )
        __snake_case   :				List[Any]        =       {}







      @property
      def 						UpperCAmelCase		( self   )       ->   Tuple:






        '''simple docstring'''






        return True







      @property
      def 						UpperCAmelCase		( self   )       ->   List[Any]:






        '''simple docstring'''






        return len(self.encoder   )







      def 						UpperCAmelCase		( self   )       ->   int:






        '''simple docstring'''






        return dict(self.encoder				,						**self.added_tokens_encoder   )







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[str]:






        '''simple docstring'''






        __snake_case   :				List[str]        =       tuple(token[:-1]   ) + (token[-1] + "</w>",)
        if token in self.cache:
          return self.cache[token]
        __snake_case   :				Any        =       get_pairs(UpperCAmelCase   )

        if not pairs:
          return token + "</w>"

        while True:
          __snake_case   :				List[Any]        =       min(UpperCAmelCase				,						key=lambda UpperCAmelCase   : self.bpe_ranks.get(UpperCAmelCase				,						float("inf"   )   )   )
          if bigram not in self.bpe_ranks:
            break
          __snake_case    ,   __snake_case   :				List[str]        =       bigram
          __snake_case   :				Union[str, Any]        =       []
          __snake_case   :				Optional[int]        =       0
          while i < len(UpperCAmelCase   ):
            try:
              __snake_case   :				List[Any]        =       word.index(UpperCAmelCase				,						UpperCAmelCase   )
            except ValueError:
              new_word.extend(word[i:]   )
              break
            else:
              new_word.extend(word[i:j]   )
              __snake_case   :				Any        =       j

            if word[i] == first and i < len(UpperCAmelCase   ) - 1 and word[i + 1] == second:
              new_word.append(first + second   )
              i += 2
            else:
              new_word.append(word[i]   )
              i += 1
          __snake_case   :				List[Any]        =       tuple(UpperCAmelCase   )
          __snake_case   :				Optional[Any]        =       new_word
          if len(UpperCAmelCase   ) == 1:
            break
          else:
            __snake_case   :				Optional[Any]        =       get_pairs(UpperCAmelCase   )
        __snake_case   :				List[str]        =       " ".join(UpperCAmelCase   )
        if word == "\n  </w>":
          __snake_case   :				Tuple        =       "\n</w>"
        __snake_case   :				Tuple        =       word
        return word







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Union[str, Any]:






        '''simple docstring'''






        __snake_case   :				str        =       []
        if self.fix_text is None:
          # Using BERT's BasicTokenizer
          __snake_case   :				Any        =       self.nlp.tokenize(UpperCAmelCase   )
          for token in text:
            split_tokens.extend(list(self.bpe(UpperCAmelCase   ).split(" "   )   )   )
        else:
          # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
          __snake_case   :				Union[str, Any]        =       self.nlp(text_standardize(self.fix_text(UpperCAmelCase   )   )   )
          for token in text:
            split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
        return split_tokens







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Union[str, Any]:






        '''simple docstring'''






        return self.encoder.get(UpperCAmelCase				,						self.encoder.get(self.unk_token   )   )







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[Any]:






        '''simple docstring'''






        return self.decoder.get(UpperCAmelCase				,						self.unk_token   )







      def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[str]:






        '''simple docstring'''






        __snake_case   :				Union[str, Any]        =       "".join(UpperCAmelCase   ).replace("</w>"				,						" "   ).strip()
        return out_string







      def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase = None   )       ->   Tuple[str]:






        '''simple docstring'''






        if not os.path.isdir(UpperCAmelCase   ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
          return
        __snake_case   :				str        =       os.path.join(
            UpperCAmelCase				,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
        __snake_case   :				Any        =       os.path.join(
            UpperCAmelCase				,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

        with open(UpperCAmelCase				,						"w"				,						encoding="utf-8"   ) as f:
          f.write(json.dumps(self.encoder				,						indent=2				,						sort_keys=UpperCAmelCase				,						ensure_ascii=UpperCAmelCase   ) + "\n"   )

        __snake_case   :				int        =       0
        with open(UpperCAmelCase				,						"w"				,						encoding="utf-8"   ) as writer:
          writer.write("#version: 0.2\n"   )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,						key=lambda UpperCAmelCase   : kv[1]   ):
            if index != token_index:
              logger.warning(
                  F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                  " Please check that the tokenizer is not corrupted!"   )
              __snake_case   :				Tuple        =       token_index
            writer.write(" ".join(UpperCAmelCase   ) + "\n"   )
            index += 1

        return vocab_file, merge_file








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_UpperCamelCase       = {
    '''configuration_deta''': ['''DETA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DetaConfig'''],
}

try:
     if not is_vision_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _UpperCamelCase       = ['''DetaImageProcessor''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _UpperCamelCase       = [
         '''DETA_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''DetaForObjectDetection''',
         '''DetaModel''',
         '''DetaPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig

     try:
          if not is_vision_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .image_processing_deta import DetaImageProcessor

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_deta import (
              DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
              DetaForObjectDetection,
              DetaModel,
              DetaPreTrainedModel,
          )

else:
     import sys

     _UpperCamelCase       = _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

