code: 
  
  
  
  
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  _UpperCamelCase       = logging.get_logger(__name__)
  
  _UpperCamelCase       = {
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  _UpperCamelCase       = {
      '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
      '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
  }
  
  _UpperCamelCase       = {
      '''openai-gpt''': 512,
  }
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    Union[str, Any]				) ->    List[str]:
    __snake_case   :				List[str]        =       text.strip()
    if not text:
      return []
    __snake_case   :				List[Any]        =       text.split()
    return tokens
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase=True				,						UpperCAmelCase=None				,						UpperCAmelCase=True				,						UpperCAmelCase=None				,						UpperCAmelCase=True				,						)       ->   Optional[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if never_split is None:
            __snake_case   :				Tuple        =       []
          __snake_case   :				str        =       do_lower_case
          __snake_case   :				Optional[Any]        =       set(UpperCAmelCase   )
          __snake_case   :				Dict        =       tokenize_chinese_chars
          __snake_case   :				str        =       strip_accents
          __snake_case   :				int        =       do_split_on_punc
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase=None   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       self.never_split.union(set(UpperCAmelCase   )   ) if never_split else self.never_split
          __snake_case   :				Tuple        =       self._clean_text(UpperCAmelCase   )
  
          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
            __snake_case   :				int        =       self._tokenize_chinese_chars(UpperCAmelCase   )
          # prevents treating the same character with different unicode codepoints as different characters
          __snake_case   :				List[Any]        =       unicodedata.normalize("NFC"				,						UpperCAmelCase   )
          __snake_case   :				Tuple        =       whitespace_tokenize(UpperCAmelCase   )
          __snake_case   :				Union[str, Any]        =       []
          for token in orig_tokens:
            if token not in never_split:
              if self.do_lower_case:
                __snake_case   :				Optional[int]        =       token.lower()
                if self.strip_accents is not False:
                  __snake_case   :				Optional[int]        =       self._run_strip_accents(UpperCAmelCase   )
              elif self.strip_accents:
                __snake_case   :				List[str]        =       self._run_strip_accents(UpperCAmelCase   )
            split_tokens.extend(self._run_split_on_punc(UpperCAmelCase				,						UpperCAmelCase   )   )
  
          __snake_case   :				Optional[Any]        =       whitespace_tokenize(" ".join(UpperCAmelCase   )   )
          return output_tokens
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       unicodedata.normalize("NFD"				,						UpperCAmelCase   )
          __snake_case   :				List[Any]        =       []
          for char in text:
            __snake_case   :				Tuple        =       unicodedata.category(UpperCAmelCase   )
            if cat == "Mn":
              continue
            output.append(UpperCAmelCase   )
          return "".join(UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase=None   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if not self.do_split_on_punc or (never_split is not None and text in never_split):
            return [text]
          __snake_case   :				Union[str, Any]        =       list(UpperCAmelCase   )
          __snake_case   :				Optional[Any]        =       0
          __snake_case   :				Optional[int]        =       True
          __snake_case   :				Any        =       []
          while i < len(UpperCAmelCase   ):
            __snake_case   :				str        =       chars[i]
            if _is_punctuation(UpperCAmelCase   ):
              output.append([char]   )
              __snake_case   :				Any        =       True
            else:
              if start_new_word:
                output.append([]   )
              __snake_case   :				Dict        =       False
              output[-1].append(UpperCAmelCase   )
            i += 1
  
          return ["".join(UpperCAmelCase   ) for x in output]
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       []
          for char in text:
            __snake_case   :				Tuple        =       ord(UpperCAmelCase   )
            if self._is_chinese_char(UpperCAmelCase   ):
              output.append(" "   )
              output.append(UpperCAmelCase   )
              output.append(" "   )
            else:
              output.append(UpperCAmelCase   )
          return "".join(UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if (
              (cp >= 0x4E00 and cp <= 0x9FFF)
              or (cp >= 0x3400 and cp <= 0x4DBF)  #
              or (cp >= 0x20000 and cp <= 0x2A6DF)  #
              or (cp >= 0x2A700 and cp <= 0x2B73F)  #
              or (cp >= 0x2B740 and cp <= 0x2B81F)  #
              or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
              or (cp >= 0xF900 and cp <= 0xFAFF)
              or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
          ):  #
            return True
  
          return False
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       []
          for char in text:
            __snake_case   :				Union[str, Any]        =       ord(UpperCAmelCase   )
            if cp == 0 or cp == 0xFFFD or _is_control(UpperCAmelCase   ):
              continue
            if _is_whitespace(UpperCAmelCase   ):
              output.append(" "   )
            else:
              output.append(UpperCAmelCase   )
          return "".join(UpperCAmelCase   )
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    int				) ->    Dict:
    __snake_case   :				int        =       set()
    __snake_case   :				Dict        =       word[0]
    for char in word[1:]:
      pairs.add((prev_char, char)				)
      __snake_case   :				List[str]        =       char
    return pairs
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    Union[str, Any]				) ->    Union[str, Any]:
    __snake_case   :				List[Any]        =       text.replace("—"   ,     "-"				)
    __snake_case   :				Optional[Any]        =       text.replace("–"   ,     "-"				)
    __snake_case   :				List[Any]        =       text.replace("―"   ,     "-"				)
    __snake_case   :				Any        =       text.replace("…"   ,     "..."				)
    __snake_case   :				str        =       text.replace("´"   ,     "'"				)
    __snake_case   :				str        =       re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"   ,     R" \1 "   ,     lowercase				)
    __snake_case   :				Union[str, Any]        =       re.sub(R"\s*\n\s*"   ,     " \n "   ,     lowercase				)
    __snake_case   :				Any        =       re.sub(R"[^\S\n]+"   ,     " "   ,     lowercase				)
    return text.strip()
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        UpperCAmelCase_						:      List[str]       =VOCAB_FILES_NAMES
        UpperCAmelCase_						:      int       =PRETRAINED_VOCAB_FILES_MAP
        UpperCAmelCase_						:      int       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        UpperCAmelCase_						:      Union[str, Any]       =["input_ids", "attention_mask"]
  
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase="<unk>"				,						**UpperCAmelCase   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          super().__init__(unk_token=UpperCAmelCase				,						**UpperCAmelCase   )
  
          try:
            import ftfy
            from spacy.lang.en import English
  
            __snake_case   :				List[Any]        =       English()
            __snake_case   :				Optional[int]        =       _nlp.tokenizer
            __snake_case   :				Tuple        =       ftfy.fix_text
          except ImportError:
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
            __snake_case   :				Dict        =       BasicTokenizer(do_lower_case=UpperCAmelCase   )
            __snake_case   :				str        =       None
  
          with open(UpperCAmelCase				,						encoding="utf-8"   ) as vocab_handle:
            __snake_case   :				str        =       json.load(UpperCAmelCase   )
          __snake_case   :				Union[str, Any]        =       {v: k for k, v in self.encoder.items()}
          with open(UpperCAmelCase				,						encoding="utf-8"   ) as merges_handle:
            __snake_case   :				Dict        =       merges_handle.read().split("\n"   )[1:-1]
          __snake_case   :				Dict        =       [tuple(merge.split()   ) for merge in merges]
          __snake_case   :				str        =       dict(zip(UpperCAmelCase				,						range(len(UpperCAmelCase   )   )   )   )
          __snake_case   :				List[Any]        =       {}
  
  
  
  
  
  
  
        @property
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return True
  
  
  
  
  
  
  
        @property
        def 						UpperCAmelCase		( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return len(self.encoder   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return dict(self.encoder				,						**self.added_tokens_encoder   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[str]        =       tuple(token[:-1]   ) + (token[-1] + "</w>",)
          if token in self.cache:
            return self.cache[token]
          __snake_case   :				Any        =       get_pairs(UpperCAmelCase   )
  
          if not pairs:
            return token + "</w>"
  
          while True:
            __snake_case   :				List[Any]        =       min(UpperCAmelCase				,						key=lambda UpperCAmelCase   : self.bpe_ranks.get(UpperCAmelCase				,						float("inf"   )   )   )
            if bigram not in self.bpe_ranks:
              break
            __snake_case    ,   __snake_case   :				List[str]        =       bigram
            __snake_case   :				Union[str, Any]        =       []
            __snake_case   :				Optional[int]        =       0
            while i < len(UpperCAmelCase   ):
              try:
                __snake_case   :				List[Any]        =       word.index(UpperCAmelCase				,						UpperCAmelCase   )
              except ValueError:
                new_word.extend(word[i:]   )
                break
              else:
                new_word.extend(word[i:j]   )
                __snake_case   :				Any        =       j
  
              if word[i] == first and i < len(UpperCAmelCase   ) - 1 and word[i + 1] == second:
                new_word.append(first + second   )
                i += 2
              else:
                new_word.append(word[i]   )
                i += 1
            __snake_case   :				List[Any]        =       tuple(UpperCAmelCase   )
            __snake_case   :				Optional[Any]        =       new_word
            if len(UpperCAmelCase   ) == 1:
              break
            else:
              __snake_case   :				Optional[Any]        =       get_pairs(UpperCAmelCase   )
          __snake_case   :				List[str]        =       " ".join(UpperCAmelCase   )
          if word == "\n  </w>":
            __snake_case   :				Tuple        =       "\n</w>"
          __snake_case   :				Tuple        =       word
          return word
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       []
          if self.fix_text is None:
            # Using BERT's BasicTokenizer
            __snake_case   :				Any        =       self.nlp.tokenize(UpperCAmelCase   )
            for token in text:
              split_tokens.extend(list(self.bpe(UpperCAmelCase   ).split(" "   )   )   )
          else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            __snake_case   :				Union[str, Any]        =       self.nlp(text_standardize(self.fix_text(UpperCAmelCase   )   )   )
            for token in text:
              split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
          return split_tokens
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return self.encoder.get(UpperCAmelCase				,						self.encoder.get(self.unk_token   )   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return self.decoder.get(UpperCAmelCase				,						self.unk_token   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       "".join(UpperCAmelCase   ).replace("</w>"				,						" "   ).strip()
          return out_string
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase = None   )       ->   Tuple[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if not os.path.isdir(UpperCAmelCase   ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
            return
          __snake_case   :				str        =       os.path.join(
              UpperCAmelCase				,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
          __snake_case   :				Any        =       os.path.join(
              UpperCAmelCase				,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )
  
          with open(UpperCAmelCase				,						"w"				,						encoding="utf-8"   ) as f:
            f.write(json.dumps(self.encoder				,						indent=2				,						sort_keys=UpperCAmelCase				,						ensure_ascii=UpperCAmelCase   ) + "\n"   )
  
          __snake_case   :				int        =       0
          with open(UpperCAmelCase				,						"w"				,						encoding="utf-8"   ) as writer:
            writer.write("#version: 0.2\n"   )
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,						key=lambda UpperCAmelCase   : kv[1]   ):
              if index != token_index:
                logger.warning(
                    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                    " Please check that the tokenizer is not corrupted!"   )
                __snake_case   :				Tuple        =       token_index
              writer.write(" ".join(UpperCAmelCase   ) + "\n"   )
              index += 1
  
          return vocab_file, merge_file
  
  
style_context: 
  
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available
  
  
  _UpperCamelCase       = {
      '''configuration_deta''': ['''DETA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DetaConfig'''],
  }
  
  try:
       if not is_vision_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       _UpperCamelCase       = ['''DetaImageProcessor''']
  
  try:
       if not is_torch_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       _UpperCamelCase       = [
           '''DETA_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''DetaForObjectDetection''',
           '''DetaModel''',
           '''DetaPreTrainedModel''',
       ]
  
  
  if TYPE_CHECKING:
       from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig
  
       try:
            if not is_vision_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .image_processing_deta import DetaImageProcessor
  
       try:
            if not is_torch_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .modeling_deta import (
                DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
                DetaForObjectDetection,
                DetaModel,
                DetaPreTrainedModel,
            )
  
  else:
       import sys
  
       _UpperCamelCase       = _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
  
  