"""simple docstring"""






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__A         =				logging.get_logger(__name__)

__A         =				{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__A         =				{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

__A         =				{
    "openai-gpt": 5_1_2,
}



def SCREAMING_SNAKE_CASE__    (     __UpperCAmelCase				)						->						Any:
   lowercase__:     List[str]      =   text.strip()
   if not text:
      return []
   lowercase__:     Dict      =   text.split()
   return tokens


class 					UpperCAmelCase		(_UpperCAmelCase						):



    """simple docstring"""

    def __init__(     self					,    _UpperCAmelCase=True					,    _UpperCAmelCase=None					,    _UpperCAmelCase=True					,    _UpperCAmelCase=None					,    _UpperCAmelCase=True					,    ):
       if never_split is None:
          lowercase__:     List[str]      =   []
       lowercase__:     Dict      =   do_lower_case
       lowercase__:     Optional[Any]      =   set(_UpperCAmelCase							)
       lowercase__:     List[Any]      =   tokenize_chinese_chars
       lowercase__:     List[Any]      =   strip_accents
       lowercase__:     List[Any]      =   do_split_on_punc






    def _snake_case    (     self					,    _UpperCAmelCase					,    _UpperCAmelCase=None							):
       lowercase__:     str      =   self.never_split.union(set(_UpperCAmelCase							)							) if never_split else self.never_split
       lowercase__:     Optional[int]      =   self._clean_text(_UpperCAmelCase							)

       # This was added on November 1st, 2018 for the multilingual and Chinese
       # models. This is also applied to the English models now, but it doesn't
       # matter since the English models were not trained on any Chinese data
       # and generally don't have any Chinese data in them (there are Chinese
       # characters in the vocabulary because Wikipedia does have some Chinese
       # words in the English Wikipedia.).
       if self.tokenize_chinese_chars:
          lowercase__:     str      =   self._tokenize_chinese_chars(_UpperCAmelCase							)
       # prevents treating the same character with different unicode codepoints as different characters
       lowercase__:     str      =   unicodedata.normalize('''NFC'''					,    _UpperCAmelCase							)
       lowercase__:     Optional[Any]      =   whitespace_tokenize(_UpperCAmelCase							)
       lowercase__:     Dict      =   []
       for token in orig_tokens:
          if token not in never_split:
             if self.do_lower_case:
                lowercase__:     Optional[Any]      =   token.lower()
                if self.strip_accents is not False:
                   lowercase__:     Tuple      =   self._run_strip_accents(_UpperCAmelCase							)
             elif self.strip_accents:
                lowercase__:     Optional[int]      =   self._run_strip_accents(_UpperCAmelCase							)
          split_tokens.extend(self._run_split_on_punc(_UpperCAmelCase					,    _UpperCAmelCase							)							)

       lowercase__:     List[str]      =   whitespace_tokenize(''' '''.join(_UpperCAmelCase							)							)
       return output_tokens






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     Optional[int]      =   unicodedata.normalize('''NFD'''					,    _UpperCAmelCase							)
       lowercase__:     Optional[int]      =   []
       for char in text:
          lowercase__:     Union[str, Any]      =   unicodedata.category(_UpperCAmelCase							)
          if cat == "Mn":
             continue
          output.append(_UpperCAmelCase							)
       return "".join(_UpperCAmelCase							)






    def _snake_case    (     self					,    _UpperCAmelCase					,    _UpperCAmelCase=None							):
       if not self.do_split_on_punc or (never_split is not None and text in never_split):
          return [text]
       lowercase__:     List[str]      =   list(_UpperCAmelCase							)
       lowercase__:     Union[str, Any]      =   0
       lowercase__:     List[str]      =   True
       lowercase__:     Optional[int]      =   []
       while i < len(_UpperCAmelCase							):
          lowercase__:     int      =   chars[i]
          if _is_punctuation(_UpperCAmelCase							):
             output.append([char]							)
             lowercase__:     Union[str, Any]      =   True
          else:
             if start_new_word:
                output.append([]							)
             lowercase__:     Optional[int]      =   False
             output[-1].append(_UpperCAmelCase							)
          i += 1

       return ["".join(_UpperCAmelCase							) for x in output]






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     Optional[Any]      =   []
       for char in text:
          lowercase__:     int      =   ord(_UpperCAmelCase							)
          if self._is_chinese_char(_UpperCAmelCase							):
             output.append(''' '''							)
             output.append(_UpperCAmelCase							)
             output.append(''' '''							)
          else:
             output.append(_UpperCAmelCase							)
       return "".join(_UpperCAmelCase							)






    def _snake_case    (     self					,    _UpperCAmelCase							):
       # This defines a "chinese character" as anything in the CJK Unicode block:
       #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
       #
       # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
       # despite its name. The modern Korean Hangul alphabet is a different block,
       # as is Japanese Hiragana and Katakana. Those alphabets are used to write
       # space-separated words, so they are not treated specially and handled
       # like the all of the other languages.
       if (
           (cp >= 0X4E00 and cp <= 0X9FFF)
           or (cp >= 0X3400 and cp <= 0X4DBF)  #
           or (cp >= 0X2_0000 and cp <= 0X2_A6DF)  #
           or (cp >= 0X2_A700 and cp <= 0X2_B73F)  #
           or (cp >= 0X2_B740 and cp <= 0X2_B81F)  #
           or (cp >= 0X2_B820 and cp <= 0X2_CEAF)  #
           or (cp >= 0XF900 and cp <= 0XFAFF)
           or (cp >= 0X2_F800 and cp <= 0X2_FA1F)  #
       ):  #
          return True

       return False






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     int      =   []
       for char in text:
          lowercase__:     Any      =   ord(_UpperCAmelCase							)
          if cp == 0 or cp == 0XFFFD or _is_control(_UpperCAmelCase							):
             continue
          if _is_whitespace(_UpperCAmelCase							):
             output.append(''' '''							)
          else:
             output.append(_UpperCAmelCase							)
       return "".join(_UpperCAmelCase							)



def SCREAMING_SNAKE_CASE__    (     __UpperCAmelCase				)						->						Union[str, Any]:
   lowercase__:     int      =   set()
   lowercase__:     int      =   word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)				)
      lowercase__:     Dict      =   char
   return pairs



def SCREAMING_SNAKE_CASE__    (     __UpperCAmelCase				)						->						Tuple:
   lowercase__:     Optional[int]      =   text.replace('''—''' ,					'''-'''				)
   lowercase__:     Dict      =   text.replace('''–''' ,					'''-'''				)
   lowercase__:     List[Any]      =   text.replace('''―''' ,					'''-'''				)
   lowercase__:     Tuple      =   text.replace('''…''' ,					'''...'''				)
   lowercase__:     str      =   text.replace('''´''' ,					'''\''''				)
   lowercase__:     str      =   re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)''' ,					R''' \1 ''' ,					__UpperCAmelCase				)
   lowercase__:     Optional[Any]      =   re.sub(R'''\s*\n\s*''' ,					''' \n ''' ,					__UpperCAmelCase				)
   lowercase__:     Optional[int]      =   re.sub(R'''[^\S\n]+''' ,					''' ''' ,					__UpperCAmelCase				)
   return text.strip()




class 					UpperCAmelCase		(_UpperCAmelCase						):



    """simple docstring"""

    _UpperCAmelCase			:Any							=				VOCAB_FILES_NAMES
    _UpperCAmelCase			:Optional[Any]							=				PRETRAINED_VOCAB_FILES_MAP
    _UpperCAmelCase			:Dict							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _UpperCAmelCase			:List[str]							=				["input_ids", "attention_mask"]






    def __init__(     self					,    _UpperCAmelCase					,    _UpperCAmelCase					,    _UpperCAmelCase="<unk>"					,    **_UpperCAmelCase							):
       super().__init__(unk_token=_UpperCAmelCase					,    **_UpperCAmelCase							)

       try:
          import ftfy
          from spacy.lang.en import English

          lowercase__:     Any      =   English()
          lowercase__:     Dict      =   _nlp.tokenizer
          lowercase__:     Optional[int]      =   ftfy.fix_text
       except ImportError:
          logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''							)
          lowercase__:     int      =   BasicTokenizer(do_lower_case=_UpperCAmelCase							)
          lowercase__:     int      =   None

       with open(_UpperCAmelCase					,    encoding='''utf-8'''							) as vocab_handle:
          lowercase__:     int      =   json.load(_UpperCAmelCase							)
       lowercase__:     List[Any]      =   {v: k for k, v in self.encoder.items()}
       with open(_UpperCAmelCase					,    encoding='''utf-8'''							) as merges_handle:
          lowercase__:     Union[str, Any]      =   merges_handle.read().split('''\n'''							)[1:-1]
       lowercase__:     Optional[int]      =   [tuple(merge.split()							) for merge in merges]
       lowercase__:     Union[str, Any]      =   dict(zip(_UpperCAmelCase					,    range(len(_UpperCAmelCase							)							)							)							)
       lowercase__:     Union[str, Any]      =   {}






    @property
    def _snake_case    (     self							):
       return True






    @property
    def _snake_case    (     self							):
       return len(self.encoder							)






    def _snake_case    (     self							):
       return dict(self.encoder					,    **self.added_tokens_encoder							)






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     int      =   tuple(token[:-1]							) + (token[-1] + '''</w>''',)
       if token in self.cache:
          return self.cache[token]
       lowercase__:     Dict      =   get_pairs(_UpperCAmelCase							)

       if not pairs:
          return token + "</w>"

       while True:
          lowercase__:     Tuple      =   min(_UpperCAmelCase					,    key=lambda _UpperCAmelCase							: self.bpe_ranks.get(_UpperCAmelCase					,    float('''inf'''							)							)							)
          if bigram not in self.bpe_ranks:
             break
          lowercase__,				lowercase__:     Optional[int]      =   bigram
          lowercase__:     int      =   []
          lowercase__:     Optional[int]      =   0
          while i < len(_UpperCAmelCase							):
             try:
                lowercase__:     Dict      =   word.index(_UpperCAmelCase					,    _UpperCAmelCase							)
             except ValueError:
                new_word.extend(word[i:]							)
                break
             else:
                new_word.extend(word[i:j]							)
                lowercase__:     Dict      =   j

             if word[i] == first and i < len(_UpperCAmelCase							) - 1 and word[i + 1] == second:
                new_word.append(first + second							)
                i += 2
             else:
                new_word.append(word[i]							)
                i += 1
          lowercase__:     List[str]      =   tuple(_UpperCAmelCase							)
          lowercase__:     Any      =   new_word
          if len(_UpperCAmelCase							) == 1:
             break
          else:
             lowercase__:     Dict      =   get_pairs(_UpperCAmelCase							)
       lowercase__:     Any      =   ''' '''.join(_UpperCAmelCase							)
       if word == "\n  </w>":
          lowercase__:     Optional[Any]      =   '''\n</w>'''
       lowercase__:     int      =   word
       return word






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     Optional[int]      =   []
       if self.fix_text is None:
          # Using BERT's BasicTokenizer
          lowercase__:     Union[str, Any]      =   self.nlp.tokenize(_UpperCAmelCase							)
          for token in text:
             split_tokens.extend(list(self.bpe(_UpperCAmelCase							).split(''' '''							)							)							)
       else:
          # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
          lowercase__:     List[str]      =   self.nlp(text_standardize(self.fix_text(_UpperCAmelCase							)							)							)
          for token in text:
             split_tokens.extend(list(self.bpe(token.text.lower()							).split(''' '''							)							)							)
       return split_tokens






    def _snake_case    (     self					,    _UpperCAmelCase							):
       return self.encoder.get(_UpperCAmelCase					,    self.encoder.get(self.unk_token							)							)






    def _snake_case    (     self					,    _UpperCAmelCase							):
       return self.decoder.get(_UpperCAmelCase					,    self.unk_token							)






    def _snake_case    (     self					,    _UpperCAmelCase							):
       lowercase__:     List[Any]      =   ''''''.join(_UpperCAmelCase							).replace('''</w>'''					,    ''' '''							).strip()
       return out_string






    def _snake_case    (     self					,    _UpperCAmelCase					,    _UpperCAmelCase = None							):
       if not os.path.isdir(_UpperCAmelCase							):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
          return
       lowercase__:     Union[str, Any]      =   os.path.join(
           _UpperCAmelCase					,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
       lowercase__:     Dict      =   os.path.join(
           _UpperCAmelCase					,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']							)

       with open(_UpperCAmelCase					,    '''w'''					,    encoding='''utf-8'''							) as f:
          f.write(json.dumps(self.encoder					,    indent=2					,    sort_keys=_UpperCAmelCase					,    ensure_ascii=_UpperCAmelCase							) + '''\n'''							)

       lowercase__:     Tuple      =   0
       with open(_UpperCAmelCase					,    '''w'''					,    encoding='''utf-8'''							) as writer:
          writer.write('''#version: 0.2\n'''							)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,    key=lambda _UpperCAmelCase							: kv[1]							):
             if index != token_index:
                logger.warning(
                    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                    ''' Please check that the tokenizer is not corrupted!'''							)
                lowercase__:     Optional[int]      =   token_index
             writer.write(''' '''.join(_UpperCAmelCase							) + '''\n'''							)
             index += 1

       return vocab_file, merge_file

"""simple docstring"""






def SCREAMING_SNAKE_CASE__    (     __UpperCAmelCase				)						->						int:
   if not isinstance(__UpperCAmelCase ,					__UpperCAmelCase				):
      lowercase__:     Optional[int]      =   F"""Input value of [number={number}] must be an integer"""
      raise TypeError(__UpperCAmelCase				)
   if number < 1:
      raise ValueError('''Input must be a positive integer'''				)
   return number * (2 * number - 1)


if __name__ == "__main__":
       import doctest

       doctest.testmod()
