





"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__					:     Optional[Any]							=				logging.get_logger(__name__)

a__					:     Any							=				{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a__					:     Dict							=				{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

a__					:     str							=				{
    """openai-gpt""": 5_1_2,
}





def 							a__			(  _UpperCAmelCase	)    ->  List[Any]:

 '''simple docstring'''






 snake_case__    							=				text.strip()
 if not text:
  return []
 snake_case__    							=				text.split()
 return tokens



class      lowercase_				(     __lowerCAmelCase							):

       def __init__(			self   ,					__SCREAMING_SNAKE_CASE=True   ,					__SCREAMING_SNAKE_CASE=None   ,					__SCREAMING_SNAKE_CASE=True   ,					__SCREAMING_SNAKE_CASE=None   ,					__SCREAMING_SNAKE_CASE=True   ,					):






        '''simple docstring'''

        if never_split is None:
         snake_case__    							=				[]
        snake_case__    							=				do_lower_case
        snake_case__    							=				set(__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				tokenize_chinese_chars
        snake_case__    							=				strip_accents
        snake_case__    							=				do_split_on_punc

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE=None     ):






        '''simple docstring'''

        snake_case__    							=				self.never_split.union(set(__SCREAMING_SNAKE_CASE     )     ) if never_split else self.never_split
        snake_case__    							=				self._clean_text(__SCREAMING_SNAKE_CASE     )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
         snake_case__    							=				self._tokenize_chinese_chars(__SCREAMING_SNAKE_CASE     )
        # prevents treating the same character with different unicode codepoints as different characters
        snake_case__    							=				unicodedata.normalize("NFC"   ,					__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				whitespace_tokenize(__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				[]
        for token in orig_tokens:
         if token not in never_split:
          if self.do_lower_case:
           snake_case__    							=				token.lower()
           if self.strip_accents is not False:
            snake_case__    							=				self._run_strip_accents(__SCREAMING_SNAKE_CASE     )
          elif self.strip_accents:
           snake_case__    							=				self._run_strip_accents(__SCREAMING_SNAKE_CASE     )
         split_tokens.extend(self._run_split_on_punc(__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE     )     )

        snake_case__    							=				whitespace_tokenize(" ".join(__SCREAMING_SNAKE_CASE     )     )
        return output_tokens

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				unicodedata.normalize("NFD"   ,					__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				[]
        for char in text:
         snake_case__    							=				unicodedata.category(__SCREAMING_SNAKE_CASE     )
         if cat == "Mn":
          continue
         output.append(__SCREAMING_SNAKE_CASE     )
        return "".join(__SCREAMING_SNAKE_CASE     )

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE=None     ):






        '''simple docstring'''

        if not self.do_split_on_punc or (never_split is not None and text in never_split):
         return [text]
        snake_case__    							=				list(__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				0
        snake_case__    							=				True
        snake_case__    							=				[]
        while i < len(__SCREAMING_SNAKE_CASE     ):
         snake_case__    							=				chars[i]
         if _is_punctuation(__SCREAMING_SNAKE_CASE     ):
          output.append([char]     )
          snake_case__    							=				True
         else:
          if start_new_word:
           output.append([]     )
          snake_case__    							=				False
          output[-1].append(__SCREAMING_SNAKE_CASE     )
         i += 1

        return ["".join(__SCREAMING_SNAKE_CASE     ) for x in output]

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				[]
        for char in text:
         snake_case__    							=				ord(__SCREAMING_SNAKE_CASE     )
         if self._is_chinese_char(__SCREAMING_SNAKE_CASE     ):
          output.append(" "     )
          output.append(__SCREAMING_SNAKE_CASE     )
          output.append(" "     )
         else:
          output.append(__SCREAMING_SNAKE_CASE     )
        return "".join(__SCREAMING_SNAKE_CASE     )

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        if (
            (cp >= 0X4e00 and cp <= 0X9fff)
            or (cp >= 0X3400 and cp <= 0X4dbf)  #
            or (cp >= 0X20000 and cp <= 0X2a6df)  #
            or (cp >= 0X2a700 and cp <= 0X2b73f)  #
            or (cp >= 0X2b740 and cp <= 0X2b81f)  #
            or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
            or (cp >= 0Xf900 and cp <= 0Xfaff)
            or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
        ):  #
         return True

        return False

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				[]
        for char in text:
         snake_case__    							=				ord(__SCREAMING_SNAKE_CASE     )
         if cp == 0 or cp == 0Xfffd or _is_control(__SCREAMING_SNAKE_CASE     ):
          continue
         if _is_whitespace(__SCREAMING_SNAKE_CASE     ):
          output.append(" "     )
         else:
          output.append(__SCREAMING_SNAKE_CASE     )
        return "".join(__SCREAMING_SNAKE_CASE     )





def 							a__			(  _UpperCAmelCase	)    ->  Union[str, Any]:

 '''simple docstring'''






 snake_case__    							=				set()
 snake_case__    							=				word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)	)
  snake_case__    							=				char
 return pairs





def 							a__			(  _UpperCAmelCase	)    ->  int:

 '''simple docstring'''






 snake_case__    							=				text.replace("—"	,     "-"	)
 snake_case__    							=				text.replace("–"	,     "-"	)
 snake_case__    							=				text.replace("―"	,     "-"	)
 snake_case__    							=				text.replace("…"	,     "..."	)
 snake_case__    							=				text.replace("´"	,     "'"	)
 snake_case__    							=				re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"	,     R" \1 "	,     _UpperCAmelCase	)
 snake_case__    							=				re.sub(R"\s*\n\s*"	,     " \n "	,     _UpperCAmelCase	)
 snake_case__    							=				re.sub(R"[^\S\n]+"	,     " "	,     _UpperCAmelCase	)
 return text.strip()



class      lowercase_				(     __lowerCAmelCase							):
       snake_case__				:							int											=     VOCAB_FILES_NAMES
       snake_case__				:							List[str]											=     PRETRAINED_VOCAB_FILES_MAP
       snake_case__				:							Union[str, Any]											=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       snake_case__				:							int											=     ["""input_ids""", """attention_mask"""]

       def __init__(			self   ,					__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE="<unk>"   ,					**__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        super().__init__(unk_token=__SCREAMING_SNAKE_CASE   ,					**__SCREAMING_SNAKE_CASE     )

        try:
         import ftfy
         from spacy.lang.en import English

         snake_case__    							=				English()
         snake_case__    							=				_nlp.tokenizer
         snake_case__    							=				ftfy.fix_text
        except ImportError:
         logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."     )
         snake_case__    							=				BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     )
         snake_case__    							=				None

        with open(__SCREAMING_SNAKE_CASE   ,					encoding="utf-8"     ) as vocab_handle:
         snake_case__    							=				json.load(__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				{v: k for k, v in self.encoder.items()}
        with open(__SCREAMING_SNAKE_CASE   ,					encoding="utf-8"     ) as merges_handle:
         snake_case__    							=				merges_handle.read().split("\n"     )[1:-1]
        snake_case__    							=				[tuple(merge.split()     ) for merge in merges]
        snake_case__    							=				dict(zip(__SCREAMING_SNAKE_CASE   ,					range(len(__SCREAMING_SNAKE_CASE     )     )     )     )
        snake_case__    							=				{}

       @property
       def 		_SCREAMING_SNAKE_CASE(			self     ):






        '''simple docstring'''

        return True

       @property
       def 		_SCREAMING_SNAKE_CASE(			self     ):






        '''simple docstring'''

        return len(self.encoder     )

       def 		_SCREAMING_SNAKE_CASE(			self     ):






        '''simple docstring'''

        return dict(self.encoder   ,					**self.added_tokens_encoder     )

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				tuple(token[:-1]     ) + (token[-1] + "</w>",)
        if token in self.cache:
         return self.cache[token]
        snake_case__    							=				get_pairs(__SCREAMING_SNAKE_CASE     )

        if not pairs:
         return token + "</w>"

        while True:
         snake_case__    							=				min(__SCREAMING_SNAKE_CASE   ,					key=lambda __SCREAMING_SNAKE_CASE     : self.bpe_ranks.get(__SCREAMING_SNAKE_CASE   ,					float("inf"     )     )     )
         if bigram not in self.bpe_ranks:
          break
         snake_case__							,   snake_case__    							=				bigram
         snake_case__    							=				[]
         snake_case__    							=				0
         while i < len(__SCREAMING_SNAKE_CASE     ):
          try:
           snake_case__    							=				word.index(__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE     )
          except ValueError:
           new_word.extend(word[i:]     )
           break
          else:
           new_word.extend(word[i:j]     )
           snake_case__    							=				j

          if word[i] == first and i < len(__SCREAMING_SNAKE_CASE     ) - 1 and word[i + 1] == second:
           new_word.append(first + second     )
           i += 2
          else:
           new_word.append(word[i]     )
           i += 1
         snake_case__    							=				tuple(__SCREAMING_SNAKE_CASE     )
         snake_case__    							=				new_word
         if len(__SCREAMING_SNAKE_CASE     ) == 1:
          break
         else:
          snake_case__    							=				get_pairs(__SCREAMING_SNAKE_CASE     )
        snake_case__    							=				" ".join(__SCREAMING_SNAKE_CASE     )
        if word == "\n  </w>":
         snake_case__    							=				"\n</w>"
        snake_case__    							=				word
        return word

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				[]
        if self.fix_text is None:
         # Using BERT's BasicTokenizer
         snake_case__    							=				self.nlp.tokenize(__SCREAMING_SNAKE_CASE     )
         for token in text:
          split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE     ).split(" "     )     )     )
        else:
         # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
         snake_case__    							=				self.nlp(text_standardize(self.fix_text(__SCREAMING_SNAKE_CASE     )     )     )
         for token in text:
          split_tokens.extend(list(self.bpe(token.text.lower()     ).split(" "     )     )     )
        return split_tokens

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        return self.encoder.get(__SCREAMING_SNAKE_CASE   ,					self.encoder.get(self.unk_token     )     )

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        return self.decoder.get(__SCREAMING_SNAKE_CASE   ,					self.unk_token     )

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE     ):






        '''simple docstring'''

        snake_case__    							=				"".join(__SCREAMING_SNAKE_CASE     ).replace("</w>"   ,					" "     ).strip()
        return out_string

       def 		_SCREAMING_SNAKE_CASE(			self   ,					__SCREAMING_SNAKE_CASE   ,					__SCREAMING_SNAKE_CASE = None     ):






        '''simple docstring'''

        if not os.path.isdir(__SCREAMING_SNAKE_CASE     ):
         logger.error(F"Vocabulary path ({save_directory}) should be a directory"     )
         return
        snake_case__    							=				os.path.join(
            __SCREAMING_SNAKE_CASE   ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
        snake_case__    							=				os.path.join(
            __SCREAMING_SNAKE_CASE   ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]     )

        with open(__SCREAMING_SNAKE_CASE   ,					"w"   ,					encoding="utf-8"     ) as f:
         f.write(json.dumps(self.encoder   ,					indent=2   ,					sort_keys=__SCREAMING_SNAKE_CASE   ,					ensure_ascii=__SCREAMING_SNAKE_CASE     ) + "\n"     )

        snake_case__    							=				0
        with open(__SCREAMING_SNAKE_CASE   ,					"w"   ,					encoding="utf-8"     ) as writer:
         writer.write("#version: 0.2\n"     )
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,					key=lambda __SCREAMING_SNAKE_CASE     : kv[1]     ):
          if index != token_index:
           logger.warning(
               F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
               " Please check that the tokenizer is not corrupted!"     )
           snake_case__    							=				token_index
          writer.write(" ".join(__SCREAMING_SNAKE_CASE     ) + "\n"     )
          index += 1

        return vocab_file, merge_file








"""simple docstring"""


def 							a__			(  _UpperCAmelCase	)    ->  int:

 '''simple docstring'''






 return int(_UpperCAmelCase	) if x - int(_UpperCAmelCase	) <= 0 else int(_UpperCAmelCase	) + 1


if __name__ == "__main__":
   import doctest

   doctest.testmod()

