






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


A_	:	Any				=				logging.get_logger(__name__)

A_	:	List[str]				=				{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A_	:	Union[str, Any]				=				{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

A_	:	List[str]				=				{
    'openai-gpt': 512,
}

def 						snake_case      (UpperCAmelCase__						) ->       Union[str, Any]:
  UpperCamelCase_:      Optional[Any]					   =       text.strip()
  if not text:
    return []
  UpperCamelCase_:      str					   =       text.split()
  return tokens




class        _lowerCAmelCase( A__      ):





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase=True						,    _lowerCamelCase=None						,    _lowerCamelCase=True						,    _lowerCamelCase=None						,    _lowerCamelCase=True						,    ):
         if never_split is None:
           UpperCamelCase_:      int					   =       []
         UpperCamelCase_:      List[Any]					   =       do_lower_case
         UpperCamelCase_:      Dict					   =       set(_lowerCAmelCase							)
         UpperCamelCase_:      int					   =       tokenize_chinese_chars
         UpperCamelCase_:      Optional[Any]					   =       strip_accents
         UpperCamelCase_:      List[Any]					   =       do_split_on_punc





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase=None							):
         UpperCamelCase_:      Optional[Any]					   =       self.never_split.union(set(_lowerCAmelCase							)							) if never_split else self.never_split
         UpperCamelCase_:      Optional[Any]					   =       self._clean_text(_lowerCAmelCase							)

         # This was added on November 1st, 2018 for the multilingual and Chinese
         # models. This is also applied to the English models now, but it doesn't
         # matter since the English models were not trained on any Chinese data
         # and generally don't have any Chinese data in them (there are Chinese
         # characters in the vocabulary because Wikipedia does have some Chinese
         # words in the English Wikipedia.).
         if self.tokenize_chinese_chars:
           UpperCamelCase_:      List[str]					   =       self._tokenize_chinese_chars(_lowerCAmelCase							)
         # prevents treating the same character with different unicode codepoints as different characters
         UpperCamelCase_:      Any					   =       unicodedata.normalize('NFC'						,    _lowerCAmelCase							)
         UpperCamelCase_:      Any					   =       whitespace_tokenize(_lowerCAmelCase							)
         UpperCamelCase_:      Any					   =       []
         for token in orig_tokens:
           if token not in never_split:
             if self.do_lower_case:
               UpperCamelCase_:      Optional[Any]					   =       token.lower()
               if self.strip_accents is not False:
                 UpperCamelCase_:      Dict					   =       self._run_strip_accents(_lowerCAmelCase							)
             elif self.strip_accents:
               UpperCamelCase_:      str					   =       self._run_strip_accents(_lowerCAmelCase							)
           split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase						,    _lowerCAmelCase							)							)

         UpperCamelCase_:      List[Any]					   =       whitespace_tokenize(' '.join(_lowerCAmelCase							)							)
         return output_tokens





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      Optional[int]					   =       unicodedata.normalize('NFD'						,    _lowerCAmelCase							)
         UpperCamelCase_:      Dict					   =       []
         for char in text:
           UpperCamelCase_:      Any					   =       unicodedata.category(_lowerCAmelCase							)
           if cat == "Mn":
             continue
           output.append(_lowerCAmelCase							)
         return "".join(_lowerCAmelCase							)





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase=None							):
         if not self.do_split_on_punc or (never_split is not None and text in never_split):
           return [text]
         UpperCamelCase_:      Optional[Any]					   =       list(_lowerCAmelCase							)
         UpperCamelCase_:      Optional[int]					   =       0
         UpperCamelCase_:      List[str]					   =       True
         UpperCamelCase_:      str					   =       []
         while i < len(_lowerCAmelCase							):
           UpperCamelCase_:      Tuple					   =       chars[i]
           if _is_punctuation(_lowerCAmelCase							):
             output.append([char]							)
             UpperCamelCase_:      Optional[int]					   =       True
           else:
             if start_new_word:
               output.append([]							)
             UpperCamelCase_:      List[Any]					   =       False
             output[-1].append(_lowerCAmelCase							)
           i += 1

         return ["".join(_lowerCAmelCase							) for x in output]





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      Tuple					   =       []
         for char in text:
           UpperCamelCase_:      str					   =       ord(_lowerCAmelCase							)
           if self._is_chinese_char(_lowerCAmelCase							):
             output.append(' '							)
             output.append(_lowerCAmelCase							)
             output.append(' '							)
           else:
             output.append(_lowerCAmelCase							)
         return "".join(_lowerCAmelCase							)





       def     _a					(      self						,    _lowerCamelCase							):
         # This defines a "chinese character" as anything in the CJK Unicode block:
         #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
         #
         # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
         # despite its name. The modern Korean Hangul alphabet is a different block,
         # as is Japanese Hiragana and Katakana. Those alphabets are used to write
         # space-separated words, so they are not treated specially and handled
         # like the all of the other languages.
         if (
             (cp >= 0X4_e00 and cp <= 0X9_fff)
             or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
             or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
             or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
             or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
             or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
             or (cp >= 0Xf_900 and cp <= 0Xf_aff)
             or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
         ):  #
           return True

         return False





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      str					   =       []
         for char in text:
           UpperCamelCase_:      Tuple					   =       ord(_lowerCAmelCase							)
           if cp == 0 or cp == 0Xf_ffd or _is_control(_lowerCAmelCase							):
             continue
           if _is_whitespace(_lowerCAmelCase							):
             output.append(' '							)
           else:
             output.append(_lowerCAmelCase							)
         return "".join(_lowerCAmelCase							)







def 						snake_case      (UpperCAmelCase__						) ->       str:
  UpperCamelCase_:      Optional[Any]					   =       set()
  UpperCamelCase_:      List[str]					   =       word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)						)
    UpperCamelCase_:      List[str]					   =       char
  return pairs

def 						snake_case      (UpperCAmelCase__						) ->       List[str]:
  UpperCamelCase_:      int					   =       text.replace('—'			,					'-'						)
  UpperCamelCase_:      List[str]					   =       text.replace('–'			,					'-'						)
  UpperCamelCase_:      List[Any]					   =       text.replace('―'			,					'-'						)
  UpperCamelCase_:      Union[str, Any]					   =       text.replace('…'			,					'...'						)
  UpperCamelCase_:      Dict					   =       text.replace('´'			,					'\''						)
  UpperCamelCase_:      int					   =       re.sub(R'(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'			,					R' \1 '			,					UpperCAmelCase__						)
  UpperCamelCase_:      List[Any]					   =       re.sub(R'\s*\n\s*'			,					' \n '			,					UpperCAmelCase__						)
  UpperCamelCase_:      int					   =       re.sub(R'[^\S\n]+'			,					' '			,					UpperCAmelCase__						)
  return text.strip()




class        _lowerCAmelCase( A__      ):





       """simple docstring"""







       a       :	Union[str, Any]								=VOCAB_FILES_NAMES
       a       :	int								=PRETRAINED_VOCAB_FILES_MAP
       a       :	Optional[Any]								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a       :	List[str]								=['''input_ids''', '''attention_mask''']





       def __init__(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase="<unk>"						,    **_lowerCamelCase							):
         super().__init__(unk_token=_lowerCAmelCase						,    **_lowerCAmelCase							)

         try:
           import ftfy
           from spacy.lang.en import English

           UpperCamelCase_:      Optional[Any]					   =       English()
           UpperCamelCase_:      Dict					   =       _nlp.tokenizer
           UpperCamelCase_:      List[str]					   =       ftfy.fix_text
         except ImportError:
           logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'							)
           UpperCamelCase_:      str					   =       BasicTokenizer(do_lower_case=_lowerCAmelCase							)
           UpperCamelCase_:      str					   =       None

         with open(_lowerCAmelCase						,    encoding='utf-8'							) as vocab_handle:
           UpperCamelCase_:      str					   =       json.load(_lowerCAmelCase							)
         UpperCamelCase_:      int					   =       {v: k for k, v in self.encoder.items()}
         with open(_lowerCAmelCase						,    encoding='utf-8'							) as merges_handle:
           UpperCamelCase_:      Union[str, Any]					   =       merges_handle.read().split('\n'							)[1:-1]
         UpperCamelCase_:      Tuple					   =       [tuple(merge.split()							) for merge in merges]
         UpperCamelCase_:      str					   =       dict(zip(_lowerCAmelCase						,    range(len(_lowerCAmelCase							)							)							)							)
         UpperCamelCase_:      Tuple					   =       {}





       @property
       def     _a					(      self							):
         return True





       @property
       def     _a					(      self							):
         return len(self.encoder							)





       def     _a					(      self							):
         return dict(self.encoder						,    **self.added_tokens_encoder							)





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      str					   =       tuple(token[:-1]							) + (token[-1] + '</w>',)
         if token in self.cache:
           return self.cache[token]
         UpperCamelCase_:      Tuple					   =       get_pairs(_lowerCAmelCase							)

         if not pairs:
           return token + "</w>"

         while True:
           UpperCamelCase_:      Optional[Any]					   =       min(_lowerCAmelCase						,    key=lambda _lowerCamelCase							: self.bpe_ranks.get(_lowerCAmelCase						,    float('inf'							)							)							)
           if bigram not in self.bpe_ranks:
             break
           UpperCamelCase_   ,UpperCamelCase_:      Dict					   =       bigram
           UpperCamelCase_:      Union[str, Any]					   =       []
           UpperCamelCase_:      Optional[int]					   =       0
           while i < len(_lowerCAmelCase							):
             try:
               UpperCamelCase_:      List[str]					   =       word.index(_lowerCAmelCase						,    _lowerCAmelCase							)
             except ValueError:
               new_word.extend(word[i:]							)
               break
             else:
               new_word.extend(word[i:j]							)
               UpperCamelCase_:      Optional[Any]					   =       j

             if word[i] == first and i < len(_lowerCAmelCase							) - 1 and word[i + 1] == second:
               new_word.append(first + second							)
               i += 2
             else:
               new_word.append(word[i]							)
               i += 1
           UpperCamelCase_:      str					   =       tuple(_lowerCAmelCase							)
           UpperCamelCase_:      Tuple					   =       new_word
           if len(_lowerCAmelCase							) == 1:
             break
           else:
             UpperCamelCase_:      Union[str, Any]					   =       get_pairs(_lowerCAmelCase							)
         UpperCamelCase_:      Dict					   =       ' '.join(_lowerCAmelCase							)
         if word == "\n  </w>":
           UpperCamelCase_:      str					   =       '\n</w>'
         UpperCamelCase_:      Union[str, Any]					   =       word
         return word





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      List[str]					   =       []
         if self.fix_text is None:
           # Using BERT's BasicTokenizer
           UpperCamelCase_:      int					   =       self.nlp.tokenize(_lowerCAmelCase							)
           for token in text:
             split_tokens.extend(list(self.bpe(_lowerCAmelCase							).split(' '							)							)							)
         else:
           # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
           UpperCamelCase_:      Any					   =       self.nlp(text_standardize(self.fix_text(_lowerCAmelCase							)							)							)
           for token in text:
             split_tokens.extend(list(self.bpe(token.text.lower()							).split(' '							)							)							)
         return split_tokens





       def     _a					(      self						,    _lowerCamelCase							):
         return self.encoder.get(_lowerCAmelCase						,    self.encoder.get(self.unk_token							)							)





       def     _a					(      self						,    _lowerCamelCase							):
         return self.decoder.get(_lowerCAmelCase						,    self.unk_token							)





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      List[str]					   =       ''.join(_lowerCAmelCase							).replace('</w>'						,    ' '							).strip()
         return out_string





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase = None							):
         if not os.path.isdir(_lowerCAmelCase							):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
           return
         UpperCamelCase_:      Dict					   =       os.path.join(
             _lowerCAmelCase						,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_:      Optional[int]					   =       os.path.join(
             _lowerCAmelCase						,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']							)

         with open(_lowerCAmelCase						,    'w'						,    encoding='utf-8'							) as f:
           f.write(json.dumps(self.encoder						,    indent=2						,    sort_keys=_lowerCAmelCase						,    ensure_ascii=_lowerCAmelCase							) + '\n'							)

         UpperCamelCase_:      int					   =       0
         with open(_lowerCAmelCase						,    'w'						,    encoding='utf-8'							) as writer:
           writer.write('#version: 0.2\n'							)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,    key=lambda _lowerCamelCase							: kv[1]							):
             if index != token_index:
               logger.warning(
                   f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                   ' Please check that the tokenizer is not corrupted!'							)
               UpperCamelCase_:      Optional[int]					   =       token_index
             writer.write(' '.join(_lowerCAmelCase							) + '\n'							)
             index += 1

         return vocab_file, merge_file







def 						snake_case      (UpperCAmelCase__ = 4_0_0_0_0_0_0						) ->       int:

  if n <= 1:
    return 0
  UpperCamelCase_:      Dict					   =       0
  UpperCamelCase_:      Union[str, Any]					   =       2
  UpperCamelCase_:      int					   =       0
  while 4 * b + a <= n:
    UpperCamelCase_   ,UpperCamelCase_:      List[str]					   =       b, 4 * b + a
    count += a
  return count + b


if __name__ == "__main__":
   print(F'''{solution() = }''')