
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_A       =					logging.get_logger(__name__)

_A       =					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_A       =					{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

_A       =					{
    "openai-gpt": 5_12,
}



def 		lowercase_     (  A__						)				->				Dict:
    """simple docstring"""

    snake_case						=   text.strip()
    if not text:
        return []
    snake_case						=   text.split()
    return tokens







class 	lowerCamelCase						(    A_    ):





     def __init__(self :       int							,       _A :       Optional[int]=True							,       _A :       int=None							,       _A :       int=True							,       _A :       Union[str, Any]=None							,       _A :       List[str]=True							,       )		->		List[str]:
         if never_split is None:
             snake_case						=   []
         snake_case						=   do_lower_case
         snake_case						=   set(_A  )
         snake_case						=   tokenize_chinese_chars
         snake_case						=   strip_accents
         snake_case						=   do_split_on_punc





     def 						UpperCAmelCase(self :       int							,       _A :       Optional[Any]							,       _A :       str=None  )		->		Dict:
         snake_case						=   self.never_split.union(set(_A  )  ) if never_split else self.never_split
         snake_case						=   self._clean_text(_A  )

         # This was added on November 1st, 2018 for the multilingual and Chinese
         # models. This is also applied to the English models now, but it doesn't
         # matter since the English models were not trained on any Chinese data
         # and generally don't have any Chinese data in them (there are Chinese
         # characters in the vocabulary because Wikipedia does have some Chinese
         # words in the English Wikipedia.).
         if self.tokenize_chinese_chars:
             snake_case						=   self._tokenize_chinese_chars(_A  )
         # prevents treating the same character with different unicode codepoints as different characters
         snake_case						=   unicodedata.normalize("NFC"							,       _A  )
         snake_case						=   whitespace_tokenize(_A  )
         snake_case						=   []
         for token in orig_tokens:
             if token not in never_split:
                 if self.do_lower_case:
                     snake_case						=   token.lower()
                     if self.strip_accents is not False:
                         snake_case						=   self._run_strip_accents(_A  )
                 elif self.strip_accents:
                     snake_case						=   self._run_strip_accents(_A  )
             split_tokens.extend(self._run_split_on_punc(_A							,       _A  )  )

         snake_case						=   whitespace_tokenize(" ".join(_A  )  )
         return output_tokens





     def 						UpperCAmelCase(self :       Any							,       _A :       Dict  )		->		int:
         snake_case						=   unicodedata.normalize("NFD"							,       _A  )
         snake_case						=   []
         for char in text:
             snake_case						=   unicodedata.category(_A  )
             if cat == "Mn":
                 continue
             output.append(_A  )
         return "".join(_A  )





     def 						UpperCAmelCase(self :       List[Any]							,       _A :       List[str]							,       _A :       Any=None  )		->		List[Any]:
         if not self.do_split_on_punc or (never_split is not None and text in never_split):
             return [text]
         snake_case						=   list(_A  )
         snake_case						=   0
         snake_case						=   True
         snake_case						=   []
         while i < len(_A  ):
             snake_case						=   chars[i]
             if _is_punctuation(_A  ):
                 output.append([char]  )
                 snake_case						=   True
             else:
                 if start_new_word:
                     output.append([]  )
                 snake_case						=   False
                 output[-1].append(_A  )
             i += 1

         return ["".join(_A  ) for x in output]





     def 						UpperCAmelCase(self :       Any							,       _A :       Optional[int]  )		->		List[Any]:
         snake_case						=   []
         for char in text:
             snake_case						=   ord(_A  )
             if self._is_chinese_char(_A  ):
                 output.append(" "  )
                 output.append(_A  )
                 output.append(" "  )
             else:
                 output.append(_A  )
         return "".join(_A  )





     def 						UpperCAmelCase(self :       List[Any]							,       _A :       Union[str, Any]  )		->		int:
         # This defines a "chinese character" as anything in the CJK Unicode block:
         #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
         #
         # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
         # despite its name. The modern Korean Hangul alphabet is a different block,
         # as is Japanese Hiragana and Katakana. Those alphabets are used to write
         # space-separated words, so they are not treated specially and handled
         # like the all of the other languages.
         if (
             (cp >= 0X4e_00 and cp <= 0X9f_ff)
             or (cp >= 0X34_00 and cp <= 0X4d_bf)  #
             or (cp >= 0X2_00_00 and cp <= 0X2_a6_df)  #
             or (cp >= 0X2_a7_00 and cp <= 0X2_b7_3f)  #
             or (cp >= 0X2_b7_40 and cp <= 0X2_b8_1f)  #
             or (cp >= 0X2_b8_20 and cp <= 0X2_ce_af)  #
             or (cp >= 0Xf9_00 and cp <= 0Xfa_ff)
             or (cp >= 0X2_f8_00 and cp <= 0X2_fa_1f)  #
         ):  #
             return True

         return False





     def 						UpperCAmelCase(self :       Dict							,       _A :       List[str]  )		->		Union[str, Any]:
         snake_case						=   []
         for char in text:
             snake_case						=   ord(_A  )
             if cp == 0 or cp == 0Xff_fd or _is_control(_A  ):
                 continue
             if _is_whitespace(_A  ):
                 output.append(" "  )
             else:
                 output.append(_A  )
         return "".join(_A  )



def 		lowercase_     (  A__						)				->				List[str]:
    """simple docstring"""

    snake_case						=   set()
    snake_case						=   word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        snake_case						=   char
    return pairs



def 		lowercase_     (  A__						)				->				List[Any]:
    """simple docstring"""

    snake_case						=   text.replace("—"					,						"-"						)
    snake_case						=   text.replace("–"					,						"-"						)
    snake_case						=   text.replace("―"					,						"-"						)
    snake_case						=   text.replace("…"					,						"..."						)
    snake_case						=   text.replace("´"					,						"'"						)
    snake_case						=   re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"					,						r" \1 "					,						A__						)
    snake_case						=   re.sub(r"\s*\n\s*"					,						" \n "					,						A__						)
    snake_case						=   re.sub(r"[^\S\n]+"					,						" "					,						A__						)
    return text.strip()







class 	lowerCamelCase						(    A_    ):
     UpperCAmelCase__							:				List[str] 	=						VOCAB_FILES_NAMES
     UpperCAmelCase__							:				Union[str, Any] 	=						PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__							:				Optional[Any] 	=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__							:				List[str] 	=						["input_ids", "attention_mask"]





     def __init__(self :       Tuple							,       _A :       Dict							,       _A :       Union[str, Any]							,       _A :       int="<unk>"							,       **_A :       Optional[int]  )		->		List[Any]:
         super().__init__(unk_token=_A							,       **_A  )

         try:
             import ftfy
             from spacy.lang.en import English

             snake_case						=   English()
             snake_case						=   _nlp.tokenizer
             snake_case						=   ftfy.fix_text
         except ImportError:
             logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."  )
             snake_case						=   BasicTokenizer(do_lower_case=_A  )
             snake_case						=   None

         with open(_A							,       encoding="utf-8"  ) as vocab_handle:
             snake_case						=   json.load(_A  )
         snake_case						=   {v: k for k, v in self.encoder.items()}
         with open(_A							,       encoding="utf-8"  ) as merges_handle:
             snake_case						=   merges_handle.read().split("\n"  )[1:-1]
         snake_case						=   [tuple(merge.split()  ) for merge in merges]
         snake_case						=   dict(zip(_A							,       range(len(_A  )  )  )  )
         snake_case						=   {}





     @property
     def 						UpperCAmelCase(self :       Optional[Any]  )		->		Optional[int]:
         return True





     @property
     def 						UpperCAmelCase(self :       Optional[int]  )		->		Tuple:
         return len(self.encoder  )





     def 						UpperCAmelCase(self :       int  )		->		Any:
         return dict(self.encoder							,       **self.added_tokens_encoder  )





     def 						UpperCAmelCase(self :       int							,       _A :       str  )		->		Dict:
         snake_case						=   tuple(token[:-1]  ) + (token[-1] + "</w>",)
         if token in self.cache:
             return self.cache[token]
         snake_case						=   get_pairs(_A  )

         if not pairs:
             return token + "</w>"

         while True:
             snake_case						=   min(_A							,       key=lambda _A  : self.bpe_ranks.get(_A							,       float("inf"  )  )  )
             if bigram not in self.bpe_ranks:
                 break
             snake_case	,				snake_case						=   bigram
             snake_case						=   []
             snake_case						=   0
             while i < len(_A  ):
                 try:
                     snake_case						=   word.index(_A							,       _A  )
                 except ValueError:
                     new_word.extend(word[i:]  )
                     break
                 else:
                     new_word.extend(word[i:j]  )
                     snake_case						=   j

                 if word[i] == first and i < len(_A  ) - 1 and word[i + 1] == second:
                     new_word.append(first + second  )
                     i += 2
                 else:
                     new_word.append(word[i]  )
                     i += 1
             snake_case						=   tuple(_A  )
             snake_case						=   new_word
             if len(_A  ) == 1:
                 break
             else:
                 snake_case						=   get_pairs(_A  )
         snake_case						=   " ".join(_A  )
         if word == "\n  </w>":
             snake_case						=   "\n</w>"
         snake_case						=   word
         return word





     def 						UpperCAmelCase(self :       List[str]							,       _A :       List[Any]  )		->		Union[str, Any]:
         snake_case						=   []
         if self.fix_text is None:
             # Using BERT's BasicTokenizer
             snake_case						=   self.nlp.tokenize(_A  )
             for token in text:
                 split_tokens.extend(list(self.bpe(_A  ).split(" "  )  )  )
         else:
             # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
             snake_case						=   self.nlp(text_standardize(self.fix_text(_A  )  )  )
             for token in text:
                 split_tokens.extend(list(self.bpe(token.text.lower()  ).split(" "  )  )  )
         return split_tokens





     def 						UpperCAmelCase(self :       int							,       _A :       Union[str, Any]  )		->		int:
         return self.encoder.get(_A							,       self.encoder.get(self.unk_token  )  )





     def 						UpperCAmelCase(self :       Dict							,       _A :       Optional[Any]  )		->		Union[str, Any]:
         return self.decoder.get(_A							,       self.unk_token  )





     def 						UpperCAmelCase(self :       List[Any]							,       _A :       Optional[int]  )		->		List[str]:
         snake_case						=   "".join(_A  ).replace("</w>"							,       " "  ).strip()
         return out_string





     def 						UpperCAmelCase(self :       List[str]							,       _A :       str							,       _A :       Optional[str] = None  )		->		Tuple[str]:
         if not os.path.isdir(_A  ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'  )
             return
         snake_case						=   os.path.join(
             _A							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
         snake_case						=   os.path.join(
             _A							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]  )

         with open(_A							,       "w"							,       encoding="utf-8"  ) as f:
             f.write(json.dumps(self.encoder							,       indent=2							,       sort_keys=_A							,       ensure_ascii=_A  ) + "\n"  )

         snake_case						=   0
         with open(_A							,       "w"							,       encoding="utf-8"  ) as writer:
             writer.write("#version: 0.2\n"  )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,       key=lambda _A  : kv[1]  ):
                 if index != token_index:
                     logger.warning(
                         f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                         " Please check that the tokenizer is not corrupted!"  )
                     snake_case						=   token_index
                 writer.write(" ".join(_A  ) + "\n"  )
                 index += 1

         return vocab_file, merge_file



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A       =					logging.get_logger(__name__)

_A       =					{
    "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/config.json"
    # See all CPMAnt models at https://huggingface.co/models?filter=cpmant
}







class 	lowerCamelCase						(    A_    ):
     UpperCAmelCase__							:				List[Any] 	=						"cpmant"





     def __init__(self :       List[Any]							,       _A :       int = 3_0_7_2_0							,       _A :       int = 4_0_9_6							,       _A :       int = 3_2							,       _A :       int = 1_2_8							,       _A :       int = 1_0_2_4_0							,       _A :       int = 4_8							,       _A :       int = 0.0							,       _A :       int = 5_1_2							,       _A :       int = 2_0_4_8							,       _A :       int = 1E-6							,       _A :       float = 1.0							,       _A :       int = 3_2							,       _A :       int = 3_2							,       _A :       int = 3_2							,       _A :       bool = True							,       **_A :       Dict							,       )		->		Dict:
         super().__init__(**_A  )
         snake_case						=   prompt_types
         snake_case						=   prompt_length
         snake_case						=   segment_types
         snake_case						=   hidden_size
         snake_case						=   num_attention_heads
         snake_case						=   dim_head
         snake_case						=   dim_ff
         snake_case						=   num_hidden_layers
         snake_case						=   position_bias_num_buckets
         snake_case						=   position_bias_max_distance
         snake_case						=   dropout_p
         snake_case						=   eps
         snake_case						=   use_cache
         snake_case						=   vocab_size
         snake_case						=   init_std

