
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase           =logging.get_logger(__name__)

_lowerCamelCase           ={
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_lowerCamelCase           ={
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
}

_lowerCamelCase           ={
    """roberta-base""": 5_1_2,
    """roberta-large""": 5_1_2,
    """roberta-large-mnli""": 5_1_2,
    """distilroberta-base""": 5_1_2,
    """roberta-base-openai-detector""": 5_1_2,
    """roberta-large-openai-detector""": 5_1_2,
}





@lru_cache()
def 					_a      (		):
 lowerCamelCase	:			Optional[int]           =   (
     list(range(ord("""!"""							),			ord("""~"""							) + 1							)							) + list(range(ord("""¡"""							),			ord("""¬"""							) + 1							)							) + list(range(ord("""®"""							),			ord("""ÿ"""							) + 1							)							)
 )
 lowerCamelCase	:			List[Any]           =   bs[:]
 lowerCamelCase	:			Optional[int]           =   0
 for b in range(2**8							):
  if b not in bs:
   bs.append(_A							)
   cs.append(2**8 + n							)
   n += 1
 lowerCamelCase	:			List[str]           =   [chr(_A							) for n in cs]
 return dict(zip(_A,			_A							)							)





def 					_a      (		lowerCamelCase							):
 lowerCamelCase	:			Union[str, Any]           =   set()
 lowerCamelCase	:			Tuple           =   word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)							)
  lowerCamelCase	:			Any           =   char
 return pairs





class 							A__	(			lowerCamelCase__):
      _UpperCAmelCase       :					List[Any]					 =    VOCAB_FILES_NAMES
      _UpperCAmelCase       :					List[Any]					 =    PRETRAINED_VOCAB_FILES_MAP
      _UpperCAmelCase       :					Tuple					 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _UpperCAmelCase       :					Any					 =    ['input_ids', 'attention_mask']
      def __init__(   self					,						__magic_name__					,						__magic_name__					,						__magic_name__="replace"					,						__magic_name__="<s>"					,						__magic_name__="</s>"					,						__magic_name__="</s>"					,						__magic_name__="<s>"					,						__magic_name__="<unk>"					,						__magic_name__="<pad>"					,						__magic_name__="<mask>"					,						__magic_name__=False					,						**__magic_name__					,						):
       lowerCamelCase	:			Optional[Any]           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else bos_token
       lowerCamelCase	:			Union[str, Any]           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else eos_token
       lowerCamelCase	:			Optional[int]           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else sep_token
       lowerCamelCase	:			Optional[Any]           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else cls_token
       lowerCamelCase	:			Tuple           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else unk_token
       lowerCamelCase	:			Any           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       lowerCamelCase	:			int           =   AddedToken(lowercase_					,						lstrip=lowercase_					,						rstrip=lowercase_   ) if isinstance(lowercase_					,						lowercase_   ) else mask_token

       super().__init__(
           errors=lowercase_					,						bos_token=lowercase_					,						eos_token=lowercase_					,						unk_token=lowercase_					,						sep_token=lowercase_					,						cls_token=lowercase_					,						pad_token=lowercase_					,						mask_token=lowercase_					,						add_prefix_space=lowercase_					,						**lowercase_					,						)

       with open(lowercase_					,						encoding="""utf-8"""   ) as vocab_handle:
        lowerCamelCase	:			Any           =   json.load(lowercase_   )
       lowerCamelCase	:			str           =   {v: k for k, v in self.encoder.items()}
       lowerCamelCase	:			Any           =   errors  # how to handle errors in decoding
       lowerCamelCase	:			Any           =   bytes_to_unicode()
       lowerCamelCase	:			Union[str, Any]           =   {v: k for k, v in self.byte_encoder.items()}
       with open(lowercase_					,						encoding="""utf-8"""   ) as merges_handle:
        lowerCamelCase	:			int           =   merges_handle.read().split("""\n"""   )[1:-1]
       lowerCamelCase	:			List[str]           =   [tuple(merge.split()   ) for merge in bpe_merges]
       lowerCamelCase	:			Optional[Any]           =   dict(zip(lowercase_					,						range(len(lowercase_   )   )   )   )
       lowerCamelCase	:			Union[str, Any]           =   {}
       lowerCamelCase	:			List[Any]           =   add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       lowerCamelCase	:			Any           =   re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""   )
      @property
      def 		UpperCamelCase__							(   self   ):
       return len(self.encoder   )
      def 		UpperCamelCase__							(   self   ):
       return dict(self.encoder					,						**self.added_tokens_encoder   )
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):






       if token in self.cache:
        return self.cache[token]
       lowerCamelCase	:			str           =   tuple(lowercase_   )
       lowerCamelCase	:			Any           =   get_pairs(lowercase_   )

       if not pairs:
        return token

       while True:
        lowerCamelCase	:			str           =   min(lowercase_					,						key=lambda __magic_name__   : self.bpe_ranks.get(lowercase_					,						float("""inf"""   )   )   )
        if bigram not in self.bpe_ranks:
         break
        lowerCamelCase    ,					lowerCamelCase	:			List[Any]           =   bigram
        lowerCamelCase	:			List[Any]           =   []
        lowerCamelCase	:			Dict           =   0
        while i < len(lowercase_   ):
         try:
          lowerCamelCase	:			Dict           =   word.index(lowercase_					,						lowercase_   )
         except ValueError:
          new_word.extend(word[i:]   )
          break
         else:
          new_word.extend(word[i:j]   )
          lowerCamelCase	:			str           =   j

         if word[i] == first and i < len(lowercase_   ) - 1 and word[i + 1] == second:
          new_word.append(first + second   )
          i += 2
         else:
          new_word.append(word[i]   )
          i += 1
        lowerCamelCase	:			Optional[int]           =   tuple(lowercase_   )
        lowerCamelCase	:			List[str]           =   new_word
        if len(lowercase_   ) == 1:
         break
        else:
         lowerCamelCase	:			Optional[int]           =   get_pairs(lowercase_   )
       lowerCamelCase	:			Any           =   """ """.join(lowercase_   )
       lowerCamelCase	:			List[Any]           =   word
       return word
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       lowerCamelCase	:			int           =   []
       for token in re.findall(self.pat					,						lowercase_   ):
        lowerCamelCase	:			List[str]           =   """""".join(
            self.byte_encoder[b] for b in token.encode("""utf-8"""   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_   ).split(""" """   )   )
       return bpe_tokens
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       return self.encoder.get(lowercase_					,						self.encoder.get(self.unk_token   )   )
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       return self.decoder.get(lowercase_   )
      def 		UpperCamelCase__							(   self					,						__magic_name__   ):
       lowerCamelCase	:			Tuple           =   """""".join(lowercase_   )
       lowerCamelCase	:			List[Any]           =   bytearray([self.byte_decoder[c] for c in text]   ).decode("""utf-8"""					,						errors=self.errors   )
       return text
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):






       if not os.path.isdir(lowercase_   ):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
        return
       lowerCamelCase	:			List[str]           =   os.path.join(
           lowercase_					,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
       lowerCamelCase	:			Tuple           =   os.path.join(
           lowercase_					,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

       with open(lowercase_					,						"""w"""					,						encoding="""utf-8"""   ) as f:
        f.write(json.dumps(self.encoder					,						indent=2					,						sort_keys=lowercase_					,						ensure_ascii=lowercase_   ) + """\n"""   )

       lowerCamelCase	:			Union[str, Any]           =   0
       with open(lowercase_					,						"""w"""					,						encoding="""utf-8"""   ) as writer:
        writer.write("""#version: 0.2\n"""   )
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,						key=lambda __magic_name__   : kv[1]   ):
         if index != token_index:
          logger.warning(
              F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              """ Please check that the tokenizer is not corrupted!"""   )
          lowerCamelCase	:			Tuple           =   token_index
         writer.write(""" """.join(lowercase_   ) + """\n"""   )
         index += 1

       return vocab_file, merge_file
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):






       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       lowerCamelCase	:			Union[str, Any]           =   [self.cls_token_id]
       lowerCamelCase	:			List[Any]           =   [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None					,						__magic_name__ = False   ):






       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowercase_					,						token_ids_a=lowercase_					,						already_has_special_tokens=lowercase_   )

       if token_ids_a is None:
        return [1] + ([0] * len(lowercase_   )) + [1]
       return [1] + ([0] * len(lowercase_   )) + [1, 1] + ([0] * len(lowercase_   )) + [1]
      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__ = None   ):
       lowerCamelCase	:			Dict           =   [self.sep_token_id]
       lowerCamelCase	:			Any           =   [self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep   ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






      def 		UpperCamelCase__							(   self					,						__magic_name__					,						__magic_name__=False					,						**__magic_name__   ):
       lowerCamelCase	:			Dict           =   kwargs.pop("""add_prefix_space"""					,						self.add_prefix_space   )
       if (is_split_into_words or add_prefix_space) and (len(lowercase_   ) > 0 and not text[0].isspace()):
        lowerCamelCase	:			int           =   """ """ + text
       return (text, kwargs)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_keras_nlp_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCamelCase           ={
    """configuration_gpt2""": ["""GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPT2Config""", """GPT2OnnxConfig"""],
    """tokenization_gpt2""": ["""GPT2Tokenizer"""],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =["""GPT2TokenizerFast"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =[
          """GPT2_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """GPT2DoubleHeadsModel""",
          """GPT2ForQuestionAnswering""",
          """GPT2ForSequenceClassification""",
          """GPT2ForTokenClassification""",
          """GPT2LMHeadModel""",
          """GPT2Model""",
          """GPT2PreTrainedModel""",
          """load_tf_weights_in_gpt2""",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =[
          """TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """TFGPT2DoubleHeadsModel""",
          """TFGPT2ForSequenceClassification""",
          """TFGPT2LMHeadModel""",
          """TFGPT2MainLayer""",
          """TFGPT2Model""",
          """TFGPT2PreTrainedModel""",
      ]

try:
      if not is_keras_nlp_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =["""TFGPT2Tokenizer"""]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _lowerCamelCase           =["""FlaxGPT2LMHeadModel""", """FlaxGPT2Model""", """FlaxGPT2PreTrainedModel"""]

if TYPE_CHECKING:
      from .configuration_gpta import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTaConfig, GPTaOnnxConfig
      from .tokenization_gpta import GPTaTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_gpta_fast import GPTaTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_gpta import (
                GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
                GPTaDoubleHeadsModel,
                GPTaForQuestionAnswering,
                GPTaForSequenceClassification,
                GPTaForTokenClassification,
                GPTaLMHeadModel,
                GPTaModel,
                GPTaPreTrainedModel,
                load_tf_weights_in_gpta,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_gpta import (
                TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFGPTaDoubleHeadsModel,
                TFGPTaForSequenceClassification,
                TFGPTaLMHeadModel,
                TFGPTaMainLayer,
                TFGPTaModel,
                TFGPTaPreTrainedModel,
            )

      try:
            if not is_keras_nlp_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_gpta_tf import TFGPTaTokenizer

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel, FlaxGPTaPreTrainedModel

else:
      import sys

      _lowerCamelCase           =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




