


"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowerCAmelCase__      				=   logging.get_logger(__name__)

lowerCAmelCase__      				=   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase__      				=   {
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase__      				=   {
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}


class       SCREAMING_SNAKE_CASE__				(		lowercase				):
       """simple docstring"""


       a  :				int   =VOCAB_FILES_NAMES
       a  :				Tuple   =PRETRAINED_VOCAB_FILES_MAP
       a  :				List[str]   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a  :				Optional[int]   =["input_ids", "attention_mask"]
       a  :				int   =RobertaTokenizer

       def __init__(						self		,     snake_case__=None		,     snake_case__=None		,     snake_case__=None		,     snake_case__="replace"		,     snake_case__="<s>"		,     snake_case__="</s>"		,     snake_case__="</s>"		,     snake_case__="<s>"		,     snake_case__="<unk>"		,     snake_case__="<pad>"		,     snake_case__="<mask>"		,     snake_case__=False		,     snake_case__=True		,     **snake_case__		,     ):






         """simple docstring"""





         super().__init__(
             snake_case__		,     snake_case__		,     tokenizer_file=snake_case__		,     errors=snake_case__		,     bos_token=snake_case__		,     eos_token=snake_case__		,     sep_token=snake_case__		,     cls_token=snake_case__		,     unk_token=snake_case__		,     pad_token=snake_case__		,     mask_token=snake_case__		,     add_prefix_space=snake_case__		,     trim_offsets=snake_case__		,     **snake_case__		,     )

         lowerCAmelCase     :     List[Any]								=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
         if pre_tok_state.get("add_prefix_space"		,     snake_case__						) != add_prefix_space:
           lowerCAmelCase     :     Optional[int]								=							getattr(snake_case__		,     pre_tok_state.pop("type"						)						)
           lowerCAmelCase     :     Union[str, Any]								=							add_prefix_space
           lowerCAmelCase     :     Union[str, Any]								=							pre_tok_class(**snake_case__						)

         lowerCAmelCase     :     int								=							add_prefix_space

         lowerCAmelCase     :     Union[str, Any]								=							"post_processor"
         lowerCAmelCase     :     List[str]								=							getattr(self.backend_tokenizer		,     snake_case__		,     snake_case__						)
         if tokenizer_component_instance:
           lowerCAmelCase     :     Union[str, Any]								=							json.loads(tokenizer_component_instance.__getstate__()						)

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
             lowerCAmelCase     :     Tuple								=							tuple(state["sep"]						)
           if "cls" in state:
             lowerCAmelCase     :     List[Any]								=							tuple(state["cls"]						)

           lowerCAmelCase     :     int								=							False

           if state.get("add_prefix_space"		,     snake_case__						) != add_prefix_space:
             lowerCAmelCase     :     Union[str, Any]								=							add_prefix_space
             lowerCAmelCase     :     List[Any]								=							True

           if state.get("trim_offsets"		,     snake_case__						) != trim_offsets:
             lowerCAmelCase     :     List[str]								=							trim_offsets
             lowerCAmelCase     :     Optional[int]								=							True

           if changes_to_apply:
             lowerCAmelCase     :     Union[str, Any]								=							getattr(snake_case__		,     state.pop("type"						)						)
             lowerCAmelCase     :     Optional[Any]								=							component_class(**snake_case__						)
             setattr(self.backend_tokenizer		,     snake_case__		,     snake_case__						)

       @property
       def        lowercase__							(						self						):






         """simple docstring"""





         if self._mask_token is None:
           if self.verbose:
             logger.error("Using mask_token, but it is not set yet."						)
           return None
         return str(self._mask_token						)

       @mask_token.setter
       def        lowercase__							(						self		,     snake_case__						):






         """simple docstring"""





         lowerCAmelCase     :     List[Any]								=							AddedToken(snake_case__		,     lstrip=snake_case__		,     rstrip=snake_case__						) if isinstance(snake_case__		,     snake_case__						) else value
         lowerCAmelCase     :     Optional[Any]								=							value

       def        lowercase__							(						self		,     *snake_case__		,     **snake_case__						):






         """simple docstring"""





         lowerCAmelCase     :     Any								=							kwargs.get("is_split_into_words"		,     snake_case__						)
         assert self.add_prefix_space or not is_split_into_words, (
             f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*snake_case__		,     **snake_case__						)

       def        lowercase__							(						self		,     *snake_case__		,     **snake_case__						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							kwargs.get("is_split_into_words"		,     snake_case__						)

         assert self.add_prefix_space or not is_split_into_words, (
             f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*snake_case__		,     **snake_case__						)

       def        lowercase__							(						self		,     snake_case__		,     snake_case__ = None						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							self._tokenizer.model.save(snake_case__		,     name=snake_case__						)
         return tuple(snake_case__						)

       def        lowercase__							(						self		,     snake_case__		,     snake_case__=None						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[Any]								=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
           return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







       def        lowercase__							(						self		,     snake_case__		,     snake_case__ = None						):






         """simple docstring"""





         lowerCAmelCase     :     List[Any]								=							[self.sep_token_id]
         lowerCAmelCase     :     str								=							[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase				:		Union[str, Any]     =   {
    "configuration_convbert": ["CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvBertConfig", "ConvBertOnnxConfig"],
    "tokenization_convbert": ["ConvBertTokenizer"],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Optional[Any]     =   ["ConvBertTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Dict     =   [
     "CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "ConvBertForMaskedLM",
     "ConvBertForMultipleChoice",
     "ConvBertForQuestionAnswering",
     "ConvBertForSequenceClassification",
     "ConvBertForTokenClassification",
     "ConvBertLayer",
     "ConvBertModel",
     "ConvBertPreTrainedModel",
     "load_tf_weights_in_convbert",
 ]


try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		Tuple     =   [
     "TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFConvBertForMaskedLM",
     "TFConvBertForMultipleChoice",
     "TFConvBertForQuestionAnswering",
     "TFConvBertForSequenceClassification",
     "TFConvBertForTokenClassification",
     "TFConvBertLayer",
     "TFConvBertModel",
     "TFConvBertPreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
 from .tokenization_convbert import ConvBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_convbert_fast import ConvBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_convbert import (
      CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      ConvBertForMaskedLM,
      ConvBertForMultipleChoice,
      ConvBertForQuestionAnswering,
      ConvBertForSequenceClassification,
      ConvBertForTokenClassification,
      ConvBertLayer,
      ConvBertModel,
      ConvBertPreTrainedModel,
      load_tf_weights_in_convbert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_convbert import (
      TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertLayer,
      TFConvBertModel,
      TFConvBertPreTrainedModel,
  )


else:
 import sys

 lowerCamelCase				:		Optional[int]     =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



