





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


UpperCAmelCase_     							= logging.get_logger(__name__)


UpperCAmelCase_     							= {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase_     							= {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_     							= {
    'facebook/bart-base': 1024,
    'facebook/bart-large': 1024,
    'facebook/bart-large-mnli': 1024,
    'facebook/bart-large-cnn': 1024,
    'facebook/bart-large-xsum': 1024,
    'yjernite/bart_eli5': 1024,
}






class     lowercase__       (	__lowerCamelCase				):

   '''simple docstring'''


   a						:							Optional[int]   		=		VOCAB_FILES_NAMES
   a						:							Dict   		=		PRETRAINED_VOCAB_FILES_MAP
   a						:							List[Any]   		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a						:							int   		=		["input_ids", "attention_mask"]
   a						:							Optional[int]   		=		BartTokenizer




   def __init__(    self,		__magic_name__=None,		__magic_name__=None,		__magic_name__=None,		__magic_name__="replace",		__magic_name__="<s>",		__magic_name__="</s>",		__magic_name__="</s>",		__magic_name__="<s>",		__magic_name__="<unk>",		__magic_name__="<pad>",		__magic_name__="<mask>",		__magic_name__=False,		__magic_name__=True,		**__magic_name__,		) ->      int:






       """simple docstring"""
       super().__init__(
           __magic_name__,		__magic_name__,		tokenizer_file=__magic_name__,		errors=__magic_name__,		bos_token=__magic_name__,		eos_token=__magic_name__,		sep_token=__magic_name__,		cls_token=__magic_name__,		unk_token=__magic_name__,		pad_token=__magic_name__,		mask_token=__magic_name__,		add_prefix_space=__magic_name__,		trim_offsets=__magic_name__,		**__magic_name__,		)

       UpperCamelCase__							:				Optional[Any]          =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',		__magic_name__							) != add_prefix_space:
           UpperCamelCase__							:				List[Any]          =      getattr(__magic_name__,		pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:				int          =      add_prefix_space
           UpperCamelCase__							:				List[Any]          =      pre_tok_class(**__magic_name__							)

       UpperCamelCase__							:				Union[str, Any]          =      add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       UpperCamelCase__							:				int          =      '''post_processor'''
       UpperCamelCase__							:				Union[str, Any]          =      getattr(self.backend_tokenizer,		__magic_name__,		__magic_name__							)
       if tokenizer_component_instance:
           UpperCamelCase__							:				int          =      json.loads(tokenizer_component_instance.__getstate__()							)

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
               UpperCamelCase__							:				Union[str, Any]          =      tuple(state['''sep''']							)
           if "cls" in state:
               UpperCamelCase__							:				List[Any]          =      tuple(state['''cls''']							)

           UpperCamelCase__							:				Optional[int]          =      False

           if state.get('''add_prefix_space''',		__magic_name__							) != add_prefix_space:
               UpperCamelCase__							:				Dict          =      add_prefix_space
               UpperCamelCase__							:				Optional[Any]          =      True

           if state.get('''trim_offsets''',		__magic_name__							) != trim_offsets:
               UpperCamelCase__							:				Union[str, Any]          =      trim_offsets
               UpperCamelCase__							:				Optional[int]          =      True

           if changes_to_apply:
               UpperCamelCase__							:				str          =      getattr(__magic_name__,		state.pop('''type'''							)							)
               UpperCamelCase__							:				Dict          =      component_class(**__magic_name__							)
               setattr(self.backend_tokenizer,		__magic_name__,		__magic_name__							)




   @property
   def       UpperCamelCase__  (    self							) ->      str:






       """simple docstring"""
       if self._mask_token is None:
           if self.verbose:
               logger.error('''Using mask_token, but it is not set yet.'''							)
           return None
       return str(self._mask_token							)




   @mask_token.setter
   def       UpperCamelCase__  (    self,		__magic_name__							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      AddedToken(__magic_name__,		lstrip=__magic_name__,		rstrip=__magic_name__							) if isinstance(__magic_name__,		__magic_name__							) else value
       UpperCamelCase__							:				Dict          =      value




   def       UpperCamelCase__  (    self,		*__magic_name__,		**__magic_name__							) ->      BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:				List[Any]          =      kwargs.get('''is_split_into_words''',		__magic_name__							)

       if is_split_into_words and not self.add_prefix_space:
           raise ValueError(
               f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
               '''to use it with pretokenized inputs.'''							)

       return super()._batch_encode_plus(*__magic_name__,		**__magic_name__							)




   def       UpperCamelCase__  (    self,		*__magic_name__,		**__magic_name__							) ->      BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:				List[str]          =      kwargs.get('''is_split_into_words''',		__magic_name__							)

       if is_split_into_words and not self.add_prefix_space:
           raise ValueError(
               f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
               '''to use it with pretokenized inputs.'''							)

       return super()._encode_plus(*__magic_name__,		**__magic_name__							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__ = None							) ->      Tuple[str]:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self._tokenizer.model.save(__magic_name__,		name=__magic_name__							)
       return tuple(__magic_name__							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__=None							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				int          =      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
           return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__ = None							) ->      List[int]:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      [self.sep_token_id]
       UpperCamelCase__							:				Dict          =      [self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep							) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase_     							= {
    'configuration_convbert': ['CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ConvBertConfig', 'ConvBertOnnxConfig'],
    'tokenization_convbert': ['ConvBertTokenizer'],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_     							= ['ConvBertTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_     							= [
      'CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
      'ConvBertForMaskedLM',
      'ConvBertForMultipleChoice',
      'ConvBertForQuestionAnswering',
      'ConvBertForSequenceClassification',
      'ConvBertForTokenClassification',
      'ConvBertLayer',
      'ConvBertModel',
      'ConvBertPreTrainedModel',
      'load_tf_weights_in_convbert',
  ]


try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_     							= [
      'TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFConvBertForMaskedLM',
      'TFConvBertForMultipleChoice',
      'TFConvBertForQuestionAnswering',
      'TFConvBertForSequenceClassification',
      'TFConvBertForTokenClassification',
      'TFConvBertLayer',
      'TFConvBertModel',
      'TFConvBertPreTrainedModel',
  ]


if TYPE_CHECKING:
  from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
  from .tokenization_convbert import ConvBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_convbert_fast import ConvBertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_convbert import (
        CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        ConvBertForMaskedLM,
        ConvBertForMultipleChoice,
        ConvBertForQuestionAnswering,
        ConvBertForSequenceClassification,
        ConvBertForTokenClassification,
        ConvBertLayer,
        ConvBertModel,
        ConvBertPreTrainedModel,
        load_tf_weights_in_convbert,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_convbert import (
        TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFConvBertForMaskedLM,
        TFConvBertForMultipleChoice,
        TFConvBertForQuestionAnswering,
        TFConvBertForSequenceClassification,
        TFConvBertForTokenClassification,
        TFConvBertLayer,
        TFConvBertModel,
        TFConvBertPreTrainedModel,
    )


else:
  import sys

  UpperCAmelCase_     							= _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

