

import warnings
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_UpperCAmelCase		:   Union[str, Any]										=       logging.get_logger(__name__)

_UpperCAmelCase		:   Optional[int]										=       {
    """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/config.json""",
    # See all BART models at https://huggingface.co/models?filter=bart
}







class  lowercase				(       _UpperCAmelCase			):
				__SCREAMING_SNAKE_CASE					: str       			=			"""bart"""
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			["""past_key_values"""]
				__SCREAMING_SNAKE_CASE					: Optional[Any]       			=			{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}




				def __init__(					self      ,   snake_case=5_0265      ,   snake_case=1024      ,   snake_case=12      ,   snake_case=4096      ,   snake_case=16      ,   snake_case=12      ,   snake_case=4096      ,   snake_case=16      ,   snake_case=0.0      ,   snake_case=0.0      ,   snake_case="gelu"      ,   snake_case=1024      ,   snake_case=0.1      ,   snake_case=0.0      ,   snake_case=0.0      ,   snake_case=0.02      ,   snake_case=0.0      ,   snake_case=False      ,   snake_case=True      ,   snake_case=3      ,   snake_case=1      ,   snake_case=0      ,   snake_case=2      ,   snake_case=True      ,   snake_case=2      ,   snake_case=2      ,   **snake_case      ,   ):
											snake_case_							=    vocab_size
											snake_case_							=    max_position_embeddings
											snake_case_							=    d_model
											snake_case_							=    encoder_ffn_dim
											snake_case_							=    encoder_layers
											snake_case_							=    encoder_attention_heads
											snake_case_							=    decoder_ffn_dim
											snake_case_							=    decoder_layers
											snake_case_							=    decoder_attention_heads
											snake_case_							=    dropout
											snake_case_							=    attention_dropout
											snake_case_							=    activation_dropout
											snake_case_							=    activation_function
											snake_case_							=    init_std
											snake_case_							=    encoder_layerdrop
											snake_case_							=    decoder_layerdrop
											snake_case_							=    classifier_dropout
											snake_case_							=    use_cache
											snake_case_							=    encoder_layers
											snake_case_							=    scale_embedding  # scale factor will be sqrt(d_model) if True

											super().__init__(
											    num_labels=UpperCAmelCase_      ,   pad_token_id=UpperCAmelCase_      ,   bos_token_id=UpperCAmelCase_      ,   eos_token_id=UpperCAmelCase_      ,   is_encoder_decoder=UpperCAmelCase_      ,   decoder_start_token_id=UpperCAmelCase_      ,   forced_eos_token_id=UpperCAmelCase_      ,   **UpperCAmelCase_      ,   )

											# ensure backward compatibility for BART CNN models
											if self.forced_bos_token_id is None and kwargs.get('force_bos_token_to_be_generated'      ,   UpperCAmelCase_       ):
																		snake_case_							=    self.bos_token_id
																		warnings.warn(
																		    F'''Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. '''
																		    'The config can simply be saved and uploaded again to be fixed.'       )







class  lowercase				(       _UpperCAmelCase			):




				@property
				def     a							(					self       ):
											if self.task in ["default", "seq2seq-lm"]:
																		snake_case_							=    OrderedDict(
																		    [
																		        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																		        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																		    ]       )

																		if self.use_past:
																									snake_case_							=    {0: 'batch'}
																									snake_case_							=    {0: 'batch', 1: 'past_decoder_sequence + sequence'}
																		else:
																									snake_case_							=    {0: 'batch', 1: 'decoder_sequence'}
																									snake_case_							=    {0: 'batch', 1: 'decoder_sequence'}

																		if self.use_past:
																									self.fill_with_past_key_values_(UpperCAmelCase_      ,   direction='inputs'       )
											elif self.task == "causal-lm":
																		# TODO: figure this case out.
																		snake_case_							=    OrderedDict(
																		    [
																		        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																		        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																		    ]       )
																		if self.use_past:
																									snake_case_				,    snake_case_							=    self.num_layers
																									for i in range(UpperCAmelCase_       ):
																																snake_case_							=    {0: 'batch', 2: 'past_sequence + sequence'}
																																snake_case_							=    {0: 'batch', 2: 'past_sequence + sequence'}
											else:
																		snake_case_							=    OrderedDict(
																		    [
																		        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																		        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																		        ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
																		        ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
																		    ]       )

											return common_inputs




				@property
				def     a							(					self       ):
											if self.task in ["default", "seq2seq-lm"]:
																		snake_case_							=    super().outputs
											else:
																		snake_case_							=    super(UpperCAmelCase_      ,   self       ).outputs
																		if self.use_past:
																									snake_case_				,    snake_case_							=    self.num_layers
																									for i in range(UpperCAmelCase_       ):
																																snake_case_							=    {0: 'batch', 2: 'past_sequence + sequence'}
																																snake_case_							=    {0: 'batch', 2: 'past_sequence + sequence'}
											return common_outputs




				def     a							(					self      ,   snake_case      ,   snake_case = -1      ,   snake_case = -1      ,   snake_case = False      ,   snake_case = None      ,   ):
											snake_case_							=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_       )

											# Generate decoder inputs
											snake_case_							=    seq_length if not self.use_past else 1
											snake_case_							=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_       )
											snake_case_							=    {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
											snake_case_							=    dict(**UpperCAmelCase_      ,   **UpperCAmelCase_       )

											if self.use_past:
																		if not is_torch_available():
																									raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'       )
																		else:
																									import torch
																		snake_case_				,    snake_case_							=    common_inputs['input_ids'].shape
																		snake_case_							=    common_inputs['decoder_input_ids'].shape[1]
																		snake_case_				,    snake_case_							=    self.num_attention_heads
																		snake_case_							=    (
																		    batch,
																		    num_encoder_attention_heads,
																		    encoder_seq_length,
																		    self._config.hidden_size // num_encoder_attention_heads,
																		)
																		snake_case_							=    decoder_seq_length + 3
																		snake_case_							=    (
																		    batch,
																		    num_decoder_attention_heads,
																		    decoder_past_length,
																		    self._config.hidden_size // num_decoder_attention_heads,
																		)

																		snake_case_							=    torch.cat(
																		    [common_inputs['decoder_attention_mask'], torch.ones(UpperCAmelCase_      ,   UpperCAmelCase_       )]      ,   dim=1       )

																		snake_case_							=    []
																		# If the number of encoder and decoder layers are present in the model configuration, both are considered
																		snake_case_				,    snake_case_							=    self.num_layers
																		snake_case_							=    min(UpperCAmelCase_      ,   UpperCAmelCase_       )
																		snake_case_							=    max(UpperCAmelCase_      ,   UpperCAmelCase_       ) - min_num_layers
																		snake_case_							=    'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

																		for _ in range(UpperCAmelCase_       ):
																									common_inputs["past_key_values"].append(
																									    (
																									        torch.zeros(UpperCAmelCase_       ),
																									        torch.zeros(UpperCAmelCase_       ),
																									        torch.zeros(UpperCAmelCase_       ),
																									        torch.zeros(UpperCAmelCase_       ),
																									    )       )
																		# TODO: test this.
																		snake_case_							=    encoder_shape if remaining_side_name == 'encoder' else decoder_shape
																		for _ in range(UpperCAmelCase_      ,   UpperCAmelCase_       ):
																									common_inputs["past_key_values"].append((torch.zeros(UpperCAmelCase_       ), torch.zeros(UpperCAmelCase_       ))       )
											return common_inputs




				def     a							(					self      ,   snake_case      ,   snake_case = -1      ,   snake_case = -1      ,   snake_case = False      ,   snake_case = None      ,   ):
											snake_case_							=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_       )

											if self.use_past:
																		if not is_torch_available():
																									raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'       )
																		else:
																									import torch
																		snake_case_				,    snake_case_							=    common_inputs['input_ids'].shape
																		# Not using the same length for past_key_values
																		snake_case_							=    seqlen + 2
																		snake_case_				,    snake_case_							=    self.num_layers
																		snake_case_				,    snake_case_							=    self.num_attention_heads
																		snake_case_							=    (
																		    batch,
																		    num_encoder_attention_heads,
																		    past_key_values_length,
																		    self._config.hidden_size // num_encoder_attention_heads,
																		)

																		snake_case_							=    common_inputs['attention_mask'].dtype
																		snake_case_							=    torch.cat(
																		    [common_inputs['attention_mask'], torch.ones(UpperCAmelCase_      ,   UpperCAmelCase_      ,   dtype=UpperCAmelCase_       )]      ,   dim=1       )
																		snake_case_							=    [
																		    (torch.zeros(UpperCAmelCase_       ), torch.zeros(UpperCAmelCase_       )) for _ in range(UpperCAmelCase_       )
																		]
											return common_inputs




				def     a							(					self      ,   snake_case      ,   snake_case = -1      ,   snake_case = -1      ,   snake_case = False      ,   snake_case = None      ,   ):
											# Copied from OnnxConfig.generate_dummy_inputs
											# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
											# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
											snake_case_							=    compute_effective_axis_dimension(
											    UpperCAmelCase_      ,   fixed_dimension=OnnxConfig.default_fixed_batch      ,   num_token_to_add=0       )

											# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
											snake_case_							=    tokenizer.num_special_tokens_to_add(UpperCAmelCase_       )
											snake_case_							=    compute_effective_axis_dimension(
											    UpperCAmelCase_      ,   fixed_dimension=OnnxConfig.default_fixed_sequence      ,   num_token_to_add=UpperCAmelCase_       )

											# Generate dummy inputs according to compute batch and sequence
											snake_case_							=    [' '.join([tokenizer.unk_token]       ) * seq_length] * batch_size
											snake_case_							=    dict(tokenizer(UpperCAmelCase_      ,   return_tensors=UpperCAmelCase_       )       )
											return common_inputs




				def     a							(					self      ,   snake_case      ,   snake_case = -1      ,   snake_case = -1      ,   snake_case = False      ,   snake_case = None      ,   ):
											if self.task in ["default", "seq2seq-lm"]:
																		snake_case_							=    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
																		    UpperCAmelCase_      ,   batch_size=UpperCAmelCase_      ,   seq_length=UpperCAmelCase_      ,   is_pair=UpperCAmelCase_      ,   framework=UpperCAmelCase_       )

											elif self.task == "causal-lm":
																		snake_case_							=    self._generate_dummy_inputs_for_causal_lm(
																		    UpperCAmelCase_      ,   batch_size=UpperCAmelCase_      ,   seq_length=UpperCAmelCase_      ,   is_pair=UpperCAmelCase_      ,   framework=UpperCAmelCase_       )
											else:
																		snake_case_							=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
																		    UpperCAmelCase_      ,   batch_size=UpperCAmelCase_      ,   seq_length=UpperCAmelCase_      ,   is_pair=UpperCAmelCase_      ,   framework=UpperCAmelCase_       )

											return common_inputs




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											if self.task in ["default", "seq2seq-lm"]:
																		snake_case_							=    super()._flatten_past_key_values_(UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_       )
											else:
																		snake_case_							=    super(UpperCAmelCase_      ,   self       )._flatten_past_key_values_(
																		    UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_      ,   UpperCAmelCase_       )






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__snake_case								=  logging.get_logger(__name__)


__snake_case								=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
__snake_case								=  {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

__snake_case								=  {
    """facebook/bart-base""": 10_24,
    """facebook/bart-large""": 10_24,
    """facebook/bart-large-mnli""": 10_24,
    """facebook/bart-large-cnn""": 10_24,
    """facebook/bart-large-xsum""": 10_24,
    """yjernite/bart_eli5""": 10_24,
}




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   Tuple				       =VOCAB_FILES_NAMES
	A__    :   Any				       =PRETRAINED_VOCAB_FILES_MAP
	A__    :   str				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A__    :   Tuple				       =["""input_ids""", """attention_mask"""]
	A__    :   Optional[int]				       =BartTokenizer
	def __init__(			self							:   str       ,   UpperCAmelCase_							:   Optional[Any]=None       ,   UpperCAmelCase_							:   int=None       ,   UpperCAmelCase_							:   str=None       ,   UpperCAmelCase_							:   Optional[Any]="replace"       ,   UpperCAmelCase_							:   int="<s>"       ,   UpperCAmelCase_							:   List[Any]="</s>"       ,   UpperCAmelCase_							:   int="</s>"       ,   UpperCAmelCase_							:   Tuple="<s>"       ,   UpperCAmelCase_							:   Any="<unk>"       ,   UpperCAmelCase_							:   Any="<pad>"       ,   UpperCAmelCase_							:   Optional[Any]="<mask>"       ,   UpperCAmelCase_							:   Optional[int]=False       ,   UpperCAmelCase_							:   List[str]=True       ,   **UpperCAmelCase_							:   List[Any]       ,   ):
						super().__init__(
						    UpperCAmelCase_       ,   UpperCAmelCase_       ,   tokenizer_file=UpperCAmelCase_       ,   errors=UpperCAmelCase_       ,   bos_token=UpperCAmelCase_       ,   eos_token=UpperCAmelCase_       ,   sep_token=UpperCAmelCase_       ,   cls_token=UpperCAmelCase_       ,   unk_token=UpperCAmelCase_       ,   pad_token=UpperCAmelCase_       ,   mask_token=UpperCAmelCase_       ,   add_prefix_space=UpperCAmelCase_       ,   trim_offsets=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )

						SCREAMING_SNAKE_CASE__													= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
						if pre_tok_state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
											SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   pre_tok_state.pop('type'							)							)
											SCREAMING_SNAKE_CASE__													= add_prefix_space
											SCREAMING_SNAKE_CASE__													= pre_tok_class(**UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						SCREAMING_SNAKE_CASE__													= 'post_processor'
						SCREAMING_SNAKE_CASE__													= getattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
						if tokenizer_component_instance:
											SCREAMING_SNAKE_CASE__													= json.loads(tokenizer_component_instance.__getstate__()							)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['sep']							)
											if "cls" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['cls']							)

											SCREAMING_SNAKE_CASE__													= False

											if state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
																SCREAMING_SNAKE_CASE__													= add_prefix_space
																SCREAMING_SNAKE_CASE__													= True

											if state.get('trim_offsets'       ,   UpperCAmelCase_							) != trim_offsets:
																SCREAMING_SNAKE_CASE__													= trim_offsets
																SCREAMING_SNAKE_CASE__													= True

											if changes_to_apply:
																SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   state.pop('type'							)							)
																SCREAMING_SNAKE_CASE__													= component_class(**UpperCAmelCase_							)
																setattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
	@property
	def      A_				(			self							:   Tuple							):
						if self._mask_token is None:
											if self.verbose:
																logger.error('Using mask_token, but it is not set yet.'							)
											return None
						return str(self._mask_token							)
	@mask_token.setter
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= AddedToken(UpperCAmelCase_       ,   lstrip=UpperCAmelCase_       ,   rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_       ,   UpperCAmelCase_							) else value
						SCREAMING_SNAKE_CASE__													= value
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Tuple       ,   **UpperCAmelCase_							:   str							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._batch_encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Dict       ,   **UpperCAmelCase_							:   int							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   Optional[str] = None							):
						SCREAMING_SNAKE_CASE__													= self._tokenizer.model.save(UpperCAmelCase_       ,   name=UpperCAmelCase_							)
						return tuple(UpperCAmelCase_							)
	def      A_				(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Union[str, Any]=None							):
						SCREAMING_SNAKE_CASE__													= [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
											return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[int]       ,   UpperCAmelCase_							:   Optional[List[int]] = None							):
						SCREAMING_SNAKE_CASE__													= [self.sep_token_id]
						SCREAMING_SNAKE_CASE__													= [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





