
from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


__snake_case								=  logging.get_logger(__name__)






def 							_lowercase							(     UpperCamelCase_		)		->		List[int]:
					'''simple docstring'''


					if isinstance(UpperCamelCase_	,	np.ndarray		):
										return list(tensor.shape		)

					SCREAMING_SNAKE_CASE__													= tf.shape(UpperCamelCase_		)

					if tensor.shape == tf.TensorShape(UpperCamelCase_		):
										return dynamic

					SCREAMING_SNAKE_CASE__													= tensor.shape.as_list()

					return [dynamic[i] if s is None else s for i, s in enumerate(UpperCamelCase_		)]






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_ = None	,	UpperCamelCase_ = None		)		->		tf.Tensor:
					'''simple docstring'''


					return tf.nn.softmax(logits=logits + 1e-9	,	axis=UpperCamelCase_	,	name=UpperCamelCase_		)






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_	,	UpperCamelCase_	,	UpperCamelCase_=1e-5	,	UpperCamelCase_=-1		)		->		List[Any]:
					'''simple docstring'''


					if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(UpperCamelCase_	,	UpperCamelCase_		):
										raise NotImplementedError('Only 1D weight and bias tensors are supported for now, with only a single axis.'		)

					# Get mean and variance on the axis to be normalized
					SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= tf.nn.moments(UpperCamelCase_	,	axes=[axis]	,	keepdims=UpperCamelCase_		)

					if axis != -1:
										# Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
										# on every dimension except axis
										SCREAMING_SNAKE_CASE__													= [1] * inputs.shape.rank
										SCREAMING_SNAKE_CASE__													= shape_list(UpperCamelCase_		)[axis]
										SCREAMING_SNAKE_CASE__													= tf.reshape(UpperCamelCase_	,	UpperCamelCase_		)
										SCREAMING_SNAKE_CASE__													= tf.reshape(UpperCamelCase_	,	UpperCamelCase_		)

					# Compute layer normalization using the batch_normalization
					# function.
					SCREAMING_SNAKE_CASE__													= tf.nn.batch_normalization(
					    UpperCamelCase_	,	UpperCamelCase_	,	UpperCamelCase_	,	offset=UpperCamelCase_	,	scale=UpperCamelCase_	,	variance_epsilon=UpperCamelCase_	,	)
					return outputs






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_=0	,	UpperCamelCase_=-1		)		->		Optional[int]:
					'''simple docstring'''


					if end_dim < 0:
										end_dim += input.shape.rank
					if start_dim < 0:
										start_dim += input.shape.rank

					if start_dim == end_dim:
										return input

					SCREAMING_SNAKE_CASE__													= tf.shape(UpperCamelCase_		)
					SCREAMING_SNAKE_CASE__													= tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]		)
					SCREAMING_SNAKE_CASE__													= tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]]	,	axis=0		)
					return tf.reshape(UpperCamelCase_	,	UpperCamelCase_		)






def 							_lowercase							(     UpperCamelCase_		)		->		tf.Tensor:
					'''simple docstring'''


					if not isinstance(UpperCamelCase_	,	tf.Tensor		):
										SCREAMING_SNAKE_CASE__													= tf.convert_to_tensor(UpperCamelCase_		)  # Catches stray NumPy inputs
					if encoder_attention_mask.shape.rank == 3:
										SCREAMING_SNAKE_CASE__													= encoder_attention_mask[:, None, :, :]
					if encoder_attention_mask.shape.rank == 2:
										SCREAMING_SNAKE_CASE__													= encoder_attention_mask[:, None, None, :]
					# T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
					# Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
					# /transformer/transformer_layers.py#L270
					# encoder_extended_attention_mask = (encoder_extended_attention_mask ==
					# encoder_extended_attention_mask.transpose(-1, -2))
					SCREAMING_SNAKE_CASE__													= (
					    tf.cast(1	,	encoder_attention_mask.dtype		) - encoder_extended_attention_mask
					) * encoder_extended_attention_mask.dtype.min

					return encoder_extended_attention_mask






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_	,	UpperCamelCase_ = "input_ids"		)		->		None:
					'''simple docstring'''


					tf.debugging.assert_less(
					    UpperCamelCase_	,	tf.cast(UpperCamelCase_	,	dtype=tensor.dtype		)	,	message=(
					        F'The maximum value of {tensor_name} ({tf.math.reduce_max(UpperCamelCase_		)}) must be smaller than the embedding '
					        F'layer\'s input dimension ({embed_dim}). The likely cause is some problem at tokenization time.'
					    )	,	)






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_	,	UpperCamelCase_		)		->		Optional[int]:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= 64512
					# Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
					# because in that case even chunking the array would not make the saving
					# possible.
					SCREAMING_SNAKE_CASE__													= [x for x in data if len(UpperCamelCase_		) > HDF5_OBJECT_HEADER_LIMIT]

					# Expecting this to never be true.
					if bad_attributes:
										raise RuntimeError(
										    'The following attributes cannot be saved to HDF5 file because '
										    F'they are larger than {HDF5_OBJECT_HEADER_LIMIT} '
										    F'bytes: {bad_attributes}'		)

					SCREAMING_SNAKE_CASE__													= np.asarray(UpperCamelCase_		)

					SCREAMING_SNAKE_CASE__													= 1
					SCREAMING_SNAKE_CASE__													= np.array_split(UpperCamelCase_	,	UpperCamelCase_		)

					# This will never loop forever thanks to the test above.
					while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data		):
										num_chunks += 1
										SCREAMING_SNAKE_CASE__													= np.array_split(UpperCamelCase_	,	UpperCamelCase_		)

					if num_chunks > 1:
										for chunk_id, chunk_data in enumerate(UpperCamelCase_		):
															SCREAMING_SNAKE_CASE__													= chunk_data
					else:
										SCREAMING_SNAKE_CASE__													= data






def 							_lowercase							(     UpperCamelCase_	,	UpperCamelCase_		)		->		List[str]:
					'''simple docstring'''


					if name in group.attrs:
										SCREAMING_SNAKE_CASE__													= [n.decode('utf8'		) if hasattr(UpperCamelCase_	,	'decode'		) else n for n in group.attrs[name]]
					else:
										SCREAMING_SNAKE_CASE__													= []
										SCREAMING_SNAKE_CASE__													= 0
										while "%s%d" % (name, chunk_id) in group.attrs:
															data.extend(
															    [n.decode('utf8'		) if hasattr(UpperCamelCase_	,	'decode'		) else n for n in group.attrs['%s%d' % (name, chunk_id)]]		)
															chunk_id += 1
					return data






def 							_lowercase							(     UpperCamelCase_		)		->		List[str]:
					'''simple docstring'''


					def _expand_single_ad_tensor(UpperCamelCase_		):
										if isinstance(UpperCamelCase_	,	tf.Tensor		) and t.shape.rank == 1:
															return tf.expand_dims(UpperCamelCase_	,	axis=-1		)
										return t

					return tf.nest.map_structure(_expand_single_ad_tensor	,	UpperCamelCase_		)







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__snake_case								=  logging.get_logger(__name__)


__snake_case								=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
__snake_case								=  {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

__snake_case								=  {
    """facebook/bart-base""": 10_24,
    """facebook/bart-large""": 10_24,
    """facebook/bart-large-mnli""": 10_24,
    """facebook/bart-large-cnn""": 10_24,
    """facebook/bart-large-xsum""": 10_24,
    """yjernite/bart_eli5""": 10_24,
}




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   Tuple				       =VOCAB_FILES_NAMES
	A__    :   Any				       =PRETRAINED_VOCAB_FILES_MAP
	A__    :   str				       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A__    :   Tuple				       =["""input_ids""", """attention_mask"""]
	A__    :   Optional[int]				       =BartTokenizer
	def __init__(			self							:   str       ,   UpperCAmelCase_							:   Optional[Any]=None       ,   UpperCAmelCase_							:   int=None       ,   UpperCAmelCase_							:   str=None       ,   UpperCAmelCase_							:   Optional[Any]="replace"       ,   UpperCAmelCase_							:   int="<s>"       ,   UpperCAmelCase_							:   List[Any]="</s>"       ,   UpperCAmelCase_							:   int="</s>"       ,   UpperCAmelCase_							:   Tuple="<s>"       ,   UpperCAmelCase_							:   Any="<unk>"       ,   UpperCAmelCase_							:   Any="<pad>"       ,   UpperCAmelCase_							:   Optional[Any]="<mask>"       ,   UpperCAmelCase_							:   Optional[int]=False       ,   UpperCAmelCase_							:   List[str]=True       ,   **UpperCAmelCase_							:   List[Any]       ,   ):
						super().__init__(
						    UpperCAmelCase_       ,   UpperCAmelCase_       ,   tokenizer_file=UpperCAmelCase_       ,   errors=UpperCAmelCase_       ,   bos_token=UpperCAmelCase_       ,   eos_token=UpperCAmelCase_       ,   sep_token=UpperCAmelCase_       ,   cls_token=UpperCAmelCase_       ,   unk_token=UpperCAmelCase_       ,   pad_token=UpperCAmelCase_       ,   mask_token=UpperCAmelCase_       ,   add_prefix_space=UpperCAmelCase_       ,   trim_offsets=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )

						SCREAMING_SNAKE_CASE__													= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
						if pre_tok_state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
											SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   pre_tok_state.pop('type'							)							)
											SCREAMING_SNAKE_CASE__													= add_prefix_space
											SCREAMING_SNAKE_CASE__													= pre_tok_class(**UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						SCREAMING_SNAKE_CASE__													= 'post_processor'
						SCREAMING_SNAKE_CASE__													= getattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
						if tokenizer_component_instance:
											SCREAMING_SNAKE_CASE__													= json.loads(tokenizer_component_instance.__getstate__()							)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['sep']							)
											if "cls" in state:
																SCREAMING_SNAKE_CASE__													= tuple(state['cls']							)

											SCREAMING_SNAKE_CASE__													= False

											if state.get('add_prefix_space'       ,   UpperCAmelCase_							) != add_prefix_space:
																SCREAMING_SNAKE_CASE__													= add_prefix_space
																SCREAMING_SNAKE_CASE__													= True

											if state.get('trim_offsets'       ,   UpperCAmelCase_							) != trim_offsets:
																SCREAMING_SNAKE_CASE__													= trim_offsets
																SCREAMING_SNAKE_CASE__													= True

											if changes_to_apply:
																SCREAMING_SNAKE_CASE__													= getattr(UpperCAmelCase_       ,   state.pop('type'							)							)
																SCREAMING_SNAKE_CASE__													= component_class(**UpperCAmelCase_							)
																setattr(self.backend_tokenizer       ,   UpperCAmelCase_       ,   UpperCAmelCase_							)
	@property
	def      A_				(			self							:   Tuple							):
						if self._mask_token is None:
											if self.verbose:
																logger.error('Using mask_token, but it is not set yet.'							)
											return None
						return str(self._mask_token							)
	@mask_token.setter
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   List[Any]							):
						SCREAMING_SNAKE_CASE__													= AddedToken(UpperCAmelCase_       ,   lstrip=UpperCAmelCase_       ,   rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_       ,   UpperCAmelCase_							) else value
						SCREAMING_SNAKE_CASE__													= value
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Tuple       ,   **UpperCAmelCase_							:   str							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._batch_encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   *UpperCAmelCase_							:   Dict       ,   **UpperCAmelCase_							:   int							):
						SCREAMING_SNAKE_CASE__													= kwargs.get('is_split_into_words'       ,   UpperCAmelCase_							)

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'							)

						return super()._encode_plus(*UpperCAmelCase_       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   Optional[str] = None							):
						SCREAMING_SNAKE_CASE__													= self._tokenizer.model.save(UpperCAmelCase_       ,   name=UpperCAmelCase_							)
						return tuple(UpperCAmelCase_							)
	def      A_				(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Union[str, Any]=None							):
						SCREAMING_SNAKE_CASE__													= [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
											return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[int]       ,   UpperCAmelCase_							:   Optional[List[int]] = None							):
						SCREAMING_SNAKE_CASE__													= [self.sep_token_id]
						SCREAMING_SNAKE_CASE__													= [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





