




'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a__         =     logging.get_logger(__name__)

a__         =     {
    '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json''',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					Optional[Any]    =  """m2m_100"""
						UpperCAmelCase_		:					Any    =  ["""past_key_values"""]
						UpperCAmelCase_		:					Optional[Any]    =  {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}






						def __init__(   self						:						int					,   __UpperCamelCase						:						List[str]=1_2_8_1_1_2					,   __UpperCamelCase						:						List[str]=1_0_2_4					,   __UpperCamelCase						:						List[str]=1_2					,   __UpperCamelCase						:						int=4_0_9_6					,   __UpperCamelCase						:						Optional[Any]=1_6					,   __UpperCamelCase						:						Optional[Any]=1_2					,   __UpperCamelCase						:						int=4_0_9_6					,   __UpperCamelCase						:						Dict=1_6					,   __UpperCamelCase						:						List[Any]=0.05					,   __UpperCamelCase						:						List[str]=0.05					,   __UpperCamelCase						:						List[str]=True					,   __UpperCamelCase						:						int=True					,   __UpperCamelCase						:						List[Any]="relu"					,   __UpperCamelCase						:						Tuple=1_0_2_4					,   __UpperCamelCase						:						Dict=0.1					,   __UpperCamelCase						:						List[str]=0.1					,   __UpperCamelCase						:						List[Any]=0.0					,   __UpperCamelCase						:						str=0.02					,   __UpperCamelCase						:						Any=2					,   __UpperCamelCase						:						Any=True					,   __UpperCamelCase						:						Dict=1					,   __UpperCamelCase						:						Tuple=0					,   __UpperCamelCase						:						List[Any]=2					,   **__UpperCamelCase						:						Union[str, Any]					,   ):



									'''simple docstring'''






									snake_case__          =						vocab_size
									snake_case__          =						max_position_embeddings
									snake_case__          =						d_model
									snake_case__          =						encoder_ffn_dim
									snake_case__          =						encoder_layers
									snake_case__          =						encoder_attention_heads
									snake_case__          =						decoder_ffn_dim
									snake_case__          =						decoder_layers
									snake_case__          =						decoder_attention_heads
									snake_case__          =						dropout
									snake_case__          =						attention_dropout
									snake_case__          =						activation_dropout
									snake_case__          =						activation_function
									snake_case__          =						init_std
									snake_case__          =						encoder_layerdrop
									snake_case__          =						decoder_layerdrop
									snake_case__          =						use_cache
									snake_case__          =						encoder_layers
									snake_case__          =						scale_embedding  # scale factor will be sqrt(d_model) if True

									super().__init__(
									    pad_token_id=__UpperCamelCase					,   bos_token_id=__UpperCamelCase					,   eos_token_id=__UpperCamelCase					,   is_encoder_decoder=__UpperCamelCase					,   decoder_start_token_id=__UpperCamelCase					,   **__UpperCamelCase					,   )







class __magic_name__(						__lowerCAmelCase      ):






						@property
						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						OrderedDict(
									    [
									        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
									        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
									    ]       )

									if self.use_past:
												snake_case__          =						{0: """batch"""}
												snake_case__          =						{0: """batch""", 1: """past_decoder_sequence + sequence"""}
									else:
												snake_case__          =						{0: """batch""", 1: """decoder_sequence"""}
												snake_case__          =						{0: """batch""", 1: """decoder_sequence"""}

									if self.use_past:
												self.fill_with_past_key_values_(__UpperCamelCase					,   direction="""inputs"""       )
									return common_inputs






						def       __lowerCAmelCase(   self						:						int					,   __UpperCamelCase						:						PreTrainedTokenizer					,   __UpperCamelCase						:						int = -1					,   __UpperCamelCase						:						int = -1					,   __UpperCamelCase						:						bool = False					,   __UpperCamelCase						:						Optional[TensorType] = None					,   ):



									'''simple docstring'''






									snake_case__          =						compute_effective_axis_dimension(
									    __UpperCamelCase					,   fixed_dimension=OnnxConfig.default_fixed_batch					,   num_token_to_add=0       )

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									snake_case__          =						tokenizer.num_special_tokens_to_add(__UpperCamelCase       )
									snake_case__          =						compute_effective_axis_dimension(
									    __UpperCamelCase					,   fixed_dimension=OnnxConfig.default_fixed_sequence					,   num_token_to_add=__UpperCamelCase       )

									# Generate dummy inputs according to compute batch and sequence
									snake_case__          =						[""" """.join([tokenizer.unk_token]       ) * seq_length] * batch_size
									snake_case__          =						dict(tokenizer(__UpperCamelCase					,   return_tensors=__UpperCamelCase       )       )
									return common_inputs






						def       __lowerCAmelCase(   self						:						int					,   __UpperCamelCase						:						PreTrainedTokenizer					,   __UpperCamelCase						:						int = -1					,   __UpperCamelCase						:						int = -1					,   __UpperCamelCase						:						bool = False					,   __UpperCamelCase						:						Optional[TensorType] = None					,   ):



									'''simple docstring'''






									snake_case__          =						self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )

									# Generate decoder inputs
									snake_case__          =						seq_length if not self.use_past else 1
									snake_case__          =						self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )
									snake_case__          =						{f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
									snake_case__          =						dict(**__UpperCamelCase					,   **__UpperCamelCase       )

									if self.use_past:
												if not is_torch_available():
															raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""       )
												else:
															import torch
												snake_case__    ,			snake_case__          =						common_inputs["""input_ids"""].shape
												snake_case__          =						common_inputs["""decoder_input_ids"""].shape[1]
												snake_case__    ,			snake_case__          =						self.num_attention_heads
												snake_case__          =						(
												    batch,
												    num_encoder_attention_heads,
												    encoder_seq_length,
												    self._config.hidden_size // num_encoder_attention_heads,
												)
												snake_case__          =						decoder_seq_length + 3
												snake_case__          =						(
												    batch,
												    num_decoder_attention_heads,
												    decoder_past_length,
												    self._config.hidden_size // num_decoder_attention_heads,
												)

												snake_case__          =						torch.cat(
												    [common_inputs["""decoder_attention_mask"""], torch.ones(__UpperCamelCase					,   __UpperCamelCase       )]					,   dim=1       )

												snake_case__          =						[]
												# If the number of encoder and decoder layers are present in the model configuration, both are considered
												snake_case__    ,			snake_case__          =						self.num_layers
												snake_case__          =						min(__UpperCamelCase					,   __UpperCamelCase       )
												snake_case__          =						max(__UpperCamelCase					,   __UpperCamelCase       ) - min_num_layers
												snake_case__          =						"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

												for _ in range(__UpperCamelCase       ):
															common_inputs["past_key_values"].append(
															    (
															        torch.zeros(__UpperCamelCase       ),
															        torch.zeros(__UpperCamelCase       ),
															        torch.zeros(__UpperCamelCase       ),
															        torch.zeros(__UpperCamelCase       ),
															    )       )
												# TODO: test this.
												snake_case__          =						encoder_shape if remaining_side_name == """encoder""" else decoder_shape
												for _ in range(__UpperCamelCase					,   __UpperCamelCase       ):
															common_inputs["past_key_values"].append((torch.zeros(__UpperCamelCase       ), torch.zeros(__UpperCamelCase       ))       )
									return common_inputs







						UpperCAmelCase_		:					Optional[Any]    =  _generate_dummy_inputs_for_default_and_seqaseq_lm





import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class       _UpperCamelCase   (					tf.keras.layers.Layer		):
      def __init__(      self :int			,   lowerCamelCase :List			,   lowerCamelCase :bool			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :str = "longest"			,   lowerCamelCase :bool = True			,   lowerCamelCase :int = 512			,   lowerCamelCase :int = None			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   **lowerCamelCase :Optional[Any]			,   )  ->      List[str]:
       super().__init__()
       if use_fast_bert_tokenizer:
        UpperCAmelCase__							=    FastBertTokenizer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case_nfd_strip_accents=lowerCamelCase			,   **lowerCamelCase   )
       else:
        UpperCAmelCase__							=    tf.lookup.StaticVocabularyTable(
            tf.lookup.KeyValueTensorInitializer(
                keys=lowerCamelCase			,   key_dtype=tf.string			,   values=tf.range(tf.size(lowerCamelCase			,   out_type=tf.intaa   )			,   dtype=tf.intaa   )			,   value_dtype=tf.intaa			,   )			,   num_oov_buckets=1			,   )
        UpperCAmelCase__							=    BertTokenizerLayer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case=lowerCamelCase			,   **lowerCamelCase   )

       UpperCAmelCase__							=    vocab_list
       UpperCAmelCase__							=    do_lower_case
       UpperCAmelCase__							=    cls_token_id or vocab_list.index("[CLS]"   )
       UpperCAmelCase__							=    sep_token_id or vocab_list.index("[SEP]"   )
       UpperCAmelCase__							=    pad_token_id or vocab_list.index("[PAD]"   )
       UpperCAmelCase__							=    ShrinkLongestTrimmer(max_length - 3			,   axis=1   )  # Allow room for special tokens
       UpperCAmelCase__							=    max_length
       UpperCAmelCase__							=    padding
       UpperCAmelCase__							=    truncation
       UpperCAmelCase__							=    pad_to_multiple_of
       UpperCAmelCase__							=    return_token_type_ids
       UpperCAmelCase__							=    return_attention_mask
      @classmethod
      def 						UpperCAmelCase_    (      cls :List[str]			,   lowerCamelCase :"PreTrainedTokenizerBase"			,   **lowerCamelCase :Dict   )  ->      Dict:  # noqa: F821
       UpperCAmelCase__							=    kwargs.pop("do_lower_case"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
       UpperCAmelCase__							=    kwargs.pop("cls_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
       UpperCAmelCase__							=    kwargs.pop("sep_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
       UpperCAmelCase__							=    kwargs.pop("pad_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

       UpperCAmelCase__							=    tokenizer.get_vocab()
       UpperCAmelCase__							=    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,   key=lambda lowerCamelCase   : x[1]   )
       UpperCAmelCase__							=    [entry[0] for entry in vocab]
       return cls(
           vocab_list=lowerCamelCase			,   do_lower_case=lowerCamelCase			,   cls_token_id=lowerCamelCase			,   sep_token_id=lowerCamelCase			,   pad_token_id=lowerCamelCase			,   **lowerCamelCase			,   )
      @classmethod
      def 						UpperCAmelCase_    (      cls :Any			,   lowerCamelCase :Union[str, os.PathLike]			,   *lowerCamelCase :str			,   **lowerCamelCase :List[str]   )  ->      Tuple:
       try:
        UpperCAmelCase__							=    BertTokenizer.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       except:  # noqa: E722
        from .tokenization_bert_fast import BertTokenizerFast

        UpperCAmelCase__							=    BertTokenizerFast.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       return cls.from_tokenizer(lowerCamelCase			,   **lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :Optional[int]   )  ->      List[str]:
       if self.do_lower_case:
        UpperCAmelCase__							=    case_fold_utfa(lowerCamelCase   )
       UpperCAmelCase__							=    self.tf_tokenizer.tokenize(lowerCamelCase   )
       return tokens.merge_dims(1			,   -1   )
      def 						UpperCAmelCase_    (      self :Optional[Any]			,   lowerCamelCase :Tuple			,   lowerCamelCase :str=None			,   lowerCamelCase :Tuple=None			,   lowerCamelCase :Any=None			,   lowerCamelCase :Optional[int]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Tuple=None			,   )  ->      str:
       if padding is None:
        UpperCAmelCase__							=    self.padding
       if padding not in ("longest", "max_length"):
        raise ValueError("Padding must be either 'longest' or 'max_length'!"   )
       if max_length is not None and text_pair is not None:
        # Because we have to instantiate a Trimmer to do it properly
        raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"   )
       if max_length is None:
        UpperCAmelCase__							=    self.max_length
       if truncation is None:
        UpperCAmelCase__							=    self.truncation
       if pad_to_multiple_of is None:
        UpperCAmelCase__							=    self.pad_to_multiple_of
       if return_token_type_ids is None:
        UpperCAmelCase__							=    self.return_token_type_ids
       if return_attention_mask is None:
        UpperCAmelCase__							=    self.return_attention_mask
       if not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None and not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None:
        if text.shape.rank > 1:
         raise ValueError("text argument should not be multidimensional when a text pair is supplied!"   )
        if text_pair.shape.rank > 1:
         raise ValueError("text_pair should not be multidimensional!"   )
       if text.shape.rank == 2:
        UpperCAmelCase__			,			UpperCAmelCase__							=    text[:, 0], text[:, 1]
       UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
       if text_pair is None:  # Unpaired text
        if truncation:
         UpperCAmelCase__							=    text[:, : max_length - 2]  # Allow room for special tokens
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text,)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       else:  # Paired text
        UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
        if truncation:
         UpperCAmelCase__			,			UpperCAmelCase__							=    self.paired_trimmer.trim([text, text_pair]   )
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text, text_pair)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       if padding == "longest":
        UpperCAmelCase__							=    input_ids.bounding_shape(axis=1   )
        if pad_to_multiple_of is not None:
         # No ceiling division in tensorflow, so we negate floordiv instead
         UpperCAmelCase__							=    pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,   lowerCamelCase   ))
       else:
        UpperCAmelCase__							=    max_length

       UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
       UpperCAmelCase__							=    {"input_ids": input_ids}
       if return_attention_mask:
        UpperCAmelCase__							=    attention_mask
       if return_token_type_ids:
        UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(
            lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
        UpperCAmelCase__							=    token_type_ids
       return output







      def 						UpperCAmelCase_    (      self :Tuple   )  ->      Tuple:
       return {
           "vocab_list": self.vocab_list,
           "do_lower_case": self.do_lower_case,
           "cls_token_id": self.cls_token_id,
           "sep_token_id": self.sep_token_id,
           "pad_token_id": self.pad_token_id,
       }





