



'''simple docstring'''






import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

A_				:					List[str]							=      logging.get_logger(__name__)

A_				:					str							=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

A_				:					List[str]							=      {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

A_				:					Optional[int]							=      {
    """microsoft/deberta-base""": 5_1_2,
    """microsoft/deberta-large""": 5_1_2,
    """microsoft/deberta-xlarge""": 5_1_2,
    """microsoft/deberta-base-mnli""": 5_1_2,
    """microsoft/deberta-large-mnli""": 5_1_2,
    """microsoft/deberta-xlarge-mnli""": 5_1_2,
}

A_				:					Tuple							=      {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}




def        snake_case_   (	)->						Optional[int]:





       '''simple docstring'''


       _UpperCAmelCase    :		List[str]     	=					(
           list(range(ord("""!"""						)			,  ord("""~"""						) + 1						)						) + list(range(ord("""¡"""						)			,  ord("""¬"""						) + 1						)						) + list(range(ord("""®"""						)			,  ord("""ÿ"""						) + 1						)						)
       )
       _UpperCAmelCase    :		str     	=					bs[:]
       _UpperCAmelCase    :		Optional[int]     	=					0
       for b in range(2**8						):
              if b not in bs:
                     bs.append(lowerCAmelCase_						)
                     cs.append(2**8 + n						)
                     n += 1
       _UpperCAmelCase    :		Optional[int]     	=					[chr(lowerCAmelCase_						) for n in cs]
       return dict(zip(lowerCAmelCase_			,  lowerCAmelCase_						)						)




def        snake_case_   (	lowerCAmelCase_						)->						Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase    :		str     	=					set()
       _UpperCAmelCase    :		Any     	=					word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)						)
              _UpperCAmelCase    :		List[Any]     	=					char
       return pairs



class 				lowercase			(    _lowerCamelCase							):


 """simple docstring"""


 UpperCAmelCase		      =    VOCAB_FILES_NAMES
 UpperCAmelCase		      =    PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase		      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase		      =    ["""input_ids""", """attention_mask""", """token_type_ids"""]







 def __init__(				self    ,a_    ,a_    ,a_="replace"    ,a_="[CLS]"    ,a_="[SEP]"    ,a_="[SEP]"    ,a_="[CLS]"    ,a_="[UNK]"    ,a_="[PAD]"    ,a_="[MASK]"    ,a_=False    ,a_=False    ,**a_    ,)						->  List[Any]:
        _UpperCAmelCase    :		Dict     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else bos_token
        _UpperCAmelCase    :		Optional[int]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else eos_token
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else sep_token
        _UpperCAmelCase    :		Tuple     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else cls_token
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else unk_token
        _UpperCAmelCase    :		Dict     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        _UpperCAmelCase    :		List[str]     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else mask_token

        super().__init__(
            errors=a_    ,bos_token=a_    ,eos_token=a_    ,unk_token=a_    ,sep_token=a_    ,cls_token=a_    ,pad_token=a_    ,mask_token=a_    ,add_prefix_space=a_    ,add_bos_token=a_    ,**a_    ,)
        _UpperCAmelCase    :		Any     	=					add_bos_token

        with open(a_    ,encoding="""utf-8"""				) as vocab_handle:
               _UpperCAmelCase    :		Optional[int]     	=					json.load(a_				)
        _UpperCAmelCase    :		Any     	=					{v: k for k, v in self.encoder.items()}
        _UpperCAmelCase    :		str     	=					errors  # how to handle errors in decoding
        _UpperCAmelCase    :		Any     	=					bytes_to_unicode()
        _UpperCAmelCase    :		List[Any]     	=					{v: k for k, v in self.byte_encoder.items()}
        with open(a_    ,encoding="""utf-8"""				) as merges_handle:
               _UpperCAmelCase    :		int     	=					merges_handle.read().split("""\n"""				)[1:-1]
        _UpperCAmelCase    :		str     	=					[tuple(merge.split()				) for merge in bpe_merges]
        _UpperCAmelCase    :		str     	=					dict(zip(a_    ,range(len(a_				)				)				)				)
        _UpperCAmelCase    :		str     	=					{}
        _UpperCAmelCase    :		List[str]     	=					add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        _UpperCAmelCase    :		List[Any]     	=					re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)







 @property
 # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
 def _snake_case					(				self				)						->  Tuple:
        return len(self.encoder				)







 def _snake_case					(				self				)						->  Optional[Any]:
        return dict(self.encoder    ,**self.added_tokens_encoder				)







 def _snake_case					(				self    ,a_				)						->  Tuple:
        if token in self.cache:
               return self.cache[token]
        _UpperCAmelCase    :		Any     	=					tuple(a_				)
        _UpperCAmelCase    :		Tuple     	=					get_pairs(a_				)

        if not pairs:
               return token

        while True:
               _UpperCAmelCase    :		List[Any]     	=					min(a_    ,key=lambda a_				: self.bpe_ranks.get(a_    ,float("""inf"""				)				)				)
               if bigram not in self.bpe_ranks:
                      break
               _UpperCAmelCase	,_UpperCAmelCase    :		List[str]     	=					bigram
               _UpperCAmelCase    :		str     	=					[]
               _UpperCAmelCase    :		int     	=					0
               while i < len(a_				):
                      try:
                             _UpperCAmelCase    :		Tuple     	=					word.index(a_    ,a_				)
                      except ValueError:
                             new_word.extend(word[i:]				)
                             break
                      else:
                             new_word.extend(word[i:j]				)
                             _UpperCAmelCase    :		Tuple     	=					j

                      if word[i] == first and i < len(a_				) - 1 and word[i + 1] == second:
                             new_word.append(first + second				)
                             i += 2
                      else:
                             new_word.append(word[i]				)
                             i += 1
               _UpperCAmelCase    :		Optional[int]     	=					tuple(a_				)
               _UpperCAmelCase    :		Any     	=					new_word
               if len(a_				) == 1:
                      break
               else:
                      _UpperCAmelCase    :		Any     	=					get_pairs(a_				)
        _UpperCAmelCase    :		List[str]     	=					""" """.join(a_				)
        _UpperCAmelCase    :		Dict     	=					word
        return word







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  List[int]:
        if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _UpperCAmelCase    :		Optional[int]     	=					[self.cls_token_id]
        _UpperCAmelCase    :		str     	=					[self.sep_token_id]
        return cls + token_ids_a + sep + token_ids_a + sep







 def _snake_case					(				self    ,a_    ,a_ = None    ,a_ = False				)						->  List[int]:
        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=a_    ,token_ids_a=a_    ,already_has_special_tokens=a_				)

        if token_ids_a is None:
               return [1] + ([0] * len(a_				)) + [1]
        return [1] + ([0] * len(a_				)) + [1] + ([0] * len(a_				)) + [1]







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  List[int]:
        _UpperCAmelCase    :		List[Any]     	=					[self.sep_token_id]
        _UpperCAmelCase    :		Tuple     	=					[self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







 def _snake_case					(				self    ,a_				)						->  List[str]:
        _UpperCAmelCase    :		Optional[Any]     	=					[]
        for token in re.findall(self.pat    ,a_				):
               _UpperCAmelCase    :		List[Any]     	=					"""""".join(
                   self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_				).split(""" """				)				)
        return bpe_tokens







 def _snake_case					(				self    ,a_				)						->  Union[str, Any]:
        return self.encoder.get(a_    ,self.encoder.get(self.unk_token				)				)







 def _snake_case					(				self    ,a_				)						->  Dict:
        return self.decoder.get(a_				)







 def _snake_case					(				self    ,a_				)						->  Optional[int]:
        _UpperCAmelCase    :		Optional[Any]     	=					"""""".join(a_				)
        _UpperCAmelCase    :		Union[str, Any]     	=					bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8"""    ,errors=self.errors				)
        return text







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  Tuple[str]:
        if not os.path.isdir(a_				):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
               return
        _UpperCAmelCase    :		Union[str, Any]     	=					os.path.join(
            a_    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
        _UpperCAmelCase    :		Tuple     	=					os.path.join(
            a_    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

        with open(a_    ,"""w"""    ,encoding="""utf-8"""				) as f:
               f.write(json.dumps(self.encoder    ,indent=2    ,sort_keys=a_    ,ensure_ascii=a_				) + """\n"""				)

        _UpperCAmelCase    :		List[str]     	=					0
        with open(a_    ,"""w"""    ,encoding="""utf-8"""				) as writer:
               writer.write("""#version: 0.2\n"""				)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,key=lambda a_				: kv[1]				):
                      if index != token_index:
                             logger.warning(
                                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 """ Please check that the tokenizer is not corrupted!"""				)
                             _UpperCAmelCase    :		List[Any]     	=					token_index
                      writer.write(""" """.join(a_				) + """\n"""				)
                      index += 1

        return vocab_file, merge_file







 def _snake_case					(				self    ,a_    ,a_=False    ,**a_				)						->  List[str]:
        _UpperCAmelCase    :		int     	=					kwargs.pop("""add_prefix_space"""    ,self.add_prefix_space				)
        if (is_split_into_words or add_prefix_space) and (len(a_				) > 0 and not text[0].isspace()):
               _UpperCAmelCase    :		Any     	=					""" """ + text
        return (text, kwargs)







 def _snake_case					(				self    ,a_				)						->  List[int]:
        _UpperCAmelCase    :		str     	=					[]
        for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(a_    ,add_special_tokens=a_				) + [self.eos_token_id]				)
        if len(a_				) > self.model_max_length:
               _UpperCAmelCase    :		List[Any]     	=					input_ids[-self.model_max_length :]
        return input_ids








'''simple docstring'''






import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
   import tensorflow as tf

   tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
   import torch

if is_datasets_available():
   from datasets import load_dataset


A_				:					Optional[Any]							=      5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors




def        snake_case_   (	lowerCAmelCase_						)->						Dict:





       '''simple docstring'''


       return PTtoTFCommand(
           args.model_name			,  args.local_dir			,  args.max_error			,  args.new_weights			,  args.no_pr			,  args.push			,  args.extra_commit_description			,  args.override_model_class			,  )



class 				lowercase			(    _lowerCamelCase							):


 """simple docstring"""


 @staticmethod
 def _snake_case					(				a_				)						->  List[str]:
        _UpperCAmelCase    :		Union[str, Any]     	=					parser.add_parser(
            """pt-to-tf"""    ,help=(
                """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
                """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
            )    ,)
        train_parser.add_argument(
            """--model-name"""    ,type=a_    ,required=a_    ,help="""The model name, including owner/organization, as seen on the hub."""    ,)
        train_parser.add_argument(
            """--local-dir"""    ,type=a_    ,default=""""""    ,help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""    ,)
        train_parser.add_argument(
            """--max-error"""    ,type=a_    ,default=a_    ,help=(
                f'''Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'''
            )    ,)
        train_parser.add_argument(
            """--new-weights"""    ,action="""store_true"""    ,help="""Optional flag to create new TensorFlow weights, even if they already exist."""    ,)
        train_parser.add_argument(
            """--no-pr"""    ,action="""store_true"""    ,help="""Optional flag to NOT open a PR with converted weights."""				)
        train_parser.add_argument(
            """--push"""    ,action="""store_true"""    ,help="""Optional flag to push the weights directly to `main` (requires permissions)"""    ,)
        train_parser.add_argument(
            """--extra-commit-description"""    ,type=a_    ,default=""""""    ,help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""    ,)
        train_parser.add_argument(
            """--override-model-class"""    ,type=a_    ,default=a_    ,help="""If you think you know better than the auto-detector, you can specify the model class here. """
            """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""    ,)
        train_parser.set_defaults(func=a_				)







 @staticmethod
 def _snake_case					(				a_    ,a_				)						->  Union[str, Any]:
        _UpperCAmelCase    :		Optional[int]     	=					set(pt_outputs.keys()				)
        _UpperCAmelCase    :		Any     	=					set(tf_outputs.keys()				)
        if pt_out_attrs != tf_out_attrs:
               raise ValueError(
                   f'''The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'''
                   f''' {tf_out_attrs})'''				)

        # 2. For each output attribute, computes the difference
        def _find_pt_tf_differences(a_    ,a_    ,a_    ,a_=""				):
               # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
               # recursivelly, keeping the name of the attribute.
               if isinstance(a_    ,torch.Tensor				):
                      _UpperCAmelCase    :		List[str]     	=					np.max(np.abs(pt_out.numpy() - tf_out.numpy()				)				)
                      _UpperCAmelCase    :		Dict     	=					tensor_difference
               else:
                      _UpperCAmelCase    :		int     	=					attr_name
                      for i, pt_item in enumerate(a_				):
                             # If it is a named attribute, we keep the name. Otherwise, just its index.
                             if isinstance(a_    ,a_				):
                                    _UpperCAmelCase    :		Any     	=					root_name + pt_item
                                    _UpperCAmelCase    :		Optional[int]     	=					tf_out[pt_item]
                                    _UpperCAmelCase    :		Optional[int]     	=					pt_out[pt_item]
                             else:
                                    _UpperCAmelCase    :		List[Any]     	=					root_name + f'''[{i}]'''
                                    _UpperCAmelCase    :		str     	=					tf_out[i]
                             _UpperCAmelCase    :		Tuple     	=					_find_pt_tf_differences(a_    ,a_    ,a_    ,a_				)

               return differences

        return _find_pt_tf_differences(a_    ,a_    ,{}				)







 def __init__(				self    ,a_    ,a_    ,a_    ,a_    ,a_    ,a_    ,a_    ,a_    ,*a_    ,)						->  Optional[int]:
        _UpperCAmelCase    :		Optional[int]     	=					logging.get_logger("""transformers-cli/pt_to_tf"""				)
        _UpperCAmelCase    :		Optional[Any]     	=					model_name
        _UpperCAmelCase    :		str     	=					local_dir if local_dir else os.path.join("""/tmp"""    ,a_				)
        _UpperCAmelCase    :		str     	=					max_error
        _UpperCAmelCase    :		str     	=					new_weights
        _UpperCAmelCase    :		List[Any]     	=					no_pr
        _UpperCAmelCase    :		List[Any]     	=					push
        _UpperCAmelCase    :		List[Any]     	=					extra_commit_description
        _UpperCAmelCase    :		str     	=					override_model_class







 def _snake_case					(				self    ,a_    ,a_    ,a_				)						->  Tuple:

        def _get_audio_input():
               _UpperCAmelCase    :		Union[str, Any]     	=					load_dataset("""hf-internal-testing/librispeech_asr_dummy"""    ,"""clean"""    ,split="""validation"""				)
               _UpperCAmelCase    :		str     	=					ds.sort("""id"""				).select(range(2				)				)[:2]["""audio"""]
               _UpperCAmelCase    :		List[str]     	=					[x["""array"""] for x in speech_samples]
               return raw_samples

        _UpperCAmelCase    :		List[str]     	=					type(pt_model.config				)
        if model_config_class in PROCESSOR_MAPPING:
               _UpperCAmelCase    :		Optional[Any]     	=					AutoProcessor.from_pretrained(self._local_dir				)
               if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
                      _UpperCAmelCase    :		Dict     	=					processor.tokenizer.eos_token
        elif model_config_class in IMAGE_PROCESSOR_MAPPING:
               _UpperCAmelCase    :		Optional[int]     	=					AutoImageProcessor.from_pretrained(self._local_dir				)
        elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
               _UpperCAmelCase    :		Any     	=					AutoFeatureExtractor.from_pretrained(self._local_dir				)
        elif model_config_class in TOKENIZER_MAPPING:
               _UpperCAmelCase    :		List[str]     	=					AutoTokenizer.from_pretrained(self._local_dir				)
               if processor.pad_token is None:
                      _UpperCAmelCase    :		Any     	=					processor.eos_token
        else:
               raise ValueError(f'''Unknown data processing type (model config type: {model_config_class})'''				)

        _UpperCAmelCase    :		Optional[Any]     	=					set(inspect.signature(pt_model.forward				).parameters.keys()				)
        _UpperCAmelCase    :		str     	=					{}
        if "input_ids" in model_forward_signature:
               processor_inputs.update(
                   {
                       """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
                       """padding""": True,
                       """truncation""": True,
                   }				)
        if "pixel_values" in model_forward_signature:
               _UpperCAmelCase    :		Optional[Any]     	=					load_dataset("""cifar10"""    ,"""plain_text"""    ,split="""test"""				)[:2]["""img"""]
               processor_inputs.update({"""images""": sample_images}				)
        if "input_features" in model_forward_signature:
               _UpperCAmelCase    :		Optional[int]     	=					inspect.signature(processor.feature_extractor				).parameters
               # Pad to the largest input length by default but take feature extractor default
               # padding value if it exists e.g. "max_length" and is not False or None
               if "padding" in feature_extractor_signature:
                      _UpperCAmelCase    :		Optional[Any]     	=					feature_extractor_signature["""padding"""].default
                      if default_strategy is not False and default_strategy is not None:
                             _UpperCAmelCase    :		Optional[Any]     	=					default_strategy
                      else:
                             _UpperCAmelCase    :		int     	=					True
               else:
                      _UpperCAmelCase    :		str     	=					True
               processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy}				)
        if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
               processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True}				)
        _UpperCAmelCase    :		Tuple     	=					processor(**a_    ,return_tensors="""pt"""				)
        _UpperCAmelCase    :		Union[str, Any]     	=					processor(**a_    ,return_tensors="""tf"""				)

        # Extra input requirements, in addition to the input modality
        if (
            config.is_encoder_decoder
            or (hasattr(a_    ,"""encoder"""				) and hasattr(a_    ,"""decoder"""				))
            or "decoder_input_ids" in tf_dummy_inputs
        ):
               _UpperCAmelCase    :		Dict     	=					np.asarray([[1], [1]]    ,dtype=a_				) * (pt_model.config.decoder_start_token_id or 0)
               pt_input.update({"""decoder_input_ids""": torch.tensor(a_				)}				)
               tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(a_				)}				)

        return pt_input, tf_input







 def _snake_case					(				self				)						->  Optional[Any]:
        # hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
        if version.parse(huggingface_hub.__version__				) < version.parse("""0.9.0"""				):
               raise ImportError(
                   """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
                   """ installation."""				)
        else:
               from huggingface_hub import Repository, create_commit
               from huggingface_hub._commit_api import CommitOperationAdd

        # Fetch remote data
        _UpperCAmelCase    :		Tuple     	=					Repository(local_dir=self._local_dir    ,clone_from=self._model_name				)

        # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
        _UpperCAmelCase    :		Optional[int]     	=					AutoConfig.from_pretrained(self._local_dir				)
        _UpperCAmelCase    :		Any     	=					config.architectures
        if self._override_model_class is not None:
               if self._override_model_class.startswith("""TF"""				):
                      _UpperCAmelCase    :		int     	=					[self._override_model_class[2:]]
               else:
                      _UpperCAmelCase    :		int     	=					[self._override_model_class]
               try:
                      _UpperCAmelCase    :		str     	=					getattr(import_module("""transformers"""				)    ,architectures[0]				)
               except AttributeError:
                      raise ValueError(f'''Model class {self._override_model_class} not found in transformers.'''				)
               try:
                      _UpperCAmelCase    :		Any     	=					getattr(import_module("""transformers"""				)    ,"""TF""" + architectures[0]				)
               except AttributeError:
                      raise ValueError(f'''TF model class TF{self._override_model_class} not found in transformers.'''				)
        elif architectures is None:  # No architecture defined -- use auto classes
               _UpperCAmelCase    :		Tuple     	=					getattr(import_module("""transformers"""				)    ,"""AutoModel"""				)
               _UpperCAmelCase    :		Any     	=					getattr(import_module("""transformers"""				)    ,"""TFAutoModel"""				)
               self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel"""				)
        else:  # Architecture defined -- use it
               if len(a_				) > 1:
                      raise ValueError(f'''More than one architecture was found, aborting. (architectures = {architectures})'''				)
               self._logger.warning(f'''Detected architecture: {architectures[0]}'''				)
               _UpperCAmelCase    :		int     	=					getattr(import_module("""transformers"""				)    ,architectures[0]				)
               try:
                      _UpperCAmelCase    :		List[Any]     	=					getattr(import_module("""transformers"""				)    ,"""TF""" + architectures[0]				)
               except AttributeError:
                      raise AttributeError(f'''The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'''				)

        # Check the TF dummy inputs to see what keys we need in the forward pass
        _UpperCAmelCase    :		Union[str, Any]     	=					tf_class.from_config(a_				)
        _UpperCAmelCase    :		Dict     	=					tf_from_pt_model.dummy_inputs

        del tf_from_pt_model  # Try to keep only one model in memory at a time

        # Load the model and get some basic inputs
        _UpperCAmelCase    :		Union[str, Any]     	=					pt_class.from_pretrained(self._local_dir				)
        pt_model.eval()

        _UpperCAmelCase	,_UpperCAmelCase    :		Dict     	=					self.get_inputs(a_    ,a_    ,a_				)

        with torch.no_grad():
               _UpperCAmelCase    :		List[str]     	=					pt_model(**a_    ,output_hidden_states=a_				)
        del pt_model  # will no longer be used, and may have a large memory footprint

        _UpperCAmelCase    :		Optional[int]     	=					tf_class.from_pretrained(self._local_dir    ,from_pt=a_				)
        _UpperCAmelCase    :		Dict     	=					tf_from_pt_model(**a_    ,output_hidden_states=a_    ,training=a_				)

        # Confirms that cross loading PT weights into TF worked.
        _UpperCAmelCase    :		int     	=					self.find_pt_tf_differences(a_    ,a_				)
        _UpperCAmelCase    :		Any     	=					{k: v for k, v in crossload_differences.items() if """hidden""" not in k}
        _UpperCAmelCase    :		Any     	=					{k: v for k, v in crossload_differences.items() if """hidden""" in k}
        if len(a_				) == 0 and architectures is not None:
               raise ValueError(
                   f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
                   """ output was found. All outputs start with 'hidden'"""				)
        _UpperCAmelCase    :		str     	=					max(output_differences.values()				) if output_differences else 0.0
        _UpperCAmelCase    :		Any     	=					max(hidden_differences.values()				)
        if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
               raise ValueError(
                   """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
                   + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
                   + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]				)
                   + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
                   + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]				)				)

        # Save the weights in a TF format (if needed) and confirms that the results are still good
        _UpperCAmelCase    :		Union[str, Any]     	=					os.path.join(self._local_dir    ,a_				)
        _UpperCAmelCase    :		List[str]     	=					os.path.join(self._local_dir    ,a_				)
        if (not os.path.exists(a_				) and not os.path.exists(a_				)) or self._new_weights:
               tf_from_pt_model.save_pretrained(self._local_dir				)
        del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

        _UpperCAmelCase    :		Union[str, Any]     	=					tf_class.from_pretrained(self._local_dir				)
        _UpperCAmelCase    :		Optional[Any]     	=					tf_model(**a_    ,output_hidden_states=a_				)

        _UpperCAmelCase    :		Dict     	=					self.find_pt_tf_differences(a_    ,a_				)
        _UpperCAmelCase    :		Optional[Any]     	=					{k: v for k, v in conversion_differences.items() if """hidden""" not in k}
        _UpperCAmelCase    :		Tuple     	=					{k: v for k, v in conversion_differences.items() if """hidden""" in k}
        if len(a_				) == 0 and architectures is not None:
               raise ValueError(
                   f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
                   """ output was found. All outputs start with 'hidden'"""				)
        _UpperCAmelCase    :		int     	=					max(output_differences.values()				) if output_differences else 0.0
        _UpperCAmelCase    :		List[str]     	=					max(hidden_differences.values()				)
        if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
               raise ValueError(
                   """The converted TensorFlow model has different outputs, something went wrong!\n"""
                   + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
                   + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]				)
                   + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
                   + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]				)				)

        _UpperCAmelCase    :		Dict     	=					"""Update TF weights""" if self._new_weights else """Add TF weights"""
        if self._push:
               repo.git_add(auto_lfs_track=a_				)
               repo.git_commit(a_				)
               repo.git_push(blocking=a_				)  # this prints a progress bar with the upload
               self._logger.warning(f'''TF weights pushed into {self._model_name}'''				)
        elif not self._no_pr:
               self._logger.warning("""Uploading the weights into a new PR..."""				)
               _UpperCAmelCase    :		List[Any]     	=					(
                   """Model converted by the [`transformers`' `pt_to_tf`"""
                   """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
                   """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
                   f'''Maximum crossload output difference={max_crossload_output_diff:.3e}; '''
                   f'''Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'''
                   f'''Maximum conversion output difference={max_conversion_output_diff:.3e}; '''
                   f'''Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'''
               )
               if self._max_error > MAX_ERROR:
                      commit_descrition += (
                          f'''\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'''
                      )
               if self._extra_commit_description:
                      commit_descrition += "\n\n" + self._extra_commit_description

               # sharded model -> adds all related files (index and .h5 shards)
               if os.path.exists(a_				):
                      _UpperCAmelCase    :		List[Any]     	=					[
                          CommitOperationAdd(path_in_repo=a_    ,path_or_fileobj=a_				)
                      ]
                      for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5"""				):
                             operations += [
                                 CommitOperationAdd(path_in_repo=os.path.basename(a_				)    ,path_or_fileobj=a_				)
                             ]
               else:
                      _UpperCAmelCase    :		Dict     	=					[CommitOperationAdd(path_in_repo=a_    ,path_or_fileobj=a_				)]

               _UpperCAmelCase    :		Dict     	=					create_commit(
                   repo_id=self._model_name    ,operations=a_    ,commit_message=a_    ,commit_description=a_    ,repo_type="""model"""    ,create_pr=a_    ,).pr_url
               self._logger.warning(f'''PR open in {hub_pr_url}'''				)



