code: 
  
  
  '''simple docstring'''
  
  import inspect
  import os
  from argparse import ArgumentParser, Namespace
  from importlib import import_module
  
  import huggingface_hub
  import numpy as np
  from packaging import version
  
  from .. import (
      FEATURE_EXTRACTOR_MAPPING,
      IMAGE_PROCESSOR_MAPPING,
      PROCESSOR_MAPPING,
      TOKENIZER_MAPPING,
      AutoConfig,
      AutoFeatureExtractor,
      AutoImageProcessor,
      AutoProcessor,
      AutoTokenizer,
      is_datasets_available,
      is_tf_available,
      is_torch_available,
  )
  from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
  from . import BaseTransformersCLICommand
  
  
  if is_tf_available():
         import tensorflow as tf
  
         tf.config.experimental.enable_tensor_float_aa_execution(False)
  
  if is_torch_available():
         import torch
  
  if is_datasets_available():
         from datasets import load_dataset
  
  
  __lowercase:		int							=     5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors
  
  
  
  def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				Tuple   ):
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
  
  
  
     return PTtoTFCommand(
         args.model_name    ,		args.local_dir    ,		args.max_error    ,		args.new_weights    ,		args.no_pr    ,		args.push    ,		args.extra_commit_description    ,		args.override_model_class    ,		)
  
  
  
  
  
  
  
  class  UpperCAmelCase (       a_):
  
   @staticmethod
   def 		lowercase_						( a_ :	Tuple					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						parser.add_parser(
          "pt-to-tf",					help=(
              "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
              " Can also be used to validate existing weights without opening PRs, with --no-pr."
          ),					)
      train_parser.add_argument(
          "--model-name",					type=_snake_case,					required=_snake_case,					help="The model name, including owner/organization, as seen on the hub.",					)
      train_parser.add_argument(
          "--local-dir",					type=_snake_case,					default="",					help="Optional local directory of the model repository. Defaults to /tmp/{model_name}",					)
      train_parser.add_argument(
          "--max-error",					type=_snake_case,					default=_snake_case,					help=(
              f'Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'
          ),					)
      train_parser.add_argument(
          "--new-weights",					action="store_true",					help="Optional flag to create new TensorFlow weights, even if they already exist.",					)
      train_parser.add_argument(
          "--no-pr",					action="store_true",					help="Optional flag to NOT open a PR with converted weights."					)
      train_parser.add_argument(
          "--push",					action="store_true",					help="Optional flag to push the weights directly to `main` (requires permissions)",					)
      train_parser.add_argument(
          "--extra-commit-description",					type=_snake_case,					default="",					help="Optional additional commit description to use when opening a PR (e.g. to tag the owner).",					)
      train_parser.add_argument(
          "--override-model-class",					type=_snake_case,					default=_snake_case,					help="If you think you know better than the auto-detector, you can specify the model class here. "
          "Can be either an AutoModel class or a specific model class like BertForSequenceClassification.",					)
      train_parser.set_defaults(func=_snake_case					)
  
   @staticmethod
   def 		lowercase_						( a_ :	List[Any],					a_ :	Optional[int]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						set(pt_outputs.keys()					)
      UpperCamelCase__ 						=						set(tf_outputs.keys()					)
      if pt_out_attrs != tf_out_attrs:
         raise ValueError(
             f'The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'
             f' {tf_out_attrs})'					)
  
      # 2. For each output attribute, computes the difference
      def _find_pt_tf_differences(a_ :	Any,					a_ :	List[Any],					a_ :	int,					a_ :	Optional[int]=""					):
         # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
         # recursivelly, keeping the name of the attribute.
         if isinstance(_snake_case,					torch.Tensor					):
            UpperCamelCase__ 						=						np.max(np.abs(pt_out.numpy() - tf_out.numpy()					)					)
            UpperCamelCase__ 						=						tensor_difference
         else:
            UpperCamelCase__ 						=						attr_name
            for i, pt_item in enumerate(_snake_case					):
               # If it is a named attribute, we keep the name. Otherwise, just its index.
               if isinstance(_snake_case,					_snake_case					):
                  UpperCamelCase__ 						=						root_name + pt_item
                  UpperCamelCase__ 						=						tf_out[pt_item]
                  UpperCamelCase__ 						=						pt_out[pt_item]
               else:
                  UpperCamelCase__ 						=						root_name + f'[{i}]'
                  UpperCamelCase__ 						=						tf_out[i]
               UpperCamelCase__ 						=						_find_pt_tf_differences(_snake_case,					_snake_case,					_snake_case,					_snake_case					)
  
         return differences
  
      return _find_pt_tf_differences(_snake_case,					_snake_case,					{}					)
  
   def __init__( self :	List[str],					a_ :	Optional[int],					a_ :	str,					a_ :	str,					a_ :	List[Any],					a_ :	int,					a_ :	Optional[int],					a_ :	Tuple,					a_ :	Union[str, Any],					*a_ :	Any,					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						logging.get_logger("transformers-cli/pt_to_tf"					)
      UpperCamelCase__ 						=						model_name
      UpperCamelCase__ 						=						local_dir if local_dir else os.path.join("/tmp",					_snake_case					)
      UpperCamelCase__ 						=						max_error
      UpperCamelCase__ 						=						new_weights
      UpperCamelCase__ 						=						no_pr
      UpperCamelCase__ 						=						push
      UpperCamelCase__ 						=						extra_commit_description
      UpperCamelCase__ 						=						override_model_class
  
   def 		lowercase_						( self :	List[str],					a_ :	Dict,					a_ :	int,					a_ :	Dict					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      def _get_audio_input():
         UpperCamelCase__ 						=						load_dataset("hf-internal-testing/librispeech_asr_dummy",					"clean",					split="validation"					)
         UpperCamelCase__ 						=						ds.sort("id"					).select(range(2					)					)[:2]['''audio''']
         UpperCamelCase__ 						=						[x['''array'''] for x in speech_samples]
         return raw_samples
  
      UpperCamelCase__ 						=						type(pt_model.config					)
      if model_config_class in PROCESSOR_MAPPING:
         UpperCamelCase__ 						=						AutoProcessor.from_pretrained(self._local_dir					)
         if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
            UpperCamelCase__ 						=						processor.tokenizer.eos_token
      elif model_config_class in IMAGE_PROCESSOR_MAPPING:
         UpperCamelCase__ 						=						AutoImageProcessor.from_pretrained(self._local_dir					)
      elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
         UpperCamelCase__ 						=						AutoFeatureExtractor.from_pretrained(self._local_dir					)
      elif model_config_class in TOKENIZER_MAPPING:
         UpperCamelCase__ 						=						AutoTokenizer.from_pretrained(self._local_dir					)
         if processor.pad_token is None:
            UpperCamelCase__ 						=						processor.eos_token
      else:
         raise ValueError(f'Unknown data processing type (model config type: {model_config_class})'					)
  
      UpperCamelCase__ 						=						set(inspect.signature(pt_model.forward					).parameters.keys()					)
      UpperCamelCase__ 						=						{}
      if "input_ids" in model_forward_signature:
         processor_inputs.update(
             {
                 "text": ["Hi there!", "I am a batch with more than one row and different input lengths."],
                 "padding": True,
                 "truncation": True,
             }					)
      if "pixel_values" in model_forward_signature:
         UpperCamelCase__ 						=						load_dataset("cifar10",					"plain_text",					split="test"					)[:2]['''img''']
         processor_inputs.update({"images": sample_images}					)
      if "input_features" in model_forward_signature:
         UpperCamelCase__ 						=						inspect.signature(processor.feature_extractor					).parameters
         # Pad to the largest input length by default but take feature extractor default
         # padding value if it exists e.g. "max_length" and is not False or None
         if "padding" in feature_extractor_signature:
            UpperCamelCase__ 						=						feature_extractor_signature['''padding'''].default
            if default_strategy is not False and default_strategy is not None:
               UpperCamelCase__ 						=						default_strategy
            else:
               UpperCamelCase__ 						=						True
         else:
            UpperCamelCase__ 						=						True
         processor_inputs.update({"audio": _get_audio_input(), "padding": padding_strategy}					)
      if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
         processor_inputs.update({"audio": _get_audio_input(), "padding": True}					)
      UpperCamelCase__ 						=						processor(**_snake_case,					return_tensors="pt"					)
      UpperCamelCase__ 						=						processor(**_snake_case,					return_tensors="tf"					)
  
      # Extra input requirements, in addition to the input modality
      if (
          config.is_encoder_decoder
          or (hasattr(_snake_case,					"encoder"					) and hasattr(_snake_case,					"decoder"					))
          or "decoder_input_ids" in tf_dummy_inputs
      ):
         UpperCamelCase__ 						=						np.asarray([[1], [1]],					dtype=_snake_case					) * (pt_model.config.decoder_start_token_id or 0)
         pt_input.update({"decoder_input_ids": torch.tensor(_snake_case					)}					)
         tf_input.update({"decoder_input_ids": tf.convert_to_tensor(_snake_case					)}					)
  
      return pt_input, tf_input
  
   def 		lowercase_						( self :	Optional[Any]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      if version.parse(huggingface_hub.__version__					) < version.parse("0.9.0"					):
         raise ImportError(
             "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"
             " installation."					)
      else:
         from huggingface_hub import Repository, create_commit
         from huggingface_hub._commit_api import CommitOperationAdd
  
      # Fetch remote data
      UpperCamelCase__ 						=						Repository(local_dir=self._local_dir,					clone_from=self._model_name					)
  
      # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
      UpperCamelCase__ 						=						AutoConfig.from_pretrained(self._local_dir					)
      UpperCamelCase__ 						=						config.architectures
      if self._override_model_class is not None:
         if self._override_model_class.startswith("TF"					):
            UpperCamelCase__ 						=						[self._override_model_class[2:]]
         else:
            UpperCamelCase__ 						=						[self._override_model_class]
         try:
            UpperCamelCase__ 						=						getattr(import_module("transformers"					),					architectures[0]					)
         except AttributeError:
            raise ValueError(f'Model class {self._override_model_class} not found in transformers.'					)
         try:
            UpperCamelCase__ 						=						getattr(import_module("transformers"					),					"TF" + architectures[0]					)
         except AttributeError:
            raise ValueError(f'TF model class TF{self._override_model_class} not found in transformers.'					)
      elif architectures is None:  # No architecture defined -- use auto classes
         UpperCamelCase__ 						=						getattr(import_module("transformers"					),					"AutoModel"					)
         UpperCamelCase__ 						=						getattr(import_module("transformers"					),					"TFAutoModel"					)
         self._logger.warning("No detected architecture, using AutoModel/TFAutoModel"					)
      else:  # Architecture defined -- use it
         if len(_snake_case					) > 1:
            raise ValueError(f'More than one architecture was found, aborting. (architectures = {architectures})'					)
         self._logger.warning(f'Detected architecture: {architectures[0]}'					)
         UpperCamelCase__ 						=						getattr(import_module("transformers"					),					architectures[0]					)
         try:
            UpperCamelCase__ 						=						getattr(import_module("transformers"					),					"TF" + architectures[0]					)
         except AttributeError:
            raise AttributeError(f'The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'					)
  
          # Check the TF dummy inputs to see what keys we need in the forward pass
      UpperCamelCase__ 						=						tf_class.from_config(_snake_case					)
      UpperCamelCase__ 						=						tf_from_pt_model.dummy_inputs
  
      del tf_from_pt_model  # Try to keep only one model in memory at a time
  
      # Load the model and get some basic inputs
      UpperCamelCase__ 						=						pt_class.from_pretrained(self._local_dir					)
      pt_model.eval()
  
      UpperCamelCase__ 						=						self.get_inputs(_snake_case,					_snake_case,					_snake_case					)
  
      with torch.no_grad():
         UpperCamelCase__ 						=						pt_model(**_snake_case,					output_hidden_states=_snake_case					)
      del pt_model  # will no longer be used, and may have a large memory footprint
  
      UpperCamelCase__ 						=						tf_class.from_pretrained(self._local_dir,					from_pt=_snake_case					)
      UpperCamelCase__ 						=						tf_from_pt_model(**_snake_case,					output_hidden_states=_snake_case,					training=_snake_case					)
  
      # Confirms that cross loading PT weights into TF worked.
      UpperCamelCase__ 						=						self.find_pt_tf_differences(_snake_case,					_snake_case					)
      UpperCamelCase__ 						=						{k: v for k, v in crossload_differences.items() if '''hidden''' not in k}
      UpperCamelCase__ 						=						{k: v for k, v in crossload_differences.items() if '''hidden''' in k}
      if len(_snake_case					) == 0 and architectures is not None:
         raise ValueError(
             f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
             " output was found. All outputs start with \'hidden\'"					)
      UpperCamelCase__ 						=						max(output_differences.values()					) if output_differences else 0.0
      UpperCamelCase__ 						=						max(hidden_differences.values()					)
      if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
         raise ValueError(
             "The cross-loaded TensorFlow model has different outputs, something went wrong!\n"
             + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
             + "\n".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]					)
             + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
             + "\n".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]					)					)
  
      # Save the weights in a TF format (if needed) and confirms that the results are still good
      UpperCamelCase__ 						=						os.path.join(self._local_dir,					_snake_case					)
      UpperCamelCase__ 						=						os.path.join(self._local_dir,					_snake_case					)
      if (not os.path.exists(_snake_case					) and not os.path.exists(_snake_case					)) or self._new_weights:
         tf_from_pt_model.save_pretrained(self._local_dir					)
      del tf_from_pt_model  # will no longer be used, and may have a large memory footprint
  
      UpperCamelCase__ 						=						tf_class.from_pretrained(self._local_dir					)
      UpperCamelCase__ 						=						tf_model(**_snake_case,					output_hidden_states=_snake_case					)
  
      UpperCamelCase__ 						=						self.find_pt_tf_differences(_snake_case,					_snake_case					)
      UpperCamelCase__ 						=						{k: v for k, v in conversion_differences.items() if '''hidden''' not in k}
      UpperCamelCase__ 						=						{k: v for k, v in conversion_differences.items() if '''hidden''' in k}
      if len(_snake_case					) == 0 and architectures is not None:
         raise ValueError(
             f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
             " output was found. All outputs start with \'hidden\'"					)
      UpperCamelCase__ 						=						max(output_differences.values()					) if output_differences else 0.0
      UpperCamelCase__ 						=						max(hidden_differences.values()					)
      if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
         raise ValueError(
             "The converted TensorFlow model has different outputs, something went wrong!\n"
             + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
             + "\n".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]					)
             + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
             + "\n".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]					)					)
  
      UpperCamelCase__ 						=						'''Update TF weights''' if self._new_weights else '''Add TF weights'''
      if self._push:
         repo.git_add(auto_lfs_track=_snake_case					)
         repo.git_commit(_snake_case					)
         repo.git_push(blocking=_snake_case					)  # this prints a progress bar with the upload
         self._logger.warning(f'TF weights pushed into {self._model_name}'					)
      elif not self._no_pr:
         self._logger.warning("Uploading the weights into a new PR..."					)
         UpperCamelCase__ 						=						(
             '''Model converted by the [`transformers`\' `pt_to_tf`'''
             ''' CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). '''
             '''All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n'''
             f'Maximum crossload output difference={max_crossload_output_diff:.3e}; '
             f'Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'
             f'Maximum conversion output difference={max_conversion_output_diff:.3e}; '
             f'Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'
         )
         if self._max_error > MAX_ERROR:
            commit_descrition += (
                f'\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'
            )
         if self._extra_commit_description:
            commit_descrition += "\n\n" + self._extra_commit_description
  
         # sharded model -> adds all related files (index and .h5 shards)
         if os.path.exists(_snake_case					):
            UpperCamelCase__ 						=						[
                CommitOperationAdd(path_in_repo=_snake_case,					path_or_fileobj=_snake_case					)
            ]
            for shard_path in tf.io.gfile.glob(self._local_dir + "/tf_model-*.h5"					):
               operations += [
                   CommitOperationAdd(path_in_repo=os.path.basename(_snake_case					),					path_or_fileobj=_snake_case					)
               ]
         else:
            UpperCamelCase__ 						=						[CommitOperationAdd(path_in_repo=_snake_case,					path_or_fileobj=_snake_case					)]
  
         UpperCamelCase__ 						=						create_commit(
             repo_id=self._model_name,					operations=_snake_case,					commit_message=_snake_case,					commit_description=_snake_case,					repo_type="model",					create_pr=_snake_case,					).pr_url
         self._logger.warning(f'PR open in {hub_pr_url}'					)
style_context: 
  
  
  '''simple docstring'''
  
  from transformers import SqueezeBertTokenizer, SqueezeBertTokenizerFast
  from transformers.testing_utils import require_tokenizers, slow
  
  from ..bert.test_tokenization_bert import BertTokenizationTest
  
  
  
  
  
  
  
  @require_tokenizers
  class  UpperCAmelCase (       SCREAMING_SNAKE_CASE__):
   _lowerCamelCase :					Optional[int]    					=					SqueezeBertTokenizer
   _lowerCamelCase :					List[Any]    					=					SqueezeBertTokenizerFast
   _lowerCamelCase :					int    					=					True
  
   def 		lowercase_						( self :	List[str],					**a_ :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      return SqueezeBertTokenizerFast.from_pretrained(self.tmpdirname,					**a_					)
  
   @slow
   def 		lowercase_						( self :	int					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						SqueezeBertTokenizer.from_pretrained("squeezebert/squeezebert-mnli-headless"					)
  
      UpperCamelCase__ 						=						tokenizer.encode("sequence builders",					add_special_tokens=a_					)
      UpperCamelCase__ 						=						tokenizer.encode("multi-sequence build",					add_special_tokens=a_					)
  
      UpperCamelCase__ 						=						tokenizer.build_inputs_with_special_tokens(a_					)
      UpperCamelCase__ 						=						tokenizer.build_inputs_with_special_tokens(a_,					a_					)
  
      assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
      assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
          tokenizer.sep_token_id
      ]