code: import inspect
  import os
  from argparse import ArgumentParser, Namespace
  from importlib import import_module
  
  import huggingface_hub
  import numpy as np
  from packaging import version
  
  from .. import (
      FEATURE_EXTRACTOR_MAPPING,
      IMAGE_PROCESSOR_MAPPING,
      PROCESSOR_MAPPING,
      TOKENIZER_MAPPING,
      AutoConfig,
      AutoFeatureExtractor,
      AutoImageProcessor,
      AutoProcessor,
      AutoTokenizer,
      is_datasets_available,
      is_tf_available,
      is_torch_available,
  )
  from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
  from . import BaseTransformersCLICommand
  
  
  if is_tf_available():
  		import tensorflow as tf
  
  		tf.config.experimental.enable_tensor_float_aa_execution(False)
  
  if is_torch_available():
  		import torch
  
  if is_datasets_available():
  		from datasets import load_dataset
  
  
  lowerCamelCase__           =				5E-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   List[str]:
  		return PTtoTFCommand(
  		    args.model_name						,						args.local_dir						,						args.max_error						,						args.new_weights						,						args.no_pr						,						args.push						,						args.extra_commit_description						,						args.override_model_class						,						)
  
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  
  
  
  
  					@staticmethod
  					def 							UpperCamelCase_						( _lowerCAmelCase :					ArgumentParser						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     parser.add_parser(
  							    "pt-to-tf"      ,help=(
  							        "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
  							        " Can also be used to validate existing weights without opening PRs, with --no-pr."
  							    )      ,)
  							train_parser.add_argument(
  							    "--model-name"      ,type=_lowerCAmelCase      ,required=_lowerCAmelCase      ,help="The model name, including owner/organization, as seen on the hub."      ,)
  							train_parser.add_argument(
  							    "--local-dir"      ,type=_lowerCAmelCase      ,default=""      ,help="Optional local directory of the model repository. Defaults to /tmp/{model_name}"      ,)
  							train_parser.add_argument(
  							    "--max-error"      ,type=_lowerCAmelCase      ,default=_lowerCAmelCase      ,help=(
  							        F"""Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."""
  							    )      ,)
  							train_parser.add_argument(
  							    "--new-weights"      ,action="store_true"      ,help="Optional flag to create new TensorFlow weights, even if they already exist."      ,)
  							train_parser.add_argument(
  							    "--no-pr"      ,action="store_true"      ,help="Optional flag to NOT open a PR with converted weights."						)
  							train_parser.add_argument(
  							    "--push"      ,action="store_true"      ,help="Optional flag to push the weights directly to `main` (requires permissions)"      ,)
  							train_parser.add_argument(
  							    "--extra-commit-description"      ,type=_lowerCAmelCase      ,default=""      ,help="Optional additional commit description to use when opening a PR (e.g. to tag the owner)."      ,)
  							train_parser.add_argument(
  							    "--override-model-class"      ,type=_lowerCAmelCase      ,default=_lowerCAmelCase      ,help="If you think you know better than the auto-detector, you can specify the model class here. "
  							    "Can be either an AutoModel class or a specific model class like BertForSequenceClassification."      ,)
  							train_parser.set_defaults(func=_lowerCAmelCase						)
  
  
  
  
  					@staticmethod
  					def 							UpperCamelCase_						( _lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     set(pt_outputs.keys()						)
  							__snake_case		       =     set(tf_outputs.keys()						)
  							if pt_out_attrs != tf_out_attrs:
  									raise ValueError(
  									    F"""The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:"""
  									    F""" {tf_out_attrs})"""						)
  
  							# 2. For each output attribute, computes the difference
  							def _find_pt_tf_differences(_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					int=""						):
  									# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
  									# recursivelly, keeping the name of the attribute.
  									if isinstance(_lowerCAmelCase      ,torch.Tensor						):
  											__snake_case		       =     np.max(np.abs(pt_out.numpy() - tf_out.numpy()						)						)
  											__snake_case		       =     tensor_difference
  									else:
  											__snake_case		       =     attr_name
  											for i, pt_item in enumerate(_lowerCAmelCase						):
  													# If it is a named attribute, we keep the name. Otherwise, just its index.
  													if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						):
  															__snake_case		       =     root_name + pt_item
  															__snake_case		       =     tf_out[pt_item]
  															__snake_case		       =     pt_out[pt_item]
  													else:
  															__snake_case		       =     root_name + F"""[{i}]"""
  															__snake_case		       =     tf_out[i]
  													__snake_case		       =     _find_pt_tf_differences(_lowerCAmelCase      ,_lowerCAmelCase      ,_lowerCAmelCase      ,_lowerCAmelCase						)
  
  									return differences
  
  							return _find_pt_tf_differences(_lowerCAmelCase      ,_lowerCAmelCase      ,{}						)
  
  
  
  
  					def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					float      ,_lowerCAmelCase :					bool      ,_lowerCAmelCase :					bool      ,_lowerCAmelCase :					bool      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					str      ,*_lowerCAmelCase :					List[Any]      ,):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     logging.get_logger("transformers-cli/pt_to_tf"						)
  							__snake_case		       =     model_name
  							__snake_case		       =     local_dir if local_dir else os.path.join("/tmp"      ,_lowerCAmelCase						)
  							__snake_case		       =     max_error
  							__snake_case		       =     new_weights
  							__snake_case		       =     no_pr
  							__snake_case		       =     push
  							__snake_case		       =     extra_commit_description
  							__snake_case		       =     override_model_class
  
  
  
  
  					def 							UpperCamelCase_						( self :					List[str]      ,_lowerCAmelCase :					int      ,_lowerCAmelCase :					Optional[Any]      ,_lowerCAmelCase :					Tuple						):
  
  
  							"""simple docstring"""
  
  
  
  
  							def _get_audio_input():
  									__snake_case		       =     load_dataset("hf-internal-testing/librispeech_asr_dummy"      ,"clean"      ,split="validation"						)
  									__snake_case		       =     ds.sort("id"						).select(range(2						)						)[:2]["audio"]
  									__snake_case		       =     [x["array"] for x in speech_samples]
  									return raw_samples
  
  							__snake_case		       =     type(pt_model.config						)
  							if model_config_class in PROCESSOR_MAPPING:
  									__snake_case		       =     AutoProcessor.from_pretrained(self._local_dir						)
  									if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
  											__snake_case		       =     processor.tokenizer.eos_token
  							elif model_config_class in IMAGE_PROCESSOR_MAPPING:
  									__snake_case		       =     AutoImageProcessor.from_pretrained(self._local_dir						)
  							elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
  									__snake_case		       =     AutoFeatureExtractor.from_pretrained(self._local_dir						)
  							elif model_config_class in TOKENIZER_MAPPING:
  									__snake_case		       =     AutoTokenizer.from_pretrained(self._local_dir						)
  									if processor.pad_token is None:
  											__snake_case		       =     processor.eos_token
  							else:
  									raise ValueError(F"""Unknown data processing type (model config type: {model_config_class})"""						)
  
  							__snake_case		       =     set(inspect.signature(pt_model.forward						).parameters.keys()						)
  							__snake_case		       =     {}
  							if "input_ids" in model_forward_signature:
  									processor_inputs.update(
  									    {
  									        "text": ["Hi there!", "I am a batch with more than one row and different input lengths."],
  									        "padding": True,
  									        "truncation": True,
  									    }						)
  							if "pixel_values" in model_forward_signature:
  									__snake_case		       =     load_dataset("cifar10"      ,"plain_text"      ,split="test"						)[:2]["img"]
  									processor_inputs.update({"images": sample_images}						)
  							if "input_features" in model_forward_signature:
  									__snake_case		       =     inspect.signature(processor.feature_extractor						).parameters
  									# Pad to the largest input length by default but take feature extractor default
  									# padding value if it exists e.g. "max_length" and is not False or None
  									if "padding" in feature_extractor_signature:
  											__snake_case		       =     feature_extractor_signature["padding"].default
  											if default_strategy is not False and default_strategy is not None:
  													__snake_case		       =     default_strategy
  											else:
  													__snake_case		       =     True
  									else:
  											__snake_case		       =     True
  									processor_inputs.update({"audio": _get_audio_input(), "padding": padding_strategy}						)
  							if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
  									processor_inputs.update({"audio": _get_audio_input(), "padding": True}						)
  							__snake_case		       =     processor(**_lowerCAmelCase      ,return_tensors="pt"						)
  							__snake_case		       =     processor(**_lowerCAmelCase      ,return_tensors="tf"						)
  
  							# Extra input requirements, in addition to the input modality
  							if (
  							    config.is_encoder_decoder
  							    or (hasattr(_lowerCAmelCase      ,"encoder"						) and hasattr(_lowerCAmelCase      ,"decoder"						))
  							    or "decoder_input_ids" in tf_dummy_inputs
  							):
  									__snake_case		       =     np.asarray([[1], [1]]      ,dtype=_lowerCAmelCase						) * (pt_model.config.decoder_start_token_id or 0)
  									pt_input.update({"decoder_input_ids": torch.tensor(_lowerCAmelCase						)}						)
  									tf_input.update({"decoder_input_ids": tf.convert_to_tensor(_lowerCAmelCase						)}						)
  
  							return pt_input, tf_input
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					Tuple						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if version.parse(huggingface_hub.__version__						) < version.parse("0.9.0"						):
  									raise ImportError(
  									    "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"
  									    " installation."						)
  							else:
  									from huggingface_hub import Repository, create_commit
  									from huggingface_hub._commit_api import CommitOperationAdd
  
  							# Fetch remote data
  							__snake_case		       =     Repository(local_dir=self._local_dir      ,clone_from=self._model_name						)
  
  							# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
  							__snake_case		       =     AutoConfig.from_pretrained(self._local_dir						)
  							__snake_case		       =     config.architectures
  							if self._override_model_class is not None:
  									if self._override_model_class.startswith("TF"						):
  											__snake_case		       =     [self._override_model_class[2:]]
  									else:
  											__snake_case		       =     [self._override_model_class]
  									try:
  											__snake_case		       =     getattr(import_module("transformers"						)      ,architectures[0]						)
  									except AttributeError:
  											raise ValueError(F"""Model class {self._override_model_class} not found in transformers."""						)
  									try:
  											__snake_case		       =     getattr(import_module("transformers"						)      ,"TF" + architectures[0]						)
  									except AttributeError:
  											raise ValueError(F"""TF model class TF{self._override_model_class} not found in transformers."""						)
  							elif architectures is None:  # No architecture defined -- use auto classes
  									__snake_case		       =     getattr(import_module("transformers"						)      ,"AutoModel"						)
  									__snake_case		       =     getattr(import_module("transformers"						)      ,"TFAutoModel"						)
  									self._logger.warning("No detected architecture, using AutoModel/TFAutoModel"						)
  							else:  # Architecture defined -- use it
  									if len(_lowerCAmelCase						) > 1:
  											raise ValueError(F"""More than one architecture was found, aborting. (architectures = {architectures})"""						)
  									self._logger.warning(F"""Detected architecture: {architectures[0]}"""						)
  									__snake_case		       =     getattr(import_module("transformers"						)      ,architectures[0]						)
  									try:
  											__snake_case		       =     getattr(import_module("transformers"						)      ,"TF" + architectures[0]						)
  									except AttributeError:
  											raise AttributeError(F"""The TensorFlow equivalent of {architectures[0]} doesn't exist in transformers."""						)
  
          # Check the TF dummy inputs to see what keys we need in the forward pass
  							__snake_case		       =     tf_class.from_config(_lowerCAmelCase						)
  							__snake_case		       =     tf_from_pt_model.dummy_inputs
  
  							del tf_from_pt_model  # Try to keep only one model in memory at a time
  
  							# Load the model and get some basic inputs
  							__snake_case		       =     pt_class.from_pretrained(self._local_dir						)
  							pt_model.eval()
  
  							__snake_case       ,       __snake_case		       =     self.get_inputs(_lowerCAmelCase      ,_lowerCAmelCase      ,_lowerCAmelCase						)
  
  							with torch.no_grad():
  									__snake_case		       =     pt_model(**_lowerCAmelCase      ,output_hidden_states=_lowerCAmelCase						)
  							del pt_model  # will no longer be used, and may have a large memory footprint
  
  							__snake_case		       =     tf_class.from_pretrained(self._local_dir      ,from_pt=_lowerCAmelCase						)
  							__snake_case		       =     tf_from_pt_model(**_lowerCAmelCase      ,output_hidden_states=_lowerCAmelCase      ,training=_lowerCAmelCase						)
  
  							# Confirms that cross loading PT weights into TF worked.
  							__snake_case		       =     self.find_pt_tf_differences(_lowerCAmelCase      ,_lowerCAmelCase						)
  							__snake_case		       =     {k: v for k, v in crossload_differences.items() if "hidden" not in k}
  							__snake_case		       =     {k: v for k, v in crossload_differences.items() if "hidden" in k}
  							if len(_lowerCAmelCase						) == 0 and architectures is not None:
  									raise ValueError(
  									    F"""Something went wrong -- the config file has architectures ({architectures}), but no model head"""
  									    " output was found. All outputs start with 'hidden'"						)
  							__snake_case		       =     max(output_differences.values()						) if output_differences else 0.0
  							__snake_case		       =     max(hidden_differences.values()						)
  							if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
  									raise ValueError(
  									    "The cross-loaded TensorFlow model has different outputs, something went wrong!\n"
  									    + F"""\nList of maximum output differences above the threshold ({self._max_error}):\n"""
  									    + "\n".join([F"""{k}: {v:.3e}""" for k, v in output_differences.items() if v > self._max_error]						)
  									    + F"""\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"""
  									    + "\n".join([F"""{k}: {v:.3e}""" for k, v in hidden_differences.items() if v > self._max_error]						)						)
  
  							# Save the weights in a TF format (if needed) and confirms that the results are still good
  							__snake_case		       =     os.path.join(self._local_dir      ,_lowerCAmelCase						)
  							__snake_case		       =     os.path.join(self._local_dir      ,_lowerCAmelCase						)
  							if (not os.path.exists(_lowerCAmelCase						) and not os.path.exists(_lowerCAmelCase						)) or self._new_weights:
  									tf_from_pt_model.save_pretrained(self._local_dir						)
  							del tf_from_pt_model  # will no longer be used, and may have a large memory footprint
  
  							__snake_case		       =     tf_class.from_pretrained(self._local_dir						)
  							__snake_case		       =     tf_model(**_lowerCAmelCase      ,output_hidden_states=_lowerCAmelCase						)
  
  							__snake_case		       =     self.find_pt_tf_differences(_lowerCAmelCase      ,_lowerCAmelCase						)
  							__snake_case		       =     {k: v for k, v in conversion_differences.items() if "hidden" not in k}
  							__snake_case		       =     {k: v for k, v in conversion_differences.items() if "hidden" in k}
  							if len(_lowerCAmelCase						) == 0 and architectures is not None:
  									raise ValueError(
  									    F"""Something went wrong -- the config file has architectures ({architectures}), but no model head"""
  									    " output was found. All outputs start with 'hidden'"						)
  							__snake_case		       =     max(output_differences.values()						) if output_differences else 0.0
  							__snake_case		       =     max(hidden_differences.values()						)
  							if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
  									raise ValueError(
  									    "The converted TensorFlow model has different outputs, something went wrong!\n"
  									    + F"""\nList of maximum output differences above the threshold ({self._max_error}):\n"""
  									    + "\n".join([F"""{k}: {v:.3e}""" for k, v in output_differences.items() if v > self._max_error]						)
  									    + F"""\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"""
  									    + "\n".join([F"""{k}: {v:.3e}""" for k, v in hidden_differences.items() if v > self._max_error]						)						)
  
  							__snake_case		       =     "Update TF weights" if self._new_weights else "Add TF weights"
  							if self._push:
  									repo.git_add(auto_lfs_track=_lowerCAmelCase						)
  									repo.git_commit(_lowerCAmelCase						)
  									repo.git_push(blocking=_lowerCAmelCase						)  # this prints a progress bar with the upload
  									self._logger.warning(F"""TF weights pushed into {self._model_name}"""						)
  							elif not self._no_pr:
  									self._logger.warning("Uploading the weights into a new PR..."						)
  									__snake_case		       =     (
  									    "Model converted by the [`transformers`' `pt_to_tf`"
  									    " CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). "
  									    "All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"
  									    F"""Maximum crossload output difference={max_crossload_output_diff:.3e}; """
  									    F"""Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n"""
  									    F"""Maximum conversion output difference={max_conversion_output_diff:.3e}; """
  									    F"""Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n"""
  									)
  									if self._max_error > MAX_ERROR:
  											commit_descrition += (
  											    F"""\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!"""
  											)
  									if self._extra_commit_description:
  											commit_descrition += "\n\n" + self._extra_commit_description
  
  									# sharded model -> adds all related files (index and .h5 shards)
  									if os.path.exists(_lowerCAmelCase						):
  											__snake_case		       =     [
  											    CommitOperationAdd(path_in_repo=_lowerCAmelCase      ,path_or_fileobj=_lowerCAmelCase						)
  											]
  											for shard_path in tf.io.gfile.glob(self._local_dir + "/tf_model-*.h5"						):
  													operations += [
  													    CommitOperationAdd(path_in_repo=os.path.basename(_lowerCAmelCase						)      ,path_or_fileobj=_lowerCAmelCase						)
  													]
  									else:
  											__snake_case		       =     [CommitOperationAdd(path_in_repo=_lowerCAmelCase      ,path_or_fileobj=_lowerCAmelCase						)]
  
  									__snake_case		       =     create_commit(
  									    repo_id=self._model_name      ,operations=_lowerCAmelCase      ,commit_message=_lowerCAmelCase      ,commit_description=_lowerCAmelCase      ,repo_type="model"      ,create_pr=_lowerCAmelCase      ,).pr_url
  									self._logger.warning(F"""PR open in {hub_pr_url}"""						)
  
  
  
  
  
  
style_context: import json
  import os
  from functools import lru_cache
  from typing import List, Optional, Tuple
  
  import regex as re
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  lowerCamelCase__           =				logging.get_logger(__name__)
  
  lowerCamelCase__           =				{
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  lowerCamelCase__           =				{
      'vocab_file': {
          'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
          'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
          'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
          'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
          'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
          'roberta-large-openai-detector': (
              'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
          ),
      },
      'merges_file': {
          'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
          'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
          'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
          'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
          'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
          'roberta-large-openai-detector': (
              'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
          ),
      },
  }
  
  lowerCamelCase__           =				{
      'roberta-base': 5_12,
      'roberta-large': 5_12,
      'roberta-large-mnli': 5_12,
      'distilroberta-base': 5_12,
      'roberta-base-openai-detector': 5_12,
      'roberta-large-openai-detector': 5_12,
  }
  
  
  
  @lru_cache()
  def 							_lowerCamelCase(				)  ->   List[str]:
  		__snake_case		       =     (
  		    list(range(ord("!"	)						,						ord("~"	) + 1	)	) + list(range(ord("¡"	)						,						ord("¬"	) + 1	)	) + list(range(ord("®"	)						,						ord("ÿ"	) + 1	)	)
  		)
  		__snake_case		       =     bs[:]
  		__snake_case		       =     0
  		for b in range(2**8	):
  				if b not in bs:
  						bs.append(__snake_case	)
  						cs.append(2**8 + n	)
  						n += 1
  		__snake_case		       =     [chr(__snake_case	) for n in cs]
  		return dict(zip(__snake_case						,						__snake_case	)	)
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   Optional[Any]:
  		__snake_case		       =     set()
  		__snake_case		       =     word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)	)
  				__snake_case		       =     char
  		return pairs
  
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  					__UpperCamelCase						=   VOCAB_FILES_NAMES
  					__UpperCamelCase						=   PRETRAINED_VOCAB_FILES_MAP
  					__UpperCamelCase						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					__UpperCamelCase						=   ["""input_ids""", """attention_mask"""]
  
  
  
  
  					def __init__( self :					str      ,_lowerCAmelCase :					Tuple      ,_lowerCAmelCase :					int      ,_lowerCAmelCase :					Union[str, Any]="replace"      ,_lowerCAmelCase :					List[Any]="<s>"      ,_lowerCAmelCase :					Union[str, Any]="</s>"      ,_lowerCAmelCase :					Any="</s>"      ,_lowerCAmelCase :					Optional[int]="<s>"      ,_lowerCAmelCase :					int="<unk>"      ,_lowerCAmelCase :					Tuple="<pad>"      ,_lowerCAmelCase :					Optional[Any]="<mask>"      ,_lowerCAmelCase :					List[str]=False      ,**_lowerCAmelCase :					Union[str, Any]      ,):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else bos_token
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else eos_token
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else sep_token
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else cls_token
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else unk_token
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else pad_token
  
  							# Mask token behave like a normal word, i.e. include the space before it
  							__snake_case		       =     AddedToken(_lowerCAmelCase      ,lstrip=_lowerCAmelCase      ,rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase      ,_lowerCAmelCase						) else mask_token
  
  							super().__init__(
  							    errors=_lowerCAmelCase      ,bos_token=_lowerCAmelCase      ,eos_token=_lowerCAmelCase      ,unk_token=_lowerCAmelCase      ,sep_token=_lowerCAmelCase      ,cls_token=_lowerCAmelCase      ,pad_token=_lowerCAmelCase      ,mask_token=_lowerCAmelCase      ,add_prefix_space=_lowerCAmelCase      ,**_lowerCAmelCase      ,)
  
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as vocab_handle:
  									__snake_case		       =     json.load(_lowerCAmelCase						)
  							__snake_case		       =     {v: k for k, v in self.encoder.items()}
  							__snake_case		       =     errors  # how to handle errors in decoding
  							__snake_case		       =     bytes_to_unicode()
  							__snake_case		       =     {v: k for k, v in self.byte_encoder.items()}
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as merges_handle:
  									__snake_case		       =     merges_handle.read().split("\n"						)[1:-1]
  							__snake_case		       =     [tuple(merge.split()						) for merge in bpe_merges]
  							__snake_case		       =     dict(zip(_lowerCAmelCase      ,range(len(_lowerCAmelCase						)						)						)						)
  							__snake_case		       =     {}
  							__snake_case		       =     add_prefix_space
  
  							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  							__snake_case		       =     re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"						)
  
  
  
  
  					@property
  					def 							UpperCamelCase_						( self :					int						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return len(self.encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return dict(self.encoder      ,**self.added_tokens_encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Tuple						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if token in self.cache:
  									return self.cache[token]
  							__snake_case		       =     tuple(_lowerCAmelCase						)
  							__snake_case		       =     get_pairs(_lowerCAmelCase						)
  
  							if not pairs:
  									return token
  
  							while True:
  									__snake_case		       =     min(_lowerCAmelCase      ,key=lambda _lowerCAmelCase						: self.bpe_ranks.get(_lowerCAmelCase      ,float("inf"						)						)						)
  									if bigram not in self.bpe_ranks:
  											break
  									__snake_case       ,       __snake_case		       =     bigram
  									__snake_case		       =     []
  									__snake_case		       =     0
  									while i < len(_lowerCAmelCase						):
  											try:
  													__snake_case		       =     word.index(_lowerCAmelCase      ,_lowerCAmelCase						)
  											except ValueError:
  													new_word.extend(word[i:]						)
  													break
  											else:
  													new_word.extend(word[i:j]						)
  													__snake_case		       =     j
  
  											if word[i] == first and i < len(_lowerCAmelCase						) - 1 and word[i + 1] == second:
  													new_word.append(first + second						)
  													i += 2
  											else:
  													new_word.append(word[i]						)
  													i += 1
  									__snake_case		       =     tuple(_lowerCAmelCase						)
  									__snake_case		       =     new_word
  									if len(_lowerCAmelCase						) == 1:
  											break
  									else:
  											__snake_case		       =     get_pairs(_lowerCAmelCase						)
  							__snake_case		       =     " ".join(_lowerCAmelCase						)
  							__snake_case		       =     word
  							return word
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Dict						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							for token in re.findall(self.pat      ,_lowerCAmelCase						):
  									__snake_case		       =     "".join(
  									    self.byte_encoder[b] for b in token.encode("utf-8"						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
  									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCAmelCase						).split(" "						)						)
  							return bpe_tokens
  
  
  
  
  					def 							UpperCamelCase_						( self :					List[str]      ,_lowerCAmelCase :					Dict						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.encoder.get(_lowerCAmelCase      ,self.encoder.get(self.unk_token						)						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Optional[Any]      ,_lowerCAmelCase :					Union[str, Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.decoder.get(_lowerCAmelCase						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     "".join(_lowerCAmelCase						)
  							__snake_case		       =     bytearray([self.byte_decoder[c] for c in text]						).decode("utf-8"      ,errors=self.errors						)
  							return text
  
  
  
  
  					def 							UpperCamelCase_						( self :					str      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					Optional[str] = None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if not os.path.isdir(_lowerCAmelCase						):
  									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
  									return
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)
  
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as f:
  									f.write(json.dumps(self.encoder      ,indent=2      ,sort_keys=_lowerCAmelCase      ,ensure_ascii=_lowerCAmelCase						) + "\n"						)
  
  							__snake_case		       =     0
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as writer:
  									writer.write("#version: 0.2\n"						)
  									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,key=lambda _lowerCAmelCase						: kv[1]						):
  											if index != token_index:
  													logger.warning(
  													    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  													    " Please check that the tokenizer is not corrupted!"						)
  													__snake_case		       =     token_index
  											writer.write(" ".join(_lowerCAmelCase						) + "\n"						)
  											index += 1
  
  							return vocab_file, merge_file
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					List[int]      ,_lowerCAmelCase :					Optional[List[int]] = None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if token_ids_a is None:
  									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  							__snake_case		       =     [self.cls_token_id]
  							__snake_case		       =     [self.sep_token_id]
  							return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					List[int]      ,_lowerCAmelCase :					Optional[List[int]] = None      ,_lowerCAmelCase :					bool = False						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if already_has_special_tokens:
  									return super().get_special_tokens_mask(
  									    token_ids_a=_lowerCAmelCase      ,token_ids_a=_lowerCAmelCase      ,already_has_special_tokens=_lowerCAmelCase						)
  
  							if token_ids_a is None:
  									return [1] + ([0] * len(_lowerCAmelCase						)) + [1]
  							return [1] + ([0] * len(_lowerCAmelCase						)) + [1, 1] + ([0] * len(_lowerCAmelCase						)) + [1]
  
  
  
  
  					def 							UpperCamelCase_						( self :					Tuple      ,_lowerCAmelCase :					List[int]      ,_lowerCAmelCase :					Optional[List[int]] = None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     [self.sep_token_id]
  							__snake_case		       =     [self.cls_token_id]
  
  							if token_ids_a is None:
  									return len(cls + token_ids_a + sep						) * [0]
  							return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					List[Any]      ,_lowerCAmelCase :					Optional[Any]=False      ,**_lowerCAmelCase :					List[str]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     kwargs.pop("add_prefix_space"      ,self.add_prefix_space						)
  							if (is_split_into_words or add_prefix_space) and (len(_lowerCAmelCase						) > 0 and not text[0].isspace()):
  									__snake_case		       =     " " + text
  							return (text, kwargs)
  
  
  
  
  
  