
import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
     import tensorflow as tf

     tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
     import torch

if is_datasets_available():
     from datasets import load_dataset


__magic_name__  :  str     			=   5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors





def        lowerCAmelCase       (				snake_case__   :      Namespace )->			str:
    return PTtoTFCommand(
        args.model_name				,					args.local_dir				,					args.max_error				,					args.new_weights				,					args.no_pr				,					args.push				,					args.extra_commit_description				,					args.override_model_class				,					)



class 							lowerCamelCase       (		__snake_case					):



  """simple docstring"""





  @staticmethod
  def 					lowercase_						(					__UpperCamelCase					):
      A_								=					parser.add_parser(
          "pt-to-tf"						,    help=(
              "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
              " Can also be used to validate existing weights without opening PRs, with --no-pr."
          )						,    )
      train_parser.add_argument(
          "--model-name"						,    type=__UpperCamelCase						,    required=__UpperCamelCase						,    help="The model name, including owner/organization, as seen on the hub."						,    )
      train_parser.add_argument(
          "--local-dir"						,    type=__UpperCamelCase						,    default=""						,    help="Optional local directory of the model repository. Defaults to /tmp/{model_name}"						,    )
      train_parser.add_argument(
          "--max-error"						,    type=__UpperCamelCase						,    default=__UpperCamelCase						,    help=(
              f'Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'
          )						,    )
      train_parser.add_argument(
          "--new-weights"						,    action="store_true"						,    help="Optional flag to create new TensorFlow weights, even if they already exist."						,    )
      train_parser.add_argument(
          "--no-pr"						,    action="store_true"						,    help="Optional flag to NOT open a PR with converted weights."					)
      train_parser.add_argument(
          "--push"						,    action="store_true"						,    help="Optional flag to push the weights directly to `main` (requires permissions)"						,    )
      train_parser.add_argument(
          "--extra-commit-description"						,    type=__UpperCamelCase						,    default=""						,    help="Optional additional commit description to use when opening a PR (e.g. to tag the owner)."						,    )
      train_parser.add_argument(
          "--override-model-class"						,    type=__UpperCamelCase						,    default=__UpperCamelCase						,    help="If you think you know better than the auto-detector, you can specify the model class here. "
          "Can be either an AutoModel class or a specific model class like BertForSequenceClassification."						,    )
      train_parser.set_defaults(func=__UpperCamelCase					)

  @staticmethod
  def 					lowercase_						(					__UpperCamelCase						,    __UpperCamelCase					):
      A_								=					set(pt_outputs.keys()					)
      A_								=					set(tf_outputs.keys()					)
      if pt_out_attrs != tf_out_attrs:
          raise ValueError(
              f'The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'
              f' {tf_out_attrs})'					)

      # 2. For each output attribute, computes the difference
      def _find_pt_tf_differences(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase=""					):
          # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
          # recursivelly, keeping the name of the attribute.
          if isinstance(__UpperCamelCase						,    torch.Tensor					):
              A_								=					np.max(np.abs(pt_out.numpy() - tf_out.numpy()					)					)
              A_								=					tensor_difference
          else:
              A_								=					attr_name
              for i, pt_item in enumerate(__UpperCamelCase					):
                  # If it is a named attribute, we keep the name. Otherwise, just its index.
                  if isinstance(__UpperCamelCase						,    __UpperCamelCase					):
                      A_								=					root_name + pt_item
                      A_								=					tf_out[pt_item]
                      A_								=					pt_out[pt_item]
                  else:
                      A_								=					root_name + f'[{i}]'
                      A_								=					tf_out[i]
                  A_								=					_find_pt_tf_differences(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					)

          return differences

      return _find_pt_tf_differences(__UpperCamelCase						,    __UpperCamelCase						,    {}					)

  def __init__(					self						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase						,    *__UpperCamelCase						,    ):
      A_								=					logging.get_logger("transformers-cli/pt_to_tf"					)
      A_								=					model_name
      A_								=					local_dir if local_dir else os.path.join("/tmp"						,    __UpperCamelCase					)
      A_								=					max_error
      A_								=					new_weights
      A_								=					no_pr
      A_								=					push
      A_								=					extra_commit_description
      A_								=					override_model_class

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					):

      def _get_audio_input():
          A_								=					load_dataset("hf-internal-testing/librispeech_asr_dummy"						,    "clean"						,    split="validation"					)
          A_								=					ds.sort("id"					).select(range(2					)					)[:2]["audio"]
          A_								=					[x["array"] for x in speech_samples]
          return raw_samples

      A_								=					type(pt_model.config					)
      if model_config_class in PROCESSOR_MAPPING:
          A_								=					AutoProcessor.from_pretrained(self._local_dir					)
          if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
              A_								=					processor.tokenizer.eos_token
      elif model_config_class in IMAGE_PROCESSOR_MAPPING:
          A_								=					AutoImageProcessor.from_pretrained(self._local_dir					)
      elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
          A_								=					AutoFeatureExtractor.from_pretrained(self._local_dir					)
      elif model_config_class in TOKENIZER_MAPPING:
          A_								=					AutoTokenizer.from_pretrained(self._local_dir					)
          if processor.pad_token is None:
              A_								=					processor.eos_token
      else:
          raise ValueError(f'Unknown data processing type (model config type: {model_config_class})'					)

      A_								=					set(inspect.signature(pt_model.forward					).parameters.keys()					)
      A_								=					{}
      if "input_ids" in model_forward_signature:
          processor_inputs.update(
              {
                  "text": ["Hi there!", "I am a batch with more than one row and different input lengths."],
                  "padding": True,
                  "truncation": True,
              }					)
      if "pixel_values" in model_forward_signature:
          A_								=					load_dataset("cifar10"						,    "plain_text"						,    split="test"					)[:2]["img"]
          processor_inputs.update({"images": sample_images}					)
      if "input_features" in model_forward_signature:
          A_								=					inspect.signature(processor.feature_extractor					).parameters
          # Pad to the largest input length by default but take feature extractor default
          # padding value if it exists e.g. "max_length" and is not False or None
          if "padding" in feature_extractor_signature:
              A_								=					feature_extractor_signature["padding"].default
              if default_strategy is not False and default_strategy is not None:
                  A_								=					default_strategy
              else:
                  A_								=					True
          else:
              A_								=					True
          processor_inputs.update({"audio": _get_audio_input(), "padding": padding_strategy}					)
      if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
          processor_inputs.update({"audio": _get_audio_input(), "padding": True}					)
      A_								=					processor(**__UpperCamelCase						,    return_tensors="pt"					)
      A_								=					processor(**__UpperCamelCase						,    return_tensors="tf"					)

      # Extra input requirements, in addition to the input modality
      if (
          config.is_encoder_decoder
          or (hasattr(__UpperCamelCase						,    "encoder"					) and hasattr(__UpperCamelCase						,    "decoder"					))
          or "decoder_input_ids" in tf_dummy_inputs
      ):
          A_								=					np.asarray([[1], [1]]						,    dtype=__UpperCamelCase					) * (pt_model.config.decoder_start_token_id or 0)
          pt_input.update({"decoder_input_ids": torch.tensor(__UpperCamelCase					)}					)
          tf_input.update({"decoder_input_ids": tf.convert_to_tensor(__UpperCamelCase					)}					)

      return pt_input, tf_input

  def 					lowercase_						(					self					):
      # hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
      if version.parse(huggingface_hub.__version__					) < version.parse("0.9.0"					):
          raise ImportError(
              "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"
              " installation."					)
      else:
          from huggingface_hub import Repository, create_commit
          from huggingface_hub._commit_api import CommitOperationAdd

      # Fetch remote data
      A_								=					Repository(local_dir=self._local_dir						,    clone_from=self._model_name					)

      # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
      A_								=					AutoConfig.from_pretrained(self._local_dir					)
      A_								=					config.architectures
      if self._override_model_class is not None:
          if self._override_model_class.startswith("TF"					):
              A_								=					[self._override_model_class[2:]]
          else:
              A_								=					[self._override_model_class]
          try:
              A_								=					getattr(import_module("transformers"					)						,    architectures[0]					)
          except AttributeError:
              raise ValueError(f'Model class {self._override_model_class} not found in transformers.'					)
          try:
              A_								=					getattr(import_module("transformers"					)						,    "TF" + architectures[0]					)
          except AttributeError:
              raise ValueError(f'TF model class TF{self._override_model_class} not found in transformers.'					)
      elif architectures is None:  # No architecture defined -- use auto classes
          A_								=					getattr(import_module("transformers"					)						,    "AutoModel"					)
          A_								=					getattr(import_module("transformers"					)						,    "TFAutoModel"					)
          self._logger.warning("No detected architecture, using AutoModel/TFAutoModel"					)
      else:  # Architecture defined -- use it
          if len(__UpperCamelCase					) > 1:
              raise ValueError(f'More than one architecture was found, aborting. (architectures = {architectures})'					)
          self._logger.warning(f'Detected architecture: {architectures[0]}'					)
          A_								=					getattr(import_module("transformers"					)						,    architectures[0]					)
          try:
              A_								=					getattr(import_module("transformers"					)						,    "TF" + architectures[0]					)
          except AttributeError:
              raise AttributeError(f'The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'					)

        # Check the TF dummy inputs to see what keys we need in the forward pass
      A_								=					tf_class.from_config(__UpperCamelCase					)
      A_								=					tf_from_pt_model.dummy_inputs

      del tf_from_pt_model  # Try to keep only one model in memory at a time

      # Load the model and get some basic inputs
      A_								=					pt_class.from_pretrained(self._local_dir					)
      pt_model.eval()

      A_	,	A_								=					self.get_inputs(__UpperCamelCase						,    __UpperCamelCase						,    __UpperCamelCase					)

      with torch.no_grad():
          A_								=					pt_model(**__UpperCamelCase						,    output_hidden_states=__UpperCamelCase					)
      del pt_model  # will no longer be used, and may have a large memory footprint

      A_								=					tf_class.from_pretrained(self._local_dir						,    from_pt=__UpperCamelCase					)
      A_								=					tf_from_pt_model(**__UpperCamelCase						,    output_hidden_states=__UpperCamelCase						,    training=__UpperCamelCase					)

      # Confirms that cross loading PT weights into TF worked.
      A_								=					self.find_pt_tf_differences(__UpperCamelCase						,    __UpperCamelCase					)
      A_								=					{k: v for k, v in crossload_differences.items() if "hidden" not in k}
      A_								=					{k: v for k, v in crossload_differences.items() if "hidden" in k}
      if len(__UpperCamelCase					) == 0 and architectures is not None:
          raise ValueError(
              f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
              " output was found. All outputs start with 'hidden'"					)
      A_								=					max(output_differences.values()					) if output_differences else 0.0
      A_								=					max(hidden_differences.values()					)
      if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
          raise ValueError(
              "The cross-loaded TensorFlow model has different outputs, something went wrong!\n"
              + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
              + "\n".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]					)
              + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
              + "\n".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]					)					)

      # Save the weights in a TF format (if needed) and confirms that the results are still good
      A_								=					os.path.join(self._local_dir						,    __UpperCamelCase					)
      A_								=					os.path.join(self._local_dir						,    __UpperCamelCase					)
      if (not os.path.exists(__UpperCamelCase					) and not os.path.exists(__UpperCamelCase					)) or self._new_weights:
          tf_from_pt_model.save_pretrained(self._local_dir					)
      del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

      A_								=					tf_class.from_pretrained(self._local_dir					)
      A_								=					tf_model(**__UpperCamelCase						,    output_hidden_states=__UpperCamelCase					)

      A_								=					self.find_pt_tf_differences(__UpperCamelCase						,    __UpperCamelCase					)
      A_								=					{k: v for k, v in conversion_differences.items() if "hidden" not in k}
      A_								=					{k: v for k, v in conversion_differences.items() if "hidden" in k}
      if len(__UpperCamelCase					) == 0 and architectures is not None:
          raise ValueError(
              f'Something went wrong -- the config file has architectures ({architectures}), but no model head'
              " output was found. All outputs start with 'hidden'"					)
      A_								=					max(output_differences.values()					) if output_differences else 0.0
      A_								=					max(hidden_differences.values()					)
      if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
          raise ValueError(
              "The converted TensorFlow model has different outputs, something went wrong!\n"
              + f'\nList of maximum output differences above the threshold ({self._max_error}):\n'
              + "\n".join([f'{k}: {v:.3e}' for k, v in output_differences.items() if v > self._max_error]					)
              + f'\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'
              + "\n".join([f'{k}: {v:.3e}' for k, v in hidden_differences.items() if v > self._max_error]					)					)

      A_								=					"Update TF weights" if self._new_weights else "Add TF weights"
      if self._push:
          repo.git_add(auto_lfs_track=__UpperCamelCase					)
          repo.git_commit(__UpperCamelCase					)
          repo.git_push(blocking=__UpperCamelCase					)  # this prints a progress bar with the upload
          self._logger.warning(f'TF weights pushed into {self._model_name}'					)
      elif not self._no_pr:
          self._logger.warning("Uploading the weights into a new PR..."					)
          A_								=					(
              "Model converted by the [`transformers`' `pt_to_tf`"
              " CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). "
              "All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"
              f'Maximum crossload output difference={max_crossload_output_diff:.3e}; '
              f'Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'
              f'Maximum conversion output difference={max_conversion_output_diff:.3e}; '
              f'Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'
          )
          if self._max_error > MAX_ERROR:
              commit_descrition += (
                  f'\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'
              )
          if self._extra_commit_description:
              commit_descrition += "\n\n" + self._extra_commit_description

          # sharded model -> adds all related files (index and .h5 shards)
          if os.path.exists(__UpperCamelCase					):
              A_								=					[
                  CommitOperationAdd(path_in_repo=__UpperCamelCase						,    path_or_fileobj=__UpperCamelCase					)
              ]
              for shard_path in tf.io.gfile.glob(self._local_dir + "/tf_model-*.h5"					):
                  operations += [
                      CommitOperationAdd(path_in_repo=os.path.basename(__UpperCamelCase					)						,    path_or_fileobj=__UpperCamelCase					)
                  ]
          else:
              A_								=					[CommitOperationAdd(path_in_repo=__UpperCamelCase						,    path_or_fileobj=__UpperCamelCase					)]

          A_								=					create_commit(
              repo_id=self._model_name						,    operations=__UpperCamelCase						,    commit_message=__UpperCamelCase						,    commit_description=__UpperCamelCase						,    repo_type="model"						,    create_pr=__UpperCamelCase						,    ).pr_url
          self._logger.warning(f'PR open in {hub_pr_url}'					)








import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__  :  List[Any]     			=   logging.get_logger(__name__)

__magic_name__  :  Any     			=   {
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

__magic_name__  :  Dict     			=   {
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

__magic_name__  :  Any     			=   {'stas/tiny-wmt19-en-de': 1_0_2_4}
__magic_name__  :  int     			=   {
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 1_0_2_4,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}





def        lowerCAmelCase       (				snake_case__   :      Union[str, Any] )->			Dict:
    A_								=					set()
    A_								=					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char) )
        A_								=					char
    return pairs





def        lowerCAmelCase       (				snake_case__   :      Optional[int] )->			List[Any]:
    A_								=					text.replace("，"				,					"," )
    A_								=					re.sub(r"。\s*"				,					". "				,					snake_case__ )
    A_								=					text.replace("、"				,					"," )
    A_								=					text.replace("”"				,					"\"" )
    A_								=					text.replace("“"				,					"\"" )
    A_								=					text.replace("∶"				,					":" )
    A_								=					text.replace("："				,					":" )
    A_								=					text.replace("？"				,					"?" )
    A_								=					text.replace("《"				,					"\"" )
    A_								=					text.replace("》"				,					"\"" )
    A_								=					text.replace("）"				,					")" )
    A_								=					text.replace("！"				,					"!" )
    A_								=					text.replace("（"				,					"(" )
    A_								=					text.replace("；"				,					";" )
    A_								=					text.replace("１"				,					"1" )
    A_								=					text.replace("」"				,					"\"" )
    A_								=					text.replace("「"				,					"\"" )
    A_								=					text.replace("０"				,					"0" )
    A_								=					text.replace("３"				,					"3" )
    A_								=					text.replace("２"				,					"2" )
    A_								=					text.replace("５"				,					"5" )
    A_								=					text.replace("６"				,					"6" )
    A_								=					text.replace("９"				,					"9" )
    A_								=					text.replace("７"				,					"7" )
    A_								=					text.replace("８"				,					"8" )
    A_								=					text.replace("４"				,					"4" )
    A_								=					re.sub(r"．\s*"				,					". "				,					snake_case__ )
    A_								=					text.replace("～"				,					"~" )
    A_								=					text.replace("’"				,					"'" )
    A_								=					text.replace("…"				,					"..." )
    A_								=					text.replace("━"				,					"-" )
    A_								=					text.replace("〈"				,					"<" )
    A_								=					text.replace("〉"				,					">" )
    A_								=					text.replace("【"				,					"[" )
    A_								=					text.replace("】"				,					"]" )
    A_								=					text.replace("％"				,					"%" )
    return text





def        lowerCAmelCase       (				snake_case__   :      List[Any] )->			List[Any]:
    A_								=					[]
    for char in text:
        A_								=					unicodedata.category(snake_case__ )
        if cat.startswith("C" ):
            continue
        output.append(snake_case__ )
    return "".join(snake_case__ )



class 							lowerCamelCase       (		__snake_case					):



  """simple docstring"""





  lowerCAmelCase_								=		VOCAB_FILES_NAMES
  lowerCAmelCase_								=		PRETRAINED_VOCAB_FILES_MAP
  lowerCAmelCase_								=		PRETRAINED_INIT_CONFIGURATION
  lowerCAmelCase_								=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCAmelCase_								=		["""input_ids""", """attention_mask"""]

  def __init__(					self						,    __UpperCamelCase=None						,    __UpperCamelCase=None						,    __UpperCamelCase=None						,    __UpperCamelCase=None						,    __UpperCamelCase=False						,    __UpperCamelCase="<unk>"						,    __UpperCamelCase="<s>"						,    __UpperCamelCase="</s>"						,    __UpperCamelCase="<pad>"						,    **__UpperCamelCase						,    ):
      super().__init__(
          langs=__UpperCamelCase						,    src_vocab_file=__UpperCamelCase						,    tgt_vocab_file=__UpperCamelCase						,    merges_file=__UpperCamelCase						,    do_lower_case=__UpperCamelCase						,    unk_token=__UpperCamelCase						,    bos_token=__UpperCamelCase						,    sep_token=__UpperCamelCase						,    pad_token=__UpperCamelCase						,    **__UpperCamelCase						,    )

      try:
          import sacremoses
      except ImportError:
          raise ImportError(
              "You need to install sacremoses to use XLMTokenizer. "
              "See https://pypi.org/project/sacremoses/ for installation."					)

      A_								=					sacremoses

      A_								=					src_vocab_file
      A_								=					tgt_vocab_file
      A_								=					merges_file
      A_								=					do_lower_case

      # cache of sm.MosesPunctNormalizer instance
      A_								=					{}
      # cache of sm.MosesTokenizer instance
      A_								=					{}
      A_								=					{}

      if langs and len(__UpperCamelCase					) == 2:
          A_	,	A_								=					langs
      else:
          raise ValueError(
              f'arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '
              "Usually that means that tokenizer can't find a mapping for the given model path "
              "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."					)

      with open(__UpperCamelCase						,    encoding="utf-8"					) as src_vocab_handle:
          A_								=					json.load(__UpperCamelCase					)
      with open(__UpperCamelCase						,    encoding="utf-8"					) as tgt_vocab_handle:
          A_								=					json.load(__UpperCamelCase					)
          A_								=					{v: k for k, v in tgt_vocab.items()}
      with open(__UpperCamelCase						,    encoding="utf-8"					) as merges_handle:
          A_								=					merges_handle.read().split("\n"					)[:-1]
      A_								=					[tuple(merge.split()[:2]					) for merge in merges]
      A_								=					dict(zip(__UpperCamelCase						,    range(len(__UpperCamelCase					)					)					)					)
      A_								=					{}

  def 					lowercase_						(					self					):
      return self.get_src_vocab()

  @property
  def 					lowercase_						(					self					):
      return self.src_vocab_size

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase					):
      if lang not in self.cache_moses_punct_normalizer:
          A_								=					self.sm.MosesPunctNormalizer(lang=__UpperCamelCase					)
          A_								=					punct_normalizer
      return self.cache_moses_punct_normalizer[lang].normalize(__UpperCamelCase					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase					):
      if lang not in self.cache_moses_tokenizer:
          A_								=					self.sm.MosesTokenizer(lang=__UpperCamelCase					)
          A_								=					moses_tokenizer
      return self.cache_moses_tokenizer[lang].tokenize(
          __UpperCamelCase						,    aggressive_dash_splits=__UpperCamelCase						,    return_str=__UpperCamelCase						,    escape=__UpperCamelCase					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase					):
      if lang not in self.cache_moses_detokenizer:
          A_								=					self.sm.MosesDetokenizer(lang=__UpperCamelCase					)
          A_								=					moses_detokenizer
      return self.cache_moses_detokenizer[lang].detokenize(__UpperCamelCase					)

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase					):
      A_								=					replace_unicode_punct(__UpperCamelCase					)
      A_								=					self.moses_punct_norm(__UpperCamelCase						,    __UpperCamelCase					)
      A_								=					remove_non_printing_char(__UpperCamelCase					)
      return text

  @property
  def 					lowercase_						(					self					):
      return len(self.encoder					)

  @property
  def 					lowercase_						(					self					):
      return len(self.decoder					)

  def 					lowercase_						(					self					):
      return dict(self.encoder						,    **self.added_tokens_encoder					)

  def 					lowercase_						(					self					):
      return dict(self.decoder						,    **self.added_tokens_decoder					)

  def 					lowercase_						(					self						,    __UpperCamelCase					):
      A_								=					tuple(token[:-1]					) + (token[-1] + "</w>",)
      if token in self.cache:
          return self.cache[token]
      A_								=					get_pairs(__UpperCamelCase					)

      if not pairs:
          return token + "</w>"

      while True:
          A_								=					min(__UpperCamelCase						,    key=lambda __UpperCamelCase					: self.bpe_ranks.get(__UpperCamelCase						,    float("inf"					)					)					)
          if bigram not in self.bpe_ranks:
              break
          A_	,	A_								=					bigram
          A_								=					[]
          A_								=					0
          while i < len(__UpperCamelCase					):
              try:
                  A_								=					word.index(__UpperCamelCase						,    __UpperCamelCase					)
              except ValueError:
                  new_word.extend(word[i:]					)
                  break
              else:
                  new_word.extend(word[i:j]					)
                  A_								=					j

              if word[i] == first and i < len(__UpperCamelCase					) - 1 and word[i + 1] == second:
                  new_word.append(first + second					)
                  i += 2
              else:
                  new_word.append(word[i]					)
                  i += 1
          A_								=					tuple(__UpperCamelCase					)
          A_								=					new_word
          if len(__UpperCamelCase					) == 1:
              break
          else:
              A_								=					get_pairs(__UpperCamelCase					)
      A_								=					" ".join(__UpperCamelCase					)
      if word == "\n  </w>":
          A_								=					"\n</w>"
      A_								=					word
      return word

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase="en"						,    __UpperCamelCase=False					):
      A_								=					self.src_lang

      if self.do_lower_case:
          A_								=					text.lower()

      if bypass_tokenizer:
          A_								=					text.split()
      else:
          A_								=					self.moses_pipeline(__UpperCamelCase						,    lang=__UpperCamelCase					)
          A_								=					self.moses_tokenize(__UpperCamelCase						,    lang=__UpperCamelCase					)

      A_								=					[]
      for token in text:
          if token:
              split_tokens.extend(list(self.bpe(__UpperCamelCase					).split(" "					)					)					)

      return split_tokens

  def 					lowercase_						(					self						,    __UpperCamelCase					):
      return self.encoder.get(__UpperCamelCase						,    self.encoder.get(self.unk_token					)					)

  def 					lowercase_						(					self						,    __UpperCamelCase					):
      return self.decoder.get(__UpperCamelCase						,    self.unk_token					)

  def 					lowercase_						(					self						,    __UpperCamelCase					):
      A_								=					[t.replace(" "						,    ""					).replace("</w>"						,    " "					) for t in tokens]
      A_								=					"".join(__UpperCamelCase					).split()
      # detokenize
      A_								=					self.moses_detokenize(__UpperCamelCase						,    self.tgt_lang					)
      return text

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase = None					):
      A_								=					[self.sep_token_id]

      # no bos used in fairseq
      if token_ids_a is None:
          return token_ids_a + sep
      return token_ids_a + sep + token_ids_a + sep

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase = None						,    __UpperCamelCase = False					):

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__UpperCamelCase						,    token_ids_a=__UpperCamelCase						,    already_has_special_tokens=__UpperCamelCase					)
      # no bos used in fairseq
      if token_ids_a is not None:
          return ([0] * len(__UpperCamelCase					)) + [1] + ([0] * len(__UpperCamelCase					)) + [1]
      return ([0] * len(__UpperCamelCase					)) + [1]

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase = None					):
      A_								=					[self.sep_token_id]

      # no bos used in fairseq
      if token_ids_a is None:
          return len(token_ids_a + sep					) * [0]
      return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]

  def 					lowercase_						(					self						,    __UpperCamelCase						,    __UpperCamelCase = None					):
      if not os.path.isdir(__UpperCamelCase					):
          logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
          return

      A_								=					os.path.join(
          __UpperCamelCase						,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["src_vocab_file"]					)
      A_								=					os.path.join(
          __UpperCamelCase						,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["tgt_vocab_file"]					)
      A_								=					os.path.join(
          __UpperCamelCase						,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

      with open(__UpperCamelCase						,    "w"						,    encoding="utf-8"					) as f:
          f.write(json.dumps(self.encoder						,    indent=2						,    sort_keys=__UpperCamelCase						,    ensure_ascii=__UpperCamelCase					) + "\n"					)

      with open(__UpperCamelCase						,    "w"						,    encoding="utf-8"					) as f:
          A_								=					{v: k for k, v in self.decoder.items()}
          f.write(json.dumps(__UpperCamelCase						,    indent=2						,    sort_keys=__UpperCamelCase						,    ensure_ascii=__UpperCamelCase					) + "\n"					)

      A_								=					0
      with open(__UpperCamelCase						,    "w"						,    encoding="utf-8"					) as writer:
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,    key=lambda __UpperCamelCase					: kv[1]					):
              if index != token_index:
                  logger.warning(
                      f'Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'
                      " Please check that the tokenizer is not corrupted!"					)
                  A_								=					token_index
              writer.write(" ".join(__UpperCamelCase					) + "\n"					)
              index += 1

      return src_vocab_file, tgt_vocab_file, merges_file

  def __getstate__(					self					):
      A_								=					self.__dict__.copy()
      A_								=					None
      return state

  def __setstate__(					self						,    __UpperCamelCase					):
      A_								=					d

      try:
          import sacremoses
      except ImportError:
          raise ImportError(
              "You need to install sacremoses to use XLMTokenizer. "
              "See https://pypi.org/project/sacremoses/ for installation."					)

      A_								=					sacremoses






