



'''simple docstring'''





import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
   from cookiecutter.main import cookiecutter

   _UpperCamelCase											=						True
except ImportError:
   _UpperCamelCase											=						False

_UpperCamelCase											=						logging.get_logger(__name__)  # pylint: disable=invalid-name

def        a_     (							_lowerCAmelCase       )      ->      List[str]:
  return AddNewModelCommand(args.testing       ,args.testing_file       ,path=args.path       )






class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_     ):
  """simple docstring"""




  @staticmethod
  def 						_lowercase     ( _a      :	ArgumentParser   )					->					Any:
    __lowerCamelCase       :       List[str]           = parser.add_parser('add-new-model'   )
    add_new_model_parser.add_argument('--testing'			,	action='store_true'			,	help='If in testing mode.'   )
    add_new_model_parser.add_argument('--testing_file'			,	type=_a			,	help='Configuration file on which to run.'   )
    add_new_model_parser.add_argument(
        '--path'			,	type=_a			,	help='Path to cookiecutter. Should only be used for testing purposes.'   )
    add_new_model_parser.set_defaults(func=_a   )




  def __init__( self      :	Union[str, Any]			,	_a      :	bool			,	_a      :	str			,	_a      :	List[str]=None			,	*_a      :	Optional[Any]   )					->					str:
    __lowerCamelCase       :       str           = testing
    __lowerCamelCase       :       List[Any]           = testing_file
    __lowerCamelCase       :       Tuple           = path




  def 						_lowercase     ( self      :	List[Any]   )					->					int:
    warnings.warn(
        'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
        'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
        'checks, you should use `transformers-cli add-new-model-like` instead.'   )
    if not _has_cookiecutter:
      raise ImportError(
          'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
          'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'   )
    # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
    __lowerCamelCase       :       Any           = [directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:22]]
    if len(_a   ) > 0:
      raise ValueError(
          'Several directories starting with `cookiecutter-template-` in current working directory. '
          'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
          'change your working directory.'   )

    __lowerCamelCase       :       List[Any]           = (
        Path(_a   ).parent.parent.parent.parent if self._path is None else Path(self._path   ).parent.parent
    )
    __lowerCamelCase       :       List[str]           = path_to_transformer_root / 'templates' / 'adding_a_new_model'

    # Execute cookiecutter
    if not self._testing:
      cookiecutter(str(_a   )   )
    else:
      with open(self._testing_file			,	'r'   ) as configuration_file:
        __lowerCamelCase       :       List[Any]           = json.load(_a   )

      cookiecutter(
          str(path_to_cookiecutter if self._path is None else self._path   )			,	no_input=_a			,	extra_context=_a			,	)

    __lowerCamelCase       :       List[str]           = [directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:22]][0]

    # Retrieve configuration
    with open(directory + '/configuration.json'			,	'r'   ) as configuration_file:
      __lowerCamelCase       :       Optional[int]           = json.load(_a   )

    __lowerCamelCase       :       Tuple           = configuration['lowercase_modelname']
    __lowerCamelCase       :       List[Any]           = configuration['generate_tensorflow_pytorch_and_flax']
    os.remove(f'{directory}/configuration.json'   )

    __lowerCamelCase       :       Optional[Any]           = 'PyTorch' in generate_tensorflow_pytorch_and_flax
    __lowerCamelCase       :       List[Any]           = 'TensorFlow' in generate_tensorflow_pytorch_and_flax
    __lowerCamelCase       :       Optional[Any]           = 'Flax' in generate_tensorflow_pytorch_and_flax

    __lowerCamelCase       :       Tuple           = f'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
    os.makedirs(_a			,	exist_ok=_a   )
    os.makedirs(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}'			,	exist_ok=_a   )

    # Tests require submodules as they have parent imports
    with open(f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'			,	'w'   ):
      pass

    shutil.move(
        f'{directory}/__init__.py'			,	f'{model_dir}/__init__.py'			,	)
    shutil.move(
        f'{directory}/configuration_{lowercase_model_name}.py'			,	f'{model_dir}/configuration_{lowercase_model_name}.py'			,	)

    def remove_copy_lines(_a      :	List[str]   ):
      with open(_a			,	'r'   ) as f:
        __lowerCamelCase       :       List[str]           = f.readlines()
      with open(_a			,	'w'   ) as f:
        for line in lines:
          if "# Copied from transformers." not in line:
            f.write(_a   )

    if output_pytorch:
      if not self._testing:
        remove_copy_lines(f'{directory}/modeling_{lowercase_model_name}.py'   )

      shutil.move(
          f'{directory}/modeling_{lowercase_model_name}.py'			,	f'{model_dir}/modeling_{lowercase_model_name}.py'			,	)

      shutil.move(
          f'{directory}/test_modeling_{lowercase_model_name}.py'			,	f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'			,	)
    else:
      os.remove(f'{directory}/modeling_{lowercase_model_name}.py'   )
      os.remove(f'{directory}/test_modeling_{lowercase_model_name}.py'   )

    if output_tensorflow:
      if not self._testing:
        remove_copy_lines(f'{directory}/modeling_tf_{lowercase_model_name}.py'   )

      shutil.move(
          f'{directory}/modeling_tf_{lowercase_model_name}.py'			,	f'{model_dir}/modeling_tf_{lowercase_model_name}.py'			,	)

      shutil.move(
          f'{directory}/test_modeling_tf_{lowercase_model_name}.py'			,	f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'			,	)
    else:
      os.remove(f'{directory}/modeling_tf_{lowercase_model_name}.py'   )
      os.remove(f'{directory}/test_modeling_tf_{lowercase_model_name}.py'   )

    if output_flax:
      if not self._testing:
        remove_copy_lines(f'{directory}/modeling_flax_{lowercase_model_name}.py'   )

      shutil.move(
          f'{directory}/modeling_flax_{lowercase_model_name}.py'			,	f'{model_dir}/modeling_flax_{lowercase_model_name}.py'			,	)

      shutil.move(
          f'{directory}/test_modeling_flax_{lowercase_model_name}.py'			,	f'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'			,	)
    else:
      os.remove(f'{directory}/modeling_flax_{lowercase_model_name}.py'   )
      os.remove(f'{directory}/test_modeling_flax_{lowercase_model_name}.py'   )

    shutil.move(
        f'{directory}/{lowercase_model_name}.md'			,	f'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'			,	)

    shutil.move(
        f'{directory}/tokenization_{lowercase_model_name}.py'			,	f'{model_dir}/tokenization_{lowercase_model_name}.py'			,	)

    shutil.move(
        f'{directory}/tokenization_fast_{lowercase_model_name}.py'			,	f'{model_dir}/tokenization_{lowercase_model_name}_fast.py'			,	)

    from os import fdopen, remove
    from shutil import copymode, move
    from tempfile import mkstemp

    def replace(_a      :	str			,	_a      :	str			,	_a      :	List[str]   ):
      # Create temp file
      __lowerCamelCase       ,__lowerCamelCase       :       Any           = mkstemp()
      __lowerCamelCase       :       int           = False
      with fdopen(_a			,	'w'   ) as new_file:
        with open(_a   ) as old_file:
          for line in old_file:
            new_file.write(_a   )
            if line_to_copy_below in line:
              __lowerCamelCase       :       int           = True
              for line_to_copy in lines_to_copy:
                new_file.write(_a   )

      if not line_found:
        raise ValueError(f'Line {line_to_copy_below} was not found in file.'   )

      # Copy the file permissions from the old file to the new file
      copymode(_a			,	_a   )
      # Remove original file
      remove(_a   )
      # Move new file
      move(_a			,	_a   )

    def skip_units(_a      :	Union[str, Any]   ):
      return (
          ("generating PyTorch" in line and not output_pytorch)
          or ("generating TensorFlow" in line and not output_tensorflow)
          or ("generating Flax" in line and not output_flax)
      )

    def replace_in_files(_a      :	Dict   ):
      with open(_a   ) as datafile:
        __lowerCamelCase       :       int           = []
        __lowerCamelCase       :       Optional[Any]           = False
        __lowerCamelCase       :       Optional[int]           = False
        for line in datafile:
          if "# To replace in: " in line and "##" not in line:
            __lowerCamelCase       :       Any           = line.split('"'   )[1]
            __lowerCamelCase       :       List[str]           = skip_units(_a   )
          elif "# Below: " in line and "##" not in line:
            __lowerCamelCase       :       str           = line.split('"'   )[1]
            __lowerCamelCase       :       Any           = skip_units(_a   )
          elif "# End." in line and "##" not in line:
            if not skip_file and not skip_snippet:
              replace(_a			,	_a			,	_a   )

            __lowerCamelCase       :       str           = []
          elif "# Replace with" in line and "##" not in line:
            __lowerCamelCase       :       int           = []
          elif "##" not in line:
            lines_to_copy.append(_a   )

      remove(_a   )

    replace_in_files(f'{directory}/to_replace_{lowercase_model_name}.py'   )
    os.rmdir(_a   )











'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_mbart import MBartTokenizer
else:
   _UpperCamelCase											=						None


_UpperCamelCase											=						logging.get_logger(__name__)


_UpperCamelCase											=						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase											=						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

_UpperCamelCase											=						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
_UpperCamelCase											=						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']






class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_     ):
  """simple docstring"""




  a_												=VOCAB_FILES_NAMES
  a_												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_												=PRETRAINED_VOCAB_FILES_MAP
  a_												=["""input_ids""", """attention_mask"""]
  a_												=MBartTokenizer

  a_												=[]
  a_												=[]




  def __init__( self      :	Optional[Any]			,	_a      :	Optional[int]=None			,	_a      :	Any=None			,	_a      :	Any="<s>"			,	_a      :	Optional[Any]="</s>"			,	_a      :	List[str]="</s>"			,	_a      :	List[Any]="<s>"			,	_a      :	Union[str, Any]="<unk>"			,	_a      :	str="<pad>"			,	_a      :	Any="<mask>"			,	_a      :	Optional[Any]=None			,	_a      :	str=None			,	_a      :	Tuple=None			,	**_a      :	Dict			,	)					->					List[str]:
    # Mask token behave like a normal word, i.e. include the space before it
    __lowerCamelCase       :       List[Any]           = AddedToken(_a			,	lstrip=_a			,	rstrip=_a   ) if isinstance(_a			,	_a   ) else mask_token

    super().__init__(
        vocab_file=_a			,	tokenizer_file=_a			,	bos_token=_a			,	eos_token=_a			,	sep_token=_a			,	cls_token=_a			,	unk_token=_a			,	pad_token=_a			,	mask_token=_a			,	src_lang=_a			,	tgt_lang=_a			,	additional_special_tokens=_a			,	**_a			,	)

    __lowerCamelCase       :       Optional[Any]           = vocab_file
    __lowerCamelCase       :       List[str]           = False if not self.vocab_file else True

    __lowerCamelCase       :       str           = FAIRSEQ_LANGUAGE_CODES.copy()

    if additional_special_tokens is not None:
      # Only add those special tokens if they are not already there.
      _additional_special_tokens.extend(
          [t for t in additional_special_tokens if t not in _additional_special_tokens]   )

    self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}   )
    __lowerCamelCase       :       Optional[Any]           = {
        lang_code: self.convert_tokens_to_ids(_a   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
    }

    __lowerCamelCase       :       Optional[Any]           = src_lang if src_lang is not None else 'en_XX'
    __lowerCamelCase       :       int           = self.convert_tokens_to_ids(self._src_lang   )
    __lowerCamelCase       :       List[str]           = tgt_lang
    self.set_src_lang_special_tokens(self._src_lang   )




  @property
  def 						_lowercase     ( self      :	List[Any]   )					->					str:
    return self._src_lang




  @src_lang.setter
  def 						_lowercase     ( self      :	Union[str, Any]			,	_a      :	str   )					->					None:
    __lowerCamelCase       :       Union[str, Any]           = new_src_lang
    self.set_src_lang_special_tokens(self._src_lang   )




  def 						_lowercase     ( self      :	List[Any]			,	_a      :	List[int]			,	_a      :	Optional[List[int]] = None   )					->					List[int]:
    if token_ids_a is None:
      return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




  def 						_lowercase     ( self      :	int			,	_a      :	List[int]			,	_a      :	Optional[List[int]] = None   )					->					List[int]:
    __lowerCamelCase       :       Optional[int]           = [self.sep_token_id]
    __lowerCamelCase       :       Union[str, Any]           = [self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep   ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]




  def 						_lowercase     ( self      :	Optional[Any]			,	_a      :	Optional[Any]			,	_a      :	str			,	_a      :	Optional[str]			,	_a      :	Optional[str]			,	**_a      :	Optional[int]   )					->					Any:
    if src_lang is None or tgt_lang is None:
      raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
    __lowerCamelCase       :       Optional[Any]           = src_lang
    __lowerCamelCase       :       Dict           = self(_a			,	add_special_tokens=_a			,	return_tensors=_a			,	**_a   )
    __lowerCamelCase       :       Tuple           = self.convert_tokens_to_ids(_a   )
    __lowerCamelCase       :       Optional[Any]           = tgt_lang_id
    return inputs




  def 						_lowercase     ( self      :	Any			,	_a      :	List[str]			,	_a      :	str = "en_XX"			,	_a      :	Optional[List[str]] = None			,	_a      :	str = "ro_RO"			,	**_a      :	Tuple			,	)					->					BatchEncoding:
    __lowerCamelCase       :       List[Any]           = src_lang
    __lowerCamelCase       :       str           = tgt_lang
    return super().prepare_seqaseq_batch(_a			,	_a			,	**_a   )




  def 						_lowercase     ( self      :	List[Any]   )					->					Any:
    return self.set_src_lang_special_tokens(self.src_lang   )




  def 						_lowercase     ( self      :	Dict   )					->					Optional[Any]:
    return self.set_tgt_lang_special_tokens(self.tgt_lang   )




  def 						_lowercase     ( self      :	Tuple			,	_a      :	List[str]   )					->					None:
    __lowerCamelCase       :       Tuple           = self.convert_tokens_to_ids(_a   )
    __lowerCamelCase       :       Optional[Any]           = []
    __lowerCamelCase       :       List[str]           = [self.eos_token_id, self.cur_lang_code]

    __lowerCamelCase       :       Dict           = self.convert_ids_to_tokens(self.prefix_tokens   )
    __lowerCamelCase       :       Optional[Any]           = self.convert_ids_to_tokens(self.suffix_tokens   )

    __lowerCamelCase       :       Union[str, Any]           = processors.TemplateProcessing(
        single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,	pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,	self.prefix_tokens + self.suffix_tokens   )   )			,	)




  def 						_lowercase     ( self      :	Optional[Any]			,	_a      :	str   )					->					None:
    __lowerCamelCase       :       Union[str, Any]           = self.convert_tokens_to_ids(_a   )
    __lowerCamelCase       :       int           = []
    __lowerCamelCase       :       List[str]           = [self.eos_token_id, self.cur_lang_code]

    __lowerCamelCase       :       int           = self.convert_ids_to_tokens(self.prefix_tokens   )
    __lowerCamelCase       :       Any           = self.convert_ids_to_tokens(self.suffix_tokens   )

    __lowerCamelCase       :       Optional[int]           = processors.TemplateProcessing(
        single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,	pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,	self.prefix_tokens + self.suffix_tokens   )   )			,	)




  def 						_lowercase     ( self      :	Any			,	_a      :	str			,	_a      :	Optional[str] = None   )					->					Tuple[str]:
    if not self.can_save_slow_tokenizer:
      raise ValueError(
          'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
          'tokenizer.'   )

    if not os.path.isdir(_a   ):
      logger.error(f'Vocabulary path ({save_directory}) should be a directory.'   )
      return
    __lowerCamelCase       :       List[str]           = os.path.join(
        _a			,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

    if os.path.abspath(self.vocab_file   ) != os.path.abspath(_a   ):
      copyfile(self.vocab_file			,	_a   )

    return (out_vocab_file,)






