



'''simple docstring'''





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCamelCase											=						logging.get_logger(__name__)


_UpperCamelCase											=						{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}

def        a_     (							_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase       )      ->      Optional[Any]:
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()       )}.'       )

  if tokenizer_name is None:
    __lowerCamelCase       :       int           = TOKENIZER_CLASSES
  else:
    __lowerCamelCase       :       Optional[int]           = {tokenizer_name: getattr(_lowerCAmelCase       ,tokenizer_name + 'Fast'       )}

  logger.info(F'Loading tokenizer classes: {tokenizer_names}'       )

  for tokenizer_name in tokenizer_names:
    __lowerCamelCase       :       Optional[int]           = TOKENIZER_CLASSES[tokenizer_name]

    __lowerCamelCase       :       Tuple           = True
    if checkpoint_name is None:
      __lowerCamelCase       :       Tuple           = list(tokenizer_class.max_model_input_sizes.keys()       )
    else:
      __lowerCamelCase       :       str           = [checkpoint_name]

    logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'       )

    for checkpoint in checkpoint_names:
      logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'       )

      # Load tokenizer
      __lowerCamelCase       :       int           = tokenizer_class.from_pretrained(_lowerCAmelCase       ,force_download=_lowerCAmelCase       )

      # Save fast tokenizer
      logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'       )

      # For organization names we create sub-directories
      if "/" in checkpoint:
        __lowerCamelCase       ,__lowerCamelCase       :       Optional[Any]           = checkpoint.split('/'       )
        __lowerCamelCase       :       Any           = os.path.join(_lowerCAmelCase       ,_lowerCAmelCase       )
      elif add_prefix:
        __lowerCamelCase       :       Tuple           = checkpoint
        __lowerCamelCase       :       Tuple           = dump_path
      else:
        __lowerCamelCase       :       Dict           = None
        __lowerCamelCase       :       Optional[Any]           = dump_path

      logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'       )

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()       )[0]:
        __lowerCamelCase       :       int           = list(tokenizer.pretrained_vocab_files_map.values()       )[0][checkpoint]
        __lowerCamelCase       :       Optional[Any]           = file_path.split(_lowerCAmelCase       )[-1][0]
        if next_char == "/":
          __lowerCamelCase       :       int           = os.path.join(_lowerCAmelCase       ,_lowerCAmelCase       )
          __lowerCamelCase       :       List[Any]           = None

        logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'       )

      __lowerCamelCase       :       Tuple           = tokenizer.save_pretrained(
          _lowerCAmelCase       ,legacy_format=_lowerCAmelCase       ,filename_prefix=_lowerCAmelCase       )
      logger.info(F'=> File names {file_names}'       )

      for file_name in file_names:
        if not file_name.endswith('tokenizer.json'       ):
          os.remove(_lowerCAmelCase       )
          logger.info(F'=> removing {file_name}'       )


if __name__ == "__main__":
   _UpperCamelCase											=						argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
   )
   parser.add_argument(
       '--tokenizer_name',
       default=None,
       type=str,
       help=(
           f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
           'download and convert all the checkpoints from AWS.'
       ),
   )
   parser.add_argument(
       '--checkpoint_name',
       default=None,
       type=str,
       help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
   )
   parser.add_argument(
       '--force_download',
       action='store_true',
       help='Re-download checkpoints.',
   )
   _UpperCamelCase											=						parser.parse_args()

   convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)











'''simple docstring'''





import inspect
import warnings
from typing import Any, Dict, Optional, Union

from packaging import version

def        a_     (							*_lowerCAmelCase       ,_lowerCAmelCase = None       ,_lowerCAmelCase=True       ,_lowerCAmelCase=2       )      ->      List[str]:
  from .. import __version__

  __lowerCamelCase       :       Any           = take_from
  __lowerCamelCase       :       Optional[int]           = ()
  if not isinstance(args[0]       ,_lowerCAmelCase       ):
    __lowerCamelCase       :       Optional[Any]           = (args,)

  for attribute, version_name, message in args:
    if version.parse(version.parse(_lowerCAmelCase       ).base_version       ) >= version.parse(_lowerCAmelCase       ):
      raise ValueError(
          F'The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers\''
          F' version {__version__} is >= {version_name}'       )

    __lowerCamelCase       :       Union[str, Any]           = None
    if isinstance(_lowerCAmelCase       ,_lowerCAmelCase       ) and attribute in deprecated_kwargs:
      values += (deprecated_kwargs.pop(_lowerCAmelCase       ),)
      __lowerCamelCase       :       Optional[Any]           = F'The `{attribute}` argument is deprecated and will be removed in version {version_name}.'
    elif hasattr(_lowerCAmelCase       ,_lowerCAmelCase       ):
      values += (getattr(_lowerCAmelCase       ,_lowerCAmelCase       ),)
      __lowerCamelCase       :       List[str]           = F'The `{attribute}` attribute is deprecated and will be removed in version {version_name}.'
    elif deprecated_kwargs is None:
      __lowerCamelCase       :       Optional[Any]           = F'`{attribute}` is deprecated and will be removed in version {version_name}.'

    if warning is not None:
      __lowerCamelCase       :       Optional[int]           = warning + ' ' if standard_warn else ''
      warnings.warn(warning + message       ,_lowerCAmelCase       ,stacklevel=_lowerCAmelCase       )

  if isinstance(_lowerCAmelCase       ,_lowerCAmelCase       ) and len(_lowerCAmelCase       ) > 0:
    __lowerCamelCase       :       Optional[Any]           = inspect.getouterframes(inspect.currentframe()       )[1]
    __lowerCamelCase       :       List[str]           = call_frame.filename
    __lowerCamelCase       :       int           = call_frame.lineno
    __lowerCamelCase       :       Union[str, Any]           = call_frame.function
    __lowerCamelCase       ,__lowerCamelCase       :       Union[str, Any]           = next(iter(deprecated_kwargs.items()       )       )
    raise TypeError(F'{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`'       )

  if len(_lowerCAmelCase       ) == 0:
    return
  elif len(_lowerCAmelCase       ) == 1:
    return values[0]
  return values






