


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

snake_case				:    int       				=     logging.get_logger(__name__)


snake_case				:    Optional[Any]       				=     {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}
def 				__lowerCamelCase						( UpperCAmelCase_						:		Optional[int] ,  UpperCAmelCase_						:		Dict ,  UpperCAmelCase_						:		List[Any] ,  UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(F'''Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()						)}.'''						)

      if tokenizer_name is None:
            a						:int 							=       TOKENIZER_CLASSES
      else:
            a						:Optional[Any] 							=       {tokenizer_name: getattr(UpperCAmelCase_ ,  tokenizer_name + '''Fast'''						)}

      logger.info(F'''Loading tokenizer classes: {tokenizer_names}'''						)

      for tokenizer_name in tokenizer_names:
            a						:List[Any] 							=       TOKENIZER_CLASSES[tokenizer_name]

            a						:Union[str, Any] 							=       True
            if checkpoint_name is None:
                  a						:int 							=       list(tokenizer_class.max_model_input_sizes.keys()						)
            else:
                  a						:Dict 							=       [checkpoint_name]

            logger.info(F'''For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'''						)

            for checkpoint in checkpoint_names:
                  logger.info(F'''Loading {tokenizer_class.__class__.__name__} {checkpoint}'''						)

                  # Load tokenizer
                  a						:Any 							=       tokenizer_class.from_pretrained(UpperCAmelCase_ ,  force_download=UpperCAmelCase_						)

                  # Save fast tokenizer
                  logger.info(F'''Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'''						)

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        a						:Any 							=       checkpoint.split('''/'''						)
                        a						:Dict 							=       os.path.join(UpperCAmelCase_ ,  UpperCAmelCase_						)
                  elif add_prefix:
                        a						:int 							=       checkpoint
                        a						:List[Any] 							=       dump_path
                  else:
                        a						:Any 							=       None
                        a						:Union[str, Any] 							=       dump_path

                  logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''						)

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()						)[0]:
                        a						:List[str] 							=       list(tokenizer.pretrained_vocab_files_map.values()						)[0][checkpoint]
                        a						:Optional[int] 							=       file_path.split(UpperCAmelCase_						)[-1][0]
                        if next_char == "/":
                              a						:Union[str, Any] 							=       os.path.join(UpperCAmelCase_ ,  UpperCAmelCase_						)
                              a						:Optional[Any] 							=       None

                        logger.info(F'''=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'''						)

                  a						:int 							=       tokenizer.save_pretrained(
                      UpperCAmelCase_ ,  legacy_format=UpperCAmelCase_ ,  filename_prefix=UpperCAmelCase_						)
                  logger.info(F'''=> File names {file_names}'''						)

                  for file_name in file_names:
                        if not file_name.endswith('''tokenizer.json'''						):
                              os.remove(UpperCAmelCase_						)
                              logger.info(F'''=> removing {file_name}'''						)


if __name__ == "__main__":
     snake_case				:    Optional[int]       				=     argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
     )
     parser.add_argument(
         '''--tokenizer_name''',
         default=None,
         type=str,
         help=(
             F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
             '''download and convert all the checkpoints from AWS.'''
         ),
     )
     parser.add_argument(
         '''--checkpoint_name''',
         default=None,
         type=str,
         help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
     )
     parser.add_argument(
         '''--force_download''',
         action='''store_true''',
         help='''Re-download checkpoints.''',
     )
     snake_case				:    int       				=     parser.parse_args()

     convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)











import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast




@dataclass
class       __lowerCAmelCase				(							datasets.BuilderConfig):
       _a           =  None







class       __lowerCAmelCase				(							datasets.ArrowBasedBuilder):
       _a           =  PandasConfig






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         return datasets.DatasetInfo(features=self.config.features )






       def     SCREAMING_SNAKE_CASE			(      self:    str				,    _lowerCAmelCase:    List[str] ):
         if not self.config.data_files:
           raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}" )
         lowercase			:int							=  dl_manager.download_and_extract(self.config.data_files )
         if isinstance(_lowerCAmelCase				,    (str, list, tuple) ):
           lowercase			:Union[str, Any]							=  data_files
           if isinstance(_lowerCAmelCase				,    _lowerCAmelCase ):
             lowercase			:str							=  [files]
           # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
           lowercase			:Optional[int]							=  [dl_manager.iter_files(_lowerCAmelCase ) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,    gen_kwargs={"files": files} )]
         lowercase			:List[Any]							=  []
         for split_name, files in data_files.items():
           if isinstance(_lowerCAmelCase				,    _lowerCAmelCase ):
             lowercase			:Optional[Any]							=  [files]
           # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
           lowercase			:Optional[Any]							=  [dl_manager.iter_files(_lowerCAmelCase ) for file in files]
           splits.append(datasets.SplitGenerator(name=_lowerCAmelCase				,    gen_kwargs={"files": files} ) )
         return splits






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    pa.Table ):
         if self.config.features is not None:
           # more expensive cast to support nested features with keys in a different order
           # allows str <-> int/float or str to Audio for example
           lowercase			:Any							=  table_cast(_lowerCAmelCase				,    self.config.features.arrow_schema )
         return pa_table






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int]				,    _lowerCAmelCase:    Union[str, Any] ):
         for i, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase ) ):
           with open(_lowerCAmelCase				,    "rb" ) as f:
             lowercase			:int							=  pa.Table.from_pandas(pd.read_pickle(_lowerCAmelCase ) )
             yield i, self._cast_table(_lowerCAmelCase )



