



'''simple docstring'''


import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand







class 						UpperCAmelCase						(		snake_case__):


 '''simple docstring'''




 @staticmethod
 def    lowercase_	( lowerCAmelCase_)			->   Optional[Any]:



      """simple docstring"""







      a_									=parser.add_parser("login"				,	help="Log in using the same credentials as on huggingface.co")
      login_parser.set_defaults(func=lambda lowerCAmelCase_: LoginCommand(__SCREAMING_SNAKE_CASE))
      a_									=parser.add_parser("whoami"				,	help="Find out which huggingface.co account you are logged in as.")
      whoami_parser.set_defaults(func=lambda lowerCAmelCase_: WhoamiCommand(__SCREAMING_SNAKE_CASE))
      a_									=parser.add_parser("logout"				,	help="Log out")
      logout_parser.set_defaults(func=lambda lowerCAmelCase_: LogoutCommand(__SCREAMING_SNAKE_CASE))

      # new system: git-based repo system
      a_									=parser.add_parser(
          "repo"				,	help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos."				,	)
      a_									=repo_parser.add_subparsers(
          help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands")
      a_									=repo_subparsers.add_parser(
          "create"				,	help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co")
      repo_create_parser.add_argument(
          "name"				,	type=__SCREAMING_SNAKE_CASE				,	help="Name for your model's repo. Will be namespaced under your username to build the model id."				,	)
      repo_create_parser.add_argument("--organization"				,	type=__SCREAMING_SNAKE_CASE				,	help="Optional: organization namespace.")
      repo_create_parser.add_argument("-y"				,	"--yes"				,	action="store_true"				,	help="Optional: answer Yes to the prompt")
      repo_create_parser.set_defaults(func=lambda lowerCAmelCase_: RepoCreateCommand(__SCREAMING_SNAKE_CASE))







class 						UpperCAmelCase						:


 '''simple docstring'''




 __magic_name__       :			int	=   "\u001b[1m"
 __magic_name__       :			Dict	=   "\u001b[31m"
 __magic_name__       :			Optional[int]	=   "\u001b[90m"
 __magic_name__       :			Optional[int]	=   "\u001b[0m"




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_)			->   Dict:



      """simple docstring"""







      return f"""{cls._bold}{s}{cls._reset}"""




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_)			->   List[str]:



      """simple docstring"""







      return f"""{cls._bold}{cls._red}{s}{cls._reset}"""




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_)			->   Tuple:



      """simple docstring"""







      return f"""{cls._gray}{s}{cls._reset}"""






def 			UpperCAmelCase_							(		lowercase__							,						lowercase__							):


     '''simple docstring'''







     a_									=[max(len(str(A_							)							) for x in col							) for col in zip(*A_							,						A_							)]
     a_									=("{{:{}}} " * len(A_							)).format(*A_							)
     a_									=[]
     lines.append(row_format.format(*A_							)							)
     lines.append(row_format.format(*["-" * w for w in col_widths]							)							)
     for row in rows:
          lines.append(row_format.format(*A_							)							)
     return "\n".join(A_							)







class 						UpperCAmelCase						:


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_)			->   List[Any]:



      """simple docstring"""







      a_									=args







class 						UpperCAmelCase						(		snake_case__):


 '''simple docstring'''




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      print(
          ANSI.red(
              "ERROR! `huggingface-cli login` uses an outdated login mechanism "
              "that is not compatible with the Hugging Face Hub backend anymore. "
              "Please use `huggingface-cli login instead."))







class 						UpperCAmelCase						(		snake_case__):


 '''simple docstring'''




 def    lowercase_	( self)			->   Dict:



      """simple docstring"""







      print(
          ANSI.red(
              "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
              "`huggingface-cli whoami` instead."))
      a_									=HfFolder.get_token()
      if token is None:
           print("Not logged in")
           exit()
      try:
           a_									=whoami(__SCREAMING_SNAKE_CASE)
           print(__SCREAMING_SNAKE_CASE)
           if orgs:
                print(ANSI.bold("orgs: ")				,	",".join(__SCREAMING_SNAKE_CASE))
      except HTTPError as e:
           print(__SCREAMING_SNAKE_CASE)
           print(ANSI.red(e.response.text))
           exit(1)







class 						UpperCAmelCase						(		snake_case__):


 '''simple docstring'''




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      print(
          ANSI.red(
              "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
              "that is not compatible with the Hugging Face Hub backend anymore. "
              "Please use `huggingface-cli logout instead."))







class 						UpperCAmelCase						(		snake_case__):


 '''simple docstring'''




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      print(
          ANSI.red(
              "WARNING! Managing repositories through transformers-cli is deprecated. "
              "Please use `huggingface-cli` instead."))
      a_									=HfFolder.get_token()
      if token is None:
           print("Not logged in")
           exit(1)
      try:
           a_									=subprocess.check_output(["git", "--version"]).decode("utf-8")
           print(ANSI.gray(stdout.strip()))
      except FileNotFoundError:
           print("Looks like you do not have git installed, please install.")

      try:
           a_									=subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
           print(ANSI.gray(stdout.strip()))
      except FileNotFoundError:
           print(
               ANSI.red(
                   "Looks like you do not have git-lfs installed, please install."
                   " You can install from https://git-lfs.github.com/."
                   " Then run `git lfs install` (you only have to do this once)."))
      print("")

      a_									=whoami(__SCREAMING_SNAKE_CASE)
      a_									=self.args.organization if self.args.organization is not None else user
      a_									=f"""{namespace}/{self.args.name}"""
      print(f"""You are about to create {ANSI.bold(__SCREAMING_SNAKE_CASE)}""")

      if not self.args.yes:
           a_									=input("Proceed? [Y/n] ").lower()
           if not (choice == "" or choice == "y" or choice == "yes"):
                print("Abort")
                exit()
      try:
           a_									=create_repo(__SCREAMING_SNAKE_CASE				,	name=self.args.name				,	organization=self.args.organization)
      except HTTPError as e:
           print(__SCREAMING_SNAKE_CASE)
           print(ANSI.red(e.response.text))
           exit(1)
      print("\nYour repo now lives at:")
      print(f"""  {ANSI.bold(__SCREAMING_SNAKE_CASE)}""")
      print("\nYou can clone it locally with the command below, and commit/push as usual.")
      print(f"""\n  git clone {url}""")
      print("")






from arguments import TokenizerTrainingArguments
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, HfArgumentParser
from transformers.models.gpta.tokenization_gpta import bytes_to_unicode
def 	_lowerCamelCase  (		A_  :	List[Any]=1_0      )			->			List[str]:

   '''simple docstring'''






   for _ in tqdm(range(0							,  args.n_examples							,  A_      )      ):
      yield [next(A_      )[args.text_column] for _ in range(A_      )]


# Configuration
__UpperCAmelCase      			=  HfArgumentParser(TokenizerTrainingArguments)
__UpperCAmelCase      			=  parser.parse_args()

# Base tokenizer
__UpperCAmelCase      			=  AutoTokenizer.from_pretrained(args.base_tokenizer)
__UpperCAmelCase      			=  list(bytes_to_unicode().values())

# Load dataset
__UpperCAmelCase      			=  load_dataset(args.dataset_name, split="""train""", streaming=True)
__UpperCAmelCase      			=  iter(dataset)


# Training and saving
__UpperCAmelCase      			=  tokenizer.train_new_from_iterator(
    batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
)
new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)

