code: 
  import subprocess
  from argparse import ArgumentParser
  from typing import List, Union
  
  from huggingface_hub.hf_api import HfFolder, create_repo, whoami
  from requests.exceptions import HTTPError
  
  from . import BaseTransformersCLICommand
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	@staticmethod
  	def    snake_case__    (    lowercase			:ArgumentParser )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''login'''      ,    help='''Log in using the same credentials as on huggingface.co''' )
  				login_parser.set_defaults(func=lambda lowercase : LoginCommand(lowercase ) )
  				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''whoami'''      ,    help='''Find out which huggingface.co account you are logged in as.''' )
  				whoami_parser.set_defaults(func=lambda lowercase : WhoamiCommand(lowercase ) )
  				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''logout'''      ,    help='''Log out''' )
  				logout_parser.set_defaults(func=lambda lowercase : LogoutCommand(lowercase ) )
  
  				# new system: git-based repo system
  				SCREAMING_SNAKE_CASE    	=   parser.add_parser(
  				    '''repo'''      ,    help='''Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.'''      ,    )
  				SCREAMING_SNAKE_CASE    	=   repo_parser.add_subparsers(
  				    help='''Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands''' )
  				SCREAMING_SNAKE_CASE    	=   repo_subparsers.add_parser(
  				    '''create'''      ,    help='''Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co''' )
  				repo_create_parser.add_argument(
  				    '''name'''      ,    type=lowercase      ,    help='''Name for your model\'s repo. Will be namespaced under your username to build the model id.'''      ,    )
  				repo_create_parser.add_argument('''--organization'''      ,    type=lowercase      ,    help='''Optional: organization namespace.''' )
  				repo_create_parser.add_argument('''-y'''      ,    '''--yes'''      ,    action='''store_true'''      ,    help='''Optional: answer Yes to the prompt''' )
  				repo_create_parser.set_defaults(func=lambda lowercase : RepoCreateCommand(lowercase ) )
  
  
  
  
  
  class 						lowerCamelCase			:
  	UpperCamelCase_     :				List[str]								=  '\u001b[1m'
  	UpperCamelCase_     :				Union[str, Any]								=  '\u001b[31m'
  	UpperCamelCase_     :				Any								=  '\u001b[90m'
  	UpperCamelCase_     :				List[str]								=  '\u001b[0m'
  
  
  
  
  
  
  
  	@classmethod
  	def    snake_case__    (    cls			:List[Any]      ,    lowercase			:Any )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				return f"""{cls._bold}{s}{cls._reset}"""
  
  
  
  
  
  
  
  	@classmethod
  	def    snake_case__    (    cls			:Optional[Any]      ,    lowercase			:str )		->			Tuple:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				return f"""{cls._bold}{cls._red}{s}{cls._reset}"""
  
  
  
  
  
  
  
  	@classmethod
  	def    snake_case__    (    cls			:List[str]      ,    lowercase			:Any )		->			List[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				return f"""{cls._gray}{s}{cls._reset}"""
  
  
  
  def       a			(   a		,	a  )    ->str:
  
  
  
  
  			'''simple docstring'''
  
  
  			SCREAMING_SNAKE_CASE    	=   [max(len(str(a  )  ) for x in col  ) for col in zip(*a		,	a  )]
  			SCREAMING_SNAKE_CASE    	=   ('''{{:{}}} ''' * len(a  )).format(*a  )
  			SCREAMING_SNAKE_CASE    	=   []
  			lines.append(row_format.format(*a  )  )
  			lines.append(row_format.format(*['''-''' * w for w in col_widths]  )  )
  			for row in rows:
  						lines.append(row_format.format(*a  )  )
  			return "\n".join(a  )
  
  
  
  
  
  class 						lowerCamelCase			:
  
  
  
  
  
  
  
  	def __init__(    self			:List[str]      ,    lowercase			:Optional[Any] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   args
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				print(
  				    ANSI.red(
  				        '''ERROR! `huggingface-cli login` uses an outdated login mechanism '''
  				        '''that is not compatible with the Hugging Face Hub backend anymore. '''
  				        '''Please use `huggingface-cli login instead.''' ) )
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[Any] )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				print(
  				    ANSI.red(
  				        '''WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '''
  				        '''`huggingface-cli whoami` instead.''' ) )
  				SCREAMING_SNAKE_CASE    	=   HfFolder.get_token()
  				if token is None:
  							print('''Not logged in''' )
  							exit()
  				try:
  							SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   whoami(lowercase )
  							print(lowercase )
  							if orgs:
  										print(ANSI.bold('''orgs: ''' )      ,    ''','''.join(lowercase ) )
  				except HTTPError as e:
  							print(lowercase )
  							print(ANSI.red(e.response.text ) )
  							exit(1 )
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				print(
  				    ANSI.red(
  				        '''ERROR! `transformers-cli logout` uses an outdated logout mechanism '''
  				        '''that is not compatible with the Hugging Face Hub backend anymore. '''
  				        '''Please use `huggingface-cli logout instead.''' ) )
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				print(
  				    ANSI.red(
  				        '''WARNING! Managing repositories through transformers-cli is deprecated. '''
  				        '''Please use `huggingface-cli` instead.''' ) )
  				SCREAMING_SNAKE_CASE    	=   HfFolder.get_token()
  				if token is None:
  							print('''Not logged in''' )
  							exit(1 )
  				try:
  							SCREAMING_SNAKE_CASE    	=   subprocess.check_output(['''git''', '''--version'''] ).decode('''utf-8''' )
  							print(ANSI.gray(stdout.strip() ) )
  				except FileNotFoundError:
  							print('''Looks like you do not have git installed, please install.''' )
  
  				try:
  							SCREAMING_SNAKE_CASE    	=   subprocess.check_output(['''git-lfs''', '''--version'''] ).decode('''utf-8''' )
  							print(ANSI.gray(stdout.strip() ) )
  				except FileNotFoundError:
  							print(
  							    ANSI.red(
  							        '''Looks like you do not have git-lfs installed, please install.'''
  							        ''' You can install from https://git-lfs.github.com/.'''
  							        ''' Then run `git lfs install` (you only have to do this once).''' ) )
  				print('''''' )
  
  				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   whoami(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.args.organization if self.args.organization is not None else user
  				SCREAMING_SNAKE_CASE    	=   f"""{namespace}/{self.args.name}"""
  				print(f"""You are about to create {ANSI.bold(lowercase )}""" )
  
  				if not self.args.yes:
  							SCREAMING_SNAKE_CASE    	=   input('''Proceed? [Y/n] ''' ).lower()
  							if not (choice == "" or choice == "y" or choice == "yes"):
  										print('''Abort''' )
  										exit()
  				try:
  							SCREAMING_SNAKE_CASE    	=   create_repo(lowercase      ,    name=self.args.name      ,    organization=self.args.organization )
  				except HTTPError as e:
  							print(lowercase )
  							print(ANSI.red(e.response.text ) )
  							exit(1 )
  				print('''\nYour repo now lives at:''' )
  				print(f"""  {ANSI.bold(lowercase )}""" )
  				print('''\nYou can clone it locally with the command below, and commit/push as usual.''' )
  				print(f"""\n  git clone {url}""" )
  				print('''''' )
style_context: 
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class 						lowerCamelCase			(					__lowerCamelCase   ,		unittest.TestCase				):
  	UpperCamelCase_     :				List[str]								=  GPTNeoXJapaneseTokenizer
  	UpperCamelCase_     :				List[Any]								=  False
  	UpperCamelCase_     :				int								=  {'do_clean_text': False, 'add_prefix_space': False}
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().setUp()
  
  				SCREAMING_SNAKE_CASE    	=   [
  				    '''こん''',
  				    '''こんに''',
  				    '''にちは''',
  				    '''ばんは''',
  				    '''世界,㔺界''',
  				    '''、''',
  				    '''。''',
  				    '''<BR>''',
  				    '''<SP>''',
  				    '''<TAB>''',
  				    '''<URL>''',
  				    '''<EMAIL>''',
  				    '''<TEL>''',
  				    '''<DATE>''',
  				    '''<PRICE>''',
  				    '''<BLOCK>''',
  				    '''<KIGOU>''',
  				    '''<U2000U2BFF>''',
  				    '''<|emoji1|>''',
  				    '''<unk>''',
  				    '''<|startoftext|>''',
  				    '''<|endoftext|>''',
  				]
  				SCREAMING_SNAKE_CASE    	=   {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
  				SCREAMING_SNAKE_CASE    	=   {'''unk_token''': '''<unk>'''}
  
  				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''vocab_file'''] )
  				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''emoji_file'''] )
  				with open(self.vocab_file      ,    '''w'''      ,    encoding='''utf-8''' ) as vocab_writer:
  							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
  				with open(self.emoji_file      ,    '''w''' ) as emoji_writer:
  							emoji_writer.write(json.dumps(lowercase ) )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict      ,    **lowercase			:Optional[int] )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				kwargs.update(self.special_tokens_map )
  				return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,    **lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict      ,    lowercase			:Union[str, Any] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、世界。😀'''
  				return input_text, output_text
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:Union[str, Any] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   self.get_input_output_texts(lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode(lowercase      ,    add_special_tokens=lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.decode(lowercase      ,    clean_up_tokenization_spaces=lowercase )
  				return text, ids
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.get_tokenizer()
  
  				# Testing tokenization
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。　こんばんは、㔺界。'''
  				SCREAMING_SNAKE_CASE    	=   ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
  				SCREAMING_SNAKE_CASE    	=   tokenizer.tokenize(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  				# Testing conversion to ids without special tokens
  				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
  				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  				# Testing conversion to ids with special tokens
  				SCREAMING_SNAKE_CASE    	=   tokens + [tokenizer.unk_token]
  				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
  				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  
  
  
  
  
  
  	@slow
  	def    snake_case__    (    self			:str )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b''' )
  
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''ありがとう。'''      ,    add_special_tokens=lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''どういたしまして。'''      ,    add_special_tokens=lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase      ,    lowercase )
  
  				assert encoded_sentence == ids_a
  				assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[Any] )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Any )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass