




"""simple docstring"""





from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_lowercase						   =     logging.get_logger(__name__)

_lowercase						   =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase						   =     {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

_lowercase						   =     {
    """openai-gpt""": 5_12,
}

class    __lowercase							(      snake_case_						):


				'''simple docstring'''


				_lowerCAmelCase						      =VOCAB_FILES_NAMES
				_lowerCAmelCase						      =PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase						      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase						      =["input_ids", "attention_mask"]
				_lowerCAmelCase						      =OpenAIGPTTokenizer

				def __init__( self :  Dict   ,SCREAMING_SNAKE_CASE_ :  List[Any]=None   ,SCREAMING_SNAKE_CASE_ :  Union[str, Any]=None   ,SCREAMING_SNAKE_CASE_ :  List[str]=None   ,SCREAMING_SNAKE_CASE_ :  Dict="<unk>"   ,**SCREAMING_SNAKE_CASE_ :  Optional[int]   ):
										super().__init__(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,tokenizer_file=SCREAMING_SNAKE_CASE_   ,unk_token=SCREAMING_SNAKE_CASE_   ,**SCREAMING_SNAKE_CASE_   )

				@property
				def    _lowerCamelCase       ( self :  List[str]   ):
										return True





				def    _lowerCamelCase       ( self :  int   ,SCREAMING_SNAKE_CASE_ :  str   ,SCREAMING_SNAKE_CASE_ :  Optional[str] = None   ):
										snake_case__							:       Tuple				      =       self._tokenizer.model.save(SCREAMING_SNAKE_CASE_   ,name=SCREAMING_SNAKE_CASE_   )
										return tuple(SCREAMING_SNAKE_CASE_   )








"""simple docstring"""





import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_multi_gpu,
    require_single_gpu,
    test_sync,
)
from accelerate.utils import patch_environment

class    __lowercase							(      unittest.TestCase						):


				'''simple docstring'''


				def    _lowerCamelCase       ( self :  Any   ):
										snake_case__							:       Dict				      =       inspect.getfile(accelerate.test_utils   )
										snake_case__							:       Tuple				      =       os.path.sep.join(mod_file.split(os.path.sep   )[:-1] + ['''scripts''', '''test_sync.py''']   )

				@require_cpu
				def    _lowerCamelCase       ( self :  Optional[Any]   ):
										debug_launcher(test_sync.main   ,num_processes=1   )

				@require_cpu
				def    _lowerCamelCase       ( self :  List[str]   ):
										debug_launcher(test_sync.main   )

				@require_single_gpu
				def    _lowerCamelCase       ( self :  Union[str, Any]   ):
										test_sync.main()





				@require_multi_gpu
				def    _lowerCamelCase       ( self :  List[str]   ):
										print(F'''Found {torch.cuda.device_count()} devices.'''   )
										snake_case__							:       Dict				      =       ['''torchrun''', F'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
										with patch_environment(omp_num_threads=1   ):
																execute_subprocess_async(SCREAMING_SNAKE_CASE_   ,env=os.environ.copy()   )


