






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


UpperCAmelCase_ :						int      =      logging.get_logger(__name__)

UpperCAmelCase_ :						List[Any]      =      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase_ :						Union[str, Any]      =      {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

UpperCAmelCase_ :						Optional[int]      =      {
    '''gpt2''': 10_24,
    '''gpt2-medium''': 10_24,
    '''gpt2-large''': 10_24,
    '''gpt2-xl''': 10_24,
    '''distilgpt2''': 10_24,
}

class     _SCREAMING_SNAKE_CASE       (			__lowercase      ):
	snake_case__							:     int     	=  VOCAB_FILES_NAMES
	snake_case__							:     Optional[int]     	=  PRETRAINED_VOCAB_FILES_MAP
	snake_case__							:     List[Any]     	=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__							:     int     	=  ["input_ids", "attention_mask"]
	snake_case__							:     Union[str, Any]     	=  GPTaTokenizer





	def __init__(		self				:					Optional[int] ,  __lowerCamelCase				:					List[Any]=None ,  __lowerCamelCase				:					List[str]=None ,  __lowerCamelCase				:					Dict=None ,  __lowerCamelCase				:					Dict="<|endoftext|>" ,  __lowerCamelCase				:					List[str]="<|endoftext|>" ,  __lowerCamelCase				:					str="<|endoftext|>" ,  __lowerCamelCase				:					List[Any]=False ,  **__lowerCamelCase				:					Dict ,  ):
		super().__init__(
		    snake_case_ ,  snake_case_ ,  tokenizer_file=snake_case_ ,  unk_token=snake_case_ ,  bos_token=snake_case_ ,  eos_token=snake_case_ ,  add_prefix_space=snake_case_ ,  **snake_case_ ,  )

		UpperCamelCase     :Dict          =	kwargs.pop("""add_bos_token""" ,  snake_case_	)

		UpperCamelCase     :Optional[int]          =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
		if pre_tok_state.get("""add_prefix_space""" ,  snake_case_	) != add_prefix_space:
			UpperCamelCase     :Dict          =	getattr(snake_case_ ,  pre_tok_state.pop("""type"""	)	)
			UpperCamelCase     :List[Any]          =	add_prefix_space
			UpperCamelCase     :Optional[Any]          =	pre_tok_class(**snake_case_	)

		UpperCamelCase     :Optional[Any]          =	add_prefix_space





	def 						_A (		self				:					int ,  *__lowerCamelCase				:					Optional[int] ,  **__lowerCamelCase				:					Optional[int]	):
		UpperCamelCase     :Optional[int]          =	kwargs.get("""is_split_into_words""" ,  snake_case_	)
		assert self.add_prefix_space or not is_split_into_words, (
		    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
		    "to use it with pretokenized inputs."
		)

		return super()._batch_encode_plus(*snake_case_ ,  **snake_case_	)





	def 						_A (		self				:					List[Any] ,  *__lowerCamelCase				:					Tuple ,  **__lowerCamelCase				:					List[Any]	):
		UpperCamelCase     :Any          =	kwargs.get("""is_split_into_words""" ,  snake_case_	)

		assert self.add_prefix_space or not is_split_into_words, (
		    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
		    "to use it with pretokenized inputs."
		)

		return super()._encode_plus(*snake_case_ ,  **snake_case_	)





	def 						_A (		self				:					Union[str, Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Optional[str] = None	):
		UpperCamelCase     :Union[str, Any]          =	self._tokenizer.model.save(snake_case_ ,  name=snake_case_	)
		return tuple(snake_case_	)





	def 						_A (		self				:					Optional[Any] ,  __lowerCamelCase				:					"Conversation"	):
		UpperCamelCase     :Union[str, Any]          =	[]
		for is_user, text in conversation.iter_texts():
			input_ids.extend(self.encode(snake_case_ ,  add_special_tokens=snake_case_	) + [self.eos_token_id]	)

		if len(snake_case_	) > self.model_max_length:
			UpperCamelCase     :Tuple          =	input_ids[-self.model_max_length :]
		return input_ids


import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowercase__       ={
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

lowercase__       =logging.WARNING




def    __UpperCamelCase     (				):
 __a :  Optional[Any]						=  os.getenv('''DATASETS_VERBOSITY'''  ,   lowerCAmelCase__  )
 if env_level_str:
  if env_level_str in log_levels:
   return log_levels[env_level_str]
  else:
   logging.getLogger().warning(
       f"Unknown option DATASETS_VERBOSITY={env_level_str}, "
       f"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
 return _default_log_level




def    __UpperCamelCase     (				):
 return __name__.split('''.'''  )[0]




def    __UpperCamelCase     (				):
 return logging.getLogger(_get_library_name()  )




def    __UpperCamelCase     (				):
 # Apply our default configuration to the library root logger.
 __a :  str						=  _get_library_root_logger()
 library_root_logger.setLevel(_get_default_logging_level()  )




def    __UpperCamelCase     (				):
 __a :  Any						=  _get_library_root_logger()
 library_root_logger.setLevel(logging.NOTSET  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[str] = None  ):
 if name is None:
  __a :  Union[str, Any]						=  _get_library_name()
 return logging.getLogger(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return _get_library_root_logger().getEffectiveLevel()




def    __UpperCamelCase     (				lowerCAmelCase__							:				int  ):
 _get_library_root_logger().setLevel(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 __a :  Union[str, Any]						=  False




def    __UpperCamelCase     (				):
 __a :  Tuple						=  True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()



class 		UpperCamelCase__  :
   def __init__(self :					str					,				*snake_case_ :					str					,				**snake_case_ :					Union[str, Any]   ):  # pylint: disable=unused-argument
    __a :  Optional[Any]						=  args[0] if args else None
   def __iter__(self :					List[str]   ):
    return iter(self._iterator   )
   def __getattr__(self :					str					,				snake_case_ :					Optional[Any]   ):

    def empty_fn(*snake_case_ :					int					,				**snake_case_ :					int   ):  # pylint: disable=unused-argument
     return

    return empty_fn
   def __enter__(self :					Union[str, Any]   ):
    return self







   def __exit__(self :					str					,				snake_case_ :					List[str]					,				snake_case_ :					int					,				snake_case_ :					Optional[Any]   ):
    return





lowercase__       =True



class 		UpperCamelCase__  :
   def __call__(self :					Tuple					,				*snake_case_ :					str					,				snake_case_ :					str=False					,				**snake_case_ :					Dict   ):
    if _tqdm_active and not disable:
     return tqdm_lib.tqdm(*snake_case_					,				**snake_case_   )
    else:
     return EmptyTqdm(*snake_case_					,				**snake_case_   )
   def       lowerCAmelCase			(self :					Optional[Any]					,				*snake_case_ :					Union[str, Any]					,				**snake_case_ :					Optional[Any]   ):
    __a :  List[Any]						=  None
    if _tqdm_active:
     return tqdm_lib.tqdm.set_lock(*snake_case_					,				**snake_case_   )







   def       lowerCAmelCase			(self :					str   ):
    if _tqdm_active:
     return tqdm_lib.tqdm.get_lock()





lowercase__       =_tqdm_cls()




def    __UpperCamelCase     (				):
 global _tqdm_active
 return bool(_tqdm_active  )




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Dict						=  True




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Union[str, Any]						=  False
