code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  from collections import OrderedDict
  from typing import Any, List, Mapping, Optional
  
  from ... import PreTrainedTokenizer, TensorType, is_torch_available
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfigWithPast, PatchingSpec
  from ...utils import logging
  
  
  _A							    =	logging.get_logger(__name__)
  
  _A							    =	{
      'gpt2': 'https://huggingface.co/gpt2/resolve/main/config.json',
      'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/config.json',
      'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/config.json',
      'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/config.json',
      'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/config.json',
  }
  
  
  class 							_lowerCAmelCase		(     UpperCamelCase__	):
  				lowerCamelCase__									=    'gpt2'
  				lowerCamelCase__									=    ['past_key_values']
  				lowerCamelCase__									=    {
  				    'hidden_size': 'n_embd',
  				    'max_position_embeddings': 'n_positions',
  				    'num_attention_heads': 'n_head',
  				    'num_hidden_layers': 'n_layer',
  				}
  
  
  
  
  				def __init__( self		,		snake_case_=50_257		,		snake_case_=1_024		,		snake_case_=768		,		snake_case_=12		,		snake_case_=12		,		snake_case_=None		,		snake_case_="gelu_new"		,		snake_case_=0.1		,		snake_case_=0.1		,		snake_case_=0.1		,		snake_case_=1E-5		,		snake_case_=0.02		,		snake_case_="cls_index"		,		snake_case_=True		,		snake_case_=None		,		snake_case_=True		,		snake_case_=0.1		,		snake_case_=True		,		snake_case_=True		,		snake_case_=50_256		,		snake_case_=50_256		,		snake_case_=False		,		snake_case_=False		,		**snake_case_		,		)    ->      Union[str, Any]:
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =vocab_size
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =n_positions
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =n_embd
  											SCREAMING_SNAKE_CASE			: Tuple	     =n_layer
  											SCREAMING_SNAKE_CASE			: Any	     =n_head
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =n_inner
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =activation_function
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =resid_pdrop
  											SCREAMING_SNAKE_CASE			: Dict	     =embd_pdrop
  											SCREAMING_SNAKE_CASE			: List[Any]	     =attn_pdrop
  											SCREAMING_SNAKE_CASE			: Tuple	     =layer_norm_epsilon
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =initializer_range
  											SCREAMING_SNAKE_CASE			: List[str]	     =summary_type
  											SCREAMING_SNAKE_CASE			: Tuple	     =summary_use_proj
  											SCREAMING_SNAKE_CASE			: List[Any]	     =summary_activation
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =summary_first_dropout
  											SCREAMING_SNAKE_CASE			: int	     =summary_proj_to_labels
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =scale_attn_weights
  											SCREAMING_SNAKE_CASE			: Any	     =use_cache
  											SCREAMING_SNAKE_CASE			: Tuple	     =scale_attn_by_inverse_layer_idx
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =reorder_and_upcast_attn
  
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =bos_token_id
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =eos_token_id
  
  											super().__init__(bos_token_id=snake_case_		,		eos_token_id=snake_case_		,		**snake_case_							)
  
  
  
  
  
  
  
  class 							_lowerCAmelCase		(     UpperCamelCase__	):
  
  
  
  
  				def __init__( self		,		snake_case_		,		snake_case_ = "default"		,		snake_case_ = None		,		snake_case_ = False		,		)    ->      str:
  											super().__init__(snake_case_		,		task=snake_case_		,		patching_specs=snake_case_		,		use_past=snake_case_							)
  											if not getattr(self._config		,		'''pad_token_id'''		,		snake_case_							):
  																		# TODO: how to do that better?
  																		SCREAMING_SNAKE_CASE			: Optional[int]	     =0
  
  
  
  
  				@property
  				def     __a		( self							)    ->      Mapping[str, Mapping[int, str]]:
  											SCREAMING_SNAKE_CASE			: List[str]	     =OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}							)
  											if self.use_past:
  																		self.fill_with_past_key_values_(snake_case_		,		direction='''inputs'''							)
  																		SCREAMING_SNAKE_CASE			: Union[str, Any]	     ={0: '''batch''', 1: '''past_sequence + sequence'''}
  											else:
  																		SCREAMING_SNAKE_CASE			: Tuple	     ={0: '''batch''', 1: '''sequence'''}
  
  											return common_inputs
  
  
  
  
  				@property
  				def     __a		( self							)    ->      int:
  											return self._config.n_layer
  
  
  
  
  				@property
  				def     __a		( self							)    ->      int:
  											return self._config.n_head
  
  
  
  
  				def     __a		( self		,		snake_case_		,		snake_case_ = -1		,		snake_case_ = -1		,		snake_case_ = False		,		snake_case_ = None		,		)    ->      Mapping[str, Any]:
  											SCREAMING_SNAKE_CASE			: int	     =super(snake_case_		,		self							).generate_dummy_inputs(
  											    snake_case_		,		batch_size=snake_case_		,		seq_length=snake_case_		,		is_pair=snake_case_		,		framework=snake_case_							)
  
  											# We need to order the input in the way they appears in the forward()
  											SCREAMING_SNAKE_CASE			: List[Any]	     =OrderedDict({'''input_ids''': common_inputs['''input_ids''']}							)
  
  											# Need to add the past_keys
  											if self.use_past:
  																		if not is_torch_available():
  																									raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''							)
  																		else:
  																									import torch
  
  																									SCREAMING_SNAKE_CASE			: Optional[Any]	     =common_inputs['''input_ids'''].shape
  																									# Not using the same length for past_key_values
  																									SCREAMING_SNAKE_CASE			: Optional[Any]	     =seqlen + 2
  																									SCREAMING_SNAKE_CASE			: Union[str, Any]	     =(
  																									    batch,
  																									    self.num_attention_heads,
  																									    past_key_values_length,
  																									    self._config.hidden_size // self.num_attention_heads,
  																									)
  																									SCREAMING_SNAKE_CASE			: List[Any]	     =[
  																									    (torch.zeros(snake_case_							), torch.zeros(snake_case_							)) for _ in range(self.num_layers							)
  																									]
  
  											SCREAMING_SNAKE_CASE			: int	     =common_inputs['''attention_mask''']
  											if self.use_past:
  																		SCREAMING_SNAKE_CASE			: int	     =ordered_inputs['''attention_mask'''].dtype
  																		SCREAMING_SNAKE_CASE			: Dict	     =torch.cat(
  																		    [ordered_inputs['''attention_mask'''], torch.ones(snake_case_		,		snake_case_		,		dtype=snake_case_							)]		,		dim=1							)
  
  											return ordered_inputs
  
  
  
  
  
  
  
  				@property
  				def     __a		( self							)    ->      int:
  											return 13
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
  from ...utils import logging
  
  
  _A							    =	logging.get_logger(__name__)
  
  _A							    =	{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}
  
  _A							    =	{
      """vocab_file""": {
          """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
      },
      """tokenizer_file""": {
          """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json""",
      },
  }
  
  
  class 							_lowerCAmelCase		(     UpperCamelCase__	):
  
  
  
  
  				def __init__( self		,		snake_case_=None		,		snake_case_=None		,		snake_case_=False		,		snake_case_=True		,		snake_case_=False		,		snake_case_="<s>"		,		snake_case_="</s>"		,		snake_case_="<unk>"		,		snake_case_="<sep>"		,		snake_case_="<pad>"		,		snake_case_="<cls>"		,		snake_case_="<mask>"		,		snake_case_=["<eop>", "<eod>"]		,		**snake_case_		,		)    ->      str:
  											SCREAMING_SNAKE_CASE			: Optional[int]	     =AddedToken(snake_case_		,		lstrip=snake_case_		,		rstrip=snake_case_							) if isinstance(snake_case_		,		snake_case_							) else mask_token
  
  											super().__init__(
  											    vocab_file=snake_case_		,		tokenizer_file=snake_case_		,		do_lower_case=snake_case_		,		remove_space=snake_case_		,		keep_accents=snake_case_		,		bos_token=snake_case_		,		eos_token=snake_case_		,		unk_token=snake_case_		,		sep_token=snake_case_		,		pad_token=snake_case_		,		cls_token=snake_case_		,		mask_token=snake_case_		,		additional_special_tokens=snake_case_		,		**snake_case_		,		)
  
  											SCREAMING_SNAKE_CASE			: Dict	     =3
  											SCREAMING_SNAKE_CASE			: Dict	     =do_lower_case
  											SCREAMING_SNAKE_CASE			: int	     =remove_space
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =keep_accents
  											SCREAMING_SNAKE_CASE			: Tuple	     =vocab_file
  											SCREAMING_SNAKE_CASE			: Optional[Any]	     =False if not self.vocab_file else True
  
  											try:
  																		import jieba
  											except ModuleNotFoundError as error:
  																		raise error.__class__(
  																		    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
  																		    '''See https://pypi.org/project/jieba/ for installation.'''							)
  											SCREAMING_SNAKE_CASE			: str	     =jieba
  											SCREAMING_SNAKE_CASE			: Dict	     =str.maketrans(''' \n'''		,		'''\u2582\u2583'''							)
  
  
  
  
  				def     __a		( self		,		snake_case_		,		snake_case_ = None							)    ->      List[int]:
  											SCREAMING_SNAKE_CASE			: Dict	     =[self.sep_token_id]
  											SCREAMING_SNAKE_CASE			: Any	     =[self.cls_token_id]
  											if token_ids_a is None:
  																		return token_ids_a + sep + cls
  											return token_ids_a + sep + token_ids_a + sep + cls
  
  
  
  
  				def     __a		( self		,		snake_case_		,		snake_case_ = None							)    ->      List[int]:
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =[self.sep_token_id]
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =[2]
  
  											if token_ids_a is None:
  																		return len(token_ids_a + sep							) * [0] + cls_segment_id
  											return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id
  
  
  
  
  				def     __a		( self		,		snake_case_		,		snake_case_ = None							)    ->      Tuple[str]:
  											if not self.can_save_slow_tokenizer:
  																		raise ValueError(
  																		    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
  																		    '''tokenizer.'''							)
  
  											if not os.path.isdir(snake_case_							):
  																		logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
  																		return
  											SCREAMING_SNAKE_CASE			: Union[str, Any]	     =os.path.join(
  											    snake_case_		,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
  
  											if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case_							):
  																		copyfile(self.vocab_file		,		snake_case_							)
  
  											return (out_vocab_file,)
  
  
  
  
  				def     __a		( self		,		snake_case_		,		*snake_case_		,		**snake_case_							)    ->      Optional[Any]:
  											SCREAMING_SNAKE_CASE			: int	     =[
  											    ''' '''.join([x.translate(self.translator							) for x in self.jieba.cut(snake_case_		,		cut_all=snake_case_							)]							)
  											    for text in batch_text_or_text_pairs
  											]
  											return super()._batch_encode_plus(snake_case_		,		*snake_case_		,		**snake_case_							)
  
  
  
  
  
  
  
  				def     __a		( self		,		*snake_case_		,		**snake_case_							)    ->      str:
  											SCREAMING_SNAKE_CASE			: List[Any]	     =super()._decode(*snake_case_		,		**snake_case_							)
  											SCREAMING_SNAKE_CASE			: List[Any]	     =text.replace(''' '''		,		''''''							).replace('''\u2582'''		,		''' '''							).replace('''\u2583'''		,		'''\n'''							)
  											return text
  
  
  
  
  
  
  