



"""simple docstring"""







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
     from ... import FeatureExtractionMixin, TensorType

lowercase__		:				int          =	logging.get_logger(__name__)

lowercase__		:				Optional[Any]          =	{
    """openai/imagegpt-small""": """""",
    """openai/imagegpt-medium""": """""",
    """openai/imagegpt-large""": """""",
}







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     """imagegpt"""
      _SCREAMING_SNAKE_CASE  				=     ["""past_key_values"""]
      _SCREAMING_SNAKE_CASE  				=     {
          """hidden_size""": """n_embd""",
          """max_position_embeddings""": """n_positions""",
          """num_attention_heads""": """n_head""",
          """num_hidden_layers""": """n_layer""",
      }



      def __init__(      self				:      Union[str, Any]    ,			SCREAMING_SNAKE_CASE_				:      Union[str, Any]=5_1_2 + 1    ,			SCREAMING_SNAKE_CASE_				:      str=3_2 * 3_2    ,			SCREAMING_SNAKE_CASE_				:      int=5_1_2    ,			SCREAMING_SNAKE_CASE_				:      List[str]=2_4    ,			SCREAMING_SNAKE_CASE_				:      Any=8    ,			SCREAMING_SNAKE_CASE_				:      str=None    ,			SCREAMING_SNAKE_CASE_				:      Dict="quick_gelu"    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=0.1    ,			SCREAMING_SNAKE_CASE_				:      Union[str, Any]=0.1    ,			SCREAMING_SNAKE_CASE_				:      str=0.1    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=1E-5    ,			SCREAMING_SNAKE_CASE_				:      int=0.02    ,			SCREAMING_SNAKE_CASE_				:      List[Any]=True    ,			SCREAMING_SNAKE_CASE_				:      Dict=True    ,			SCREAMING_SNAKE_CASE_				:      Any=False    ,			SCREAMING_SNAKE_CASE_				:      Dict=False    ,			SCREAMING_SNAKE_CASE_				:      Tuple=False    ,			**SCREAMING_SNAKE_CASE_				:      List[Any]    ,			):
        lowerCAmelCase_   :				Optional[Any]								= vocab_size
        lowerCAmelCase_   :				int								= n_positions
        lowerCAmelCase_   :				Optional[int]								= n_embd
        lowerCAmelCase_   :				Dict								= n_layer
        lowerCAmelCase_   :				int								= n_head
        lowerCAmelCase_   :				List[str]								= n_inner
        lowerCAmelCase_   :				Any								= activation_function
        lowerCAmelCase_   :				Union[str, Any]								= resid_pdrop
        lowerCAmelCase_   :				Union[str, Any]								= embd_pdrop
        lowerCAmelCase_   :				List[str]								= attn_pdrop
        lowerCAmelCase_   :				Dict								= layer_norm_epsilon
        lowerCAmelCase_   :				int								= initializer_range
        lowerCAmelCase_   :				Optional[int]								= scale_attn_weights
        lowerCAmelCase_   :				Union[str, Any]								= use_cache
        lowerCAmelCase_   :				List[str]								= scale_attn_by_inverse_layer_idx
        lowerCAmelCase_   :				str								= reorder_and_upcast_attn
        lowerCAmelCase_   :				List[str]								= tie_word_embeddings

        super().__init__(tie_word_embeddings=SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      @property
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        return OrderedDict(
            [
                ('input_ids', {0: 'batch', 1: 'sequence'}),
            ]       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			SCREAMING_SNAKE_CASE_				:      "FeatureExtractionMixin"    ,			SCREAMING_SNAKE_CASE_				:      int = 1    ,			SCREAMING_SNAKE_CASE_				:      int = -1    ,			SCREAMING_SNAKE_CASE_				:      bool = False    ,			SCREAMING_SNAKE_CASE_				:      Optional["TensorType"] = None    ,			SCREAMING_SNAKE_CASE_				:      int = 3    ,			SCREAMING_SNAKE_CASE_				:      int = 3_2    ,			SCREAMING_SNAKE_CASE_				:      int = 3_2    ,			):
        lowerCAmelCase_   :				Any								= self._generate_dummy_images(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				int								= dict(preprocessor(images=SCREAMING_SNAKE_CASE_    ,			return_tensors=SCREAMING_SNAKE_CASE_       )       )

        return inputs










"""simple docstring"""







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


lowercase__		:				Optional[Any]          =	logging.get_logger(__name__)

lowercase__		:				List[str]          =	{"""tokenizer_file""": """tokenizer.json"""}

lowercase__		:				List[str]          =	{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE  				=     PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE  				=     ["""input_ids""", """attention_mask"""]
      _SCREAMING_SNAKE_CASE  				=     None



      def __init__(      self				:      Union[str, Any]    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]=None    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=None    ,			SCREAMING_SNAKE_CASE_				:      Tuple=None    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]="<unk>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="<s>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="</s>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="<pad>"    ,			SCREAMING_SNAKE_CASE_				:      List[str]=False    ,			SCREAMING_SNAKE_CASE_				:      Any=False    ,			**SCREAMING_SNAKE_CASE_				:      Any    ,			):
        super().__init__(
            SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			tokenizer_file=SCREAMING_SNAKE_CASE_    ,			unk_token=SCREAMING_SNAKE_CASE_    ,			bos_token=SCREAMING_SNAKE_CASE_    ,			eos_token=SCREAMING_SNAKE_CASE_    ,			pad_token=SCREAMING_SNAKE_CASE_    ,			add_prefix_space=SCREAMING_SNAKE_CASE_    ,			clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_    ,			)
        lowerCAmelCase_   :				Optional[Any]								= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
        if pre_tok_state.get('add_prefix_space'    ,			SCREAMING_SNAKE_CASE_       ) != add_prefix_space:
          lowerCAmelCase_   :				str								= getattr(SCREAMING_SNAKE_CASE_    ,			pre_tok_state.pop('type'       )       )
          lowerCAmelCase_   :				str								= add_prefix_space
          lowerCAmelCase_   :				str								= pre_tok_class(**SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				str								= add_prefix_space



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        lowerCAmelCase_   :				List[Any]								= kwargs.get('is_split_into_words'    ,			SCREAMING_SNAKE_CASE_       )
        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
              ' pretokenized inputs.'       )

        return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        lowerCAmelCase_   :				List[Any]								= kwargs.get('is_split_into_words'    ,			SCREAMING_SNAKE_CASE_       )

        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
              ' pretokenized inputs.'       )

        return super()._encode_plus(*SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]    ,			SCREAMING_SNAKE_CASE_				:      str    ,			SCREAMING_SNAKE_CASE_				:      Optional[str] = None       ):
        lowerCAmelCase_   :				Optional[int]								= self._tokenizer.model.save(SCREAMING_SNAKE_CASE_    ,			name=SCREAMING_SNAKE_CASE_       )
        return tuple(SCREAMING_SNAKE_CASE_       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      int    ,			SCREAMING_SNAKE_CASE_				:      "Conversation"       ):
        lowerCAmelCase_   :				int								= []
        for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       ) + [self.eos_token_id]       )

        if len(SCREAMING_SNAKE_CASE_       ) > self.model_max_length:
          lowerCAmelCase_   :				Any								= input_ids[-self.model_max_length :]
        return input_ids





