


"""simple docstring"""







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_   :   Any									=  logging.get_logger(__name__)

lowerCamelCase_   :   List[str]									=  {
    """asapp/sew-d-tiny-100k""": """https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json""",
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}






class       __A				(       _SCREAMING_SNAKE_CASE			):






 """simple docstring"""




 __lowerCAmelCase 					=  "sew-d"


 def __init__(		self ,    __A=32 ,    __A=768 ,    __A=12 ,    __A=12 ,    __A=3072 ,    __A=2 ,    __A=512 ,    __A=256 ,    __A=True ,    __A=True ,    __A=("p2c", "c2p") ,    __A="layer_norm" ,    __A="gelu_python" ,    __A=0.1 ,    __A=0.1 ,    __A=0.1 ,    __A=0.0 ,    __A=0.1 ,    __A=0.02 ,    __A=1E-7 ,    __A=1E-5 ,    __A="group" ,    __A="gelu" ,    __A=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512) ,    __A=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1) ,    __A=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1) ,    __A=False ,    __A=128 ,    __A=16 ,    __A=True ,    __A=0.05 ,    __A=10 ,    __A=2 ,    __A=0.0 ,    __A=10 ,    __A=0 ,    __A="mean" ,    __A=False ,    __A=False ,    __A=256 ,    __A=0 ,    __A=1 ,    __A=2 ,    **__A ,    )					->     Optional[Any]:
     super().__init__(**__A ,    pad_token_id=__A ,    bos_token_id=__A ,    eos_token_id=__A						)
     a 		=hidden_size
     a 		=feat_extract_norm
     a 		=feat_extract_activation
     a 		=list(__A						)
     a 		=list(__A						)
     a 		=list(__A						)
     a 		=conv_bias
     a 		=num_conv_pos_embeddings
     a 		=num_conv_pos_embedding_groups
     a 		=len(self.conv_dim						)
     a 		=num_hidden_layers
     a 		=intermediate_size
     a 		=squeeze_factor
     a 		=max_position_embeddings
     a 		=position_buckets
     a 		=share_att_key
     a 		=relative_attention
     a 		=norm_rel_ebd
     a 		=list(__A						)
     a 		=hidden_act
     a 		=num_attention_heads
     a 		=hidden_dropout
     a 		=attention_dropout
     a 		=activation_dropout
     a 		=feat_proj_dropout
     a 		=final_dropout
     a 		=layer_norm_eps
     a 		=feature_layer_norm_eps
     a 		=initializer_range
     a 		=vocab_size

     if (
         (len(self.conv_stride						) != self.num_feat_extract_layers)
         or (len(self.conv_kernel						) != self.num_feat_extract_layers)
         or (len(self.conv_dim						) != self.num_feat_extract_layers)
     ):
         raise ValueError(
             '''Configuration for convolutional layers is incorrect.'''
             '''It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,'''
             f'''but is `len(config.conv_dim) = {len(self.conv_dim						)}`, `len(config.conv_stride)'''
             f'''= {len(self.conv_stride						)}`, `len(config.conv_kernel) = {len(self.conv_kernel						)}`.'''						)

     # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
     a 		=apply_spec_augment
     a 		=mask_time_prob
     a 		=mask_time_length
     a 		=mask_time_min_masks
     a 		=mask_feature_prob
     a 		=mask_feature_length
     a 		=mask_feature_min_masks

     # ctc loss
     a 		=ctc_loss_reduction
     a 		=ctc_zero_infinity

     # sequence classification
     a 		=use_weighted_layer_sum
     a 		=classifier_proj_size





 @property
 def       SCREAMING_SNAKE_CASE							(		self						)					->     Dict:
     return functools.reduce(operator.mul ,    self.conv_stride ,    1						)




"""simple docstring"""







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


lowercase__		:				Optional[Any]          =	logging.get_logger(__name__)

lowercase__		:				List[str]          =	{"""tokenizer_file""": """tokenizer.json"""}

lowercase__		:				List[str]          =	{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE  				=     PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE  				=     ["""input_ids""", """attention_mask"""]
      _SCREAMING_SNAKE_CASE  				=     None



      def __init__(      self				:      Union[str, Any]    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]=None    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=None    ,			SCREAMING_SNAKE_CASE_				:      Tuple=None    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]="<unk>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="<s>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="</s>"    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]="<pad>"    ,			SCREAMING_SNAKE_CASE_				:      List[str]=False    ,			SCREAMING_SNAKE_CASE_				:      Any=False    ,			**SCREAMING_SNAKE_CASE_				:      Any    ,			):
        super().__init__(
            SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			tokenizer_file=SCREAMING_SNAKE_CASE_    ,			unk_token=SCREAMING_SNAKE_CASE_    ,			bos_token=SCREAMING_SNAKE_CASE_    ,			eos_token=SCREAMING_SNAKE_CASE_    ,			pad_token=SCREAMING_SNAKE_CASE_    ,			add_prefix_space=SCREAMING_SNAKE_CASE_    ,			clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_    ,			)
        lowerCAmelCase_   :				Optional[Any]								= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
        if pre_tok_state.get('add_prefix_space'    ,			SCREAMING_SNAKE_CASE_       ) != add_prefix_space:
          lowerCAmelCase_   :				str								= getattr(SCREAMING_SNAKE_CASE_    ,			pre_tok_state.pop('type'       )       )
          lowerCAmelCase_   :				str								= add_prefix_space
          lowerCAmelCase_   :				str								= pre_tok_class(**SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				str								= add_prefix_space



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      List[Any]    ,			**SCREAMING_SNAKE_CASE_				:      Tuple       ):
        lowerCAmelCase_   :				List[Any]								= kwargs.get('is_split_into_words'    ,			SCREAMING_SNAKE_CASE_       )
        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
              ' pretokenized inputs.'       )

        return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			*SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			**SCREAMING_SNAKE_CASE_				:      int       ):
        lowerCAmelCase_   :				List[Any]								= kwargs.get('is_split_into_words'    ,			SCREAMING_SNAKE_CASE_       )

        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
              ' pretokenized inputs.'       )

        return super()._encode_plus(*SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]    ,			SCREAMING_SNAKE_CASE_				:      str    ,			SCREAMING_SNAKE_CASE_				:      Optional[str] = None       ):
        lowerCAmelCase_   :				Optional[int]								= self._tokenizer.model.save(SCREAMING_SNAKE_CASE_    ,			name=SCREAMING_SNAKE_CASE_       )
        return tuple(SCREAMING_SNAKE_CASE_       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      int    ,			SCREAMING_SNAKE_CASE_				:      "Conversation"       ):
        lowerCAmelCase_   :				int								= []
        for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       ) + [self.eos_token_id]       )

        if len(SCREAMING_SNAKE_CASE_       ) > self.model_max_length:
          lowerCAmelCase_   :				Any								= input_ids[-self.model_max_length :]
        return input_ids





