



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case						:Union[str, Any]      		=     logging.get_logger(__name__)

__snake_case						:Any      		=     {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						'''switch_transformers'''
       UpperCamelCase__  :    Optional[Any]											=						['''past_key_values''']
       UpperCamelCase__  :    Optional[Any]											=						{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str=32_128     ,		__SCREAMING_SNAKE_CASE							:     int=768     ,		__SCREAMING_SNAKE_CASE							:     Any=64     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=2_048     ,		__SCREAMING_SNAKE_CASE							:     List[str]=64     ,		__SCREAMING_SNAKE_CASE							:     int=12     ,		__SCREAMING_SNAKE_CASE							:     Any=3     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=12     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=3     ,		__SCREAMING_SNAKE_CASE							:     Any=12     ,		__SCREAMING_SNAKE_CASE							:     Tuple=8     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=False     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=0.01     ,		__SCREAMING_SNAKE_CASE							:     Dict="float32"     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=False     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=32     ,		__SCREAMING_SNAKE_CASE							:     int=128     ,		__SCREAMING_SNAKE_CASE							:     Any=0.1     ,		__SCREAMING_SNAKE_CASE							:     int=1E-6     ,		__SCREAMING_SNAKE_CASE							:     Dict=0.0_01     ,		__SCREAMING_SNAKE_CASE							:     List[str]=0.0_01     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=1.0     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="relu"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=False     ,		__SCREAMING_SNAKE_CASE							:     str=True     ,		__SCREAMING_SNAKE_CASE							:     int=0     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=1     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           __a						       =				vocab_size
           __a						       =				d_model
           __a						       =				d_kv
           __a						       =				d_ff

           __a						       =				num_sparse_encoder_layers

           __a						       =				num_layers
           __a						       =				(
               num_decoder_layers if num_decoder_layers is not None else self.num_layers
           )  # default = symmetry
           __a						       =				num_sparse_decoder_layers

           # This tells us, each how many encoder layer we'll have to set a sparse layer.
           if self.num_sparse_encoder_layers > 0:
               __a						       =				self.num_layers // self.num_sparse_encoder_layers
           else:
               __a						       =				self.num_layers  # HACK: this will create 0 sparse layers

           # This tells us, each how many encoder layer we'll have to set a sparse layer.
           if self.num_sparse_decoder_layers > 0:
               __a						       =				self.num_decoder_layers // self.num_sparse_decoder_layers
           else:
               __a						       =				self.num_decoder_layers  # HACK: this will create 0 sparse layers

           __a						       =				num_heads
           __a						       =				num_experts
           __a						       =				expert_capacity
           __a						       =				router_bias
           __a						       =				router_jitter_noise
           if router_dtype not in ["float32", "float16", "bfloat16"]:
               raise ValueError(F'`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}')
           __a						       =				router_dtype

           __a						       =				router_ignore_padding_tokens
           __a						       =				relative_attention_num_buckets
           __a						       =				relative_attention_max_distance

           __a						       =				dropout_rate
           __a						       =				layer_norm_epsilon
           __a						       =				initializer_factor
           __a						       =				feed_forward_proj
           __a						       =				use_cache
           __a						       =				add_router_probs

           __a						       =				router_z_loss_coef
           __a						       =				router_aux_loss_coef

           __a						       =				self.feed_forward_proj.split('''-''')
           __a						       =				act_info[-1]
           __a						       =				act_info[0] == '''gated'''

           if len(__SCREAMING_SNAKE_CASE) > 1 and act_info[0] != "gated" or len(__SCREAMING_SNAKE_CASE) > 2:
               raise ValueError(
                   F'`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'
                   '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
                   '''\'gated-gelu\' or \'relu\'''')

           # for backwards compatibility
           if feed_forward_proj == "gated-gelu":
               __a						       =				'''gelu_new'''

           super().__init__(
               pad_token_id=__SCREAMING_SNAKE_CASE     ,		eos_token_id=__SCREAMING_SNAKE_CASE     ,		is_encoder_decoder=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)













"""simple docstring"""





import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class      a						(					__snake_case		):






    def __init__(       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							str = "▁"       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "<unk>"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "</s>"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "<pad>"       ,						)     ->		Optional[int]:
      lowerCamelCase_         =						{
          'pad': {'id': 0, 'token': pad_token},
          'eos': {'id': 1, 'token': eos_token},
          'unk': {'id': 2, 'token': unk_token},
      }

      lowerCamelCase_         =						[None] * len(self.special_tokens					)
      for token_dict in self.special_tokens.values():
        lowerCamelCase_         =						token_dict['token']

      lowerCamelCase_         =						Tokenizer(Unigram()					)

      lowerCamelCase_         =						normalizers.Sequence(
          [
              normalizers.Nmt(),
              normalizers.NFKC(),
              normalizers.Replace(Regex(' {2,}'					)       ,						' '					),
              normalizers.Lowercase(),
          ]					)
      lowerCamelCase_         =						pre_tokenizers.Sequence(
          [
              pre_tokenizers.Metaspace(replacement=__SCREAMING_SNAKE_CASE       ,						add_prefix_space=__SCREAMING_SNAKE_CASE					),
              pre_tokenizers.Digits(individual_digits=__SCREAMING_SNAKE_CASE					),
              pre_tokenizers.Punctuation(),
          ]					)
      lowerCamelCase_         =						decoders.Metaspace(replacement=__SCREAMING_SNAKE_CASE       ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						TemplateProcessing(
          single=F'''$A {self.special_tokens["eos"]["token"]}'''       ,						special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]       ,						)

      lowerCamelCase_         =						{
          'model': 'SentencePieceUnigram',
          'replacement': replacement,
          'add_prefix_space': add_prefix_space,
      }

      super().__init__(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Union[str, List[str]]       ,						__SCREAMING_SNAKE_CASE     :							int = 8000       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						)     ->		Tuple:
      lowerCamelCase_         =						trainers.UnigramTrainer(
          vocab_size=__SCREAMING_SNAKE_CASE       ,						special_tokens=self.special_tokens_list       ,						show_progress=__SCREAMING_SNAKE_CASE       ,						)

      if isinstance(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						[files]
      self._tokenizer.train(__SCREAMING_SNAKE_CASE       ,						trainer=__SCREAMING_SNAKE_CASE					)

      self.add_unk_id()






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Union[Iterator[str], Iterator[Iterator[str]]]       ,						__SCREAMING_SNAKE_CASE     :							int = 8000       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						)     ->		str:
      lowerCamelCase_         =						trainers.UnigramTrainer(
          vocab_size=__SCREAMING_SNAKE_CASE       ,						special_tokens=self.special_tokens_list       ,						show_progress=__SCREAMING_SNAKE_CASE       ,						)

      self._tokenizer.train_from_iterator(__SCREAMING_SNAKE_CASE       ,						trainer=__SCREAMING_SNAKE_CASE					)

      self.add_unk_id()






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Dict:
      lowerCamelCase_         =						json.loads(self._tokenizer.to_str()					)

      lowerCamelCase_         =						self.special_tokens['unk']['id']

      lowerCamelCase_         =						Tokenizer.from_str(json.dumps(__SCREAMING_SNAKE_CASE					)					)




