



'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCAmelCase      						=	logging.get_logger(__name__)

_lowerCAmelCase      						=	{
    '''facebook/xlm-roberta-xl''': '''https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json''',
    '''facebook/xlm-roberta-xxl''': '''https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json''',
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      int				=      '''xlm-roberta-xl'''




  def __init__(    self  ,__UpperCAmelCase=25_0880  ,__UpperCAmelCase=2560  ,__UpperCAmelCase=36  ,__UpperCAmelCase=32  ,__UpperCAmelCase=1_0240  ,__UpperCAmelCase="gelu"  ,__UpperCAmelCase=0.1  ,__UpperCAmelCase=0.1  ,__UpperCAmelCase=514  ,__UpperCAmelCase=1  ,__UpperCAmelCase=0.0_2  ,__UpperCAmelCase=1E-05  ,__UpperCAmelCase=1  ,__UpperCAmelCase=0  ,__UpperCAmelCase=2  ,__UpperCAmelCase="absolute"  ,__UpperCAmelCase=True  ,__UpperCAmelCase=None  ,**__UpperCAmelCase  ,)    ->		str:
      super().__init__(pad_token_id=__UpperCAmelCase  ,bos_token_id=__UpperCAmelCase  ,eos_token_id=__UpperCAmelCase  ,**__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					vocab_size
      lowerCAmelCase__ :  int							      =					hidden_size
      lowerCAmelCase__ :  int							      =					num_hidden_layers
      lowerCAmelCase__ :  str							      =					num_attention_heads
      lowerCAmelCase__ :  int							      =					hidden_act
      lowerCAmelCase__ :  Dict							      =					intermediate_size
      lowerCAmelCase__ :  List[Any]							      =					hidden_dropout_prob
      lowerCAmelCase__ :  str							      =					attention_probs_dropout_prob
      lowerCAmelCase__ :  Optional[int]							      =					max_position_embeddings
      lowerCAmelCase__ :  List[str]							      =					type_vocab_size
      lowerCAmelCase__ :  List[Any]							      =					initializer_range
      lowerCAmelCase__ :  Tuple							      =					layer_norm_eps
      lowerCAmelCase__ :  int							      =					position_embedding_type
      lowerCAmelCase__ :  Optional[Any]							      =					use_cache
      lowerCAmelCase__ :  Optional[Any]							      =					classifier_dropout







class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  @property
  def 							UpperCAmelCase_	(    self      )    ->		Mapping[str, Mapping[int, str]]:
      if self.task == "multiple-choice":
          lowerCAmelCase__ :  Dict							      =					{0: """batch""", 1: """choice""", 2: """sequence"""}
      else:
          lowerCAmelCase__ :  Any							      =					{0: """batch""", 1: """sequence"""}
      return OrderedDict(
          [
              ("""input_ids""", dynamic_axis),
              ("""attention_mask""", dynamic_axis),
          ]      )









'''simple docstring'''




import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class  SCREAMING_SNAKE_CASE    (a__							):


     def __init__(						self						,	_UpperCAmelCase = "▁"						,	_UpperCAmelCase = True						,	_UpperCAmelCase = "<unk>"						,	_UpperCAmelCase = "</s>"						,	_UpperCAmelCase = "<pad>"						,	):

          '''simple docstring'''






          __A						: Dict      =       {
              'pad': {'id': 0, 'token': pad_token},
              'eos': {'id': 1, 'token': eos_token},
              'unk': {'id': 2, 'token': unk_token},
          }

          __A						: List[Any]      =       [None] * len(self.special_tokens)
          for token_dict in self.special_tokens.values():
               __A						: List[str]      =       token_dict['token']

          __A						: str      =       Tokenizer(Unigram())

          __A						: Dict      =       normalizers.Sequence(
              [
                  normalizers.Nmt(),
                  normalizers.NFKC(),
                  normalizers.Replace(Regex(' {2,}')						,	' '),
                  normalizers.Lowercase(),
              ])
          __A						: Union[str, Any]      =       pre_tokenizers.Sequence(
              [
                  pre_tokenizers.Metaspace(replacement=_UpperCAmelCase						,	add_prefix_space=_UpperCAmelCase),
                  pre_tokenizers.Digits(individual_digits=_UpperCAmelCase),
                  pre_tokenizers.Punctuation(),
              ])
          __A						: Any      =       decoders.Metaspace(replacement=_UpperCAmelCase						,	add_prefix_space=_UpperCAmelCase)

          __A						: Dict      =       TemplateProcessing(
              single=F'$A {self.special_tokens["eos"]["token"]}'						,	special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]						,	)

          __A						: Any      =       {
              'model': 'SentencePieceUnigram',
              'replacement': replacement,
              'add_prefix_space': add_prefix_space,
          }

          super().__init__(_UpperCAmelCase						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase = 8000						,	_UpperCAmelCase = True						,	):

          '''simple docstring'''






          __A						: str      =       trainers.UnigramTrainer(
              vocab_size=_UpperCAmelCase						,	special_tokens=self.special_tokens_list						,	show_progress=_UpperCAmelCase						,	)

          if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
               __A						: Union[str, Any]      =       [files]
          self._tokenizer.train(_UpperCAmelCase						,	trainer=_UpperCAmelCase)

          self.add_unk_id()


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase = 8000						,	_UpperCAmelCase = True						,	):

          '''simple docstring'''






          __A						: Dict      =       trainers.UnigramTrainer(
              vocab_size=_UpperCAmelCase						,	special_tokens=self.special_tokens_list						,	show_progress=_UpperCAmelCase						,	)

          self._tokenizer.train_from_iterator(_UpperCAmelCase						,	trainer=_UpperCAmelCase)

          self.add_unk_id()


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Any      =       json.loads(self._tokenizer.to_str())

          __A						: Union[str, Any]      =       self.special_tokens['unk']['id']

          __A						: str      =       Tokenizer.from_str(json.dumps(_UpperCAmelCase))