"""simple docstring"""






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A			   = logging.get_logger(__name__)

__A			   = {
    "xlm-mlm-en-2048": "https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json",
    "xlm-mlm-ende-1024": "https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json",
    "xlm-mlm-enfr-1024": "https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json",
    "xlm-mlm-enro-1024": "https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json",
    "xlm-mlm-tlm-xnli15-1024": "https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json",
    "xlm-mlm-xnli15-1024": "https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json",
    "xlm-clm-enfr-1024": "https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json",
    "xlm-clm-ende-1024": "https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json",
    "xlm-mlm-17-1280": "https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json",
    "xlm-mlm-100-1280": "https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json",
}


class     UpperCAmelCase						(_UpperCAmelCase  ):



    """simple docstring"""

    _UpperCAmelCase	:Optional[Any]						    = "xlm"
    _UpperCAmelCase	:List[str]						    = {
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }






    def __init__(   self    ,			_UpperCAmelCase=30145    ,			_UpperCAmelCase=2048    ,			_UpperCAmelCase=12    ,			_UpperCAmelCase=16    ,			_UpperCAmelCase=0.1    ,			_UpperCAmelCase=0.1    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase=False    ,			_UpperCAmelCase=False    ,			_UpperCAmelCase=False    ,			_UpperCAmelCase=1    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase=512    ,			_UpperCAmelCase=2048**-0.5    ,			_UpperCAmelCase=1e-1_2    ,			_UpperCAmelCase=0.02    ,			_UpperCAmelCase=0    ,			_UpperCAmelCase=1    ,			_UpperCAmelCase=2    ,			_UpperCAmelCase=3    ,			_UpperCAmelCase=5    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase="first"    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase=None    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase=0.1    ,			_UpperCAmelCase=5    ,			_UpperCAmelCase=5    ,			_UpperCAmelCase=0    ,			_UpperCAmelCase=0    ,			_UpperCAmelCase=2    ,			_UpperCAmelCase=0    ,			**_UpperCAmelCase    ,			):
       lowercase__:   List[Any]					=      vocab_size
       lowercase__:   Optional[int]					=      emb_dim
       lowercase__:   Optional[Any]					=      n_layers
       lowercase__:   List[Any]					=      n_heads
       lowercase__:   str					=      dropout
       lowercase__:   List[str]					=      attention_dropout
       lowercase__:   int					=      gelu_activation
       lowercase__:   Optional[int]					=      sinusoidal_embeddings
       lowercase__:   Dict					=      causal
       lowercase__:   str					=      asm
       lowercase__:   Optional[int]					=      n_langs
       lowercase__:   Optional[Any]					=      use_lang_emb
       lowercase__:   str					=      layer_norm_eps
       lowercase__:   Tuple					=      bos_index
       lowercase__:   Any					=      eos_index
       lowercase__:   Union[str, Any]					=      pad_index
       lowercase__:   Dict					=      unk_index
       lowercase__:   Union[str, Any]					=      mask_index
       lowercase__:   str					=      is_encoder
       lowercase__:   Any					=      max_position_embeddings
       lowercase__:   int					=      embed_init_std
       lowercase__:   Union[str, Any]					=      init_std
       lowercase__:   Optional[int]					=      summary_type
       lowercase__:   Tuple					=      summary_use_proj
       lowercase__:   Optional[Any]					=      summary_activation
       lowercase__:   Union[str, Any]					=      summary_proj_to_labels
       lowercase__:   Tuple					=      summary_first_dropout
       lowercase__:   str					=      start_n_top
       lowercase__:   Optional[Any]					=      end_n_top
       lowercase__:   Union[str, Any]					=      mask_token_id
       lowercase__:   Dict					=      lang_id

       if "n_words" in kwargs:
          lowercase__:   Tuple					=      kwargs['''n_words''']

       super().__init__(pad_token_id=_UpperCAmelCase    ,			bos_token_id=_UpperCAmelCase    ,			**_UpperCAmelCase							)



class     UpperCAmelCase						(_UpperCAmelCase  ):



    """simple docstring"""

    @property
    def _snake_case			(   self							):
       if self.task == "multiple-choice":
          lowercase__:   Optional[int]					=      {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
       else:
          lowercase__:   str					=      {0: '''batch''', 1: '''sequence'''}
       return OrderedDict(
           [
               ('''input_ids''', dynamic_axis),
               ('''attention_mask''', dynamic_axis),
               ('''token_type_ids''', dynamic_axis),
           ]							)

"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__A			   = logging.get_logger(__name__)

__A			   = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A			   = {
    "vocab_file": {
        "yjernite/retribert-base-uncased": (
            "https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "yjernite/retribert-base-uncased": (
            "https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A			   = {
    "yjernite/retribert-base-uncased": 5_1_2,
}


__A			   = {
    "yjernite/retribert-base-uncased": {"do_lower_case": True},
}


class     UpperCAmelCase						(_UpperCAmelCase  ):



    """simple docstring"""

    _UpperCAmelCase	:Dict						    = VOCAB_FILES_NAMES
    _UpperCAmelCase	:str						    = PRETRAINED_VOCAB_FILES_MAP
    _UpperCAmelCase	:Optional[int]						    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _UpperCAmelCase	:Optional[int]						    = PRETRAINED_INIT_CONFIGURATION
    _UpperCAmelCase	:str						    = RetriBertTokenizer
    _UpperCAmelCase	:List[str]						    = ["input_ids", "attention_mask"]






    def __init__(   self    ,			_UpperCAmelCase=None    ,			_UpperCAmelCase=None    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase="[UNK]"    ,			_UpperCAmelCase="[SEP]"    ,			_UpperCAmelCase="[PAD]"    ,			_UpperCAmelCase="[CLS]"    ,			_UpperCAmelCase="[MASK]"    ,			_UpperCAmelCase=True    ,			_UpperCAmelCase=None    ,			**_UpperCAmelCase    ,			):
       super().__init__(
           _UpperCAmelCase    ,			tokenizer_file=_UpperCAmelCase    ,			do_lower_case=_UpperCAmelCase    ,			unk_token=_UpperCAmelCase    ,			sep_token=_UpperCAmelCase    ,			pad_token=_UpperCAmelCase    ,			cls_token=_UpperCAmelCase    ,			mask_token=_UpperCAmelCase    ,			tokenize_chinese_chars=_UpperCAmelCase    ,			strip_accents=_UpperCAmelCase    ,			**_UpperCAmelCase    ,			)

       lowercase__:   Optional[Any]					=      json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
       if (
           normalizer_state.get('''lowercase'''    ,			_UpperCAmelCase							) != do_lower_case
           or normalizer_state.get('''strip_accents'''    ,			_UpperCAmelCase							) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''    ,			_UpperCAmelCase							) != tokenize_chinese_chars
       ):
          lowercase__:   str					=      getattr(_UpperCAmelCase    ,			normalizer_state.pop('''type'''							)							)
          lowercase__:   Optional[Any]					=      do_lower_case
          lowercase__:   Tuple					=      strip_accents
          lowercase__:   str					=      tokenize_chinese_chars
          lowercase__:   Union[str, Any]					=      normalizer_class(**_UpperCAmelCase							)

       lowercase__:   List[str]					=      do_lower_case






    def _snake_case			(   self    ,			_UpperCAmelCase    ,			_UpperCAmelCase=None							):
       lowercase__:   Union[str, Any]					=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

       return output






    def _snake_case			(   self    ,			_UpperCAmelCase    ,			_UpperCAmelCase = None							):
       lowercase__:   Optional[int]					=      [self.sep_token_id]
       lowercase__:   Union[str, Any]					=      [self.cls_token_id]
       if token_ids_a is None:
          return len(cls + token_ids_a + sep							) * [0]
       return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






    def _snake_case			(   self    ,			_UpperCAmelCase    ,			_UpperCAmelCase = None							):
       lowercase__:   List[str]					=      self._tokenizer.model.save(_UpperCAmelCase    ,			name=_UpperCAmelCase							)
       return tuple(_UpperCAmelCase							)
