



'''simple docstring'''





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase											=						logging.get_logger(__name__)

_UpperCamelCase											=						'▁'

_UpperCamelCase											=						{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

_UpperCamelCase											=						{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

_UpperCamelCase											=						{
    'facebook/m2m100_418M': 1024,
}

# fmt: off
_UpperCamelCase											=						{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}






class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_     ):
  """simple docstring"""




  a_												=VOCAB_FILES_NAMES
  a_												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_												=PRETRAINED_VOCAB_FILES_MAP
  a_												=["""input_ids""", """attention_mask"""]

  a_												=[]
  a_												=[]




  def __init__( self      :	Union[str, Any]			,	_a      :	str			,	_a      :	List[str]			,	_a      :	Dict=None			,	_a      :	Optional[Any]=None			,	_a      :	Tuple="<s>"			,	_a      :	Optional[int]="</s>"			,	_a      :	Any="</s>"			,	_a      :	List[Any]="<pad>"			,	_a      :	Optional[Any]="<unk>"			,	_a      :	List[str]="m2m100"			,	_a      :	Optional[Dict[str, Any]] = None			,	_a      :	str=8			,	**_a      :	Union[str, Any]			,	)					->					None:
    __lowerCamelCase       :       List[str]           = {} if sp_model_kwargs is None else sp_model_kwargs

    __lowerCamelCase       :       str           = language_codes
    __lowerCamelCase       :       str           = FAIRSEQ_LANGUAGE_CODES[language_codes]
    __lowerCamelCase       :       List[Any]           = {lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

    __lowerCamelCase       :       Tuple           = kwargs.get('additional_special_tokens'			,	[]   )
    kwargs["additional_special_tokens"] += [
        self.get_lang_token(_a   )
        for lang_code in fairseq_language_code
        if self.get_lang_token(_a   ) not in kwargs["additional_special_tokens"]
    ]

    super().__init__(
        src_lang=_a			,	tgt_lang=_a			,	bos_token=_a			,	eos_token=_a			,	sep_token=_a			,	unk_token=_a			,	pad_token=_a			,	language_codes=_a			,	sp_model_kwargs=self.sp_model_kwargs			,	num_madeup_words=_a			,	**_a			,	)

    __lowerCamelCase       :       Tuple           = vocab_file
    __lowerCamelCase       :       str           = load_json(_a   )
    __lowerCamelCase       :       Union[str, Any]           = {v: k for k, v in self.encoder.items()}
    __lowerCamelCase       :       Dict           = spm_file
    __lowerCamelCase       :       List[str]           = load_spm(_a			,	self.sp_model_kwargs   )

    __lowerCamelCase       :       List[Any]           = len(self.encoder   )

    __lowerCamelCase       :       Dict           = {
        self.get_lang_token(_a   ): self.encoder_size + i for i, lang_code in enumerate(_a   )
    }
    __lowerCamelCase       :       int           = {lang_code: self.encoder_size + i for i, lang_code in enumerate(_a   )}
    __lowerCamelCase       :       Optional[int]           = {v: k for k, v in self.lang_token_to_id.items()}

    __lowerCamelCase       :       int           = src_lang if src_lang is not None else 'en'
    __lowerCamelCase       :       Optional[Any]           = tgt_lang
    __lowerCamelCase       :       int           = self.get_lang_id(self._src_lang   )
    self.set_src_lang_special_tokens(self._src_lang   )

    __lowerCamelCase       :       Optional[int]           = num_madeup_words




  @property
  def 						_lowercase     ( self      :	Optional[int]   )					->					int:
    return len(self.encoder   ) + len(self.lang_token_to_id   )




  @property
  def 						_lowercase     ( self      :	Dict   )					->					str:
    return self._src_lang




  @src_lang.setter
  def 						_lowercase     ( self      :	Optional[Any]			,	_a      :	str   )					->					None:
    __lowerCamelCase       :       Optional[int]           = new_src_lang
    self.set_src_lang_special_tokens(self._src_lang   )




  def 						_lowercase     ( self      :	Union[str, Any]			,	_a      :	str   )					->					List[str]:
    return self.sp_model.encode(_a			,	out_type=_a   )




  def 						_lowercase     ( self      :	Any			,	_a      :	List[str]   )					->					Any:
    if token in self.lang_token_to_id:
      return self.lang_token_to_id[token]
    return self.encoder.get(_a			,	self.encoder[self.unk_token]   )




  def 						_lowercase     ( self      :	str			,	_a      :	int   )					->					str:
    if index in self.id_to_lang_token:
      return self.id_to_lang_token[index]
    return self.decoder.get(_a			,	self.unk_token   )




  def 						_lowercase     ( self      :	List[Any]			,	_a      :	Any   )					->					Optional[Any]:
    __lowerCamelCase       :       Any           = []
    __lowerCamelCase       :       str           = ''
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        out_string += self.sp_model.decode(_a   ) + token
        __lowerCamelCase       :       Dict           = []
      else:
        current_sub_tokens.append(_a   )
    out_string += self.sp_model.decode(_a   )
    return out_string.strip()




  def 						_lowercase     ( self      :	Tuple			,	_a      :	List[int]			,	_a      :	Optional[List[int]] = None			,	_a      :	bool = False   )					->					List[int]:

    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=_a			,	token_ids_a=_a			,	already_has_special_tokens=_a   )

    __lowerCamelCase       :       Any           = [1] * len(self.prefix_tokens   )
    __lowerCamelCase       :       Optional[int]           = [1] * len(self.suffix_tokens   )
    if token_ids_a is None:
      return prefix_ones + ([0] * len(_a   )) + suffix_ones
    return prefix_ones + ([0] * len(_a   )) + ([0] * len(_a   )) + suffix_ones




  def 						_lowercase     ( self      :	str			,	_a      :	List[int]			,	_a      :	Optional[List[int]] = None   )					->					List[int]:
    if token_ids_a is None:
      return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




  def 						_lowercase     ( self      :	Union[str, Any]   )					->					Dict:
    __lowerCamelCase       :       Any           = {self.convert_ids_to_tokens(_a   ): i for i in range(self.vocab_size   )}
    vocab.update(self.added_tokens_encoder   )
    return vocab




  def __getstate__( self      :	Dict   )					->					Dict:
    __lowerCamelCase       :       Optional[Any]           = self.__dict__.copy()
    __lowerCamelCase       :       Union[str, Any]           = None
    return state




  def __setstate__( self      :	Dict			,	_a      :	Dict   )					->					None:
    __lowerCamelCase       :       Optional[Any]           = d

    # for backward compatibility
    if not hasattr(self			,	'sp_model_kwargs'   ):
      __lowerCamelCase       :       Optional[Any]           = {}

    __lowerCamelCase       :       List[str]           = load_spm(self.spm_file			,	self.sp_model_kwargs   )




  def 						_lowercase     ( self      :	Optional[Any]			,	_a      :	str			,	_a      :	Optional[str] = None   )					->					Tuple[str]:
    __lowerCamelCase       :       int           = Path(_a   )
    if not save_dir.is_dir():
      raise OSError(f'{save_directory} should be a directory'   )
    __lowerCamelCase       :       int           = save_dir / (
        (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
    )
    __lowerCamelCase       :       Optional[int]           = save_dir / (
        (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
    )

    save_json(self.encoder			,	_a   )

    if os.path.abspath(self.spm_file   ) != os.path.abspath(_a   ) and os.path.isfile(self.spm_file   ):
      copyfile(self.spm_file			,	_a   )
    elif not os.path.isfile(self.spm_file   ):
      with open(_a			,	'wb'   ) as fi:
        __lowerCamelCase       :       Optional[Any]           = self.sp_model.serialized_model_proto()
        fi.write(_a   )

    return (str(_a   ), str(_a   ))




  def 						_lowercase     ( self      :	List[str]			,	_a      :	List[str]			,	_a      :	str = "en"			,	_a      :	Optional[List[str]] = None			,	_a      :	str = "ro"			,	**_a      :	int			,	)					->					BatchEncoding:
    __lowerCamelCase       :       Dict           = src_lang
    __lowerCamelCase       :       Optional[int]           = tgt_lang
    self.set_src_lang_special_tokens(self.src_lang   )
    return super().prepare_seqaseq_batch(_a			,	_a			,	**_a   )




  def 						_lowercase     ( self      :	Union[str, Any]			,	_a      :	Dict			,	_a      :	Optional[str]			,	_a      :	Optional[str]			,	**_a      :	Union[str, Any]   )					->					Union[str, Any]:
    if src_lang is None or tgt_lang is None:
      raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
    __lowerCamelCase       :       str           = src_lang
    __lowerCamelCase       :       List[str]           = self(_a			,	add_special_tokens=_a			,	**_a   )
    __lowerCamelCase       :       List[str]           = self.get_lang_id(_a   )
    __lowerCamelCase       :       Optional[int]           = tgt_lang_id
    return inputs




  def 						_lowercase     ( self      :	Tuple   )					->					Optional[Any]:
    self.set_src_lang_special_tokens(self.src_lang   )




  def 						_lowercase     ( self      :	Union[str, Any]   )					->					Optional[int]:
    self.set_tgt_lang_special_tokens(self.tgt_lang   )




  def 						_lowercase     ( self      :	List[str]			,	_a      :	str   )					->					None:
    __lowerCamelCase       :       Union[str, Any]           = self.get_lang_token(_a   )
    __lowerCamelCase       :       str           = self.lang_token_to_id[lang_token]
    __lowerCamelCase       :       int           = [self.cur_lang_id]
    __lowerCamelCase       :       List[Any]           = [self.eos_token_id]




  def 						_lowercase     ( self      :	Dict			,	_a      :	str   )					->					None:
    __lowerCamelCase       :       str           = self.get_lang_token(_a   )
    __lowerCamelCase       :       Any           = self.lang_token_to_id[lang_token]
    __lowerCamelCase       :       str           = [self.cur_lang_id]
    __lowerCamelCase       :       str           = [self.eos_token_id]




  def 						_lowercase     ( self      :	List[Any]			,	_a      :	str   )					->					str:
    return self.lang_code_to_token[lang]




  def 						_lowercase     ( self      :	Dict			,	_a      :	str   )					->					int:
    __lowerCamelCase       :       Tuple           = self.get_lang_token(_a   )
    return self.lang_token_to_id[lang_token]




def        a_     (							_lowerCAmelCase       ,_lowerCAmelCase       )      ->      sentencepiece.SentencePieceProcessor:
  __lowerCamelCase       :       Union[str, Any]           = sentencepiece.SentencePieceProcessor(**_lowerCAmelCase       )
  spm.Load(str(_lowerCAmelCase       )       )
  return spm

def        a_     (							_lowerCAmelCase       )      ->      Union[Dict, List]:
  with open(_lowerCAmelCase       ,'r'       ) as f:
    return json.load(_lowerCAmelCase       )


def        a_     (							_lowerCAmelCase       ,_lowerCAmelCase       )      ->      None:
  with open(_lowerCAmelCase       ,'w'       ) as f:
    json.dump(_lowerCAmelCase       ,_lowerCAmelCase       ,indent=2       )











'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_UpperCamelCase											=						logging.get_logger(__name__)

_UpperCamelCase											=						{
    'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/config.json',
    'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/config.json',
    'xlm-roberta-large-finetuned-conll02-dutch': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll02-spanish': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll03-english': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll03-german': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/config.json'
    ),
}






class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_     ):
  """simple docstring"""




  a_												="""xlm-roberta"""




  def __init__( self      :	Any			,	_a      :	Optional[Any]=3_0522			,	_a      :	Optional[Any]=768			,	_a      :	int=12			,	_a      :	Tuple=12			,	_a      :	str=3072			,	_a      :	List[Any]="gelu"			,	_a      :	int=0.1			,	_a      :	Optional[int]=0.1			,	_a      :	Optional[int]=512			,	_a      :	List[Any]=2			,	_a      :	Optional[Any]=0.02			,	_a      :	str=1e-12			,	_a      :	str=1			,	_a      :	str=0			,	_a      :	Optional[Any]=2			,	_a      :	Optional[int]="absolute"			,	_a      :	int=True			,	_a      :	Tuple=None			,	**_a      :	Any			,	)					->					Dict:
    super().__init__(pad_token_id=_a			,	bos_token_id=_a			,	eos_token_id=_a			,	**_a   )

    __lowerCamelCase       :       int           = vocab_size
    __lowerCamelCase       :       Union[str, Any]           = hidden_size
    __lowerCamelCase       :       int           = num_hidden_layers
    __lowerCamelCase       :       Optional[int]           = num_attention_heads
    __lowerCamelCase       :       List[str]           = hidden_act
    __lowerCamelCase       :       Any           = intermediate_size
    __lowerCamelCase       :       int           = hidden_dropout_prob
    __lowerCamelCase       :       List[Any]           = attention_probs_dropout_prob
    __lowerCamelCase       :       Optional[int]           = max_position_embeddings
    __lowerCamelCase       :       Tuple           = type_vocab_size
    __lowerCamelCase       :       Optional[int]           = initializer_range
    __lowerCamelCase       :       Optional[int]           = layer_norm_eps
    __lowerCamelCase       :       str           = position_embedding_type
    __lowerCamelCase       :       List[Any]           = use_cache
    __lowerCamelCase       :       int           = classifier_dropout






class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_     ):
  """simple docstring"""




  @property
  def 						_lowercase     ( self      :	Optional[Any]   )					->					Mapping[str, Mapping[int, str]]:
    if self.task == "multiple-choice":
      __lowerCamelCase       :       int           = {0: 'batch', 1: 'choice', 2: 'sequence'}
    else:
      __lowerCamelCase       :       Any           = {0: 'batch', 1: 'sequence'}
    return OrderedDict(
        [
            ('input_ids', dynamic_axis),
            ('attention_mask', dynamic_axis),
        ]   )






