"""simple docstring"""






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase    :       Tuple					     =logging.get_logger(__name__)

__lowerCAmelCase    :       Dict					     ="""▁"""

__lowerCAmelCase    :       Tuple					     ={
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

__lowerCAmelCase    :       List[Any]					     ={
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

__lowerCAmelCase    :       Tuple					     ={
    """facebook/m2m100_418M""": 1_0_2_4,
}

# fmt: off
__lowerCAmelCase    :       List[str]					     ={
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}





class  _A						(       lowerCAmelCase	):
      snake_case__			:   int							  =	VOCAB_FILES_NAMES
      snake_case__			:   str							  =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case__			:   int							  =	PRETRAINED_VOCAB_FILES_MAP
      snake_case__			:   List[Any]							  =	['input_ids', 'attention_mask']

      snake_case__			:   List[int]							  =	[]
      snake_case__			:   List[int]							  =	[]

      def __init__(						self		,       __lowerCAmelCase		,       __lowerCAmelCase		,       __lowerCAmelCase=None		,       __lowerCAmelCase=None		,       __lowerCAmelCase="<s>"		,       __lowerCAmelCase="</s>"		,       __lowerCAmelCase="</s>"		,       __lowerCAmelCase="<pad>"		,       __lowerCAmelCase="<unk>"		,       __lowerCAmelCase="m2m100"		,       __lowerCAmelCase = None		,       __lowerCAmelCase=8		,       **__lowerCAmelCase		,       ):







            """simple docstring"""



            lowercase														=	{} if sp_model_kwargs is None else sp_model_kwargs

            lowercase														=	language_codes
            lowercase														=	FAIRSEQ_LANGUAGE_CODES[language_codes]
            lowercase														=	{lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

            lowercase														=	kwargs.get("""additional_special_tokens"""		,       []						)
            kwargs["additional_special_tokens"] += [
                self.get_lang_token(__lowerCAmelCase						)
                for lang_code in fairseq_language_code
                if self.get_lang_token(__lowerCAmelCase						) not in kwargs["additional_special_tokens"]
            ]

            super().__init__(
                src_lang=__lowerCAmelCase		,       tgt_lang=__lowerCAmelCase		,       bos_token=__lowerCAmelCase		,       eos_token=__lowerCAmelCase		,       sep_token=__lowerCAmelCase		,       unk_token=__lowerCAmelCase		,       pad_token=__lowerCAmelCase		,       language_codes=__lowerCAmelCase		,       sp_model_kwargs=self.sp_model_kwargs		,       num_madeup_words=__lowerCAmelCase		,       **__lowerCAmelCase		,       )

            lowercase														=	vocab_file
            lowercase														=	load_json(__lowerCAmelCase						)
            lowercase														=	{v: k for k, v in self.encoder.items()}
            lowercase														=	spm_file
            lowercase														=	load_spm(__lowerCAmelCase		,       self.sp_model_kwargs						)

            lowercase														=	len(self.encoder						)

            lowercase														=	{
                self.get_lang_token(__lowerCAmelCase						): self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase						)
            }
            lowercase														=	{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase						)}
            lowercase														=	{v: k for k, v in self.lang_token_to_id.items()}

            lowercase														=	src_lang if src_lang is not None else """en"""
            lowercase														=	tgt_lang
            lowercase														=	self.get_lang_id(self._src_lang						)
            self.set_src_lang_special_tokens(self._src_lang						)

            lowercase														=	num_madeup_words

      @property
      def 				A__       (						self						):







            """simple docstring"""



            return len(self.encoder						) + len(self.lang_token_to_id						)

      @property
      def 				A__       (						self						):







            """simple docstring"""



            return self._src_lang

      @src_lang.setter
      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	new_src_lang
            self.set_src_lang_special_tokens(self._src_lang						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            return self.sp_model.encode(__lowerCAmelCase		,       out_type=__lowerCAmelCase						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            if token in self.lang_token_to_id:
                  return self.lang_token_to_id[token]
            return self.encoder.get(__lowerCAmelCase		,       self.encoder[self.unk_token]						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            if index in self.id_to_lang_token:
                  return self.id_to_lang_token[index]
            return self.decoder.get(__lowerCAmelCase		,       self.unk_token						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	[]
            lowercase														=	""""""
            for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                        out_string += self.sp_model.decode(__lowerCAmelCase						) + token
                        lowercase														=	[]
                  else:
                        current_sub_tokens.append(__lowerCAmelCase						)
            out_string += self.sp_model.decode(__lowerCAmelCase						)
            return out_string.strip()

      def 				A__       (						self		,       __lowerCAmelCase		,       __lowerCAmelCase = None		,       __lowerCAmelCase = False						):







            """simple docstring"""



            if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=__lowerCAmelCase		,       token_ids_a=__lowerCAmelCase		,       already_has_special_tokens=__lowerCAmelCase						)

            lowercase														=	[1] * len(self.prefix_tokens						)
            lowercase														=	[1] * len(self.suffix_tokens						)
            if token_ids_a is None:
                  return prefix_ones + ([0] * len(__lowerCAmelCase						)) + suffix_ones
            return prefix_ones + ([0] * len(__lowerCAmelCase						)) + ([0] * len(__lowerCAmelCase						)) + suffix_ones

      def 				A__       (						self		,       __lowerCAmelCase		,       __lowerCAmelCase = None						):







            """simple docstring"""



            if token_ids_a is None:
                  return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	{self.convert_ids_to_tokens(__lowerCAmelCase						): i for i in range(self.vocab_size						)}
            vocab.update(self.added_tokens_encoder						)
            return vocab

      def __getstate__(						self						):







            """simple docstring"""



            lowercase														=	self.__dict__.copy()
            lowercase														=	None
            return state

      def __setstate__(						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	d

            # for backward compatibility
            if not hasattr(self		,       """sp_model_kwargs"""						):
                  lowercase														=	{}

            lowercase														=	load_spm(self.spm_file		,       self.sp_model_kwargs						)

      def 				A__       (						self		,       __lowerCAmelCase		,       __lowerCAmelCase = None						):







            """simple docstring"""



            lowercase														=	Path(__lowerCAmelCase						)
            if not save_dir.is_dir():
                  raise OSError(f'{save_directory} should be a directory'						)
            lowercase														=	save_dir / (
                (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
            )
            lowercase														=	save_dir / (
                (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
            )

            save_json(self.encoder		,       __lowerCAmelCase						)

            if os.path.abspath(self.spm_file						) != os.path.abspath(__lowerCAmelCase						) and os.path.isfile(self.spm_file						):
                  copyfile(self.spm_file		,       __lowerCAmelCase						)
            elif not os.path.isfile(self.spm_file						):
                  with open(__lowerCAmelCase		,       """wb"""						) as fi:
                        lowercase														=	self.sp_model.serialized_model_proto()
                        fi.write(__lowerCAmelCase						)

            return (str(__lowerCAmelCase						), str(__lowerCAmelCase						))

      def 				A__       (						self		,       __lowerCAmelCase		,       __lowerCAmelCase = "en"		,       __lowerCAmelCase = None		,       __lowerCAmelCase = "ro"		,       **__lowerCAmelCase		,       ):







            """simple docstring"""



            lowercase														=	src_lang
            lowercase														=	tgt_lang
            self.set_src_lang_special_tokens(self.src_lang						)
            return super().prepare_seqaseq_batch(__lowerCAmelCase		,       __lowerCAmelCase		,       **__lowerCAmelCase						)

      def 				A__       (						self		,       __lowerCAmelCase		,       __lowerCAmelCase		,       __lowerCAmelCase		,       **__lowerCAmelCase						):







            """simple docstring"""



            if src_lang is None or tgt_lang is None:
                  raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""						)
            lowercase														=	src_lang
            lowercase														=	self(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase		,       **__lowerCAmelCase						)
            lowercase														=	self.get_lang_id(__lowerCAmelCase						)
            lowercase														=	tgt_lang_id
            return inputs

      def 				A__       (						self						):







            """simple docstring"""



            self.set_src_lang_special_tokens(self.src_lang						)

      def 				A__       (						self						):







            """simple docstring"""



            self.set_tgt_lang_special_tokens(self.tgt_lang						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	self.get_lang_token(__lowerCAmelCase						)
            lowercase														=	self.lang_token_to_id[lang_token]
            lowercase														=	[self.cur_lang_id]
            lowercase														=	[self.eos_token_id]

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	self.get_lang_token(__lowerCAmelCase						)
            lowercase														=	self.lang_token_to_id[lang_token]
            lowercase														=	[self.cur_lang_id]
            lowercase														=	[self.eos_token_id]

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            return self.lang_code_to_token[lang]







      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	self.get_lang_token(__lowerCAmelCase						)
            return self.lang_token_to_id[lang_token]






def 			UpperCAmelCase__       (				lowerCAmelCase__     :str					,		lowerCAmelCase__     :Dict[str, Any]		) ->    sentencepiece.SentencePieceProcessor:
      '''simple docstring'''


      lowercase														=	sentencepiece.SentencePieceProcessor(**lowerCAmelCase__		)
      spm.Load(str(lowerCAmelCase__		)		)
      return spm


def 			UpperCAmelCase__       (				lowerCAmelCase__     :str		) ->    Union[Dict, List]:
      '''simple docstring'''


      with open(lowerCAmelCase__					,		"""r"""		) as f:
            return json.load(lowerCAmelCase__		)






def 			UpperCAmelCase__       (				lowerCAmelCase__     :Union[str, Any]					,		lowerCAmelCase__     :str		) ->    None:
      '''simple docstring'''


      with open(lowerCAmelCase__					,		"""w"""		) as f:
            json.dump(lowerCAmelCase__					,		lowerCAmelCase__					,		indent=2		)




"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase    :       List[str]					     =logging.get_logger(__name__)

__lowerCAmelCase    :       str					     ={
    """facebook/dpr-ctx_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-ctx_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-multiset-base""": (
        """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json"""
    ),
}





class  _A						(       lowerCAmelCase	):
      snake_case__			:   Tuple							  =	'dpr'

      def __init__(						self		,       __lowerCAmelCase=3_0522		,       __lowerCAmelCase=768		,       __lowerCAmelCase=12		,       __lowerCAmelCase=12		,       __lowerCAmelCase=3072		,       __lowerCAmelCase="gelu"		,       __lowerCAmelCase=0.1		,       __lowerCAmelCase=0.1		,       __lowerCAmelCase=512		,       __lowerCAmelCase=2		,       __lowerCAmelCase=0.0_2		,       __lowerCAmelCase=1E-12		,       __lowerCAmelCase=0		,       __lowerCAmelCase="absolute"		,       __lowerCAmelCase = 0		,       **__lowerCAmelCase		,       ):







            """simple docstring"""



            super().__init__(pad_token_id=__lowerCAmelCase		,       **__lowerCAmelCase						)

            lowercase														=	vocab_size
            lowercase														=	hidden_size
            lowercase														=	num_hidden_layers
            lowercase														=	num_attention_heads
            lowercase														=	hidden_act
            lowercase														=	intermediate_size
            lowercase														=	hidden_dropout_prob
            lowercase														=	attention_probs_dropout_prob
            lowercase														=	max_position_embeddings
            lowercase														=	type_vocab_size
            lowercase														=	initializer_range
            lowercase														=	layer_norm_eps
            lowercase														=	projection_dim
            lowercase														=	position_embedding_type



