





import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    DebertaForMaskedLM,
			    DebertaForQuestionAnswering,
			    DebertaForSequenceClassification,
			    DebertaForTokenClassification,
			    DebertaModel,
			)
			from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=1_3       ,  __lowerCAmelCase=7       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=9_9       ,  __lowerCAmelCase=3_2       ,  __lowerCAmelCase=5       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=3_7       ,  __lowerCAmelCase="gelu"       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=5_1_2       ,  __lowerCAmelCase=1_6       ,  __lowerCAmelCase=2       ,  __lowerCAmelCase=0.02       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase="None"       ,  __lowerCAmelCase=3       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=None       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		parent
				lowerCamelCase__								=		batch_size
				lowerCamelCase__								=		seq_length
				lowerCamelCase__								=		is_training
				lowerCamelCase__								=		use_input_mask
				lowerCamelCase__								=		use_token_type_ids
				lowerCamelCase__								=		use_labels
				lowerCamelCase__								=		vocab_size
				lowerCamelCase__								=		hidden_size
				lowerCamelCase__								=		num_hidden_layers
				lowerCamelCase__								=		num_attention_heads
				lowerCamelCase__								=		intermediate_size
				lowerCamelCase__								=		hidden_act
				lowerCamelCase__								=		hidden_dropout_prob
				lowerCamelCase__								=		attention_probs_dropout_prob
				lowerCamelCase__								=		max_position_embeddings
				lowerCamelCase__								=		type_vocab_size
				lowerCamelCase__								=		type_sequence_label_size
				lowerCamelCase__								=		initializer_range
				lowerCamelCase__								=		num_labels
				lowerCamelCase__								=		num_choices
				lowerCamelCase__								=		relative_attention
				lowerCamelCase__								=		position_biased_input
				lowerCamelCase__								=		pos_att_type
				lowerCamelCase__								=		scope





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size							)

				lowerCamelCase__								=		None
				if self.use_input_mask:
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  vocab_size=2							)

				lowerCamelCase__								=		None
				if self.use_token_type_ids:
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size							)

				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				if self.use_labels:
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.type_sequence_label_size							)
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels							)
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.num_choices							)

				lowerCamelCase__								=		self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return DebertaConfig(
				    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  initializer_range=self.initializer_range       ,  relative_attention=self.relative_attention       ,  position_biased_input=self.position_biased_input       ,  pos_att_type=self.pos_att_type       ,  )





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_config()
				lowerCamelCase__								=		3_0_0
				return config





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				self.parent.assertListEqual(list(result.loss.size()							)       ,  []							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		DebertaModel(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)[0]
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase							)[0]
				lowerCamelCase__								=		model(__lowerCAmelCase							)[0]

				self.parent.assertListEqual(list(sequence_output.size()							)       ,  [self.batch_size, self.seq_length, self.hidden_size]							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		DebertaForMaskedLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)

				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		DebertaForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertListEqual(list(result.logits.size()							)       ,  [self.batch_size, self.num_labels]							)
				self.check_loss_output(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.num_labels
				lowerCamelCase__								=		DebertaForTokenClassification(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		DebertaForQuestionAnswering(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  token_type_ids=__lowerCAmelCase       ,  start_positions=__lowerCAmelCase       ,  end_positions=__lowerCAmelCase       ,  )
				self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length)							)
				self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.prepare_config_and_inputs()
				(
				    (
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	(
				    lowerCamelCase__
				)				,	
				)								=		config_and_inputs
				lowerCamelCase__								=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
				return config, inputs_dict






@require_torch
class 					__A    (    lowerCAmelCase			,				lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						(
			    (
			        DebertaModel,
			        DebertaForMaskedLM,
			        DebertaForSequenceClassification,
			        DebertaForTokenClassification,
			        DebertaForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCAmelCase_							     =						(
			    {
			        """feature-extraction""": DebertaModel,
			        """fill-mask""": DebertaForMaskedLM,
			        """question-answering""": DebertaForQuestionAnswering,
			        """text-classification""": DebertaForSequenceClassification,
			        """token-classification""": DebertaForTokenClassification,
			        """zero-shot""": DebertaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)

			lowerCAmelCase_							     =						True
			lowerCAmelCase_							     =						False
			lowerCAmelCase_							     =						False
			lowerCAmelCase_							     =						False
			lowerCAmelCase_							     =						False





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		DebertaModelTester(self							)
				lowerCamelCase__								=		ConfigTester(self       ,  config_class=__lowerCAmelCase       ,  hidden_size=3_7							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.config_tester.run_common_tests()





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_deberta_model(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_deberta_for_sequence_classification(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_deberta_for_masked_lm(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_deberta_for_question_answering(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_deberta_for_token_classification(*__lowerCAmelCase							)





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCamelCase__								=		DebertaModel.from_pretrained(__lowerCAmelCase							)
					self.assertIsNotNone(__lowerCAmelCase							)






@require_torch
@require_sentencepiece
@require_tokenizers
class 					__A    (    unittest.TestCase ):

			'''simple docstring'''







			@unittest.skip(reason='''Model not available yet'''							)
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				pass





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		DebertaModel.from_pretrained('''microsoft/deberta-base'''							)

				lowerCamelCase__								=		torch.tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]							)
				lowerCamelCase__								=		torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
				with torch.no_grad():
					lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase							)[0]
				# compare the actual values for a slice.
				lowerCamelCase__								=		torch.tensor(
				    [[[-0.5986, -0.8055, -0.8462], [1.4484, -0.9348, -0.8059], [0.3123, 0.0032, -1.4131]]]							)
				self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,  __lowerCAmelCase       ,  atol=1E-4							)       ,  F'{output[:, 1:4, 1:4]}'							)










import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_a 						=      logging.get_logger(__name__)

_a 						=      "▁"

_a 						=      {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_a 						=      {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_a 						=      {
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
_a 						=      {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						VOCAB_FILES_NAMES
			lowerCAmelCase_							     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							     =						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							     =						["""input_ids""", """attention_mask"""]

			lowerCAmelCase_							     =						[]
			lowerCAmelCase_							     =						[]





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase="<s>"       ,  __lowerCAmelCase="</s>"       ,  __lowerCAmelCase="</s>"       ,  __lowerCAmelCase="<pad>"       ,  __lowerCAmelCase="<unk>"       ,  __lowerCAmelCase="m2m100"       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase=8       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		{} if sp_model_kwargs is None else sp_model_kwargs

				lowerCamelCase__								=		language_codes
				lowerCamelCase__								=		FAIRSEQ_LANGUAGE_CODES[language_codes]
				lowerCamelCase__								=		{lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

				lowerCamelCase__								=		kwargs.get('''additional_special_tokens'''       ,  []							)
				kwargs["additional_special_tokens"] += [
				    self.get_lang_token(__lowerCAmelCase							)
				    for lang_code in fairseq_language_code
				    if self.get_lang_token(__lowerCAmelCase							) not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    src_lang=__lowerCAmelCase       ,  tgt_lang=__lowerCAmelCase       ,  bos_token=__lowerCAmelCase       ,  eos_token=__lowerCAmelCase       ,  sep_token=__lowerCAmelCase       ,  unk_token=__lowerCAmelCase       ,  pad_token=__lowerCAmelCase       ,  language_codes=__lowerCAmelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  num_madeup_words=__lowerCAmelCase       ,  **__lowerCAmelCase       ,  )

				lowerCamelCase__								=		vocab_file
				lowerCamelCase__								=		load_json(__lowerCAmelCase							)
				lowerCamelCase__								=		{v: k for k, v in self.encoder.items()}
				lowerCamelCase__								=		spm_file
				lowerCamelCase__								=		load_spm(__lowerCAmelCase       ,  self.sp_model_kwargs							)

				lowerCamelCase__								=		len(self.encoder							)

				lowerCamelCase__								=		{
				    self.get_lang_token(__lowerCAmelCase							): self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase							)
				}
				lowerCamelCase__								=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase							)}
				lowerCamelCase__								=		{v: k for k, v in self.lang_token_to_id.items()}

				lowerCamelCase__								=		src_lang if src_lang is not None else '''en'''
				lowerCamelCase__								=		tgt_lang
				lowerCamelCase__								=		self.get_lang_id(self._src_lang							)
				self.set_src_lang_special_tokens(self._src_lang							)

				lowerCamelCase__								=		num_madeup_words





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return len(self.encoder							) + len(self.lang_token_to_id							)





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return self._src_lang





			@src_lang.setter
			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		new_src_lang
				self.set_src_lang_special_tokens(self._src_lang							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.encode(__lowerCAmelCase       ,  out_type=__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if token in self.lang_token_to_id:
					return self.lang_token_to_id[token]
				return self.encoder.get(__lowerCAmelCase       ,  self.encoder[self.unk_token]							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if index in self.id_to_lang_token:
					return self.id_to_lang_token[index]
				return self.decoder.get(__lowerCAmelCase       ,  self.unk_token							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				lowerCamelCase__								=		''''''
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						out_string += self.sp_model.decode(__lowerCAmelCase							) + token
						lowerCamelCase__								=		[]
					else:
						current_sub_tokens.append(__lowerCAmelCase							)
				out_string += self.sp_model.decode(__lowerCAmelCase							)
				return out_string.strip()





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__lowerCAmelCase       ,  token_ids_a=__lowerCAmelCase       ,  already_has_special_tokens=__lowerCAmelCase							)

				lowerCamelCase__								=		[1] * len(self.prefix_tokens							)
				lowerCamelCase__								=		[1] * len(self.suffix_tokens							)
				if token_ids_a is None:
					return prefix_ones + ([0] * len(__lowerCAmelCase							)) + suffix_ones
				return prefix_ones + ([0] * len(__lowerCAmelCase							)) + ([0] * len(__lowerCAmelCase							)) + suffix_ones





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		{self.convert_ids_to_tokens(__lowerCAmelCase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab





			def __getstate__(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.__dict__.copy()
				lowerCamelCase__								=		None
				return state





			def __setstate__(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		d

				# for backward compatibility
				if not hasattr(self       ,  '''sp_model_kwargs'''							):
					lowerCamelCase__								=		{}

				lowerCamelCase__								=		load_spm(self.spm_file       ,  self.sp_model_kwargs							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__								=		Path(__lowerCAmelCase							)
				if not save_dir.is_dir():
					raise OSError(F'{save_directory} should be a directory'							)
				lowerCamelCase__								=		save_dir / (
				    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
				)
				lowerCamelCase__								=		save_dir / (
				    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
				)

				save_json(self.encoder       ,  __lowerCAmelCase							)

				if os.path.abspath(self.spm_file							) != os.path.abspath(__lowerCAmelCase							) and os.path.isfile(self.spm_file							):
					copyfile(self.spm_file       ,  __lowerCAmelCase							)
				elif not os.path.isfile(self.spm_file							):
					with open(__lowerCAmelCase       ,  '''wb'''							) as fi:
						lowerCamelCase__								=		self.sp_model.serialized_model_proto()
						fi.write(__lowerCAmelCase							)

				return (str(__lowerCAmelCase							), str(__lowerCAmelCase							))





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = "en"       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = "ro"       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		src_lang
				lowerCamelCase__								=		tgt_lang
				self.set_src_lang_special_tokens(self.src_lang							)
				return super().prepare_seqaseq_batch(__lowerCAmelCase       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				if src_lang is None or tgt_lang is None:
					raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''							)
				lowerCamelCase__								=		src_lang
				lowerCamelCase__								=		self(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase       ,  **__lowerCAmelCase							)
				lowerCamelCase__								=		self.get_lang_id(__lowerCAmelCase							)
				lowerCamelCase__								=		tgt_lang_id
				return inputs





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.set_src_lang_special_tokens(self.src_lang							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.set_tgt_lang_special_tokens(self.tgt_lang							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				lowerCamelCase__								=		self.lang_token_to_id[lang_token]
				lowerCamelCase__								=		[self.cur_lang_id]
				lowerCamelCase__								=		[self.eos_token_id]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				lowerCamelCase__								=		self.lang_token_to_id[lang_token]
				lowerCamelCase__								=		[self.cur_lang_id]
				lowerCamelCase__								=		[self.eos_token_id]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.lang_code_to_token[lang]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				return self.lang_token_to_id[lang_token]





def lowerCAmelCase__(__snake_case       ,__snake_case							)				->      sentencepiece.SentencePieceProcessor:







	'''simple docstring'''







	lowerCamelCase__								=		sentencepiece.SentencePieceProcessor(**__snake_case							)
	spm.Load(str(__snake_case							)							)
	return spm





def lowerCAmelCase__(__snake_case							)				->      Union[Dict, List]:







	'''simple docstring'''







	with open(__snake_case       ,'''r'''							) as f:
		return json.load(__snake_case							)





def lowerCAmelCase__(__snake_case       ,__snake_case							)				->      None:







	'''simple docstring'''







	with open(__snake_case       ,'''w'''							) as f:
		json.dump(__snake_case       ,__snake_case       ,indent=2							)



