





'''simple docstring'''

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class    UpperCAmelCase	(       UpperCamelCase__			,       unittest.TestCase				):
 __lowercase   		=				BertTokenizer
 __lowercase   		=				BertTokenizerFast
 __lowercase   		=				True
 __lowercase   		=				True
 __lowercase   		=				filter_non_english
 def      UpperCAmelCase_							(				self      :Optional[Any]							)->							int:
     super().setUp()

     A__				=			[
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     A__				=			os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES["vocab_file"]							)
     with open(self.vocab_file						,       "w"						,       encoding="utf-8"							) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)
 def      UpperCAmelCase_							(				self      :Union[str, Any]						,       lowercase_      :Union[str, Any]							)->							List[Any]:
     A__				=			"UNwant\u00E9d,running"
     A__				=			"unwanted, running"
     return input_text, output_text
 def      UpperCAmelCase_							(				self      :List[Any]							)->							List[Any]:
     A__				=			self.tokenizer_class(self.vocab_file							)

     A__				=			tokenizer.tokenize("UNwant\u00E9d,running"							)
     self.assertListEqual(lowercase_						,       ["un", "##want", "##ed", ",", "runn", "##ing"]							)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_							)						,       [9, 6, 7, 12, 10, 11]							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Union[str, Any]:
     if not self.test_rust_tokenizer:
         return

     A__				=			self.get_tokenizer()
     A__				=			self.get_rust_tokenizer()

     A__				=			"UNwant\u00E9d,running"

     A__				=			tokenizer.tokenize(lowercase_							)
     A__				=			rust_tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			self.get_rust_tokenizer()
     A__				=			tokenizer.encode(lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     # With lower casing
     A__				=			self.get_tokenizer(do_lower_case=lowercase_							)
     A__				=			self.get_rust_tokenizer(do_lower_case=lowercase_							)

     A__				=			"UNwant\u00E9d,running"

     A__				=			tokenizer.tokenize(lowercase_							)
     A__				=			rust_tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_						,       add_special_tokens=lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)

     A__				=			self.get_rust_tokenizer()
     A__				=			tokenizer.encode(lowercase_							)
     A__				=			rust_tokenizer.encode(lowercase_							)
     self.assertListEqual(lowercase_						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :str							)->							int:
     A__				=			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"							)						,       ["ah", "\u535A", "\u63A8", "zz"]							)
 def      UpperCAmelCase_							(				self      :str							)->							str:
     A__				=			BasicTokenizer(do_lower_case=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "							)						,       ["hello", "!", "how", "are", "you", "?"]							)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							)						,       ["hello"]							)
 def      UpperCAmelCase_							(				self      :Optional[int]							)->							Optional[int]:
     A__				=			BasicTokenizer(do_lower_case=lowercase_						,       strip_accents=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							)						,       ["hällo", "!", "how", "are", "you", "?"]							)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							)						,       ["h\u00E9llo"]							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							str:
     A__				=			BasicTokenizer(do_lower_case=lowercase_						,       strip_accents=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							)						,       ["hallo", "!", "how", "are", "you", "?"]							)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							)						,       ["hello"]							)
 def      UpperCAmelCase_							(				self      :List[str]							)->							List[str]:
     A__				=			BasicTokenizer(do_lower_case=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							)						,       ["hallo", "!", "how", "are", "you", "?"]							)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"							)						,       ["hello"]							)
 def      UpperCAmelCase_							(				self      :Tuple							)->							Dict:
     A__				=			BasicTokenizer(do_lower_case=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "							)						,       ["HeLLo", "!", "how", "Are", "yoU", "?"]							)
 def      UpperCAmelCase_							(				self      :Any							)->							Dict:
     A__				=			BasicTokenizer(do_lower_case=lowercase_						,       strip_accents=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							)						,       ["HäLLo", "!", "how", "Are", "yoU", "?"]							)
 def      UpperCAmelCase_							(				self      :Any							)->							Tuple:
     A__				=			BasicTokenizer(do_lower_case=lowercase_						,       strip_accents=lowercase_							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "							)						,       ["HaLLo", "!", "how", "Are", "yoU", "?"]							)
 def      UpperCAmelCase_							(				self      :int							)->							List[Any]:
     A__				=			BasicTokenizer(do_lower_case=lowercase_						,       never_split=["[UNK]"]							)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"							)						,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]							)
 def      UpperCAmelCase_							(				self      :Dict							)->							Dict:
     A__				=			BasicTokenizer()
     A__				=			"a\n'll !!to?'d of, can't."
     A__				=			["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
     self.assertListEqual(tokenizer.tokenize(lowercase_							)						,       lowercase_							)
 def      UpperCAmelCase_							(				self      :List[str]							)->							Optional[int]:
     A__				=			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

     A__				=			{}
     for i, token in enumerate(lowercase_							):
         A__				=			i
     A__				=			WordpieceTokenizer(vocab=lowercase_						,       unk_token="[UNK]"							)

     self.assertListEqual(tokenizer.tokenize(""							)						,       []							)

     self.assertListEqual(tokenizer.tokenize("unwanted running"							)						,       ["un", "##want", "##ed", "runn", "##ing"]							)

     self.assertListEqual(tokenizer.tokenize("unwantedX running"							)						,       ["[UNK]", "runn", "##ing"]							)
 def      UpperCAmelCase_							(				self      :int							)->							Union[str, Any]:
     self.assertTrue(_is_whitespace(" "							)							)
     self.assertTrue(_is_whitespace("\t"							)							)
     self.assertTrue(_is_whitespace("\r"							)							)
     self.assertTrue(_is_whitespace("\n"							)							)
     self.assertTrue(_is_whitespace("\u00A0"							)							)

     self.assertFalse(_is_whitespace("A"							)							)
     self.assertFalse(_is_whitespace("-"							)							)
 def      UpperCAmelCase_							(				self      :List[str]							)->							List[Any]:
     self.assertTrue(_is_control("\u0005"							)							)

     self.assertFalse(_is_control("A"							)							)
     self.assertFalse(_is_control(" "							)							)
     self.assertFalse(_is_control("\t"							)							)
     self.assertFalse(_is_control("\r"							)							)
 def      UpperCAmelCase_							(				self      :Optional[Any]							)->							str:
     self.assertTrue(_is_punctuation("-"							)							)
     self.assertTrue(_is_punctuation("$"							)							)
     self.assertTrue(_is_punctuation("`"							)							)
     self.assertTrue(_is_punctuation("."							)							)

     self.assertFalse(_is_punctuation("A"							)							)
     self.assertFalse(_is_punctuation(" "							)							)
 def      UpperCAmelCase_							(				self      :int							)->							Any:
     A__				=			self.get_tokenizer()
     A__				=			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(lowercase_							) for t in ["Test", "\xad", "test"]]						,       [["[UNK]"], [], ["[UNK]"]]							)

     self.assertListEqual(
         [rust_tokenizer.tokenize(lowercase_							) for t in ["Test", "\xad", "test"]]						,       [["[UNK]"], [], ["[UNK]"]]							)
 @slow
 def      UpperCAmelCase_							(				self      :Dict							)->							Tuple:
     A__				=			self.tokenizer_class.from_pretrained("bert-base-uncased"							)

     A__				=			tokenizer.encode("sequence builders"						,       add_special_tokens=lowercase_							)
     A__				=			tokenizer.encode("multi-sequence build"						,       add_special_tokens=lowercase_							)

     A__				=			tokenizer.build_inputs_with_special_tokens(lowercase_							)
     A__				=			tokenizer.build_inputs_with_special_tokens(lowercase_						,       lowercase_							)

     assert encoded_sentence == [1_01] + text + [1_02]
     assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
 def      UpperCAmelCase_							(				self      :List[str]							)->							int:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__				=			self.rust_tokenizer_class.from_pretrained(lowercase_						,       **lowercase_							)

             A__				=			F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
             A__				=			tokenizer_r.encode_plus(
                 lowercase_						,       return_attention_mask=lowercase_						,       return_token_type_ids=lowercase_						,       return_offsets_mapping=lowercase_						,       add_special_tokens=lowercase_						,       )

             A__				=			tokenizer_r.do_lower_case if hasattr(lowercase_						,       "do_lower_case"							) else False
             A__				=			(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "Allen"),
                     ((21, 23), "##NL"),
                     ((23, 24), "##P"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "allen"),
                     ((21, 23), "##nl"),
                     ((23, 24), "##p"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]						,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]							)							)
             self.assertEqual([e[0] for e in expected_results]						,       tokens["offset_mapping"]							)



 def      UpperCAmelCase_							(				self      :Tuple							)->							Optional[Any]:
     A__				=			["的", "人", "有"]
     A__				=			"".join(lowercase_							)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__				=			True
             A__				=			self.tokenizer_class.from_pretrained(lowercase_						,       **lowercase_							)
             A__				=			self.rust_tokenizer_class.from_pretrained(lowercase_						,       **lowercase_							)

             A__				=			tokenizer_p.encode(lowercase_						,       add_special_tokens=lowercase_							)
             A__				=			tokenizer_r.encode(lowercase_						,       add_special_tokens=lowercase_							)

             A__				=			tokenizer_r.convert_ids_to_tokens(lowercase_							)
             A__				=			tokenizer_p.convert_ids_to_tokens(lowercase_							)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(lowercase_						,       lowercase_							)
             self.assertListEqual(lowercase_						,       lowercase_							)

             A__				=			False
             A__				=			self.rust_tokenizer_class.from_pretrained(lowercase_						,       **lowercase_							)
             A__				=			self.tokenizer_class.from_pretrained(lowercase_						,       **lowercase_							)

             A__				=			tokenizer_r.encode(lowercase_						,       add_special_tokens=lowercase_							)
             A__				=			tokenizer_p.encode(lowercase_						,       add_special_tokens=lowercase_							)

             A__				=			tokenizer_r.convert_ids_to_tokens(lowercase_							)
             A__				=			tokenizer_p.convert_ids_to_tokens(lowercase_							)

             # it is expected that only the first Chinese character is not preceded by "##".
             A__				=			[
                 F"##{token}" if idx != 0 else token for idx, token in enumerate(lowercase_							)
             ]
             self.assertListEqual(lowercase_						,       lowercase_							)
             self.assertListEqual(lowercase_						,       lowercase_							)










'''simple docstring'''

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


__lowerCAmelCase							:	Optional[Any]			={
    "Acehnese Arabic": "ace_Arab",
    "Acehnese Latin": "ace_Latn",
    "Mesopotamian Arabic": "acm_Arab",
    "Ta'izzi-Adeni Arabic": "acq_Arab",
    "Tunisian Arabic": "aeb_Arab",
    "Afrikaans": "afr_Latn",
    "South Levantine Arabic": "ajp_Arab",
    "Akan": "aka_Latn",
    "Amharic": "amh_Ethi",
    "North Levantine Arabic": "apc_Arab",
    "Modern Standard Arabic": "arb_Arab",
    "Modern Standard Arabic Romanized": "arb_Latn",
    "Najdi Arabic": "ars_Arab",
    "Moroccan Arabic": "ary_Arab",
    "Egyptian Arabic": "arz_Arab",
    "Assamese": "asm_Beng",
    "Asturian": "ast_Latn",
    "Awadhi": "awa_Deva",
    "Central Aymara": "ayr_Latn",
    "South Azerbaijani": "azb_Arab",
    "North Azerbaijani": "azj_Latn",
    "Bashkir": "bak_Cyrl",
    "Bambara": "bam_Latn",
    "Balinese": "ban_Latn",
    "Belarusian": "bel_Cyrl",
    "Bemba": "bem_Latn",
    "Bengali": "ben_Beng",
    "Bhojpuri": "bho_Deva",
    "Banjar Arabic": "bjn_Arab",
    "Banjar Latin": "bjn_Latn",
    "Standard Tibetan": "bod_Tibt",
    "Bosnian": "bos_Latn",
    "Buginese": "bug_Latn",
    "Bulgarian": "bul_Cyrl",
    "Catalan": "cat_Latn",
    "Cebuano": "ceb_Latn",
    "Czech": "ces_Latn",
    "Chokwe": "cjk_Latn",
    "Central Kurdish": "ckb_Arab",
    "Crimean Tatar": "crh_Latn",
    "Welsh": "cym_Latn",
    "Danish": "dan_Latn",
    "German": "deu_Latn",
    "Southwestern Dinka": "dik_Latn",
    "Dyula": "dyu_Latn",
    "Dzongkha": "dzo_Tibt",
    "Greek": "ell_Grek",
    "English": "eng_Latn",
    "Esperanto": "epo_Latn",
    "Estonian": "est_Latn",
    "Basque": "eus_Latn",
    "Ewe": "ewe_Latn",
    "Faroese": "fao_Latn",
    "Fijian": "fij_Latn",
    "Finnish": "fin_Latn",
    "Fon": "fon_Latn",
    "French": "fra_Latn",
    "Friulian": "fur_Latn",
    "Nigerian Fulfulde": "fuv_Latn",
    "Scottish Gaelic": "gla_Latn",
    "Irish": "gle_Latn",
    "Galician": "glg_Latn",
    "Guarani": "grn_Latn",
    "Gujarati": "guj_Gujr",
    "Haitian Creole": "hat_Latn",
    "Hausa": "hau_Latn",
    "Hebrew": "heb_Hebr",
    "Hindi": "hin_Deva",
    "Chhattisgarhi": "hne_Deva",
    "Croatian": "hrv_Latn",
    "Hungarian": "hun_Latn",
    "Armenian": "hye_Armn",
    "Igbo": "ibo_Latn",
    "Ilocano": "ilo_Latn",
    "Indonesian": "ind_Latn",
    "Icelandic": "isl_Latn",
    "Italian": "ita_Latn",
    "Javanese": "jav_Latn",
    "Japanese": "jpn_Jpan",
    "Kabyle": "kab_Latn",
    "Jingpho": "kac_Latn",
    "Kamba": "kam_Latn",
    "Kannada": "kan_Knda",
    "Kashmiri Arabic": "kas_Arab",
    "Kashmiri Devanagari": "kas_Deva",
    "Georgian": "kat_Geor",
    "Central Kanuri Arabic": "knc_Arab",
    "Central Kanuri Latin": "knc_Latn",
    "Kazakh": "kaz_Cyrl",
    "Kabiyè": "kbp_Latn",
    "Kabuverdianu": "kea_Latn",
    "Khmer": "khm_Khmr",
    "Kikuyu": "kik_Latn",
    "Kinyarwanda": "kin_Latn",
    "Kyrgyz": "kir_Cyrl",
    "Kimbundu": "kmb_Latn",
    "Northern Kurdish": "kmr_Latn",
    "Kikongo": "kon_Latn",
    "Korean": "kor_Hang",
    "Lao": "lao_Laoo",
    "Ligurian": "lij_Latn",
    "Limburgish": "lim_Latn",
    "Lingala": "lin_Latn",
    "Lithuanian": "lit_Latn",
    "Lombard": "lmo_Latn",
    "Latgalian": "ltg_Latn",
    "Luxembourgish": "ltz_Latn",
    "Luba-Kasai": "lua_Latn",
    "Ganda": "lug_Latn",
    "Luo": "luo_Latn",
    "Mizo": "lus_Latn",
    "Standard Latvian": "lvs_Latn",
    "Magahi": "mag_Deva",
    "Maithili": "mai_Deva",
    "Malayalam": "mal_Mlym",
    "Marathi": "mar_Deva",
    "Minangkabau Arabic ": "min_Arab",
    "Minangkabau Latin": "min_Latn",
    "Macedonian": "mkd_Cyrl",
    "Plateau Malagasy": "plt_Latn",
    "Maltese": "mlt_Latn",
    "Meitei Bengali": "mni_Beng",
    "Halh Mongolian": "khk_Cyrl",
    "Mossi": "mos_Latn",
    "Maori": "mri_Latn",
    "Burmese": "mya_Mymr",
    "Dutch": "nld_Latn",
    "Norwegian Nynorsk": "nno_Latn",
    "Norwegian Bokmål": "nob_Latn",
    "Nepali": "npi_Deva",
    "Northern Sotho": "nso_Latn",
    "Nuer": "nus_Latn",
    "Nyanja": "nya_Latn",
    "Occitan": "oci_Latn",
    "West Central Oromo": "gaz_Latn",
    "Odia": "ory_Orya",
    "Pangasinan": "pag_Latn",
    "Eastern Panjabi": "pan_Guru",
    "Papiamento": "pap_Latn",
    "Western Persian": "pes_Arab",
    "Polish": "pol_Latn",
    "Portuguese": "por_Latn",
    "Dari": "prs_Arab",
    "Southern Pashto": "pbt_Arab",
    "Ayacucho Quechua": "quy_Latn",
    "Romanian": "ron_Latn",
    "Rundi": "run_Latn",
    "Russian": "rus_Cyrl",
    "Sango": "sag_Latn",
    "Sanskrit": "san_Deva",
    "Santali": "sat_Olck",
    "Sicilian": "scn_Latn",
    "Shan": "shn_Mymr",
    "Sinhala": "sin_Sinh",
    "Slovak": "slk_Latn",
    "Slovenian": "slv_Latn",
    "Samoan": "smo_Latn",
    "Shona": "sna_Latn",
    "Sindhi": "snd_Arab",
    "Somali": "som_Latn",
    "Southern Sotho": "sot_Latn",
    "Spanish": "spa_Latn",
    "Tosk Albanian": "als_Latn",
    "Sardinian": "srd_Latn",
    "Serbian": "srp_Cyrl",
    "Swati": "ssw_Latn",
    "Sundanese": "sun_Latn",
    "Swedish": "swe_Latn",
    "Swahili": "swh_Latn",
    "Silesian": "szl_Latn",
    "Tamil": "tam_Taml",
    "Tatar": "tat_Cyrl",
    "Telugu": "tel_Telu",
    "Tajik": "tgk_Cyrl",
    "Tagalog": "tgl_Latn",
    "Thai": "tha_Thai",
    "Tigrinya": "tir_Ethi",
    "Tamasheq Latin": "taq_Latn",
    "Tamasheq Tifinagh": "taq_Tfng",
    "Tok Pisin": "tpi_Latn",
    "Tswana": "tsn_Latn",
    "Tsonga": "tso_Latn",
    "Turkmen": "tuk_Latn",
    "Tumbuka": "tum_Latn",
    "Turkish": "tur_Latn",
    "Twi": "twi_Latn",
    "Central Atlas Tamazight": "tzm_Tfng",
    "Uyghur": "uig_Arab",
    "Ukrainian": "ukr_Cyrl",
    "Umbundu": "umb_Latn",
    "Urdu": "urd_Arab",
    "Northern Uzbek": "uzn_Latn",
    "Venetian": "vec_Latn",
    "Vietnamese": "vie_Latn",
    "Waray": "war_Latn",
    "Wolof": "wol_Latn",
    "Xhosa": "xho_Latn",
    "Eastern Yiddish": "ydd_Hebr",
    "Yoruba": "yor_Latn",
    "Yue Chinese": "yue_Hant",
    "Chinese Simplified": "zho_Hans",
    "Chinese Traditional": "zho_Hant",
    "Standard Malay": "zsm_Latn",
    "Zulu": "zul_Latn",
}






class    UpperCAmelCase	(       UpperCamelCase__				):
 __lowercase   		=				"""facebook/nllb-200-distilled-600M"""
 __lowercase   		=				(
     """This is a tool that translates text from a language to another. It takes three inputs: `text`, which should """
     """be the text to translate, `src_lang`, which should be the language of the text to translate and `tgt_lang`, """
     """which should be the language for the desired ouput language. Both `src_lang` and `tgt_lang` are written in """
     """plain English, such as 'Romanian', or 'Albanian'. It returns the text translated in `tgt_lang`."""
 )
 __lowercase   		=				"""translator"""
 __lowercase   		=				AutoTokenizer
 __lowercase   		=				AutoModelForSeqaSeqLM
 __lowercase   		=				LANGUAGE_CODES

 __lowercase   		=				["""text""", """text""", """text"""]
 __lowercase   		=				["""text"""]
 def      UpperCAmelCase_							(				self      :List[Any]						,       lowercase_      :Any						,       lowercase_      :List[str]						,       lowercase_      :int							)->							str:
     if src_lang not in self.lang_to_code:
         raise ValueError(F"{src_lang} is not a supported language."							)
     if tgt_lang not in self.lang_to_code:
         raise ValueError(F"{tgt_lang} is not a supported language."							)
     A__				=			self.lang_to_code[src_lang]
     A__				=			self.lang_to_code[tgt_lang]
     return self.pre_processor._build_translation_inputs(
         lowercase_						,       return_tensors="pt"						,       src_lang=lowercase_						,       tgt_lang=lowercase_							)
 def      UpperCAmelCase_							(				self      :Dict						,       lowercase_      :Any							)->							int:
     return self.model.generate(**lowercase_							)



 def      UpperCAmelCase_							(				self      :int						,       lowercase_      :Optional[Any]							)->							str:
     return self.post_processor.decode(outputs[0].tolist()						,       skip_special_tokens=lowercase_							)



