import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
__lowerCamelCase			: Any							   =		sys.version_info >= (3, 10)



def 	__SCREAMING_SNAKE_CASE  (					__UpperCamelCase    :       List[str]=None ,     __UpperCamelCase    :       Optional[int]=None	)							->					str:







  """simple docstring"""





  return field(default_factory=lambda: default ,     metadata=__UpperCamelCase	)







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							42
   lowerCAmelCase_														=							42
   lowerCAmelCase_														=							42
   lowerCAmelCase_														=							42







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							42
   lowerCAmelCase_														=							field(default="toto"   ,		metadata={"help": "help message"}		)







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							True
   lowerCAmelCase_														=							None







class __snake_case				(						lowerCamelCase_		):
   lowerCAmelCase_														=							"titi"
   lowerCAmelCase_														=							"toto"







class __snake_case				(						lowerCamelCase_		):
   lowerCAmelCase_														=							"titi"
   lowerCAmelCase_														=							"toto"
   lowerCAmelCase_														=							42







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							"toto"




   def 	__a						(    self			:				int      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      BasicEnum(self.foo      )







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							"toto"




   def 	__a						(    self			:				Optional[int]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      MixedTypeEnum(self.foo      )







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							None
   lowerCAmelCase_														=							field(default=lowerCamelCase_   ,		metadata={"help": "help message"}		)
   lowerCAmelCase_														=							None
   lowerCAmelCase_														=							list_field(default=[]		)
   lowerCAmelCase_														=							list_field(default=[]		)







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							list_field(default=[]		)
   lowerCAmelCase_														=							list_field(default=[1, 2, 3]		)
   lowerCAmelCase_														=							list_field(default=["Hallo", "Bonjour", "Hello"]		)
   lowerCAmelCase_														=							list_field(default=[0.1, 0.2, 0.3]		)







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							field()
   lowerCAmelCase_														=							field()
   lowerCAmelCase_														=							field()




   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      BasicEnum(self.required_enum      )







@dataclass
class __snake_case				:
   lowerCAmelCase_														=							42
   lowerCAmelCase_														=							field()
   lowerCAmelCase_														=							None
   lowerCAmelCase_														=							field(default="toto"   ,		metadata={"help": "help message"}		)
   lowerCAmelCase_														=							list_field(default=["Hallo", "Bonjour", "Hello"]		)


if is_python_no_less_than_3_10:







   @dataclass
   class __snake_case				:
      lowerCAmelCase_														=							False
      lowerCAmelCase_														=							True
      lowerCAmelCase_														=							None







   @dataclass
   class __snake_case				:
      lowerCAmelCase_														=							None
      lowerCAmelCase_														=							field(default=lowerCamelCase_   ,		metadata={"help": "help message"}		)
      lowerCAmelCase_														=							None
      lowerCAmelCase_														=							list_field(default=[]		)
      lowerCAmelCase_														=							list_field(default=[]		)







class __snake_case				(						unittest.TestCase		):




   def 	__a						(    self			:				Union[str, Any]       ,	_lowercase			:				argparse.ArgumentParser       ,	_lowercase			:				argparse.ArgumentParser      ):



     """simple docstring"""






     self.assertEqual(len(a._actions      )       ,	len(b._actions      )      )
     for x, y in zip(a._actions       ,	b._actions      ):
       SCREAMING_SNAKE_CASE__										=      {k: v for k, v in vars(_lowercase      ).items() if k != """container"""}
       SCREAMING_SNAKE_CASE__										=      {k: v for k, v in vars(_lowercase      ).items() if k != """container"""}

       # Choices with mixed type have custom function as "type"
       # So we need to compare results directly for equality
       if xx.get("""choices"""       ,	_lowercase      ) and yy.get("""choices"""       ,	_lowercase      ):
         for expected_choice in yy["choices"] + xx["choices"]:
           self.assertEqual(xx["""type"""](_lowercase      )       ,	yy["""type"""](_lowercase      )      )
         del xx["type"], yy["type"]

       self.assertEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument("""--bar"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument("""--baz"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument("""--flag"""       ,	type=_lowercase       ,	default=_lowercase       ,	const=_lowercase       ,	nargs="""?"""      )
     self.argparsersEqual(_lowercase       ,	_lowercase      )

     SCREAMING_SNAKE_CASE__										=      ["""--foo""", """1""", """--baz""", """quux""", """--bar""", """0.5"""]
     ((SCREAMING_SNAKE_CASE__)							,     )										=      parser.parse_args_into_dataclasses(_lowercase       ,	look_for_args_file=_lowercase      )
     self.assertFalse(example.flag      )




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo"""       ,	default=42       ,	type=_lowercase      )
     expected.add_argument("""--baz"""       ,	default="""toto"""       ,	type=_lowercase       ,	help="""help message"""      )
     self.argparsersEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo"""       ,	type=_lowercase       ,	default=_lowercase       ,	const=_lowercase       ,	nargs="""?"""      )
     expected.add_argument("""--baz"""       ,	type=_lowercase       ,	default=_lowercase       ,	const=_lowercase       ,	nargs="""?"""      )
     # A boolean no_* argument always has to come after its "default: True" regular counter-part
     # and its default must be set to False
     expected.add_argument("""--no_baz"""       ,	action="""store_false"""       ,	default=_lowercase       ,	dest="""baz"""      )
     expected.add_argument("""--opt"""       ,	type=_lowercase       ,	default=_lowercase      )

     SCREAMING_SNAKE_CASE__										=      [WithDefaultBoolExample]
     if is_python_no_less_than_3_10:
       dataclass_types.append(_lowercase      )

     for dataclass_type in dataclass_types:
       SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )
       self.argparsersEqual(_lowercase       ,	_lowercase      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args([]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	baz=_lowercase       ,	opt=_lowercase      )      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """--no_baz"""]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	baz=_lowercase       ,	opt=_lowercase      )      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """--baz"""]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	baz=_lowercase       ,	opt=_lowercase      )      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """True""", """--baz""", """True""", """--opt""", """True"""]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	baz=_lowercase       ,	opt=_lowercase      )      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """False""", """--baz""", """False""", """--opt""", """False"""]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	baz=_lowercase       ,	opt=_lowercase      )      )




   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument(
         """--foo"""       ,	default="""toto"""       ,	choices=["""titi""", """toto""", 42]       ,	type=make_choice_type_function(["""titi""", """toto""", 42]      )       ,	)
     self.argparsersEqual(_lowercase       ,	_lowercase      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args([]      )
     self.assertEqual(args.foo       ,	"""toto"""      )
     SCREAMING_SNAKE_CASE__										=      parser.parse_args_into_dataclasses([]      )[0]
     self.assertEqual(enum_ex.foo       ,	MixedTypeEnum.toto      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """titi"""]      )
     self.assertEqual(args.foo       ,	"""titi"""      )
     SCREAMING_SNAKE_CASE__										=      parser.parse_args_into_dataclasses(["""--foo""", """titi"""]      )[0]
     self.assertEqual(enum_ex.foo       ,	MixedTypeEnum.titi      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """42"""]      )
     self.assertEqual(args.foo       ,	42      )
     SCREAMING_SNAKE_CASE__										=      parser.parse_args_into_dataclasses(["""--foo""", """42"""]      )[0]
     self.assertEqual(enum_ex.foo       ,	MixedTypeEnum.fourtytwo      )




   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     @dataclass
     class __snake_case				:
        lowerCAmelCase_														=							"toto"





     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument(
         """--foo"""       ,	default="""toto"""       ,	choices=("""titi""", """toto""", 42)       ,	type=make_choice_type_function(["""titi""", """toto""", 42]      )       ,	)
     self.argparsersEqual(_lowercase       ,	_lowercase      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args([]      )
     self.assertEqual(args.foo       ,	"""toto"""      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """titi"""]      )
     self.assertEqual(args.foo       ,	"""titi"""      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args(["""--foo""", """42"""]      )
     self.assertEqual(args.foo       ,	42      )




   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo_int"""       ,	nargs="""+"""       ,	default=[]       ,	type=_lowercase      )
     expected.add_argument("""--bar_int"""       ,	nargs="""+"""       ,	default=[1, 2, 3]       ,	type=_lowercase      )
     expected.add_argument("""--foo_str"""       ,	nargs="""+"""       ,	default=["""Hallo""", """Bonjour""", """Hello"""]       ,	type=_lowercase      )
     expected.add_argument("""--foo_float"""       ,	nargs="""+"""       ,	default=[0.1, 0.2, 0.3]       ,	type=_lowercase      )

     self.argparsersEqual(_lowercase       ,	_lowercase      )

     SCREAMING_SNAKE_CASE__										=      parser.parse_args([]      )
     self.assertEqual(
         _lowercase       ,	Namespace(foo_int=[]       ,	bar_int=[1, 2, 3]       ,	foo_str=["""Hallo""", """Bonjour""", """Hello"""]       ,	foo_float=[0.1, 0.2, 0.3]      )       ,	)

     SCREAMING_SNAKE_CASE__										=      parser.parse_args("""--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7""".split()      )
     self.assertEqual(_lowercase       ,	Namespace(foo_int=[1]       ,	bar_int=[2, 3]       ,	foo_str=["""a""", """b""", """c"""]       ,	foo_float=[0.1, 0.7]      )      )




   def 	__a						(    self			:				str      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo"""       ,	default=_lowercase       ,	type=_lowercase      )
     expected.add_argument("""--bar"""       ,	default=_lowercase       ,	type=_lowercase       ,	help="""help message"""      )
     expected.add_argument("""--baz"""       ,	default=_lowercase       ,	type=_lowercase      )
     expected.add_argument("""--ces"""       ,	nargs="""+"""       ,	default=[]       ,	type=_lowercase      )
     expected.add_argument("""--des"""       ,	nargs="""+"""       ,	default=[]       ,	type=_lowercase      )

     SCREAMING_SNAKE_CASE__										=      [OptionalExample]
     if is_python_no_less_than_3_10:
       dataclass_types.append(_lowercase      )

     for dataclass_type in dataclass_types:
       SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

       self.argparsersEqual(_lowercase       ,	_lowercase      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args([]      )
       self.assertEqual(_lowercase       ,	Namespace(foo=_lowercase       ,	bar=_lowercase       ,	baz=_lowercase       ,	ces=[]       ,	des=[]      )      )

       SCREAMING_SNAKE_CASE__										=      parser.parse_args("""--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3""".split()      )
       self.assertEqual(_lowercase       ,	Namespace(foo=12       ,	bar=3.14       ,	baz="""42"""       ,	ces=["""a""", """b""", """c"""]       ,	des=[1, 2, 3]      )      )




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--required_list"""       ,	nargs="""+"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument("""--required_str"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument(
         """--required_enum"""       ,	type=make_choice_type_function(["""titi""", """toto"""]      )       ,	choices=["""titi""", """toto"""]       ,	required=_lowercase       ,	)
     self.argparsersEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      argparse.ArgumentParser()
     expected.add_argument("""--foo"""       ,	type=_lowercase       ,	required=_lowercase      )
     expected.add_argument(
         """--required_enum"""       ,	type=make_choice_type_function(["""titi""", """toto"""]      )       ,	choices=["""titi""", """toto"""]       ,	required=_lowercase       ,	)
     expected.add_argument("""--opt"""       ,	type=_lowercase       ,	default=_lowercase      )
     expected.add_argument("""--baz"""       ,	default="""toto"""       ,	type=_lowercase       ,	help="""help message"""      )
     expected.add_argument("""--foo_str"""       ,	nargs="""+"""       ,	default=["""Hallo""", """Bonjour""", """Hello"""]       ,	type=_lowercase      )
     self.argparsersEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      {
         """foo""": 12,
         """bar""": 3.14,
         """baz""": """42""",
         """flag""": True,
     }

     SCREAMING_SNAKE_CASE__										=      parser.parse_dict(_lowercase      )[0]
     SCREAMING_SNAKE_CASE__										=      BasicExample(**_lowercase      )
     self.assertEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				List[str]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      {
         """foo""": 12,
         """bar""": 3.14,
         """baz""": """42""",
         """flag""": True,
         """extra""": 42,
     }

     self.assertRaises(_lowercase       ,	parser.parse_dict       ,	_lowercase       ,	allow_extra_keys=_lowercase      )




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      {
         """foo""": 12,
         """bar""": 3.14,
         """baz""": """42""",
         """flag""": True,
     }
     with tempfile.TemporaryDirectory() as tmp_dir:
       SCREAMING_SNAKE_CASE__										=      os.path.join(_lowercase       ,	"""temp_json"""      )
       os.mkdir(_lowercase      )
       with open(temp_local_path + """.json"""       ,	"""w+"""      ) as f:
         json.dump(_lowercase       ,	_lowercase      )
       SCREAMING_SNAKE_CASE__										=      parser.parse_yaml_file(Path(temp_local_path + """.json"""      )      )[0]

     SCREAMING_SNAKE_CASE__										=      BasicExample(**_lowercase      )
     self.assertEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				str      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )

     SCREAMING_SNAKE_CASE__										=      {
         """foo""": 12,
         """bar""": 3.14,
         """baz""": """42""",
         """flag""": True,
     }
     with tempfile.TemporaryDirectory() as tmp_dir:
       SCREAMING_SNAKE_CASE__										=      os.path.join(_lowercase       ,	"""temp_yaml"""      )
       os.mkdir(_lowercase      )
       with open(temp_local_path + """.yaml"""       ,	"""w+"""      ) as f:
         yaml.dump(_lowercase       ,	_lowercase      )
       SCREAMING_SNAKE_CASE__										=      parser.parse_yaml_file(Path(temp_local_path + """.yaml"""      )      )[0]
     SCREAMING_SNAKE_CASE__										=      BasicExample(**_lowercase      )
     self.assertEqual(_lowercase       ,	_lowercase      )




   def 	__a						(    self			:				int      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      HfArgumentParser(_lowercase      )
     self.assertIsNotNone(_lowercase      )

import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase			: str							   =		get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
   import sentencepiece as sp


__lowerCamelCase			: Any							   =		5
__lowerCamelCase			: Dict							   =		10







@require_sentencepiece
@require_tokenizers
class __snake_case				(						lowerCamelCase_   ,		unittest.TestCase		):
   lowerCAmelCase_														=							SpeechaTextTokenizer
   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							True




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     super().setUp()

     SCREAMING_SNAKE_CASE__										=      sp.SentencePieceProcessor()
     spm_model.Load(_lowercase      )
     SCREAMING_SNAKE_CASE__										=      ["""<s>""", """<pad>""", """</s>""", """<unk>"""]

     vocab += [spm_model.IdToPiece(id_      ) for id_ in range(len(_lowercase      )      )]
     SCREAMING_SNAKE_CASE__										=      dict(zip(_lowercase       ,	range(len(_lowercase      )      )      )      )

     SCREAMING_SNAKE_CASE__										=      Path(self.tmpdirname      )
     save_json(_lowercase       ,	save_dir / VOCAB_FILES_NAMES["""vocab_file"""]      )
     if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
       copyfile(_lowercase       ,	save_dir / VOCAB_FILES_NAMES["""spm_file"""]      )

     SCREAMING_SNAKE_CASE__										=      SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
     tokenizer.save_pretrained(self.tmpdirname      )




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      """<pad>"""
     SCREAMING_SNAKE_CASE__										=      1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase      )       ,	_lowercase      )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase      )       ,	_lowercase      )




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      list(self.get_tokenizer().get_vocab().keys()      )

     self.assertEqual(vocab_keys[0]       ,	"""<s>"""      )
     self.assertEqual(vocab_keys[1]       ,	"""<pad>"""      )
     self.assertEqual(vocab_keys[-1]       ,	"""j"""      )
     self.assertEqual(len(_lowercase      )       ,	10_01      )




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     self.assertEqual(self.get_tokenizer().vocab_size       ,	10_01      )




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )

     SCREAMING_SNAKE_CASE__										=      tokenizer.tokenize("""This is a test"""      )
     self.assertListEqual(_lowercase       ,	["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]      )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_lowercase      )       ,	[2_89, 50, 14, 1_74, 3_86]       ,	)

     SCREAMING_SNAKE_CASE__										=      tokenizer.tokenize("""I was born in 92000, and this is falsé."""      )
     self.assertListEqual(
         _lowercase       ,	[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]       ,	)
     SCREAMING_SNAKE_CASE__										=      tokenizer.convert_tokens_to_ids(_lowercase      )
     self.assertListEqual(_lowercase       ,	[12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8]      )

     SCREAMING_SNAKE_CASE__										=      tokenizer.convert_ids_to_tokens(_lowercase      )
     self.assertListEqual(
         _lowercase       ,	[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]       ,	)




   @slow
   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      {"""input_ids""": [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_lowercase       ,	model_name="""facebook/s2t-small-mustc-en-de-st"""       ,	revision="""a14f04cf0776c02f62a8cb800cf7909e15ea23ad"""       ,	)







@require_sentencepiece
class __snake_case				(						unittest.TestCase		):
   lowerCAmelCase_														=							"valhalla/s2t_mustc_multilinguial_medium"

   lowerCAmelCase_														=							"C'est trop cool"
   lowerCAmelCase_														=							"Esto es genial"




   @classmethod
   def 	__a						(    cls			:				Any      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name      )
     return cls




   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     self.assertEqual(self.tokenizer.lang_code_to_id["""pt"""]       ,	4      )
     self.assertEqual(self.tokenizer.lang_code_to_id["""ru"""]       ,	6      )
     self.assertEqual(self.tokenizer.lang_code_to_id["""it"""]       ,	9      )
     self.assertEqual(self.tokenizer.lang_code_to_id["""de"""]       ,	11      )




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     self.assertEqual(self.tokenizer.vocab_size       ,	1_00_00      )




   def 	__a						(    self			:				int      ):



     """simple docstring"""






     self.assertIn(_lowercase       ,	self.tokenizer.all_special_ids      )
     SCREAMING_SNAKE_CASE__										=      [ES_CODE, 4, 16_01, 47, 76_47, 2]
     SCREAMING_SNAKE_CASE__										=      self.tokenizer.decode(_lowercase       ,	skip_special_tokens=_lowercase      )
     SCREAMING_SNAKE_CASE__										=      self.tokenizer.decode(generated_ids[1:]       ,	skip_special_tokens=_lowercase      )
     self.assertEqual(_lowercase       ,	_lowercase      )
     self.assertNotIn(self.tokenizer.eos_token       ,	_lowercase      )




   def 	__a						(    self			:				Any      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      """fr"""
     SCREAMING_SNAKE_CASE__										=      self.tokenizer(self.french_text      ).input_ids
     self.assertEqual(encoded[0]       ,	_lowercase      )
     self.assertEqual(encoded[-1]       ,	self.tokenizer.eos_token_id      )




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      """fr"""
     self.assertListEqual(self.tokenizer.prefix_tokens       ,	[FR_CODE]      )

     SCREAMING_SNAKE_CASE__										=      """es"""
     self.assertListEqual(self.tokenizer.prefix_tokens       ,	[ES_CODE]      )
