



"""simple docstring"""







import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
lowercase__		:				Union[str, Any]          =	sys.version_info >= (3, 1_0)

def 				UpperCamelCase_    (    lowerCAmelCase__      :				List[Any]=None    ,   lowerCAmelCase__      :				Any=None						)						->   int:

  """simple docstring"""




  return field(default_factory=lambda: default    ,   metadata=lowerCAmelCase__						)







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     42
      _SCREAMING_SNAKE_CASE  				=     42
      _SCREAMING_SNAKE_CASE  				=     42
      _SCREAMING_SNAKE_CASE  				=     42







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     42
      _SCREAMING_SNAKE_CASE  				=     field(default="""toto""",					metadata={"""help""": """help message"""}  )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     False
      _SCREAMING_SNAKE_CASE  				=     True
      _SCREAMING_SNAKE_CASE  				=     None







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     """titi"""
      _SCREAMING_SNAKE_CASE  				=     """toto"""







class       UpperCamelCase__   (	lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     """titi"""
      _SCREAMING_SNAKE_CASE  				=     """toto"""
      _SCREAMING_SNAKE_CASE  				=     42







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     "toto"



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				List[str]								= BasicEnum(self.foo       )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     "toto"



      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        lowerCAmelCase_   :				List[Any]								= MixedTypeEnum(self.foo       )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     None
      _SCREAMING_SNAKE_CASE  				=     field(default=lowercase_,					metadata={"""help""": """help message"""}  )
      _SCREAMING_SNAKE_CASE  				=     None
      _SCREAMING_SNAKE_CASE  				=     list_field(default=[]  )
      _SCREAMING_SNAKE_CASE  				=     list_field(default=[]  )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     list_field(default=[]  )
      _SCREAMING_SNAKE_CASE  				=     list_field(default=[1, 2, 3]  )
      _SCREAMING_SNAKE_CASE  				=     list_field(default=["""Hallo""", """Bonjour""", """Hello"""]  )
      _SCREAMING_SNAKE_CASE  				=     list_field(default=[0.1, 0.2, 0.3]  )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     field()
      _SCREAMING_SNAKE_CASE  				=     field()
      _SCREAMING_SNAKE_CASE  				=     field()



      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        lowerCAmelCase_   :				int								= BasicEnum(self.required_enum       )







@dataclass
class       UpperCamelCase__   :




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     42
      _SCREAMING_SNAKE_CASE  				=     field()
      _SCREAMING_SNAKE_CASE  				=     None
      _SCREAMING_SNAKE_CASE  				=     field(default="""toto""",					metadata={"""help""": """help message"""}  )
      _SCREAMING_SNAKE_CASE  				=     list_field(default=["""Hallo""", """Bonjour""", """Hello"""]  )


if is_python_no_less_than_3_10:







     @dataclass
     class       UpperCamelCase__   :




           """simple docstring"""




           _SCREAMING_SNAKE_CASE  				=     False
           _SCREAMING_SNAKE_CASE  				=     True
           _SCREAMING_SNAKE_CASE  				=     None







     @dataclass
     class       UpperCamelCase__   :




           """simple docstring"""




           _SCREAMING_SNAKE_CASE  				=     None
           _SCREAMING_SNAKE_CASE  				=     field(default=lowercase_,					metadata={"""help""": """help message"""}  )
           _SCREAMING_SNAKE_CASE  				=     None
           _SCREAMING_SNAKE_CASE  				=     list_field(default=[]  )
           _SCREAMING_SNAKE_CASE  				=     list_field(default=[]  )







class       UpperCamelCase__   (	unittest.TestCase  ):




      """simple docstring"""




      def      SCREAMING_SNAKE_CASE__ (      self				:      str    ,			SCREAMING_SNAKE_CASE_				:      argparse.ArgumentParser    ,			SCREAMING_SNAKE_CASE_				:      argparse.ArgumentParser       ):
        self.assertEqual(len(a._actions       )    ,			len(b._actions       )       )
        for x, y in zip(a._actions    ,			b._actions       ):
          lowerCAmelCase_   :				Tuple								= {k: v for k, v in vars(SCREAMING_SNAKE_CASE_       ).items() if k != 'container'}
          lowerCAmelCase_   :				List[str]								= {k: v for k, v in vars(SCREAMING_SNAKE_CASE_       ).items() if k != 'container'}

          # Choices with mixed type have custom function as "type"
          # So we need to compare results directly for equality
          if xx.get('choices'    ,			SCREAMING_SNAKE_CASE_       ) and yy.get('choices'    ,			SCREAMING_SNAKE_CASE_       ):
            for expected_choice in yy["choices"] + xx["choices"]:
              self.assertEqual(xx['type'](SCREAMING_SNAKE_CASE_       )    ,			yy['type'](SCREAMING_SNAKE_CASE_       )       )
            del xx["type"], yy["type"]

          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				int								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Tuple								= argparse.ArgumentParser()
        expected.add_argument('--foo'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--bar'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--baz'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--flag'    ,			type=SCREAMING_SNAKE_CASE_    ,			default=SCREAMING_SNAKE_CASE_    ,			const=SCREAMING_SNAKE_CASE_    ,			nargs='?'       )
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[str]								= ['--foo', '1', '--baz', 'quux', '--bar', '0.5']
        ((lowerCAmelCase_)						,)   :				Tuple								= parser.parse_args_into_dataclasses(SCREAMING_SNAKE_CASE_    ,			look_for_args_file=SCREAMING_SNAKE_CASE_       )
        self.assertFalse(example.flag       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any       ):
        lowerCAmelCase_   :				Optional[Any]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Union[str, Any]								= argparse.ArgumentParser()
        expected.add_argument('--foo'    ,			default=4_2    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--baz'    ,			default='toto'    ,			type=SCREAMING_SNAKE_CASE_    ,			help='help message'       )
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        lowerCAmelCase_   :				Dict								= argparse.ArgumentParser()
        expected.add_argument('--foo'    ,			type=SCREAMING_SNAKE_CASE_    ,			default=SCREAMING_SNAKE_CASE_    ,			const=SCREAMING_SNAKE_CASE_    ,			nargs='?'       )
        expected.add_argument('--baz'    ,			type=SCREAMING_SNAKE_CASE_    ,			default=SCREAMING_SNAKE_CASE_    ,			const=SCREAMING_SNAKE_CASE_    ,			nargs='?'       )
        # A boolean no_* argument always has to come after its "default: True" regular counter-part
        # and its default must be set to False
        expected.add_argument('--no_baz'    ,			action='store_false'    ,			default=SCREAMING_SNAKE_CASE_    ,			dest='baz'       )
        expected.add_argument('--opt'    ,			type=SCREAMING_SNAKE_CASE_    ,			default=SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Optional[Any]								= [WithDefaultBoolExample]
        if is_python_no_less_than_3_10:
          dataclass_types.append(SCREAMING_SNAKE_CASE_       )

        for dataclass_type in dataclass_types:
          lowerCAmelCase_   :				List[Any]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )
          self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				List[str]								= parser.parse_args([]       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			opt=SCREAMING_SNAKE_CASE_       )       )

          lowerCAmelCase_   :				Optional[Any]								= parser.parse_args(['--foo', '--no_baz']       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			opt=SCREAMING_SNAKE_CASE_       )       )

          lowerCAmelCase_   :				Dict								= parser.parse_args(['--foo', '--baz']       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			opt=SCREAMING_SNAKE_CASE_       )       )

          lowerCAmelCase_   :				Union[str, Any]								= parser.parse_args(['--foo', 'True', '--baz', 'True', '--opt', 'True']       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			opt=SCREAMING_SNAKE_CASE_       )       )

          lowerCAmelCase_   :				Optional[Any]								= parser.parse_args(['--foo', 'False', '--baz', 'False', '--opt', 'False']       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			opt=SCREAMING_SNAKE_CASE_       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        lowerCAmelCase_   :				Any								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Any								= argparse.ArgumentParser()
        expected.add_argument(
            '--foo'    ,			default='toto'    ,			choices=['titi', 'toto', 4_2]    ,			type=make_choice_type_function(['titi', 'toto', 4_2]       )    ,			)
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Union[str, Any]								= parser.parse_args([]       )
        self.assertEqual(args.foo    ,			'toto'       )
        lowerCAmelCase_   :				Dict								= parser.parse_args_into_dataclasses([]       )[0]
        self.assertEqual(enum_ex.foo    ,			MixedTypeEnum.toto       )

        lowerCAmelCase_   :				Optional[int]								= parser.parse_args(['--foo', 'titi']       )
        self.assertEqual(args.foo    ,			'titi'       )
        lowerCAmelCase_   :				List[str]								= parser.parse_args_into_dataclasses(['--foo', 'titi']       )[0]
        self.assertEqual(enum_ex.foo    ,			MixedTypeEnum.titi       )

        lowerCAmelCase_   :				Optional[Any]								= parser.parse_args(['--foo', '42']       )
        self.assertEqual(args.foo    ,			4_2       )
        lowerCAmelCase_   :				int								= parser.parse_args_into_dataclasses(['--foo', '42']       )[0]
        self.assertEqual(enum_ex.foo    ,			MixedTypeEnum.fourtytwo       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):







        @dataclass
        class       UpperCamelCase__   :




              """simple docstring"""




              _SCREAMING_SNAKE_CASE  				=     "toto"




        lowerCAmelCase_   :				Optional[Any]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Optional[int]								= argparse.ArgumentParser()
        expected.add_argument(
            '--foo'    ,			default='toto'    ,			choices=('titi', 'toto', 4_2)    ,			type=make_choice_type_function(['titi', 'toto', 4_2]       )    ,			)
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Dict								= parser.parse_args([]       )
        self.assertEqual(args.foo    ,			'toto'       )

        lowerCAmelCase_   :				Optional[Any]								= parser.parse_args(['--foo', 'titi']       )
        self.assertEqual(args.foo    ,			'titi'       )

        lowerCAmelCase_   :				Optional[int]								= parser.parse_args(['--foo', '42']       )
        self.assertEqual(args.foo    ,			4_2       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        lowerCAmelCase_   :				Optional[int]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				str								= argparse.ArgumentParser()
        expected.add_argument('--foo_int'    ,			nargs='+'    ,			default=[]    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--bar_int'    ,			nargs='+'    ,			default=[1, 2, 3]    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--foo_str'    ,			nargs='+'    ,			default=['Hallo', 'Bonjour', 'Hello']    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--foo_float'    ,			nargs='+'    ,			default=[0.1, 0.2, 0.3]    ,			type=SCREAMING_SNAKE_CASE_       )

        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[str]								= parser.parse_args([]       )
        self.assertEqual(
            SCREAMING_SNAKE_CASE_    ,			Namespace(foo_int=[]    ,			bar_int=[1, 2, 3]    ,			foo_str=['Hallo', 'Bonjour', 'Hello']    ,			foo_float=[0.1, 0.2, 0.3]       )    ,			)

        lowerCAmelCase_   :				Any								= parser.parse_args('--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'.split()       )
        self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo_int=[1]    ,			bar_int=[2, 3]    ,			foo_str=['a', 'b', 'c']    ,			foo_float=[0.1, 0.7]       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        lowerCAmelCase_   :				int								= argparse.ArgumentParser()
        expected.add_argument('--foo'    ,			default=SCREAMING_SNAKE_CASE_    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--bar'    ,			default=SCREAMING_SNAKE_CASE_    ,			type=SCREAMING_SNAKE_CASE_    ,			help='help message'       )
        expected.add_argument('--baz'    ,			default=SCREAMING_SNAKE_CASE_    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--ces'    ,			nargs='+'    ,			default=[]    ,			type=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--des'    ,			nargs='+'    ,			default=[]    ,			type=SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Tuple								= [OptionalExample]
        if is_python_no_less_than_3_10:
          dataclass_types.append(SCREAMING_SNAKE_CASE_       )

        for dataclass_type in dataclass_types:
          lowerCAmelCase_   :				List[str]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

          self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				int								= parser.parse_args([]       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=SCREAMING_SNAKE_CASE_    ,			bar=SCREAMING_SNAKE_CASE_    ,			baz=SCREAMING_SNAKE_CASE_    ,			ces=[]    ,			des=[]       )       )

          lowerCAmelCase_   :				Union[str, Any]								= parser.parse_args('--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'.split()       )
          self.assertEqual(SCREAMING_SNAKE_CASE_    ,			Namespace(foo=1_2    ,			bar=3.14    ,			baz='42'    ,			ces=['a', 'b', 'c']    ,			des=[1, 2, 3]       )       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        lowerCAmelCase_   :				int								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[Any]								= argparse.ArgumentParser()
        expected.add_argument('--required_list'    ,			nargs='+'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--required_str'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument(
            '--required_enum'    ,			type=make_choice_type_function(['titi', 'toto']       )    ,			choices=['titi', 'toto']    ,			required=SCREAMING_SNAKE_CASE_    ,			)
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_   :				Optional[Any]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Dict								= argparse.ArgumentParser()
        expected.add_argument('--foo'    ,			type=SCREAMING_SNAKE_CASE_    ,			required=SCREAMING_SNAKE_CASE_       )
        expected.add_argument(
            '--required_enum'    ,			type=make_choice_type_function(['titi', 'toto']       )    ,			choices=['titi', 'toto']    ,			required=SCREAMING_SNAKE_CASE_    ,			)
        expected.add_argument('--opt'    ,			type=SCREAMING_SNAKE_CASE_    ,			default=SCREAMING_SNAKE_CASE_       )
        expected.add_argument('--baz'    ,			default='toto'    ,			type=SCREAMING_SNAKE_CASE_    ,			help='help message'       )
        expected.add_argument('--foo_str'    ,			nargs='+'    ,			default=['Hallo', 'Bonjour', 'Hello']    ,			type=SCREAMING_SNAKE_CASE_       )
        self.argparsersEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				Optional[int]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[Any]								= {
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }

        lowerCAmelCase_   :				Optional[Any]								= parser.parse_dict(SCREAMING_SNAKE_CASE_       )[0]
        lowerCAmelCase_   :				Union[str, Any]								= BasicExample(**SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_   :				Union[str, Any]								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				int								= {
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
            'extra': 4_2,
        }

        self.assertRaises(SCREAMING_SNAKE_CASE_    ,			parser.parse_dict    ,			SCREAMING_SNAKE_CASE_    ,			allow_extra_keys=SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				Dict								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[Any]								= {
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase_   :				int								= os.path.join(SCREAMING_SNAKE_CASE_    ,			'temp_json'       )
          os.mkdir(SCREAMING_SNAKE_CASE_       )
          with open(temp_local_path + '.json'    ,			'w+'       ) as f:
            json.dump(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Union[str, Any]								= parser.parse_yaml_file(Path(temp_local_path + '.json'       )       )[0]

        lowerCAmelCase_   :				Tuple								= BasicExample(**SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        lowerCAmelCase_   :				Tuple								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Dict								= {
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase_   :				List[Any]								= os.path.join(SCREAMING_SNAKE_CASE_    ,			'temp_yaml'       )
          os.mkdir(SCREAMING_SNAKE_CASE_       )
          with open(temp_local_path + '.yaml'    ,			'w+'       ) as f:
            yaml.dump(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Any								= parser.parse_yaml_file(Path(temp_local_path + '.yaml'       )       )[0]
        lowerCAmelCase_   :				Tuple								= BasicExample(**SCREAMING_SNAKE_CASE_       )
        self.assertEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_   :				Dict								= HfArgumentParser(SCREAMING_SNAKE_CASE_       )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_       )










"""simple docstring"""







import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__		:				Dict          =	get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class       UpperCamelCase__   (	lowercase_,					unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     XLMRobertaTokenizer
      _SCREAMING_SNAKE_CASE  				=     XLMRobertaTokenizerFast
      _SCREAMING_SNAKE_CASE  				=     True
      _SCREAMING_SNAKE_CASE  				=     True



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        super().setUp()

        # We have a SentencePiece fixture for testing
        lowerCAmelCase_   :				Any								= XLMRobertaTokenizer(SCREAMING_SNAKE_CASE_    ,			keep_accents=SCREAMING_SNAKE_CASE_       )
        tokenizer.save_pretrained(self.tmpdirname       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				Any								= '<pad>'
        lowerCAmelCase_   :				int								= 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_       )    ,			SCREAMING_SNAKE_CASE_       )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_       )    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				int								= list(self.get_tokenizer().get_vocab().keys()       )

        self.assertEqual(vocab_keys[0]    ,			'<s>'       )
        self.assertEqual(vocab_keys[1]    ,			'<pad>'       )
        self.assertEqual(vocab_keys[-1]    ,			'<mask>'       )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			1_0_0_2       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        self.assertEqual(self.get_tokenizer().vocab_size    ,			1_0_0_2       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_   :				int								= XLMRobertaTokenizer(SCREAMING_SNAKE_CASE_    ,			keep_accents=SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Optional[Any]								= tokenizer.tokenize('This is a test'       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			['▁This', '▁is', '▁a', '▁t', 'est']       )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       )    ,			[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,			)

        lowerCAmelCase_   :				int								= tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]    ,			)
        lowerCAmelCase_   :				Any								= tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                value + tokenizer.fairseq_offset
                for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
            ]    ,			)

        lowerCAmelCase_   :				Union[str, Any]								= tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_    ,			[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]    ,			)



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        if not self.test_slow_tokenizer:
          # as we don't have a slow version, we can't compare the outputs between slow and fast versions
          return

        lowerCAmelCase_   :				List[str]								= (self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
            lowerCAmelCase_   :				str								= self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )

            lowerCAmelCase_   :				str								= tempfile.mkdtemp()

            lowerCAmelCase_   :				int								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )
            lowerCAmelCase_   :				Optional[Any]								= tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f       )
            self.assertSequenceEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				int								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )
              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )

            # Save tokenizer rust, legacy_format=True
            lowerCAmelCase_   :				str								= tempfile.mkdtemp()

            lowerCAmelCase_   :				List[str]								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_    ,			legacy_format=SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				str								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it save with the same files
            self.assertSequenceEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				str								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Union[str, Any]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )

            # Save tokenizer rust, legacy_format=False
            lowerCAmelCase_   :				int								= tempfile.mkdtemp()

            lowerCAmelCase_   :				Optional[Any]								= tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_    ,			legacy_format=SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				int								= tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_       )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )

            # Checks everything loads correctly in the same way
            lowerCAmelCase_   :				List[str]								= tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_       )
            lowerCAmelCase_   :				Optional[int]								= tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_       )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )

            shutil.rmtree(SCREAMING_SNAKE_CASE_       )



      @cached_property
      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        with tempfile.NamedTemporaryFile() as f:
          shutil.copyfile(SCREAMING_SNAKE_CASE_    ,			f.name       )
          lowerCAmelCase_   :				Tuple								= XLMRobertaTokenizer(f.name    ,			keep_accents=SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Dict								= pickle.dumps(SCREAMING_SNAKE_CASE_       )
        pickle.loads(SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        if not self.test_rust_tokenizer:
          return

        lowerCAmelCase_   :				Union[str, Any]								= self.get_tokenizer()
        lowerCAmelCase_   :				Dict								= self.get_rust_tokenizer()

        lowerCAmelCase_   :				Tuple								= 'I was born in 92000, and this is falsé.'

        lowerCAmelCase_   :				Union[str, Any]								= tokenizer.tokenize(SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Dict								= rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				Any								= tokenizer.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				List[str]								= rust_tokenizer.encode(SCREAMING_SNAKE_CASE_    ,			add_special_tokens=SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				int								= self.get_rust_tokenizer()
        lowerCAmelCase_   :				Any								= tokenizer.encode(SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				List[Any]								= rust_tokenizer.encode(SCREAMING_SNAKE_CASE_       )
        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Any       ):
        lowerCAmelCase_   :				Any								= 'Hello World!'
        lowerCAmelCase_   :				Union[str, Any]								= [0, 3_5_3_7_8, 6_6_6_1, 3_8, 2]
        # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
        # xlmr.eval()
        # xlmr.encode(symbols)

        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )



      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				Tuple								= (
            'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
            ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
        )
        lowerCAmelCase_   :				int								= [
            0,
            3_2_9_3,
            8_3,
            1_0,
            4_5_5_2,
            4_9_8_9,
            7_9_8_6,
            6_7_8,
            1_0,
            5_9_1_5,
            1_1_1,
            1_7_9_4_5_9,
            1_2_4_8_5_0,
            4,
            6_0_4_4,
            2_3_7,
            1_2,
            6,
            5,
            6,
            4,
            6_7_8_0,
            7_0_5,
            1_5,
            1_3_8_8,
            4_4,
            3_7_8,
            1_0_1_1_4,
            7_1_1,
            1_5_2,
            2_0,
            6,
            5,
            2_2_3_7_6,
            6_4_2,
            1_2_2_1,
            1_5_1_9_0,
            3_4_1_5_3,
            4_5_0,
            5_6_0_8,
            9_5_9,
            1_1_1_9,
            5_7_7_0_2,
            1_3_6,
            1_8_6,
            4_7,
            1_0_9_8,
            2_9_3_6_7,
            4_7,
            # 4426, # What fairseq tokenizes from "<unk>": "_<"
            # 3678, # What fairseq tokenizes from "<unk>": "unk"
            # 2740, # What fairseq tokenizes from "<unk>": ">"
            3,  # What we tokenize from "<unk>": "<unk>"
            6,  # Residue from the tokenization: an extra sentencepiece underline
            4,
            6_0_4_4,
            2_3_7,
            6_2_8_4,
            5_0_9_0_1,
            5_2_8,
            3_1,
            9_0,
            3_4,
            9_2_7,
            2,
        ]
        # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
        # xlmr.eval()
        # xlmr.encode(symbols)

        self.assertListEqual(SCREAMING_SNAKE_CASE_    ,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )





      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        # fmt: off
        lowerCAmelCase_   :				List[str]								= {'input_ids': [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=SCREAMING_SNAKE_CASE_    ,			model_name='xlm-roberta-base'    ,			revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'    ,			)





