



'''simple docstring'''





import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
lowercase_		       =  sys.version_info >= (3, 10)




def lowerCAmelCase  (__A=None  ,	__A=None):



     """simple docstring"""



     return field(default_factory=lambda: default  ,	metadata=__A)






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       int
 __lowerCamelCase    :       float
 __lowerCamelCase    :       str
 __lowerCamelCase    :       bool






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       int          =       42
 __lowerCamelCase    :       str          =       field(default='toto'   ,   metadata={'help': 'help message'}						)






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       bool          =       False
 __lowerCamelCase    :       bool          =       True
 __lowerCamelCase    :       Optional[bool]          =       None






class __A			(  A						):



 '''simple docstring'''







 __lowerCamelCase    :       Any          =       'titi'
 __lowerCamelCase    :       Optional[Any]          =       'toto'






class __A			(  A						):



 '''simple docstring'''







 __lowerCamelCase    :       Any          =       'titi'
 __lowerCamelCase    :       Optional[Any]          =       'toto'
 __lowerCamelCase    :       List[Any]          =       42






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       BasicEnum          =       "toto"



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      _a      =  BasicEnum(self.foo       )






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       MixedTypeEnum          =       "toto"



 def 					a__  (self       )    ->     Optional[int]:


      """simple docstring"""



      _a      =  MixedTypeEnum(self.foo       )






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       Optional[int]          =       None
 __lowerCamelCase    :       Optional[float]          =       field(default=A   ,   metadata={'help': 'help message'}						)
 __lowerCamelCase    :       Optional[str]          =       None
 __lowerCamelCase    :       Optional[List[str]]          =       list_field(default=[]						)
 __lowerCamelCase    :       Optional[List[int]]          =       list_field(default=[]						)






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       List[int]          =       list_field(default=[]						)
 __lowerCamelCase    :       List[int]          =       list_field(default=[1, 2, 3]						)
 __lowerCamelCase    :       List[str]          =       list_field(default=['Hallo', 'Bonjour', 'Hello']						)
 __lowerCamelCase    :       List[float]          =       list_field(default=[0.1, 0.2, 0.3]						)






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       List[int]          =       field()
 __lowerCamelCase    :       str          =       field()
 __lowerCamelCase    :       BasicEnum          =       field()



 def 					a__  (self       )    ->     Tuple:


      """simple docstring"""



      _a      =  BasicEnum(self.required_enum       )






@dataclass
class __A			:



 '''simple docstring'''







 __lowerCamelCase    :       int
 __lowerCamelCase    :       "BasicEnum"          =       field()
 __lowerCamelCase    :       "Optional[bool]"          =       None
 __lowerCamelCase    :       "str"          =       field(default='toto'   ,   metadata={'help': 'help message'}						)
 __lowerCamelCase    :       "List[str]"          =       list_field(default=['Hallo', 'Bonjour', 'Hello']						)


if is_python_no_less_than_3_10:






  @dataclass
  class __A			:



   '''simple docstring'''







   __lowerCamelCase    :       bool          =       False
   __lowerCamelCase    :       bool          =       True
   __lowerCamelCase    :       bool | None          =       None






  @dataclass
  class __A			:



   '''simple docstring'''







   __lowerCamelCase    :       int | None          =       None
   __lowerCamelCase    :       float | None          =       field(default=A   ,   metadata={'help': 'help message'}						)
   __lowerCamelCase    :       str | None          =       None
   __lowerCamelCase    :       list[str] | None          =       list_field(default=[]						)
   __lowerCamelCase    :       list[int] | None          =       list_field(default=[]						)






class __A			(  unittest.TestCase						):



 '''simple docstring'''







 def 					a__  (self ,   A ,   A       )    ->     str:


      """simple docstring"""



      self.assertEqual(len(a._actions       ) ,   len(b._actions       )       )
      for x, y in zip(a._actions ,   b._actions       ):
           _a      =  {k: v for k, v in vars(A       ).items() if k != '''container'''}
           _a      =  {k: v for k, v in vars(A       ).items() if k != '''container'''}

           # Choices with mixed type have custom function as "type"
           # So we need to compare results directly for equality
           if xx.get('''choices''' ,   A       ) and yy.get('''choices''' ,   A       ):
                for expected_choice in yy["choices"] + xx["choices"]:
                     self.assertEqual(xx['''type'''](A       ) ,   yy['''type'''](A       )       )
                del xx["type"], yy["type"]

           self.assertEqual(A ,   A       )



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo''' ,   type=A ,   required=A       )
      expected.add_argument('''--bar''' ,   type=A ,   required=A       )
      expected.add_argument('''--baz''' ,   type=A ,   required=A       )
      expected.add_argument('''--flag''' ,   type=A ,   default=A ,   const=A ,   nargs='''?'''       )
      self.argparsersEqual(A ,   A       )

      _a      =  ['''--foo''', '''1''', '''--baz''', '''quux''', '''--bar''', '''0.5''']
      ((_a)  ,    )      =  parser.parse_args_into_dataclasses(A ,   look_for_args_file=A       )
      self.assertFalse(example.flag       )



 def 					a__  (self       )    ->     int:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo''' ,   default=42 ,   type=A       )
      expected.add_argument('''--baz''' ,   default='''toto''' ,   type=A ,   help='''help message'''       )
      self.argparsersEqual(A ,   A       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo''' ,   type=A ,   default=A ,   const=A ,   nargs='''?'''       )
      expected.add_argument('''--baz''' ,   type=A ,   default=A ,   const=A ,   nargs='''?'''       )
      # A boolean no_* argument always has to come after its "default: True" regular counter-part
      # and its default must be set to False
      expected.add_argument('''--no_baz''' ,   action='''store_false''' ,   default=A ,   dest='''baz'''       )
      expected.add_argument('''--opt''' ,   type=A ,   default=A       )

      _a      =  [WithDefaultBoolExample]
      if is_python_no_less_than_3_10:
           dataclass_types.append(A       )

      for dataclass_type in dataclass_types:
           _a      =  HfArgumentParser(A       )
           self.argparsersEqual(A ,   A       )

           _a      =  parser.parse_args([]       )
           self.assertEqual(A ,   Namespace(foo=A ,   baz=A ,   opt=A       )       )

           _a      =  parser.parse_args(['''--foo''', '''--no_baz''']       )
           self.assertEqual(A ,   Namespace(foo=A ,   baz=A ,   opt=A       )       )

           _a      =  parser.parse_args(['''--foo''', '''--baz''']       )
           self.assertEqual(A ,   Namespace(foo=A ,   baz=A ,   opt=A       )       )

           _a      =  parser.parse_args(['''--foo''', '''True''', '''--baz''', '''True''', '''--opt''', '''True''']       )
           self.assertEqual(A ,   Namespace(foo=A ,   baz=A ,   opt=A       )       )

           _a      =  parser.parse_args(['''--foo''', '''False''', '''--baz''', '''False''', '''--opt''', '''False''']       )
           self.assertEqual(A ,   Namespace(foo=A ,   baz=A ,   opt=A       )       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument(
          '''--foo''' ,   default='''toto''' ,   choices=['''titi''', '''toto''', 42] ,   type=make_choice_type_function(['''titi''', '''toto''', 42]       ) ,   )
      self.argparsersEqual(A ,   A       )

      _a      =  parser.parse_args([]       )
      self.assertEqual(args.foo ,   '''toto'''       )
      _a      =  parser.parse_args_into_dataclasses([]       )[0]
      self.assertEqual(enum_ex.foo ,   MixedTypeEnum.toto       )

      _a      =  parser.parse_args(['''--foo''', '''titi''']       )
      self.assertEqual(args.foo ,   '''titi'''       )
      _a      =  parser.parse_args_into_dataclasses(['''--foo''', '''titi''']       )[0]
      self.assertEqual(enum_ex.foo ,   MixedTypeEnum.titi       )

      _a      =  parser.parse_args(['''--foo''', '''42''']       )
      self.assertEqual(args.foo ,   42       )
      _a      =  parser.parse_args_into_dataclasses(['''--foo''', '''42''']       )[0]
      self.assertEqual(enum_ex.foo ,   MixedTypeEnum.fourtytwo       )



 def 					a__  (self       )    ->     Union[str, Any]:


      """simple docstring"""



      @dataclass
      class __A			:



       '''simple docstring'''







       __lowerCamelCase    :       Literal["titi", "toto", 42]          =       "toto"





      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument(
          '''--foo''' ,   default='''toto''' ,   choices=('''titi''', '''toto''', 42) ,   type=make_choice_type_function(['''titi''', '''toto''', 42]       ) ,   )
      self.argparsersEqual(A ,   A       )

      _a      =  parser.parse_args([]       )
      self.assertEqual(args.foo ,   '''toto'''       )

      _a      =  parser.parse_args(['''--foo''', '''titi''']       )
      self.assertEqual(args.foo ,   '''titi'''       )

      _a      =  parser.parse_args(['''--foo''', '''42''']       )
      self.assertEqual(args.foo ,   42       )



 def 					a__  (self       )    ->     List[Any]:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo_int''' ,   nargs='''+''' ,   default=[] ,   type=A       )
      expected.add_argument('''--bar_int''' ,   nargs='''+''' ,   default=[1, 2, 3] ,   type=A       )
      expected.add_argument('''--foo_str''' ,   nargs='''+''' ,   default=['''Hallo''', '''Bonjour''', '''Hello'''] ,   type=A       )
      expected.add_argument('''--foo_float''' ,   nargs='''+''' ,   default=[0.1, 0.2, 0.3] ,   type=A       )

      self.argparsersEqual(A ,   A       )

      _a      =  parser.parse_args([]       )
      self.assertEqual(
          A ,   Namespace(foo_int=[] ,   bar_int=[1, 2, 3] ,   foo_str=['''Hallo''', '''Bonjour''', '''Hello'''] ,   foo_float=[0.1, 0.2, 0.3]       ) ,   )

      _a      =  parser.parse_args('''--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'''.split()       )
      self.assertEqual(A ,   Namespace(foo_int=[1] ,   bar_int=[2, 3] ,   foo_str=['''a''', '''b''', '''c'''] ,   foo_float=[0.1, 0.7]       )       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo''' ,   default=A ,   type=A       )
      expected.add_argument('''--bar''' ,   default=A ,   type=A ,   help='''help message'''       )
      expected.add_argument('''--baz''' ,   default=A ,   type=A       )
      expected.add_argument('''--ces''' ,   nargs='''+''' ,   default=[] ,   type=A       )
      expected.add_argument('''--des''' ,   nargs='''+''' ,   default=[] ,   type=A       )

      _a      =  [OptionalExample]
      if is_python_no_less_than_3_10:
           dataclass_types.append(A       )

      for dataclass_type in dataclass_types:
           _a      =  HfArgumentParser(A       )

           self.argparsersEqual(A ,   A       )

           _a      =  parser.parse_args([]       )
           self.assertEqual(A ,   Namespace(foo=A ,   bar=A ,   baz=A ,   ces=[] ,   des=[]       )       )

           _a      =  parser.parse_args('''--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'''.split()       )
           self.assertEqual(A ,   Namespace(foo=12 ,   bar=3.14 ,   baz='''42''' ,   ces=['''a''', '''b''', '''c'''] ,   des=[1, 2, 3]       )       )



 def 					a__  (self       )    ->     Tuple:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--required_list''' ,   nargs='''+''' ,   type=A ,   required=A       )
      expected.add_argument('''--required_str''' ,   type=A ,   required=A       )
      expected.add_argument(
          '''--required_enum''' ,   type=make_choice_type_function(['''titi''', '''toto''']       ) ,   choices=['''titi''', '''toto'''] ,   required=A ,   )
      self.argparsersEqual(A ,   A       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  argparse.ArgumentParser()
      expected.add_argument('''--foo''' ,   type=A ,   required=A       )
      expected.add_argument(
          '''--required_enum''' ,   type=make_choice_type_function(['''titi''', '''toto''']       ) ,   choices=['''titi''', '''toto'''] ,   required=A ,   )
      expected.add_argument('''--opt''' ,   type=A ,   default=A       )
      expected.add_argument('''--baz''' ,   default='''toto''' ,   type=A ,   help='''help message'''       )
      expected.add_argument('''--foo_str''' ,   nargs='''+''' ,   default=['''Hallo''', '''Bonjour''', '''Hello'''] ,   type=A       )
      self.argparsersEqual(A ,   A       )



 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  {
          '''foo''': 12,
          '''bar''': 3.14,
          '''baz''': '''42''',
          '''flag''': True,
      }

      _a      =  parser.parse_dict(A       )[0]
      _a      =  BasicExample(**A       )
      self.assertEqual(A ,   A       )



 def 					a__  (self       )    ->     Any:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  {
          '''foo''': 12,
          '''bar''': 3.14,
          '''baz''': '''42''',
          '''flag''': True,
          '''extra''': 42,
      }

      self.assertRaises(A ,   parser.parse_dict ,   A ,   allow_extra_keys=A       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  {
          '''foo''': 12,
          '''bar''': 3.14,
          '''baz''': '''42''',
          '''flag''': True,
      }
      with tempfile.TemporaryDirectory() as tmp_dir:
           _a      =  os.path.join(A ,   '''temp_json'''       )
           os.mkdir(A       )
           with open(temp_local_path + '''.json''' ,   '''w+'''       ) as f:
                json.dump(A ,   A       )
           _a      =  parser.parse_yaml_file(Path(temp_local_path + '''.json'''       )       )[0]

      _a      =  BasicExample(**A       )
      self.assertEqual(A ,   A       )



 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )

      _a      =  {
          '''foo''': 12,
          '''bar''': 3.14,
          '''baz''': '''42''',
          '''flag''': True,
      }
      with tempfile.TemporaryDirectory() as tmp_dir:
           _a      =  os.path.join(A ,   '''temp_yaml'''       )
           os.mkdir(A       )
           with open(temp_local_path + '''.yaml''' ,   '''w+'''       ) as f:
                yaml.dump(A ,   A       )
           _a      =  parser.parse_yaml_file(Path(temp_local_path + '''.yaml'''       )       )[0]
      _a      =  BasicExample(**A       )
      self.assertEqual(A ,   A       )



 def 					a__  (self       )    ->     int:


      """simple docstring"""



      _a      =  HfArgumentParser(A       )
      self.assertIsNotNone(A       )








'''simple docstring'''





import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase




def lowerCAmelCase  (__A  ,	__A  ,	__A  ,	__A):



     """simple docstring"""



     if isinstance(__A  ,	__A):
          _a      =  np.full((len(__A), sequence_length, 2)  ,	__A)
     else:
          _a      =  np.full((len(__A), sequence_length)  ,	__A)

     for i, tensor in enumerate(__A):
          if padding_side == "right":
               if isinstance(__A  ,	__A):
                    _a      =  tensor[:sequence_length]
               else:
                    _a      =  tensor[:sequence_length]
          else:
               if isinstance(__A  ,	__A):
                    _a      =  tensor[:sequence_length]
               else:
                    _a      =  tensor[:sequence_length]

     return out_tensor.tolist()




def lowerCAmelCase  (__A):



     """simple docstring"""



     _a      =  ord(__A)
     if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
          return True
     _a      =  unicodedata.category(__A)
     if cat.startswith('''P'''):
          return True
     return False






@dataclass
class __A			(  A						):



 '''simple docstring'''







 __lowerCamelCase    :       PreTrainedTokenizerBase
 __lowerCamelCase    :       Union[bool, str, PaddingStrategy]          =       True
 __lowerCamelCase    :       Optional[int]          =       None
 __lowerCamelCase    :       Optional[int]          =       None
 __lowerCamelCase    :       int          =       -100
 __lowerCamelCase    :       str          =       "pt"



 def 					a__  (self ,   A       )    ->     List[str]:


      """simple docstring"""



      import torch

      _a      =  '''label''' if '''label''' in features[0].keys() else '''labels'''
      _a      =  [feature[label_name] for feature in features] if label_name in features[0].keys() else None
      _a      =  self.tokenizer.pad(
          A ,   padding=self.padding ,   max_length=self.max_length ,   pad_to_multiple_of=self.pad_to_multiple_of ,   return_tensors='''pt''' if labels is None else None ,   )

      if labels is None:
           return batch

      _a      =  torch.tensor(batch['''entity_ids''']       ).shape[1]
      _a      =  self.tokenizer.padding_side
      if padding_side == "right":
           _a      =  [
               list(A       ) + [self.label_pad_token_id] * (sequence_length - len(A       )) for label in labels
           ]
      else:
           _a      =  [
               [self.label_pad_token_id] * (sequence_length - len(A       )) + list(A       ) for label in labels
           ]

      _a      =  [feature['''ner_tags'''] for feature in features]
      _a      =  padding_tensor(A ,   -1 ,   A ,   A       )
      _a      =  [feature['''original_entity_spans'''] for feature in features]
      _a      =  padding_tensor(A ,   (-1, -1) ,   A ,   A       )
      _a      =  {k: torch.tensor(A ,   dtype=torch.intaa       ) for k, v in batch.items()}

      return batch



