


import collections
import importlib.util
import os
import re
from pathlib import Path


UpperCAmelCase_      : int  					=       'src/transformers'


# Matches is_xxx_available()
UpperCAmelCase_      : Any  					=       re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
UpperCAmelCase_      : Any  					=       re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
UpperCAmelCase_      : Dict  					=       re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
UpperCAmelCase_      : int  					=       re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
UpperCAmelCase_      : Optional[Any]  					=       re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
UpperCAmelCase_      : List[Any]  					=       re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
UpperCAmelCase_      : int  					=       re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
UpperCAmelCase_      : Optional[Any]  					=       re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
UpperCAmelCase_      : Any  					=       re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
UpperCAmelCase_      : Tuple  					=       re.compile(R'^\s*try:')
# Catches a line with else:
UpperCAmelCase_      : Any  					=       re.compile(R'^\s*else:')







def        SCREAMING_SNAKE_CASE_						(			__A :			str				)	->		List[str]:

       """simple docstring"""

       if _re_test_backend.search(__A				) is None:
              return None
       a_					:		Dict							=  [b[0] for b in _re_backend.findall(__A				)]
       backends.sort()
       return "_and_".join(__A				)







def        SCREAMING_SNAKE_CASE_						(			__A :			Optional[Any]				)	->		str:

       """simple docstring"""

       with open(__A  ,    'r'  ,    encoding='utf-8'  ,    newline='\n'				) as f:
              a_					:		Optional[Any]							=  f.readlines()

       a_					:		Optional[int]							=  0
       while line_index < len(__A				) and not lines[line_index].startswith('_import_structure = {'				):
              line_index += 1

       # If this is a traditional init, just return.
       if line_index >= len(__A				):
              return None

       # First grab the objects without a specific backend in _import_structure
       a_					:		Optional[Any]							=  []
       while not lines[line_index].startswith('if TYPE_CHECKING'				) and find_backend(lines[line_index]				) is None:
              a_					:		Optional[int]							=  lines[line_index]
              # If we have everything on a single line, let's deal with it.
              if _re_one_line_import_struct.search(__A				):
                     a_					:		Union[str, Any]							=  _re_one_line_import_struct.search(__A				).groups()[0]
                     a_					:		int							=  re.findall('\[([^\]]+)\]'  ,    __A				)
                     for imp in imports:
                            objects.extend([obj[1:-1] for obj in imp.split(', '				)]				)
                     line_index += 1
                     continue
              a_					:		Any							=  _re_import_struct_key_value.search(__A				)
              if single_line_import_search is not None:
                     a_					:		Union[str, Any]							=  [obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '				) if len(__A				) > 0]
                     objects.extend(__A				)
              elif line.startswith(' ' * 8 + '"'				):
                     objects.append(line[9:-3]				)
              line_index += 1

       a_					:		Dict							=  {'none': objects}
       # Let's continue with backend-specific objects in _import_structure
       while not lines[line_index].startswith('if TYPE_CHECKING'				):
              # If the line is an if not is_backend_available, we grab all objects associated.
              a_					:		str							=  find_backend(lines[line_index]				)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]				) is None:
                     a_					:		Dict							=  None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]				) is None:
                            line_index += 1

                     line_index += 1

                     a_					:		Tuple							=  []
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]				) <= 1 or lines[line_index].startswith(' ' * 4				):
                            a_					:		Optional[Any]							=  lines[line_index]
                            if _re_import_struct_add_one.search(__A				) is not None:
                                   objects.append(_re_import_struct_add_one.search(__A				).groups()[0]				)
                            elif _re_import_struct_add_many.search(__A				) is not None:
                                   a_					:		Tuple							=  _re_import_struct_add_many.search(__A				).groups()[0].split(', '				)
                                   a_					:		Tuple							=  [obj[1:-1] for obj in imports if len(__A				) > 0]
                                   objects.extend(__A				)
                            elif _re_between_brackets.search(__A				) is not None:
                                   a_					:		Tuple							=  _re_between_brackets.search(__A				).groups()[0].split(', '				)
                                   a_					:		Tuple							=  [obj[1:-1] for obj in imports if len(__A				) > 0]
                                   objects.extend(__A				)
                            elif _re_quote_object.search(__A				) is not None:
                                   objects.append(_re_quote_object.search(__A				).groups()[0]				)
                            elif line.startswith(' ' * 8 + '"'				):
                                   objects.append(line[9:-3]				)
                            elif line.startswith(' ' * 12 + '"'				):
                                   objects.append(line[13:-3]				)
                            line_index += 1

                     a_					:		str							=  objects
              else:
                     line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
       a_					:		str							=  []
       while (
           line_index < len(__A				)
           and find_backend(lines[line_index]				) is None
           and not lines[line_index].startswith('else'				)
       ):
              a_					:		Optional[Any]							=  lines[line_index]
              a_					:		Any							=  _re_import.search(__A				)
              if single_line_import_search is not None:
                     objects.extend(single_line_import_search.groups()[0].split(', '				)				)
              elif line.startswith(' ' * 8				):
                     objects.append(line[8:-2]				)
              line_index += 1

       a_					:		Any							=  {'none': objects}
       # Let's continue with backend-specific objects
       while line_index < len(__A				):
              # If the line is an if is_backend_available, we grab all objects associated.
              a_					:		str							=  find_backend(lines[line_index]				)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]				) is None:
                     a_					:		List[Any]							=  None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]				) is None:
                            line_index += 1

                     line_index += 1

                     a_					:		Optional[Any]							=  []
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]				) <= 1 or lines[line_index].startswith(' ' * 8				):
                            a_					:		Optional[Any]							=  lines[line_index]
                            a_					:		List[Any]							=  _re_import.search(__A				)
                            if single_line_import_search is not None:
                                   objects.extend(single_line_import_search.groups()[0].split(', '				)				)
                            elif line.startswith(' ' * 12				):
                                   objects.append(line[12:-2]				)
                            line_index += 1

                     a_					:		int							=  objects
              else:
                     line_index += 1

       return import_dict_objects, type_hint_objects







def        SCREAMING_SNAKE_CASE_						(			__A :			List[str]  ,    __A :			Any				)	->		int:

       """simple docstring"""

       def find_duplicates(__A :			List[Any]				):
              return [k for k, v in collections.Counter(__A				).items() if v > 1]

       if list(import_dict_objects.keys()				) != list(type_hint_objects.keys()				):
              return ["Both sides of the init do not have the same backends!"]

       a_					:		Tuple							=  []
       for key in import_dict_objects.keys():
              a_					:		Dict							=  find_duplicates(import_dict_objects[key]				)
              if duplicate_imports:
                     errors.append(F"""Duplicate _import_structure definitions for: {duplicate_imports}"""				)
              a_					:		List[str]							=  find_duplicates(type_hint_objects[key]				)
              if duplicate_type_hints:
                     errors.append(F"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""				)

              if sorted(set(import_dict_objects[key]				)				) != sorted(set(type_hint_objects[key]				)				):
                     a_					:		Optional[Any]							=  'base imports' if key == 'none' else F"""{key} backend"""
                     errors.append(F"""Differences for {name}:"""				)
                     for a in type_hint_objects[key]:
                            if a not in import_dict_objects[key]:
                                   errors.append(F"""  {a} in TYPE_HINT but not in _import_structure."""				)
                     for a in import_dict_objects[key]:
                            if a not in type_hint_objects[key]:
                                   errors.append(F"""  {a} in _import_structure but not in TYPE_HINT."""				)
       return errors







def        SCREAMING_SNAKE_CASE_						(			)	->		Union[str, Any]:

       """simple docstring"""

       a_					:		Dict							=  []
       for root, _, files in os.walk(__A				):
              if "__init__.py" in files:
                     a_					:		List[Any]							=  os.path.join(__A  ,    '__init__.py'				)
                     a_					:		Optional[Any]							=  parse_init(__A				)
                     if objects is not None:
                            a_					:		List[Any]							=  analyze_results(*__A				)
                            if len(__A				) > 0:
                                   a_					:		Any							=  F"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
                                   failures.append('\n'.join(__A				)				)
       if len(__A				) > 0:
              raise ValueError('\n\n'.join(__A				)				)







def        SCREAMING_SNAKE_CASE_						(			)	->		str:

       """simple docstring"""

       a_					:		str							=  []
       for path, directories, files in os.walk(__A				):
              for folder in directories:
                     # Ignore private modules
                     if folder.startswith('_'				):
                            directories.remove(__A				)
                            continue
                     # Ignore leftovers from branches (empty folders apart from pycache)
                     if len(list((Path(__A				) / folder).glob('*.py'				)				)				) == 0:
                            continue
                     a_					:		Optional[Any]							=  str((Path(__A				) / folder).relative_to(__A				)				)
                     a_					:		List[Any]							=  short_path.replace(os.path.sep  ,    '.'				)
                     submodules.append(__A				)
              for fname in files:
                     if fname == "__init__.py":
                            continue
                     a_					:		int							=  str((Path(__A				) / fname).relative_to(__A				)				)
                     a_					:		Optional[Any]							=  short_path.replace('.py'  ,    ''				).replace(os.path.sep  ,    '.'				)
                     if len(submodule.split('.'				)				) == 1:
                            submodules.append(__A				)
       return submodules


UpperCAmelCase_      : Dict  					=       [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]







def        SCREAMING_SNAKE_CASE_						(			)	->		List[str]:

       """simple docstring"""

       a_					:		int							=  importlib.util.spec_from_file_location(
           'transformers'  ,    os.path.join(__A  ,    '__init__.py'				)  ,    submodule_search_locations=[PATH_TO_TRANSFORMERS]  ,    )
       a_					:		Dict							=  spec.loader.load_module()

       a_					:		List[str]							=  [
           module
           for module in get_transformers_submodules()
           if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
       ]
       if len(__A				) > 0:
              a_					:		Union[str, Any]							=  '\n'.join(F"""- {module}""" for module in module_not_registered				)
              raise ValueError(
                  'The following submodules are not properly registered in the main init of Transformers:\n'
                  F"""{list_of_modules}\n"""
                  'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'				)


if __name__ == "__main__":
   check_all_inits()
   check_submodules()



"""simple docstring"""




import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow
class 			UpperCamelCase__:





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> int:
													return None






class 			UpperCamelCase__:





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> Tuple:
													return None






class 			UpperCamelCase__(	unittest.TestCase       ):
						lowerCAmelCase__     :  Tuple  							=						[
						    # (model_name, model_kwargs)
						    ('bert-base-cased', {}),
						    ('gpt2', {'use_cache': False}),  # We don't support exporting GPT2 past keys anymore
						]





						@require_tf
						@slow
						def  snake_case__		(				self   )    -> int:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				self._test_export(__UpperCAmelCase						,'tf'						,12						,**__UpperCAmelCase   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> int:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				self._test_export(__UpperCAmelCase						,'pt'						,12						,**__UpperCAmelCase   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import BertModel

													A__		   =							['[UNK]', '[SEP]', '[CLS]', '[PAD]', '[MASK]', 'some', 'other', 'words']
													with NamedTemporaryFile(mode='w+t'   ) as vocab_file:
																				vocab_file.write('\n'.join(__UpperCAmelCase   )   )
																				vocab_file.flush()
																				A__		   =							BertTokenizerFast(vocab_file.name   )

													with TemporaryDirectory() as bert_save_dir:
																				A__		   =							BertModel(BertConfig(vocab_size=len(__UpperCAmelCase   )   )   )
																				model.save_pretrained(__UpperCAmelCase   )
																				self._test_export(__UpperCAmelCase						,'pt'						,12						,__UpperCAmelCase   )





						@require_tf
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				A__		   =							self._test_export(__UpperCAmelCase						,'tf'						,12						,**__UpperCAmelCase   )
																				A__		   =							quantize(Path(__UpperCAmelCase   )   )

																				# Ensure the actual quantized model is not bigger than the original one
																				if quantized_path.stat().st_size >= Path(__UpperCAmelCase   ).stat().st_size:
																											self.fail('Quantized model is bigger than initial ONNX model'   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> str:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				A__		   =							self._test_export(__UpperCAmelCase						,'pt'						,12						,**__UpperCAmelCase   )
																				A__		   =							quantize(__UpperCAmelCase   )

																				# Ensure the actual quantized model is not bigger than the original one
																				if quantized_path.stat().st_size >= Path(__UpperCAmelCase   ).stat().st_size:
																											self.fail('Quantized model is bigger than initial ONNX model'   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase=None						,**__UpperCAmelCase   )    -> Union[str, Any]:
													try:
																				# Compute path
																				with TemporaryDirectory() as tempdir:
																											A__		   =							Path(__UpperCAmelCase   ).joinpath('model.onnx'   )

																				# Remove folder if exists
																				if path.parent.exists():
																											path.parent.rmdir()

																				# Export
																				convert(__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,**__UpperCAmelCase   )

																				return path
													except Exception as e:
																				self.fail(__UpperCAmelCase   )





						@require_torch
						@require_tokenizers
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import BertModel

													A__		   =							BertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
													A__		   =							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
													self._test_infer_dynamic_axis(__UpperCAmelCase						,__UpperCAmelCase						,'pt'   )





						@require_tf
						@require_tokenizers
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import TFBertModel

													A__		   =							TFBertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
													A__		   =							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
													self._test_infer_dynamic_axis(__UpperCAmelCase						,__UpperCAmelCase						,'tf'   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> str:
													A__		   =							FeatureExtractionPipeline(__UpperCAmelCase						,__UpperCAmelCase   )

													A__		   =							['input_ids', 'token_type_ids', 'attention_mask', 'output_0', 'output_1']
													A__   ,     A__   ,     A__   ,     A__		   =							infer_shapes(__UpperCAmelCase						,__UpperCAmelCase   )

													# Assert all variables are present
													self.assertEqual(len(__UpperCAmelCase   )						,len(__UpperCAmelCase   )   )
													self.assertTrue(all(var_name in shapes for var_name in variable_names   )   )
													self.assertSequenceEqual(variable_names[:3]						,__UpperCAmelCase   )
													self.assertSequenceEqual(variable_names[3:]						,__UpperCAmelCase   )

													# Assert inputs are {0: batch, 1: sequence}
													for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
																				self.assertDictEqual(shapes[var_name]						,{0: 'batch', 1: 'sequence'}   )

													# Assert outputs are {0: batch, 1: sequence} and {0: batch}
													self.assertDictEqual(shapes['output_0']						,{0: 'batch', 1: 'sequence'}   )
													self.assertDictEqual(shapes['output_1']						,{0: 'batch'}   )





						def  snake_case__		(				self   )    -> Union[str, Any]:
													A__		   =							['input_ids', 'attention_mask', 'token_type_ids']
													A__		   =							{'input_ids': [1, 2, 3, 4], 'attention_mask': [0, 0, 0, 0], 'token_type_ids': [1, 1, 1, 1]}
													A__   ,     A__		   =							ensure_valid_input(FuncContiguousArgs()						,__UpperCAmelCase						,__UpperCAmelCase   )

													# Should have exactly the same number of args (all are valid)
													self.assertEqual(len(__UpperCAmelCase   )						,3   )

													# Should have exactly the same input names
													self.assertEqual(set(__UpperCAmelCase   )						,set(__UpperCAmelCase   )   )

													# Parameter should be reordered according to their respective place in the function:
													# (input_ids, token_type_ids, attention_mask)
													self.assertEqual(__UpperCAmelCase						,(tokens['input_ids'], tokens['token_type_ids'], tokens['attention_mask'])   )

													# Generated args are interleaved with another args (for instance parameter "past" in GPT2)
													A__   ,     A__		   =							ensure_valid_input(FuncNonContiguousArgs()						,__UpperCAmelCase						,__UpperCAmelCase   )

													# Should have exactly the one arg (all before the one not provided "some_other_args")
													self.assertEqual(len(__UpperCAmelCase   )						,1   )
													self.assertEqual(len(__UpperCAmelCase   )						,1   )

													# Should have only "input_ids"
													self.assertEqual(inputs_args[0]						,tokens['input_ids']   )
													self.assertEqual(ordered_input_names[0]						,'input_ids'   )





						def  snake_case__		(				self   )    -> Union[str, Any]:
													A__		   =							generate_identified_filename(Path('/home/something/my_fake_model.onnx'   )						,'-test'   )
													self.assertEqual('/home/something/my_fake_model-test.onnx'						,generated.as_posix()   )





