


import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class  snake_case__(							snake_case_,  unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__					:     Optional[int]    =							LongformerTokenizer
						SCREAMING_SNAKE_CASE__					:     Union[str, Any]    =							True
						SCREAMING_SNAKE_CASE__					:     List[Any]    =							LongformerTokenizerFast
						SCREAMING_SNAKE_CASE__					:     List[Any]    =							True




						def   lowercase_       ( self							)    ->			str:
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										lowerCAmelCase_       :					Union[str, Any]			 = [
										    '''l''',
										    '''o''',
										    '''w''',
										    '''e''',
										    '''r''',
										    '''s''',
										    '''t''',
										    '''i''',
										    '''d''',
										    '''n''',
										    '''\u0120''',
										    '''\u0120l''',
										    '''\u0120n''',
										    '''\u0120lo''',
										    '''\u0120low''',
										    '''er''',
										    '''\u0120lowest''',
										    '''\u0120newer''',
										    '''\u0120wider''',
										    '''<unk>''',
										]
										lowerCAmelCase_       :					List[str]			 = dict(zip(__lowercase      ,  range(len(__lowercase							)							)							)							)
										lowerCAmelCase_       :					List[Any]			 = ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
										lowerCAmelCase_       :					int			 = {'''unk_token''': '''<unk>'''}

										lowerCAmelCase_       :					Tuple			 = os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['''vocab_file''']							)
										lowerCAmelCase_       :					Optional[int]			 = os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['''merges_file''']							)
										with open(self.vocab_file      ,  '''w'''      ,  encoding='''utf-8'''							) as fp:
														fp.write(json.dumps(__lowercase							) + '''\n'''							)
										with open(self.merges_file      ,  '''w'''      ,  encoding='''utf-8'''							) as fp:
														fp.write('''\n'''.join(__lowercase							)							)




						def   lowercase_       ( self      ,  **__lowercase							)    ->			int:
										kwargs.update(self.special_tokens_map							)
										return self.tokenizer_class.from_pretrained(self.tmpdirname      ,  **__lowercase							)




						def   lowercase_       ( self      ,  **__lowercase							)    ->			int:
										kwargs.update(self.special_tokens_map							)
										return self.rust_tokenizer_class.from_pretrained(self.tmpdirname      ,  **__lowercase							)




						def   lowercase_       ( self      ,  __lowercase							)    ->			Any:
										lowerCAmelCase_       :					Dict			 = '''lower newer'''
										lowerCAmelCase_       :					List[Any]			 = '''lower newer'''
										return input_text, output_text




						def   lowercase_       ( self							)    ->			str:
										lowerCAmelCase_       :					Tuple			 = self.tokenizer_class(self.vocab_file      ,  self.merges_file      ,  **self.special_tokens_map							)
										lowerCAmelCase_       :					int			 = '''lower newer'''
										lowerCAmelCase_       :					Optional[int]			 = ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
										lowerCAmelCase_       :					int			 = tokenizer.tokenize(__lowercase							)  # , add_prefix_space=True)
										self.assertListEqual(__lowercase      ,  __lowercase							)

										lowerCAmelCase_       :					Optional[int]			 = tokens + [tokenizer.unk_token]
										lowerCAmelCase_       :					str			 = [0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase							)      ,  __lowercase							)




						def   lowercase_       ( self							)    ->			Dict:
										lowerCAmelCase_       :					Any			 = self.get_tokenizer()

										self.assertListEqual(tokenizer.encode('''Hello world!'''      ,  add_special_tokens=__lowercase							)      ,  [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]							)
										self.assertListEqual(
										    tokenizer.encode('''Hello world! cécé herlolip 418'''      ,  add_special_tokens=__lowercase							)      ,  [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]      ,  )




						@slow
						def   lowercase_       ( self							)    ->			List[str]:
										lowerCAmelCase_       :					int			 = self.tokenizer_class.from_pretrained('''allenai/longformer-base-4096'''							)

										lowerCAmelCase_       :					List[Any]			 = tokenizer.encode('''sequence builders'''      ,  add_special_tokens=__lowercase							)
										lowerCAmelCase_       :					Optional[int]			 = tokenizer.encode('''multi-sequence build'''      ,  add_special_tokens=__lowercase							)

										lowerCAmelCase_       :					Dict			 = tokenizer.encode(
										    '''sequence builders'''      ,  add_special_tokens=__lowercase      ,  add_prefix_space=__lowercase							)
										lowerCAmelCase_       :					Optional[int]			 = tokenizer.encode(
										    '''sequence builders'''      ,  '''multi-sequence build'''      ,  add_special_tokens=__lowercase      ,  add_prefix_space=__lowercase							)

										lowerCAmelCase_       :					Dict			 = tokenizer.build_inputs_with_special_tokens(__lowercase							)
										lowerCAmelCase_       :					Union[str, Any]			 = tokenizer.build_inputs_with_special_tokens(__lowercase      ,  __lowercase							)

										assert encoded_sentence == encoded_text_from_decode
										assert encoded_pair == encoded_pair_from_decode




						def   lowercase_       ( self							)    ->			str:
										lowerCAmelCase_       :					Optional[Any]			 = self.get_tokenizer()

										lowerCAmelCase_       :					Dict			 = '''Encode this sequence.'''
										lowerCAmelCase_       :					Union[str, Any]			 = tokenizer.byte_encoder[''' '''.encode('''utf-8'''							)[0]]

										# Testing encoder arguments
										lowerCAmelCase_       :					List[Any]			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase      ,  add_prefix_space=__lowercase							)
										lowerCAmelCase_       :					Optional[int]			 = tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
										self.assertNotEqual(__lowercase      ,  __lowercase							)

										lowerCAmelCase_       :					Optional[Any]			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase      ,  add_prefix_space=__lowercase							)
										lowerCAmelCase_       :					Tuple			 = tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
										self.assertEqual(__lowercase      ,  __lowercase							)

										tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}							)
										lowerCAmelCase_       :					Optional[int]			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
										lowerCAmelCase_       :					Union[str, Any]			 = tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
										self.assertNotEqual(__lowercase      ,  __lowercase							)

										# Testing spaces after special tokens
										lowerCAmelCase_       :					Union[str, Any]			 = '''<mask>'''
										tokenizer.add_special_tokens(
										    {'''mask_token''': AddedToken(__lowercase      ,  lstrip=__lowercase      ,  rstrip=__lowercase							)}							)  # mask token has a left space
										lowerCAmelCase_       :					Tuple			 = tokenizer.convert_tokens_to_ids(__lowercase							)

										lowerCAmelCase_       :					List[str]			 = '''Encode <mask> sequence'''
										lowerCAmelCase_       :					Union[str, Any]			 = '''Encode <mask>sequence'''

										lowerCAmelCase_       :					Union[str, Any]			 = tokenizer.encode(__lowercase							)
										lowerCAmelCase_       :					Dict			 = encoded.index(__lowercase							)
										lowerCAmelCase_       :					str			 = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
										self.assertEqual(__lowercase      ,  __lowercase							)

										lowerCAmelCase_       :					int			 = tokenizer.encode(__lowercase							)
										lowerCAmelCase_       :					int			 = encoded.index(__lowercase							)
										lowerCAmelCase_       :					List[str]			 = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
										self.assertNotEqual(__lowercase      ,  __lowercase							)




						def   lowercase_       ( self							)    ->			str:
										pass




						def   lowercase_       ( self							)    ->			List[Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		lowerCAmelCase_       :					List[str]			 = self.rust_tokenizer_class.from_pretrained(__lowercase      ,  **__lowercase							)
																		lowerCAmelCase_       :					int			 = self.tokenizer_class.from_pretrained(__lowercase      ,  **__lowercase							)
																		lowerCAmelCase_       :					Any			 = '''A, <mask> AllenNLP sentence.'''
																		lowerCAmelCase_       :					Optional[int]			 = tokenizer_r.encode_plus(__lowercase      ,  add_special_tokens=__lowercase      ,  return_token_type_ids=__lowercase							)
																		lowerCAmelCase_       :					List[Any]			 = tokenizer_p.encode_plus(__lowercase      ,  add_special_tokens=__lowercase      ,  return_token_type_ids=__lowercase							)

																		# token_type_ids should put 0 everywhere
																		self.assertEqual(sum(tokens_r['''token_type_ids''']							)      ,  sum(tokens_p['''token_type_ids''']							)							)

																		# attention_mask should put 1 everywhere, so sum over length should be 1
																		self.assertEqual(
																		    sum(tokens_r['''attention_mask''']							) / len(tokens_r['''attention_mask''']							)      ,  sum(tokens_p['''attention_mask''']							) / len(tokens_p['''attention_mask''']							)      ,  )

																		lowerCAmelCase_       :					Optional[int]			 = tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']							)
																		lowerCAmelCase_       :					Dict			 = tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']							)

																		# Rust correctly handles the space before the mask while python doesnt
																		self.assertSequenceEqual(tokens_p['''input_ids''']      ,  [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)
																		self.assertSequenceEqual(tokens_r['''input_ids''']      ,  [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]							)

																		self.assertSequenceEqual(
																		    __lowercase      ,  ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)
																		self.assertSequenceEqual(
																		    __lowercase      ,  ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)




						def   lowercase_       ( self							)    ->			str:
										for trim_offsets, add_prefix_space in itertools.product([True, False]      ,  repeat=2							):
														lowerCAmelCase_       :					Dict			 = self.rust_tokenizer_class.from_pretrained(
														    self.tmpdirname      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)

														lowerCAmelCase_       :					Dict			 = json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
														lowerCAmelCase_       :					List[Any]			 = json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

														self.assertEqual(pre_tokenizer_state['''add_prefix_space''']      ,  __lowercase							)

														self.assertEqual(post_processor_state['''add_prefix_space''']      ,  __lowercase							)
														self.assertEqual(post_processor_state['''trim_offsets''']      ,  __lowercase							)




						def   lowercase_       ( self							)    ->			Optional[int]:
										# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
										# `trim_offsets`
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		lowerCAmelCase_       :					Any			 = '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																		lowerCAmelCase_       :					Optional[Any]			 = f"""{text_of_1_token} {text_of_1_token}"""

																		lowerCAmelCase_       :					List[Any]			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					Any			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (len(__lowercase							) + 1, len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					Optional[Any]			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					Dict			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (len(__lowercase							) + 1, len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					Tuple			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					str			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (len(__lowercase							), len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					List[str]			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					Optional[Any]			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (len(__lowercase							), len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					int			 = f""" {text}"""

																		# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																		#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																		# )
																		# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																		# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																		# self.assertEqual(
																		#     encoding.offset_mapping[1],
																		#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																		# )

																		lowerCAmelCase_       :					List[str]			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					str			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (1, 1 + len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (1 + len(__lowercase							) + 1, 1 + len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					Union[str, Any]			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					Dict			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, 1 + len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (1 + len(__lowercase							), 1 + len(__lowercase							) + 1 + len(__lowercase							))      ,  )

																		lowerCAmelCase_       :					str			 = self.rust_tokenizer_class.from_pretrained(
																		    __lowercase      ,  use_fast=__lowercase      ,  add_prefix_space=__lowercase      ,  trim_offsets=__lowercase							)
																		lowerCAmelCase_       :					str			 = tokenizer_r(__lowercase      ,  return_offsets_mapping=__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertEqual(encoding.offset_mapping[0]      ,  (0, 1 + len(__lowercase							))							)
																		self.assertEqual(
																		    encoding.offset_mapping[1]      ,  (1 + len(__lowercase							), 1 + len(__lowercase							) + 1 + len(__lowercase							))      ,  )







import collections
import importlib.util
import os
import re
from pathlib import Path


lowercase  							=					"src/transformers"


# Matches is_xxx_available()
lowercase  							=					re.compile(r"is\_([a-z_]*)_available()")
# Catches a one-line _import_struct = {xxx}
lowercase  							=					re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowercase  							=					re.compile(r"\s+\"\S*\":\s+\[([^\]]*)\]")
# Catches a line if not is_foo_available
lowercase  							=					re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")
# Catches a line _import_struct["bla"].append("foo")
lowercase  							=					re.compile(r"^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowercase  							=					re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")
# Catches a line with an object between quotes and a comma:     "MyModel",
lowercase  							=					re.compile("^\s+\"([^\"]+)\",")
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowercase  							=					re.compile("^\s+\[([^\]]+)\]")
# Catches a line with from foo import bar, bla, boo
lowercase  							=					re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
# Catches a line with try:
lowercase  							=					re.compile(r"^\s*try:")
# Catches a line with else:
lowercase  							=					re.compile(r"^\s*else:")





def 						__UpperCAmelCase			(     a_):
							if _re_test_backend.search(a_) is None:
														return None
							snake_case_				=							[b[0] for b in _re_backend.findall(a_)]
							backends.sort()
							return "_and_".join(a_)





def 						__UpperCAmelCase			(     a_):
							with open(a_							,     'r'							,     encoding='utf-8'							,     newline='\n') as f:
														snake_case_				=							f.readlines()

							snake_case_				=							0
							while line_index < len(a_) and not lines[line_index].startswith('_import_structure = {'):
														line_index += 1

							# If this is a traditional init, just return.
							if line_index >= len(a_):
														return None

							# First grab the objects without a specific backend in _import_structure
							snake_case_				=							[]
							while not lines[line_index].startswith('if TYPE_CHECKING') and find_backend(lines[line_index]) is None:
														snake_case_				=							lines[line_index]
														# If we have everything on a single line, let's deal with it.
														if _re_one_line_import_struct.search(a_):
																					snake_case_				=							_re_one_line_import_struct.search(a_).groups()[0]
																					snake_case_				=							re.findall('\[([^\]]+)\]'							,     a_)
																					for imp in imports:
																												objects.extend([obj[1:-1] for obj in imp.split(', ')])
																					line_index += 1
																					continue
														snake_case_				=							_re_import_struct_key_value.search(a_)
														if single_line_import_search is not None:
																					snake_case_				=							[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', ') if len(a_) > 0]
																					objects.extend(a_)
														elif line.startswith(' ' * 8 + '"'):
																					objects.append(line[9:-3])
														line_index += 1

							snake_case_				=							{'none': objects}
							# Let's continue with backend-specific objects in _import_structure
							while not lines[line_index].startswith('if TYPE_CHECKING'):
														# If the line is an if not is_backend_available, we grab all objects associated.
														snake_case_				=							find_backend(lines[line_index])
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]) is None:
																					snake_case_				=							None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]) is None:
																												line_index += 1

																					line_index += 1

																					snake_case_				=							[]
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]) <= 1 or lines[line_index].startswith(' ' * 4):
																												snake_case_				=							lines[line_index]
																												if _re_import_struct_add_one.search(a_) is not None:
																																			objects.append(_re_import_struct_add_one.search(a_).groups()[0])
																												elif _re_import_struct_add_many.search(a_) is not None:
																																			snake_case_				=							_re_import_struct_add_many.search(a_).groups()[0].split(', ')
																																			snake_case_				=							[obj[1:-1] for obj in imports if len(a_) > 0]
																																			objects.extend(a_)
																												elif _re_between_brackets.search(a_) is not None:
																																			snake_case_				=							_re_between_brackets.search(a_).groups()[0].split(', ')
																																			snake_case_				=							[obj[1:-1] for obj in imports if len(a_) > 0]
																																			objects.extend(a_)
																												elif _re_quote_object.search(a_) is not None:
																																			objects.append(_re_quote_object.search(a_).groups()[0])
																												elif line.startswith(' ' * 8 + '"'):
																																			objects.append(line[9:-3])
																												elif line.startswith(' ' * 12 + '"'):
																																			objects.append(line[13:-3])
																												line_index += 1

																					snake_case_				=							objects
														else:
																					line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
							snake_case_				=							[]
							while (
							    line_index < len(a_)
							    and find_backend(lines[line_index]) is None
							    and not lines[line_index].startswith('else')
							):
														snake_case_				=							lines[line_index]
														snake_case_				=							_re_import.search(a_)
														if single_line_import_search is not None:
																					objects.extend(single_line_import_search.groups()[0].split(', '))
														elif line.startswith(' ' * 8):
																					objects.append(line[8:-2])
														line_index += 1

							snake_case_				=							{'none': objects}
							# Let's continue with backend-specific objects
							while line_index < len(a_):
														# If the line is an if is_backend_available, we grab all objects associated.
														snake_case_				=							find_backend(lines[line_index])
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]) is None:
																					snake_case_				=							None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]) is None:
																												line_index += 1

																					line_index += 1

																					snake_case_				=							[]
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]) <= 1 or lines[line_index].startswith(' ' * 8):
																												snake_case_				=							lines[line_index]
																												snake_case_				=							_re_import.search(a_)
																												if single_line_import_search is not None:
																																			objects.extend(single_line_import_search.groups()[0].split(', '))
																												elif line.startswith(' ' * 12):
																																			objects.append(line[12:-2])
																												line_index += 1

																					snake_case_				=							objects
														else:
																					line_index += 1

							return import_dict_objects, type_hint_objects





def 						__UpperCAmelCase			(     a_							,     a_):

							def find_duplicates(a_):
														return [k for k, v in collections.Counter(a_).items() if v > 1]

							if list(import_dict_objects.keys()) != list(type_hint_objects.keys()):
														return ["Both sides of the init do not have the same backends!"]

							snake_case_				=							[]
							for key in import_dict_objects.keys():
														snake_case_				=							find_duplicates(import_dict_objects[key])
														if duplicate_imports:
																					errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}''')
														snake_case_				=							find_duplicates(type_hint_objects[key])
														if duplicate_type_hints:
																					errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}''')

														if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])):
																					snake_case_				=							'base imports' if key == 'none' else f'''{key} backend'''
																					errors.append(f'''Differences for {name}:''')
																					for a in type_hint_objects[key]:
																												if a not in import_dict_objects[key]:
																																			errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.''')
																					for a in import_dict_objects[key]:
																												if a not in type_hint_objects[key]:
																																			errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.''')
							return errors





def 						__UpperCAmelCase			(     ):
							snake_case_				=							[]
							for root, _, files in os.walk(a_):
														if "__init__.py" in files:
																					snake_case_				=							os.path.join(a_							,     '__init__.py')
																					snake_case_				=							parse_init(a_)
																					if objects is not None:
																												snake_case_				=							analyze_results(*a_)
																												if len(a_) > 0:
																																			snake_case_				=							f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
																																			failures.append('\n'.join(a_))
							if len(a_) > 0:
														raise ValueError('\n\n'.join(a_))





def 						__UpperCAmelCase			(     ):
							snake_case_				=							[]
							for path, directories, files in os.walk(a_):
														for folder in directories:
																					# Ignore private modules
																					if folder.startswith('_'):
																												directories.remove(a_)
																												continue
																					# Ignore leftovers from branches (empty folders apart from pycache)
																					if len(list((Path(a_) / folder).glob('*.py'))) == 0:
																												continue
																					snake_case_				=							str((Path(a_) / folder).relative_to(a_))
																					snake_case_				=							short_path.replace(os.path.sep							,     '.')
																					submodules.append(a_)
														for fname in files:
																					if fname == "__init__.py":
																												continue
																					snake_case_				=							str((Path(a_) / fname).relative_to(a_))
																					snake_case_				=							short_path.replace('.py'							,     '').replace(os.path.sep							,     '.')
																					if len(submodule.split('.')) == 1:
																												submodules.append(a_)
							return submodules


lowercase  							=					[
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
]





def 						__UpperCAmelCase			(     ):
							# This is to make sure the transformers module imported is the one in the repo.
							snake_case_				=							importlib.util.spec_from_file_location(
							    'transformers'							,     os.path.join(a_							,     '__init__.py')							,     submodule_search_locations=[PATH_TO_TRANSFORMERS]							,     )
							snake_case_				=							spec.loader.load_module()

							snake_case_				=							[
							    module
							    for module in get_transformers_submodules()
							    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
							]
							if len(a_) > 0:
														snake_case_				=							'\n'.join(f'''- {module}''' for module in module_not_registered)
														raise ValueError(
														    'The following submodules are not properly registered in the main init of Transformers:\n'
														    f'''{list_of_modules}\n'''
														    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.')


if __name__ == "__main__":
	check_all_inits()
	check_submodules()





