






import argparse
import glob
import importlib.util
import os
import re

import black
from doc_builder.style_doc import style_docstrings_in_code


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
a_							=			'''src/diffusers'''
a_							=			'''.'''


# This is to make sure the diffusers module imported is the one in the repo.
a_							=			importlib.util.spec_from_file_location(
    '''diffusers''',
    os.path.join(DIFFUSERS_PATH, '''__init__.py'''),
    submodule_search_locations=[DIFFUSERS_PATH],
)
a_							=			spec.loader.load_module()


def 				_a     (   UpperCamelCase_    : Optional[Any]			,      UpperCamelCase_    : List[str]      )       ->      Optional[int]:
						"""simple docstring"""







						return line.startswith(__snake_case      ) or len(__snake_case      ) <= 1 or re.search(R"^\s*\)(\s*->.*:|:)\s*$"			,      __snake_case      ) is not None


def 				_a     (   UpperCamelCase_    : Optional[int]      )       ->      Tuple:
						"""simple docstring"""







						lowerCAmelCase__       					=  object_name.split("."      )
						lowerCAmelCase__       					=  0

						# First let's find the module where our object lives.
						lowerCAmelCase__       					=  parts[i]
						while i < len(__snake_case      ) and not os.path.isfile(os.path.join(__snake_case			,      F"{module}.py"      )      ):
												i += 1
												if i < len(__snake_case      ):
																		lowerCAmelCase__       					=  os.path.join(__snake_case			,      parts[i]      )
						if i >= len(__snake_case      ):
												raise ValueError(F"`object_name` should begin with the name of a module of diffusers but got {object_name}."      )

						with open(os.path.join(__snake_case			,      F"{module}.py"      )			,      "r"			,      encoding="utf-8"			,      newline="\n"      ) as f:
												lowerCAmelCase__       					=  f.readlines()

						# Now let's find the class / func in the code!
						lowerCAmelCase__       					=  ''
						lowerCAmelCase__       					=  0
						for name in parts[i + 1 :]:
												while (
												    line_index < len(__snake_case      ) and re.search(RF"^{indent}(class|def)\s+{name}(\(|\:)"			,      lines[line_index]      ) is None
												):
																		line_index += 1
												indent += "    "
												line_index += 1

						if line_index >= len(__snake_case      ):
												raise ValueError(F" {object_name} does not match any function or class in {module}."      )

						# We found the beginning of the class / func, now let's find the end (when the indent diminishes).
						lowerCAmelCase__       					=  line_index
						while line_index < len(__snake_case      ) and _should_continue(lines[line_index]			,      __snake_case      ):
												line_index += 1
						# Clean up empty lines at the end (if any).
						while len(lines[line_index - 1]      ) <= 1:
												line_index -= 1

						lowerCAmelCase__       					=  lines[start_index:line_index]
						return "".join(__snake_case      )


a_							=			re.compile(r'''^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)''')
a_							=			re.compile(r'''^\s*(\S+)->(\S+)(\s+.*|$)''')
a_							=			re.compile(r'''<FILL\s+[^>]*>''')


def 				_a     (   UpperCamelCase_    : Tuple      )       ->      Tuple:
						"""simple docstring"""







						lowerCAmelCase__       					=  code.split("\n"      )
						lowerCAmelCase__       					=  0
						while idx < len(__snake_case      ) and len(lines[idx]      ) == 0:
												idx += 1
						if idx < len(__snake_case      ):
												return re.search(R"^(\s*)\S"			,      lines[idx]      ).groups()[0]
						return ""


def 				_a     (   UpperCamelCase_    : Union[str, Any]      )       ->      Tuple:
						"""simple docstring"""







						lowerCAmelCase__       					=  len(get_indent(__snake_case      )      ) > 0
						if has_indent:
												lowerCAmelCase__       					=  F"class Bla:\n{code}"
						lowerCAmelCase__       					=  black.Mode(target_versions={black.TargetVersion.PYaa}			,      line_length=119			,      preview=__snake_case      )
						lowerCAmelCase__       					=  black.format_str(__snake_case			,      mode=__snake_case      )
						lowerCAmelCase__       					=  style_docstrings_in_code(__snake_case      )
						return result[len("class Bla:\n"      ) :] if has_indent else result


def 				_a     (   UpperCamelCase_    : List[Any]			,      UpperCamelCase_    : Union[str, Any]=False      )       ->      Union[str, Any]:
						"""simple docstring"""







						with open(__snake_case			,      "r"			,      encoding="utf-8"			,      newline="\n"      ) as f:
												lowerCAmelCase__       					=  f.readlines()
						lowerCAmelCase__       					=  []
						lowerCAmelCase__       					=  0
						# Not a for loop cause `lines` is going to change (if `overwrite=True`).
						while line_index < len(__snake_case      ):
												lowerCAmelCase__       					=  _re_copy_warning.search(lines[line_index]      )
												if search is None:
																		line_index += 1
																		continue

												# There is some copied code here, let's retrieve the original.
												lowerCAmelCase__       					=  search.groups()
												lowerCAmelCase__       					=  find_code_in_diffusers(__snake_case      )
												lowerCAmelCase__       					=  get_indent(__snake_case      )

												lowerCAmelCase__       					=  line_index + 1 if indent == theoretical_indent else line_index + 2
												lowerCAmelCase__       					=  theoretical_indent
												lowerCAmelCase__       					=  start_index

												# Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
												lowerCAmelCase__       					=  True
												while line_index < len(__snake_case      ) and should_continue:
																		line_index += 1
																		if line_index >= len(__snake_case      ):
																								break
																		lowerCAmelCase__       					=  lines[line_index]
																		lowerCAmelCase__       					=  _should_continue(__snake_case			,      __snake_case      ) and re.search(F"^{indent}# End copy"			,      __snake_case      ) is None
												# Clean up empty lines at the end (if any).
												while len(lines[line_index - 1]      ) <= 1:
																		line_index -= 1

												lowerCAmelCase__       					=  lines[start_index:line_index]
												lowerCAmelCase__       					=  ''.join(__snake_case      )

												# Remove any nested `Copied from` comments to avoid circular copies
												lowerCAmelCase__       					=  [line for line in theoretical_code.split("\n"      ) if _re_copy_warning.search(__snake_case      ) is None]
												lowerCAmelCase__       					=  '\n'.join(__snake_case      )

												# Before comparing, use the `replace_pattern` on the original code.
												if len(__snake_case      ) > 0:
																		lowerCAmelCase__       					=  replace_pattern.replace("with"			,      ""      ).split(","      )
																		lowerCAmelCase__       					=  [_re_replace_pattern.search(__snake_case      ) for p in patterns]
																		for pattern in patterns:
																								if pattern is None:
																														continue
																								lowerCAmelCase__       					=  pattern.groups()
																								lowerCAmelCase__       					=  re.sub(__snake_case			,      __snake_case			,      __snake_case      )
																								if option.strip() == "all-casing":
																														lowerCAmelCase__       					=  re.sub(obja.lower()			,      obja.lower()			,      __snake_case      )
																														lowerCAmelCase__       					=  re.sub(obja.upper()			,      obja.upper()			,      __snake_case      )

            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
																		lowerCAmelCase__       					=  blackify(lines[start_index - 1] + theoretical_code      )
																		lowerCAmelCase__       					=  theoretical_code[len(lines[start_index - 1]      ) :]

												# Test for a diff and act accordingly.
												if observed_code != theoretical_code:
																		diffs.append([object_name, start_index]      )
																		if overwrite:
																								lowerCAmelCase__       					=  lines[:start_index] + [theoretical_code] + lines[line_index:]
																								lowerCAmelCase__       					=  start_index + 1

						if overwrite and len(__snake_case      ) > 0:
												# Warn the user a file has been modified.
												print(F"Detected changes, rewriting {filename}."      )
												with open(__snake_case			,      "w"			,      encoding="utf-8"			,      newline="\n"      ) as f:
																		f.writelines(__snake_case      )
						return diffs


def 				_a     (   UpperCamelCase_    : bool = False      )       ->      List[str]:
						"""simple docstring"""







						lowerCAmelCase__       					=  glob.glob(os.path.join(__snake_case			,      "**/*.py"      )			,      recursive=__snake_case      )
						lowerCAmelCase__       					=  []
						for filename in all_files:
												lowerCAmelCase__       					=  is_copy_consistent(__snake_case			,      __snake_case      )
												diffs += [F"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]
						if not overwrite and len(__snake_case      ) > 0:
												lowerCAmelCase__       					=  '\n'.join(__snake_case      )
												raise Exception(
												    "Found the following copy inconsistencies:\n"
												    + diff
												    + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."      )


if __name__ == "__main__":
	a_							=			argparse.ArgumentParser()
	parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
	a_							=			parser.parse_args()

	check_copies(args.fix_and_overwrite)













'''simple docstring'''




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer


class  SCREAMING_SNAKE_CASE    (a__							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Any      =       tempfile.mkdtemp()
          __A						: str      =       5

          # Realm tok
          __A						: Union[str, Any]      =       [
              '[UNK]',
              '[CLS]',
              '[SEP]',
              '[PAD]',
              '[MASK]',
              'test',
              'question',
              'this',
              'is',
              'the',
              'first',
              'second',
              'third',
              'fourth',
              'fifth',
              'record',
              'want',
              '##want',
              '##ed',
              'wa',
              'un',
              'runn',
              '##ing',
              ',',
              'low',
              'lowest',
          ]
          __A						: Union[str, Any]      =       os.path.join(self.tmpdirname						,	'realm_tokenizer')
          os.makedirs(_UpperCAmelCase						,	exist_ok=_UpperCAmelCase)
          __A						: Tuple      =       os.path.join(_UpperCAmelCase						,	VOCAB_FILES_NAMES['vocab_file'])
          with open(self.vocab_file						,	'w'						,	encoding='utf-8') as vocab_writer:
               vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

          __A						: int      =       os.path.join(self.tmpdirname						,	'realm_block_records')
          os.makedirs(_UpperCAmelCase						,	exist_ok=_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname						,	'realm_tokenizer'))


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          shutil.rmtree(self.tmpdirname)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Union[str, Any]      =       RealmConfig(num_block_records=self.num_block_records)
          return config


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       Dataset.from_dict(
              {
                  'id': ['0', '1'],
                  'question': ['foo', 'bar'],
                  'answers': [['Foo', 'Bar'], ['Bar']],
              })
          return dataset


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Any      =       np.array(
              [
                  B'This is the first record',
                  B'This is the second record',
                  B'This is the third record',
                  B'This is the fourth record',
                  B'This is the fifth record',
                  B'This is a longer longer longer record',
              ]						,	dtype=_UpperCAmelCase						,	)
          return block_records


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       RealmRetriever(
              block_records=self.get_dummy_block_records()						,	tokenizer=self.get_tokenizer()						,	)
          return retriever


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[Any]      =       self.get_config()
          __A						: str      =       self.get_dummy_retriever()
          __A						: List[str]      =       retriever.tokenizer

          __A						: Dict      =       np.array([0, 3]						,	dtype='long')
          __A						: Dict      =       tokenizer(['Test question']).input_ids
          __A						: Optional[Any]      =       tokenizer(
              ['the fourth']						,	add_special_tokens=_UpperCAmelCase						,	return_token_type_ids=_UpperCAmelCase						,	return_attention_mask=_UpperCAmelCase						,	).input_ids
          __A						: str      =       config.reader_seq_len

          __A ,__A ,__A ,__A						: List[str]      =       retriever(
              _UpperCAmelCase						,	_UpperCAmelCase						,	answer_ids=_UpperCAmelCase						,	max_length=_UpperCAmelCase						,	return_tensors='np')

          self.assertEqual(len(_UpperCAmelCase)						,	2)
          self.assertEqual(len(_UpperCAmelCase)						,	2)
          self.assertEqual(len(_UpperCAmelCase)						,	2)
          self.assertEqual(concat_inputs.input_ids.shape						,	(2, 10))
          self.assertEqual(concat_inputs.attention_mask.shape						,	(2, 10))
          self.assertEqual(concat_inputs.token_type_ids.shape						,	(2, 10))
          self.assertEqual(concat_inputs.special_tokens_mask.shape						,	(2, 10))
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0])						,	['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']						,	)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1])						,	['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: int      =       self.get_config()
          __A						: Any      =       self.get_dummy_retriever()
          __A						: str      =       retriever.tokenizer

          __A						: Dict      =       np.array([0, 3, 5]						,	dtype='long')
          __A						: Tuple      =       tokenizer(['Test question']).input_ids
          __A						: Union[str, Any]      =       tokenizer(
              ['the fourth', 'longer longer']						,	add_special_tokens=_UpperCAmelCase						,	return_token_type_ids=_UpperCAmelCase						,	return_attention_mask=_UpperCAmelCase						,	).input_ids
          __A						: Dict      =       config.reader_seq_len

          __A ,__A ,__A ,__A						: str      =       retriever(
              _UpperCAmelCase						,	_UpperCAmelCase						,	answer_ids=_UpperCAmelCase						,	max_length=_UpperCAmelCase						,	return_tensors='np')

          self.assertEqual([False, True, True]						,	_UpperCAmelCase)
          self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]						,	_UpperCAmelCase)
          self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       self.get_dummy_retriever()
          retriever.save_pretrained(os.path.join(self.tmpdirname						,	'realm_block_records'))

          # Test local path
          __A						: str      =       retriever.from_pretrained(os.path.join(self.tmpdirname						,	'realm_block_records'))
          self.assertEqual(retriever.block_records[0]						,	B'This is the first record')

          # Test mocked remote path
          with patch('transformers.models.realm.retrieval_realm.hf_hub_download') as mock_hf_hub_download:
               __A						: int      =       os.path.join(
                   os.path.join(self.tmpdirname						,	'realm_block_records')						,	_REALM_BLOCK_RECORDS_FILENAME)
               __A						: Tuple      =       RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa')

          self.assertEqual(retriever.block_records[0]						,	B'This is the first record')