


"""simple docstring"""






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


lowercase  		=			"""block_records.npy"""


lowercase  		=			logging.get_logger(__name__)



def 			A__		(					_UpperCAmelCase    :      str				,    _UpperCAmelCase    :      int	)							->	np.ndarray:

	'''simple docstring'''


	import tensorflow.compat.va as tf

	snake_case__  :						str		=      tf.data.TFRecordDataset(_UpperCAmelCase				,    buffer_size=5_12 * 10_24 * 10_24	)
	snake_case__  :						str		=      blocks_dataset.batch(_UpperCAmelCase				,    drop_remainder=_UpperCAmelCase	)
	snake_case__  :						List[Any]		=      next(blocks_dataset.take(1	).as_numpy_iterator()	)

	return np_record


class 			SCREAMING_SNAKE_CASE_  :




							'''simple docstring'''





							def __init__(						self	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__=2	,					lowerCamelCase__=1_000	,					lowerCamelCase__=100	,					lowerCamelCase__=100_000	,					)   ->	str:







								'''simple docstring'''




								from scann.scann_ops.py.scann_ops_pybind import builder as Builder

								snake_case__  :						Optional[Any]		=      Builder(db=lowerCamelCase__	,					num_neighbors=lowerCamelCase__	,					distance_measure="dot_product")
								snake_case__  :						Optional[int]		=      builder.tree(
								    num_leaves=lowerCamelCase__	,					num_leaves_to_search=lowerCamelCase__	,					training_sample_size=lowerCamelCase__)
								snake_case__  :						str		=      builder.score_ah(dimensions_per_block=lowerCamelCase__)

								snake_case__  :						Union[str, Any]		=      builder.build()







							def        UpperCAmelCase (						self	,					lowerCamelCase__)   ->	str:







								'''simple docstring'''




								snake_case__,					snake_case__  :						str		=      self.searcher.search_batched(question_projection.detach().cpu())
								return retrieved_block_ids.astype("int64")


class 			SCREAMING_SNAKE_CASE_  :




							'''simple docstring'''





							def __init__(						self	,					lowerCamelCase__	,					lowerCamelCase__)   ->	str:







								'''simple docstring'''




								super().__init__()
								snake_case__  :						Tuple		=      block_records
								snake_case__  :						int		=      tokenizer




							def __call__(						self	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__=None	,					lowerCamelCase__="pt")   ->	Optional[Any]:







								'''simple docstring'''




								snake_case__  :						str		=      np.take(self.block_records	,					indices=lowerCamelCase__	,					axis=0)

								snake_case__  :						Dict		=      self.tokenizer.decode(question_input_ids[0]	,					skip_special_tokens=lowerCamelCase__)

								snake_case__  :						List[Any]		=      []
								snake_case__  :						Optional[int]		=      []
								for retrieved_block in retrieved_blocks:
									text.append(lowerCamelCase__)
									text_pair.append(retrieved_block.decode())

								snake_case__  :						str		=      self.tokenizer(
								    lowerCamelCase__	,					lowerCamelCase__	,					padding=lowerCamelCase__	,					truncation=lowerCamelCase__	,					return_special_tokens_mask=lowerCamelCase__	,					max_length=lowerCamelCase__)
								snake_case__  :						Tuple		=      concat_inputs.convert_to_tensors(lowerCamelCase__)

								if answer_ids is not None:
									return self.block_has_answer(lowerCamelCase__	,					lowerCamelCase__) + (concat_inputs_tensors,)
								else:
									return (None, None, None, concat_inputs_tensors)




							@classmethod
							def        UpperCAmelCase (						cls	,					lowerCamelCase__	,					*lowerCamelCase__	,					**lowerCamelCase__)   ->	Dict:







								'''simple docstring'''




								if os.path.isdir(lowerCamelCase__):
									snake_case__  :						Tuple		=      os.path.join(lowerCamelCase__	,					_REALM_BLOCK_RECORDS_FILENAME)
								else:
									snake_case__  :						Any		=      hf_hub_download(
									    repo_id=lowerCamelCase__	,					filename=_REALM_BLOCK_RECORDS_FILENAME	,					**lowerCamelCase__)
								snake_case__  :						int		=      np.load(lowerCamelCase__	,					allow_pickle=lowerCamelCase__)

								snake_case__  :						Union[str, Any]		=      AutoTokenizer.from_pretrained(lowerCamelCase__	,					*lowerCamelCase__	,					**lowerCamelCase__)

								return cls(lowerCamelCase__	,					lowerCamelCase__)




							def        UpperCAmelCase (						self	,					lowerCamelCase__)   ->	Tuple:







								'''simple docstring'''




								np.save(os.path.join(lowerCamelCase__	,					_REALM_BLOCK_RECORDS_FILENAME)	,					self.block_records)
								# save tokenizer
								self.tokenizer.save_pretrained(lowerCamelCase__)







							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__)   ->	int:







								'''simple docstring'''




								snake_case__  :						Optional[Any]		=      []
								snake_case__  :						int		=      []
								snake_case__  :						int		=      []
								snake_case__  :						Any		=      0

								for input_id in concat_inputs.input_ids:
									snake_case__  :						List[Any]		=      input_id.tolist()
									# Check answers between two [SEP] tokens
									snake_case__  :						Optional[int]		=      input_id_list.index(self.tokenizer.sep_token_id)
									snake_case__  :						Optional[int]		=      first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id)

									start_pos.append([])
									end_pos.append([])
									for answer in answer_ids:
										for idx in range(first_sep_idx + 1	,					lowerCamelCase__):
											if answer[0] == input_id_list[idx]:
												if input_id_list[idx : idx + len(lowerCamelCase__)] == answer:
													start_pos[-1].append(lowerCamelCase__)
													end_pos[-1].append(idx + len(lowerCamelCase__) - 1)

									if len(start_pos[-1]) == 0:
										has_answers.append(lowerCamelCase__)
									else:
										has_answers.append(lowerCamelCase__)
										if len(start_pos[-1]) > max_answers:
											snake_case__  :						List[str]		=      len(start_pos[-1])

        # Pad -1 to max_answers
								for start_pos_, end_pos_ in zip(lowerCamelCase__	,					lowerCamelCase__):
									if len(start_pos_) < max_answers:
										snake_case__  :						Union[str, Any]		=      [-1] * (max_answers - len(start_pos_))
										start_pos_ += padded
										end_pos_ += padded
								return has_answers, start_pos, end_pos









import tempfile
import unittest

from diffusers import (
    DDIMScheduler,
    DDPMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    PNDMScheduler,
    logging,
)
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.utils.testing_utils import CaptureLogger







class 					snake_case__(				UpperCAmelCase__				):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__   :	Union[str, Any]       =				"""config.json"""




						@register_to_config
						def __init__(					self ,    __lowercase=2 ,    __lowercase=5 ,    __lowercase=(2, 5) ,    __lowercase="for diffusion" ,    __lowercase=[1, 3] ,    )       ->     int:
										pass







class 					snake_case__(				UpperCAmelCase__				):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__   :	int       =				"""config.json"""




						@register_to_config
						def __init__(					self ,    __lowercase=2 ,    __lowercase=5 ,    __lowercase=(2, 5) ,    __lowercase="for diffusion" ,    __lowercase=[1, 3] ,    )       ->     Optional[Any]:
										pass







class 					snake_case__(				UpperCAmelCase__				):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__   :	int       =				"""config.json"""




						@register_to_config
						def __init__(					self ,    __lowercase=2 ,    __lowercase=5 ,    __lowercase=(2, 5) ,    __lowercase="for diffusion" ,    __lowercase=[1, 3] ,    __lowercase=[1, 3] ,    )       ->     List[str]:
										pass







class 					snake_case__(				UpperCAmelCase__				):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__   :	Optional[Any]       =				"""config.json"""




						@register_to_config
						def __init__(					self ,    __lowercase=2 ,    __lowercase=5 ,    __lowercase=(2, 5) ,    __lowercase="for diffusion" ,    __lowercase=[1, 5] ,    __lowercase=[5, 4] ,    )       ->     List[str]:
										pass







class 					snake_case__(				unittest.TestCase				):







						'''simple docstring'''




						def     lowercase_			(					self				)       ->     str:
										with self.assertRaises(__lowercase				):
														ConfigMixin.load_config('''dummy_path'''				)




						def     lowercase_			(					self				)       ->     Optional[int]:
										lowerCAmelCase_			:   List[str]     					=					SampleObject()
										lowerCAmelCase_			:   Dict     					=					obj.config
										assert config["a"] == 2
										assert config["b"] == 5
										assert config["c"] == (2, 5)
										assert config["d"] == "for diffusion"
										assert config["e"] == [1, 3]

										# init ignore private arguments
										lowerCAmelCase_			:   List[str]     					=					SampleObject(_name_or_path='''lalala'''				)
										lowerCAmelCase_			:   Dict     					=					obj.config
										assert config["a"] == 2
										assert config["b"] == 5
										assert config["c"] == (2, 5)
										assert config["d"] == "for diffusion"
										assert config["e"] == [1, 3]

										# can override default
										lowerCAmelCase_			:   List[Any]     					=					SampleObject(c=6				)
										lowerCAmelCase_			:   Optional[int]     					=					obj.config
										assert config["a"] == 2
										assert config["b"] == 5
										assert config["c"] == 6
										assert config["d"] == "for diffusion"
										assert config["e"] == [1, 3]

										# can use positional arguments.
										lowerCAmelCase_			:   Optional[Any]     					=					SampleObject(1 ,    c=6				)
										lowerCAmelCase_			:   Dict     					=					obj.config
										assert config["a"] == 1
										assert config["b"] == 5
										assert config["c"] == 6
										assert config["d"] == "for diffusion"
										assert config["e"] == [1, 3]




						def     lowercase_			(					self				)       ->     int:
										lowerCAmelCase_			:   List[Any]     					=					SampleObject()
										lowerCAmelCase_			:   str     					=					obj.config

										assert config["a"] == 2
										assert config["b"] == 5
										assert config["c"] == (2, 5)
										assert config["d"] == "for diffusion"
										assert config["e"] == [1, 3]

										with tempfile.TemporaryDirectory() as tmpdirname:
														obj.save_config(__lowercase				)
														lowerCAmelCase_			:   int     					=					SampleObject.from_config(SampleObject.load_config(__lowercase				)				)
														lowerCAmelCase_			:   Optional[Any]     					=					new_obj.config

										# unfreeze configs
										lowerCAmelCase_			:   str     					=					dict(__lowercase				)
										lowerCAmelCase_			:   Any     					=					dict(__lowercase				)

										assert config.pop('''c'''				) == (2, 5)  # instantiated as tuple
										assert new_config.pop('''c'''				) == [2, 5]  # saved & loaded as list because of json
										config.pop('''_use_default_values'''				)
										assert config == new_config




						def     lowercase_			(					self				)       ->     List[str]:
										lowerCAmelCase_			:   Optional[Any]     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   int     					=					DDIMScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler'''				)

										assert ddim.__class__ == DDIMScheduler
										# no warning should be thrown
										assert cap_logger.out == ""




						def     lowercase_			(					self				)       ->     Optional[Any]:
										lowerCAmelCase_			:   List[Any]     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   Any     					=					EulerDiscreteScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler'''				)

										assert euler.__class__ == EulerDiscreteScheduler
										# no warning should be thrown
										assert cap_logger.out == ""




						def     lowercase_			(					self				)       ->     Any:
										lowerCAmelCase_			:   Tuple     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   Optional[int]     					=					EulerAncestralDiscreteScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler'''				)

										assert euler.__class__ == EulerAncestralDiscreteScheduler
										# no warning should be thrown
										assert cap_logger.out == ""




						def     lowercase_			(					self				)       ->     List[Any]:
										lowerCAmelCase_			:   Any     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   Dict     					=					PNDMScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler'''				)

										assert pndm.__class__ == PNDMScheduler
										# no warning should be thrown
										assert cap_logger.out == ""




						def     lowercase_			(					self				)       ->     Tuple:
										lowerCAmelCase_			:   Optional[int]     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   int     					=					DDPMScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler''' ,    prediction_type='''sample''' ,    beta_end=8 ,    )

										with CaptureLogger(__lowercase				) as cap_logger_a:
														lowerCAmelCase_			:   Tuple     					=					DDPMScheduler.from_pretrained('''google/ddpm-celebahq-256''' ,    beta_start=8_8				)

										assert ddpm.__class__ == DDPMScheduler
										assert ddpm.config.prediction_type == "sample"
										assert ddpm.config.beta_end == 8
										assert ddpm_a.config.beta_start == 8_8

										# no warning should be thrown
										assert cap_logger.out == ""
										assert cap_logger_a.out == ""




						def     lowercase_			(					self				)       ->     int:
										lowerCAmelCase_			:   Tuple     					=					logging.get_logger('''diffusers.configuration_utils'''				)
										# 30 for warning
										logger.setLevel(3_0				)

										with CaptureLogger(__lowercase				) as cap_logger:
														lowerCAmelCase_			:   List[str]     					=					DPMSolverMultistepScheduler.from_pretrained(
														    '''hf-internal-testing/tiny-stable-diffusion-torch''' ,    subfolder='''scheduler'''				)

										assert dpm.__class__ == DPMSolverMultistepScheduler
										# no warning should be thrown
										assert cap_logger.out == ""




						def     lowercase_			(					self				)       ->     Optional[Any]:
										# let's first save a config that should be in the form
										#    a=2,
										#    b=5,
										#    c=(2, 5),
										#    d="for diffusion",
										#    e=[1, 3],

										lowerCAmelCase_			:   Dict     					=					SampleObject()

										lowerCAmelCase_			:   Optional[int]     					=					{k: v for k, v in config.config.items() if not k.startswith('''_'''				)}

										# make sure that default config has all keys in `_use_default_values`
										assert set(config_dict.keys()				) == set(config.config._use_default_values				)

										with tempfile.TemporaryDirectory() as tmpdirname:
														config.save_config(__lowercase				)

														# now loading it with SampleObject2 should put f into `_use_default_values`
														lowerCAmelCase_			:   List[Any]     					=					SampleObjecta.from_config(__lowercase				)

														assert "f" in config._use_default_values
														assert config.f == [1, 3]

										# now loading the config, should **NOT** use [1, 3] for `f`, but the default [1, 4] value
										# **BECAUSE** it is part of `config._use_default_values`
										lowerCAmelCase_			:   List[Any]     					=					SampleObjecta.from_config(config.config				)
										assert new_config.f == [5, 4]

										config.config._use_default_values.pop()
										lowerCAmelCase_			:   int     					=					SampleObjecta.from_config(config.config				)
										assert new_config_a.f == [1, 3]

										# Nevertheless "e" should still be correctly loaded to [1, 3] from SampleObject2 instead of defaulting to [1, 5]
										assert new_config_a.e == [1, 3]