

"""simple docstring"""




import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
	import torch


__A	:       str        =						random.Random()
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__=1.0     ,				lowercase__=None     ,				lowercase__=None ):
					"""simple docstring"""
					if rng is None:
										A     =   global_rng

					A     =   []
					for batch_idx in range(shape[0] ):
										values.append([] )
										for _ in range(shape[1] ):
															values[-1].append(rng.random() * scale )

					return values





@require_torch
class      __UpperCamelCase					(  unittest.TestCase		):


		def __init__(self					:						Tuple	,	__SCREAMING_SNAKE_CASE					:						Tuple	,	__SCREAMING_SNAKE_CASE					:						Union[str, Any]=7	,	__SCREAMING_SNAKE_CASE					:						List[Any]=4_0_0	,	__SCREAMING_SNAKE_CASE					:						str=2_0_0_0	,	__SCREAMING_SNAKE_CASE					:						List[str]=1	,	__SCREAMING_SNAKE_CASE					:						Tuple=0.0	,	__SCREAMING_SNAKE_CASE					:						Any=2_4_0_0_0	,	__SCREAMING_SNAKE_CASE					:						Optional[int]=True	,	):
							A     =   parent
							A     =   batch_size
							A     =   min_seq_length
							A     =   max_seq_length
							A     =   (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
							A     =   feature_size
							A     =   padding_value
							A     =   sampling_rate
							A     =   return_attention_mask


		def    SCREAMING_SNAKE_CASE__  (self					:						Union[str, Any]):
							return {
							    "feature_size": self.feature_size,
							    "padding_value": self.padding_value,
							    "sampling_rate": self.sampling_rate,
							    "return_attention_mask": self.return_attention_mask,
							}


		def    SCREAMING_SNAKE_CASE__  (self					:						Any	,	__SCREAMING_SNAKE_CASE					:						List[Any]=False	,	__SCREAMING_SNAKE_CASE					:						Any=False):
							def _flatten(__SCREAMING_SNAKE_CASE					:						List[Any]):
												return list(itertools.chain(*__SCREAMING_SNAKE_CASE))

							if equal_length:
												A     =   floats_list((self.batch_size, self.max_seq_length))
							else:
												# make sure that inputs increase in size
												A     =   [
												    _flatten(floats_list((x, self.feature_size)))
												    for x in range(self.min_seq_length	,	self.max_seq_length	,	self.seq_length_diff)
												]

							if numpify:
												A     =   [np.asarray(__SCREAMING_SNAKE_CASE) for x in audio_inputs]

							return audio_inputs





@require_torch
class      __UpperCamelCase					(  _A	,						unittest.TestCase		):
		SCREAMING_SNAKE_CASE														=       EncodecFeatureExtractor


		def    SCREAMING_SNAKE_CASE__  (self					:						List[Any]):
							A     =   EnCodecFeatureExtractionTester(self)


		def    SCREAMING_SNAKE_CASE__  (self					:						Any):
							# Tests that all call wrap to encode_plus and batch_encode_plus
							A     =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
							# create three inputs of length 800, 1000, and 1200
							A     =   [floats_list((1, x))[0] for x in range(8_0_0	,	1_4_0_0	,	2_0_0)]
							A     =   [np.asarray(__SCREAMING_SNAKE_CASE) for audio_input in audio_inputs]

							# Test not batched input
							A     =   feat_extract(audio_inputs[0]	,	return_tensors="np").input_values
							A     =   feat_extract(np_audio_inputs[0]	,	return_tensors="np").input_values
							self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE	,	__SCREAMING_SNAKE_CASE	,	atol=1E-3))

							# Test batched
							A     =   feat_extract(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="np").input_values
							A     =   feat_extract(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="np").input_values
							for enc_seq_a, enc_seq_a in zip(__SCREAMING_SNAKE_CASE	,	__SCREAMING_SNAKE_CASE):
												self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE	,	__SCREAMING_SNAKE_CASE	,	atol=1E-3))


		def    SCREAMING_SNAKE_CASE__  (self					:						str):
							A     =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
							A     =   np.random.rand(1_0_0).astype(np.floataa)
							A     =   np_audio_inputs.tolist()

							for inputs in [py_audio_inputs, np_audio_inputs]:
												A     =   feature_extractor.pad([{"input_values": inputs}]	,	return_tensors="np")
												self.assertTrue(np_processed.input_values.dtype == np.floataa)
												A     =   feature_extractor.pad([{"input_values": inputs}]	,	return_tensors="pt")
												self.assertTrue(pt_processed.input_values.dtype == torch.floataa)


		def    SCREAMING_SNAKE_CASE__  (self					:						List[str]	,	__SCREAMING_SNAKE_CASE					:						Optional[int]):
							from datasets import load_dataset

							A     =   load_dataset("hf-internal-testing/librispeech_asr_dummy"	,	"clean"	,	split="validation")
							# automatic decoding with librispeech
							A     =   ds.sort("id").select(range(__SCREAMING_SNAKE_CASE))[:num_samples]["audio"]

							return [x["array"] for x in audio_samples]


		def    SCREAMING_SNAKE_CASE__  (self					:						int):
							# fmt: off
							A     =   torch.tensor(
							    [2.3804E-03, 2.0752E-03, 1.9836E-03, 2.1057E-03, 1.6174E-03,
							     3.0518E-04, 9.1553E-05, 3.3569E-04, 9.7656E-04, 1.8311E-03,
							     2.0142E-03, 2.1057E-03, 1.7395E-03, 4.5776E-04, -3.9673E-04,
							     4.5776E-04, 1.0071E-03, 9.1553E-05, 4.8828E-04, 1.1597E-03,
							     7.3242E-04, 9.4604E-04, 1.8005E-03, 1.8311E-03, 8.8501E-04,
							     4.2725E-04, 4.8828E-04, 7.3242E-04, 1.0986E-03, 2.1057E-03])
							# fmt: on
							A     =   self._load_datasamples(1)
							A     =   EncodecFeatureExtractor()
							A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(input_values.shape	,	(1, 1, 9_3_6_8_0))
							self.assertTrue(torch.allclose(input_values[0, 0, :3_0]	,	__SCREAMING_SNAKE_CASE	,	atol=1E-6))


		def    SCREAMING_SNAKE_CASE__  (self					:						Optional[Any]):
							# fmt: off
							A     =   torch.tensor(
							    [2.3804E-03, 2.0752E-03, 1.9836E-03, 2.1057E-03, 1.6174E-03,
							     3.0518E-04, 9.1553E-05, 3.3569E-04, 9.7656E-04, 1.8311E-03,
							     2.0142E-03, 2.1057E-03, 1.7395E-03, 4.5776E-04, -3.9673E-04,
							     4.5776E-04, 1.0071E-03, 9.1553E-05, 4.8828E-04, 1.1597E-03,
							     7.3242E-04, 9.4604E-04, 1.8005E-03, 1.8311E-03, 8.8501E-04,
							     4.2725E-04, 4.8828E-04, 7.3242E-04, 1.0986E-03, 2.1057E-03])
							# fmt: on
							A     =   self._load_datasamples(1)
							A     =   [np.tile(input_audio[0][None]	,	reps=(2, 1))]
							input_audio[0][1] *= 0.5
							A     =   EncodecFeatureExtractor(feature_size=2)
							A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(input_values.shape	,	(1, 2, 9_3_6_8_0))
							self.assertTrue(torch.allclose(input_values[0, 0, :3_0]	,	__SCREAMING_SNAKE_CASE	,	atol=1E-6))
							self.assertTrue(torch.allclose(input_values[0, 1, :3_0]	,	EXPECTED_INPUT_VALUES * 0.5	,	atol=1E-6))


		def    SCREAMING_SNAKE_CASE__  (self					:						List[str]):
							A     =   self._load_datasamples(2)
							# would be easier if the stride was like
							A     =   EncodecFeatureExtractor(feature_size=1	,	chunk_length_s=1	,	overlap=0.0_1)

							# pad and trunc raise an error ?
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE	,	"^Both padding and truncation were set. Make sure you only set one.$"	,	):
												A     =   feature_extractor(
												    __SCREAMING_SNAKE_CASE	,	padding="max_length"	,	truncation=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values

							# truncate to chunk
							A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	truncation=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(2, 1, 7_1_5_2_0))  # 2 chunks

							# force truncate to max_length
							A     =   feature_extractor(
							    __SCREAMING_SNAKE_CASE	,	truncation=__SCREAMING_SNAKE_CASE	,	max_length=4_8_0_0_0	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(2, 1, 4_8_0_0_0))

							# pad to chunk
							A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(padded_outputs.shape	,	(2, 1, 9_5_2_8_0))

							# pad to chunk
							A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(2, 1, 9_5_2_8_0))

							# force pad to max length
							A     =   feature_extractor(
							    __SCREAMING_SNAKE_CASE	,	padding="max_length"	,	max_length=1_0_0_0_0_0	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(2, 1, 1_0_0_0_0_0))

							# force no pad
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE	,	"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"	,	):
												A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values

							A     =   feature_extractor(input_audio[0]	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(1, 1, 9_3_6_8_0))

							# no pad if no chunk_length_s
							A     =   None
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE	,	"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"	,	):
												A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values

							A     =   feature_extractor(input_audio[0]	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(1, 1, 9_3_6_8_0))

							# no pad if no overlap
							A     =   2
							A     =   None
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE	,	"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"	,	):
												A     =   feature_extractor(__SCREAMING_SNAKE_CASE	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values

							A     =   feature_extractor(input_audio[0]	,	padding=__SCREAMING_SNAKE_CASE	,	return_tensors="pt").input_values
							self.assertEquals(truncated_outputs.shape	,	(1, 1, 9_3_6_8_0))






"""simple docstring"""




import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


__lowerCamelCase									=	"block_records.npy"


__lowerCamelCase									=	logging.get_logger(__name__)






def 	UpperCAmelCase       (						UpperCamelCase__							,     UpperCamelCase__					):






							"""simple docstring"""





							import tensorflow.compat.va as tf

							A__						  =				tf.data.TFRecordDataset(UpperCamelCase__							,     buffer_size=512 * 1_024 * 1_024					)
							A__						  =				blocks_dataset.batch(UpperCamelCase__							,     drop_remainder=UpperCamelCase__					)
							A__						  =				next(blocks_dataset.take(1					).as_numpy_iterator()					)

							return np_record
class      UpperCamelCase__:





						def __init__(							self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase=2		,__UpperCAmelCase=10_00		,__UpperCAmelCase=1_00		,__UpperCAmelCase=10_00_00		,)       ->					Optional[int]:
													from scann.scann_ops.py.scann_ops_pybind import builder as Builder

													A__						  =				Builder(db=__UpperCAmelCase		,num_neighbors=__UpperCAmelCase		,distance_measure='dot_product'  )
													A__						  =				builder.tree(
													    num_leaves=__UpperCAmelCase		,num_leaves_to_search=__UpperCAmelCase		,training_sample_size=__UpperCAmelCase  )
													A__						  =				builder.score_ah(dimensions_per_block=__UpperCAmelCase  )

													A__						  =				builder.build()





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Tuple:
													A__  ,	A__						  =				self.searcher.search_batched(question_projection.detach().cpu()  )
													return retrieved_block_ids.astype('int64'  )






class      UpperCamelCase__:





						def __init__(							self		,__UpperCAmelCase		,__UpperCAmelCase  )       ->					List[str]:
													super().__init__()
													A__						  =				block_records
													A__						  =				tokenizer





						def __call__(							self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase=None		,__UpperCAmelCase="pt"  )       ->					str:
													A__						  =				np.take(self.block_records		,indices=__UpperCAmelCase		,axis=0  )

													A__						  =				self.tokenizer.decode(question_input_ids[0]		,skip_special_tokens=__UpperCAmelCase  )

													A__						  =				[]
													A__						  =				[]
													for retrieved_block in retrieved_blocks:
																				text.append(__UpperCAmelCase  )
																				text_pair.append(retrieved_block.decode()  )

													A__						  =				self.tokenizer(
													    __UpperCAmelCase		,__UpperCAmelCase		,padding=__UpperCAmelCase		,truncation=__UpperCAmelCase		,return_special_tokens_mask=__UpperCAmelCase		,max_length=__UpperCAmelCase  )
													A__						  =				concat_inputs.convert_to_tensors(__UpperCAmelCase  )

													if answer_ids is not None:
																				return self.block_has_answer(__UpperCAmelCase		,__UpperCAmelCase  ) + (concat_inputs_tensors,)
													else:
																				return (None, None, None, concat_inputs_tensors)





						@classmethod
						def 					snake_case__						(							cls		,__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase  )       ->					List[Any]:
													if os.path.isdir(__UpperCAmelCase  ):
																				A__						  =				os.path.join(__UpperCAmelCase		,_REALM_BLOCK_RECORDS_FILENAME  )
													else:
																				A__						  =				hf_hub_download(
																				    repo_id=__UpperCAmelCase		,filename=_REALM_BLOCK_RECORDS_FILENAME		,**__UpperCAmelCase  )
													A__						  =				np.load(__UpperCAmelCase		,allow_pickle=__UpperCAmelCase  )

													A__						  =				AutoTokenizer.from_pretrained(__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase  )

													return cls(__UpperCAmelCase		,__UpperCAmelCase  )





						def 					snake_case__						(							self		,__UpperCAmelCase  )       ->					Tuple:
													# save block records
													np.save(os.path.join(__UpperCAmelCase		,_REALM_BLOCK_RECORDS_FILENAME  )		,self.block_records  )
													# save tokenizer
													self.tokenizer.save_pretrained(__UpperCAmelCase  )





						def 					snake_case__						(							self		,__UpperCAmelCase		,__UpperCAmelCase  )       ->					List[Any]:
													A__						  =				[]
													A__						  =				[]
													A__						  =				[]
													A__						  =				0

													for input_id in concat_inputs.input_ids:
																				A__						  =				input_id.tolist()
																				# Check answers between two [SEP] tokens
																				A__						  =				input_id_list.index(self.tokenizer.sep_token_id  )
																				A__						  =				first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id  )

																				start_pos.append([]  )
																				end_pos.append([]  )
																				for answer in answer_ids:
																											for idx in range(first_sep_idx + 1		,__UpperCAmelCase  ):
																																		if answer[0] == input_id_list[idx]:
																																									if input_id_list[idx : idx + len(__UpperCAmelCase  )] == answer:
																																																start_pos[-1].append(__UpperCAmelCase  )
																																																end_pos[-1].append(idx + len(__UpperCAmelCase  ) - 1  )

																				if len(start_pos[-1]  ) == 0:
																											has_answers.append(__UpperCAmelCase  )
																				else:
																											has_answers.append(__UpperCAmelCase  )
																											if len(start_pos[-1]  ) > max_answers:
																																		A__						  =				len(start_pos[-1]  )

        # Pad -1 to max_answers
													for start_pos_, end_pos_ in zip(__UpperCAmelCase		,__UpperCAmelCase  ):
																				if len(start_pos_  ) < max_answers:
																											A__						  =				[-1] * (max_answers - len(start_pos_  ))
																											start_pos_ += padded
																											end_pos_ += padded
													return has_answers, start_pos, end_pos





