

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_       : Dict         =			logging.get_logger(__name__)

A_       : Tuple         =			{
    'asapp/sew-d-tiny-100k': 'https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}



class     _a						(__magic_name__				):


					'''simple docstring'''






					UpperCAmelCase__:   List[Any]  							=       '''sew-d'''






					def __init__( self    ,       A__=32    ,       A__=768    ,       A__=12    ,       A__=12    ,       A__=3072    ,       A__=2    ,       A__=512    ,       A__=256    ,       A__=True    ,       A__=True    ,       A__=("p2c", "c2p")    ,       A__="layer_norm"    ,       A__="gelu_python"    ,       A__=0.1    ,       A__=0.1    ,       A__=0.1    ,       A__=0.0    ,       A__=0.1    ,       A__=0.0_2    ,       A__=1e-7    ,       A__=1e-5    ,       A__="group"    ,       A__="gelu"    ,       A__=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)    ,       A__=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)    ,       A__=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)    ,       A__=False    ,       A__=128    ,       A__=16    ,       A__=True    ,       A__=0.0_5    ,       A__=10    ,       A__=2    ,       A__=0.0    ,       A__=10    ,       A__=0    ,       A__="mean"    ,       A__=False    ,       A__=False    ,       A__=256    ,       A__=0    ,       A__=1    ,       A__=2    ,       **A__    ,       ):
						super().__init__(**A__    ,       pad_token_id=A__    ,       bos_token_id=A__    ,       eos_token_id=A__			)
						A__							:    Optional[int]        =		hidden_size
						A__							:    Any        =		feat_extract_norm
						A__							:    Any        =		feat_extract_activation
						A__							:    Optional[int]        =		list(A__			)
						A__							:    List[str]        =		list(A__			)
						A__							:    Union[str, Any]        =		list(A__			)
						A__							:    Union[str, Any]        =		conv_bias
						A__							:    Tuple        =		num_conv_pos_embeddings
						A__							:    Optional[Any]        =		num_conv_pos_embedding_groups
						A__							:    List[Any]        =		len(self.conv_dim			)
						A__							:    List[str]        =		num_hidden_layers
						A__							:    Any        =		intermediate_size
						A__							:    int        =		squeeze_factor
						A__							:    str        =		max_position_embeddings
						A__							:    Optional[Any]        =		position_buckets
						A__							:    Optional[int]        =		share_att_key
						A__							:    Any        =		relative_attention
						A__							:    int        =		norm_rel_ebd
						A__							:    int        =		list(A__			)
						A__							:    Optional[Any]        =		hidden_act
						A__							:    Optional[int]        =		num_attention_heads
						A__							:    Dict        =		hidden_dropout
						A__							:    Tuple        =		attention_dropout
						A__							:    List[Any]        =		activation_dropout
						A__							:    List[Any]        =		feat_proj_dropout
						A__							:    Optional[Any]        =		final_dropout
						A__							:    List[str]        =		layer_norm_eps
						A__							:    List[Any]        =		feature_layer_norm_eps
						A__							:    List[Any]        =		initializer_range
						A__							:    Dict        =		vocab_size

						if (
						    (len(self.conv_stride			) != self.num_feat_extract_layers)
						    or (len(self.conv_kernel			) != self.num_feat_extract_layers)
						    or (len(self.conv_dim			) != self.num_feat_extract_layers)
						):
							raise ValueError(
							    """Configuration for convolutional layers is incorrect."""
							    """It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"""
							    F"""but is `len(config.conv_dim) = {len(self.conv_dim			)}`, `len(config.conv_stride)"""
							    F"""= {len(self.conv_stride			)}`, `len(config.conv_kernel) = {len(self.conv_kernel			)}`."""			)

						# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
						A__							:    Dict        =		apply_spec_augment
						A__							:    Any        =		mask_time_prob
						A__							:    Optional[int]        =		mask_time_length
						A__							:    Tuple        =		mask_time_min_masks
						A__							:    int        =		mask_feature_prob
						A__							:    Tuple        =		mask_feature_length
						A__							:    Union[str, Any]        =		mask_feature_min_masks

						# ctc loss
						A__							:    str        =		ctc_loss_reduction
						A__							:    Optional[int]        =		ctc_zero_infinity

						# sequence classification
						A__							:    List[Any]        =		use_weighted_layer_sum
						A__							:    Dict        =		classifier_proj_size






					@property
					def   __A			( self			):
						return functools.reduce(operator.mul    ,       self.conv_stride    ,       1			)






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A_       : Any         =			pytest.mark.integration



@require_faiss
class     _a						(__magic_name__				):


					'''simple docstring'''






					def   __A			( self			):
						A__							:    Tuple        =		Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(A__			) for x in np.arange(30			).tolist()]}			)
						return dset






					def   __A			( self			):
						import faiss

						A__							:    Dataset        =		self._create_dummy_dataset()
						A__							:    Union[str, Any]        =		dset.map(
						    lambda A__    ,       A__			: {"vecs": i * np.ones(5    ,       dtype=np.floataa			)}    ,       with_indices=A__    ,       keep_in_memory=A__			)
						A__							:    int        =		dset.add_faiss_index("""vecs"""    ,       batch_size=100    ,       metric_type=faiss.METRIC_INNER_PRODUCT			)
						A__   ,   A__							:    Optional[Any]        =		dset.get_nearest_examples("""vecs"""    ,       np.ones(5    ,       dtype=np.floataa			)			)
						self.assertEqual(examples["""filename"""][0]    ,       """my_name-train_29"""			)
						dset.drop_index("""vecs"""			)






					def   __A			( self			):
						import faiss

						A__							:    Dataset        =		self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((30, 5)			) * np.arange(30			).reshape(-1    ,       1			)    ,       index_name="""vecs"""    ,       batch_size=100    ,       metric_type=faiss.METRIC_INNER_PRODUCT    ,       )
						A__   ,   A__							:    str        =		dset.get_nearest_examples("""vecs"""    ,       np.ones(5    ,       dtype=np.floataa			)			)
						self.assertEqual(examples["""filename"""][0]    ,       """my_name-train_29"""			)






					def   __A			( self			):
						import faiss

						A__							:    Dataset        =		self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((30, 5)			) * np.arange(30			).reshape(-1    ,       1			)    ,       index_name="""vecs"""    ,       metric_type=faiss.METRIC_INNER_PRODUCT    ,       )

						# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
						# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
						# see https://bugs.python.org/issue14243 and
						# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
						with tempfile.NamedTemporaryFile(delete=A__			) as tmp_file:
							dset.save_faiss_index("""vecs"""    ,       tmp_file.name			)
							dset.load_faiss_index("""vecs2"""    ,       tmp_file.name			)
						os.unlink(tmp_file.name			)

						A__   ,   A__							:    Tuple        =		dset.get_nearest_examples("""vecs2"""    ,       np.ones(5    ,       dtype=np.floataa			)			)
						self.assertEqual(examples["""filename"""][0]    ,       """my_name-train_29"""			)






					def   __A			( self			):
						A__							:    Dataset        =		self._create_dummy_dataset()
						dset.add_faiss_index_from_external_arrays(
						    external_arrays=np.ones((30, 5)			) * np.arange(30			).reshape(-1    ,       1			)    ,       index_name="""vecs"""			)
						dset.drop_index("""vecs"""			)
						self.assertRaises(A__    ,       partial(dset.get_nearest_examples    ,       """vecs2"""    ,       np.ones(5    ,       dtype=np.floataa			)			)			)






					def   __A			( self			):
						from elasticsearch import Elasticsearch

						A__							:    Dataset        =		self._create_dummy_dataset()
						with patch("""elasticsearch.Elasticsearch.search"""			) as mocked_search, patch(
						    """elasticsearch.client.IndicesClient.create"""			) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""			) as mocked_bulk:
							A__							:    List[Any]        =		{"""acknowledged""": True}
							mocked_bulk.return_value([(True, None)] * 30			)
							A__							:    List[Any]        =		{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
							A__							:    Any        =		Elasticsearch()

							dset.add_elasticsearch_index("""filename"""    ,       es_client=A__			)
							A__   ,   A__							:    Any        =		dset.get_nearest_examples("""filename"""    ,       """my_name-train_29"""			)
							self.assertEqual(examples["""filename"""][0]    ,       """my_name-train_29"""			)



@require_faiss
class     _a						(__magic_name__				):


					'''simple docstring'''






					def   __A			( self			):
						import faiss

						A__							:    Optional[int]        =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)

						# add vectors
						index.add_vectors(np.eye(5    ,       dtype=np.floataa			)			)
						self.assertIsNotNone(index.faiss_index			)
						self.assertEqual(index.faiss_index.ntotal    ,       5			)
						index.add_vectors(np.zeros((5, 5)    ,       dtype=np.floataa			)			)
						self.assertEqual(index.faiss_index.ntotal    ,       10			)

						# single query
						A__							:    Any        =		np.zeros(5    ,       dtype=np.floataa			)
						A__							:    str        =		1
						A__   ,   A__							:    Optional[Any]        =		index.search(A__			)
						self.assertRaises(A__    ,       index.search    ,       query.reshape(-1    ,       1			)			)
						self.assertGreater(scores[0]    ,       0			)
						self.assertEqual(indices[0]    ,       1			)

						# batched queries
						A__							:    Optional[int]        =		np.eye(5    ,       dtype=np.floataa			)[::-1]
						A__   ,   A__							:    str        =		index.search_batch(A__			)
						self.assertRaises(A__    ,       index.search_batch    ,       queries[0]			)
						A__							:    str        =		[scores[0] for scores in total_scores]
						A__							:    Tuple        =		[indices[0] for indices in total_indices]
						self.assertGreater(np.min(A__			)    ,       0			)
						self.assertListEqual([4, 3, 2, 1, 0]    ,       A__			)






					def   __A			( self			):
						import faiss

						A__							:    Dict        =		FaissIndex(string_factory="""Flat"""			)
						index.add_vectors(np.eye(5    ,       dtype=np.floataa			)			)
						self.assertIsInstance(index.faiss_index    ,       faiss.IndexFlat			)
						A__							:    Dict        =		FaissIndex(string_factory="""LSH"""			)
						index.add_vectors(np.eye(5    ,       dtype=np.floataa			)			)
						self.assertIsInstance(index.faiss_index    ,       faiss.IndexLSH			)
						with self.assertRaises(A__			):
							A__							:    List[Any]        =		FaissIndex(string_factory="""Flat"""    ,       custom_index=faiss.IndexFlat(5			)			)






					def   __A			( self			):
						import faiss

						A__							:    List[Any]        =		faiss.IndexFlat(5			)
						A__							:    Union[str, Any]        =		FaissIndex(custom_index=A__			)
						index.add_vectors(np.eye(5    ,       dtype=np.floataa			)			)
						self.assertIsInstance(index.faiss_index    ,       faiss.IndexFlat			)






					def   __A			( self			):
						import faiss

						A__							:    Any        =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
						index.add_vectors(np.eye(5    ,       dtype=np.floataa			)			)

						# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
						# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
						# see https://bugs.python.org/issue14243 and
						# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
						with tempfile.NamedTemporaryFile(delete=A__			) as tmp_file:
							index.save(tmp_file.name			)
							A__							:    int        =		FaissIndex.load(tmp_file.name			)
						os.unlink(tmp_file.name			)

						A__							:    Optional[Any]        =		np.zeros(5    ,       dtype=np.floataa			)
						A__							:    Optional[int]        =		1
						A__   ,   A__							:    List[Any]        =		index.search(A__			)
						self.assertGreater(scores[0]    ,       0			)
						self.assertEqual(indices[0]    ,       1			)



@require_faiss
def 						UpperCamelCase					(lowercase_: Dict					)     ->      Optional[Any]:
	import faiss

	A__							:    Dict        =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT					)
	index.add_vectors(np.eye(5    ,      dtype=np.floataa					)					)

	A__							:    Optional[Any]        =		"""index.faiss"""
	A__							:    Any        =		f"""mock://{index_name}"""
	index.save(lowercase_    ,      storage_options=mockfs.storage_options					)
	A__							:    str        =		FaissIndex.load(lowercase_    ,      storage_options=mockfs.storage_options					)

	A__							:    int        =		np.zeros(5    ,      dtype=np.floataa					)
	A__							:    Union[str, Any]        =		1
	A__   ,   A__							:    Any        =		index.search(lowercase_					)
	assert scores[0] > 0
	assert indices[0] == 1



@require_elasticsearch
class     _a						(__magic_name__				):


					'''simple docstring'''






					def   __A			( self			):
						from elasticsearch import Elasticsearch

						with patch("""elasticsearch.Elasticsearch.search"""			) as mocked_search, patch(
						    """elasticsearch.client.IndicesClient.create"""			) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""			) as mocked_bulk:
							A__							:    List[str]        =		Elasticsearch()
							A__							:    List[Any]        =		{"""acknowledged""": True}
							A__							:    int        =		ElasticSearchIndex(es_client=A__			)
							mocked_bulk.return_value([(True, None)] * 3			)
							index.add_documents(["""foo""", """bar""", """foobar"""]			)

							# single query
							A__							:    Dict        =		"""foo"""
							A__							:    Tuple        =		{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
							A__   ,   A__							:    Tuple        =		index.search(A__			)
							self.assertEqual(scores[0]    ,       1			)
							self.assertEqual(indices[0]    ,       0			)

							# single query with timeout
							A__							:    List[str]        =		"""foo"""
							A__							:    str        =		{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
							A__   ,   A__							:    Dict        =		index.search(A__    ,       request_timeout=30			)
							self.assertEqual(scores[0]    ,       1			)
							self.assertEqual(indices[0]    ,       0			)

							# batched queries
							A__							:    Union[str, Any]        =		["""foo""", """bar""", """foobar"""]
							A__							:    Tuple        =		{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
							A__   ,   A__							:    Dict        =		index.search_batch(A__			)
							A__							:    Tuple        =		[scores[0] for scores in total_scores]
							A__							:    Optional[int]        =		[indices[0] for indices in total_indices]
							self.assertGreater(np.min(A__			)    ,       0			)
							self.assertListEqual([1, 1, 1]    ,       A__			)

							# batched queries with timeout
							A__							:    int        =		["""foo""", """bar""", """foobar"""]
							A__							:    List[Any]        =		{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
							A__   ,   A__							:    Tuple        =		index.search_batch(A__    ,       request_timeout=30			)
							A__							:    List[Any]        =		[scores[0] for scores in total_scores]
							A__							:    Optional[int]        =		[indices[0] for indices in total_indices]
							self.assertGreater(np.min(A__			)    ,       0			)
							self.assertListEqual([1, 1, 1]    ,       A__			)



