

'''simple docstring'''

import csv
import os

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.csv import CsvDatasetReader, CsvDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 				__UpperCamelCase							(	lowercase__						:  List[str],					lowercase__						:  Union[str, Any] ):






     '''simple docstring'''





     assert isinstance(lowercase__,					lowercase__ )
     assert dataset.num_rows == 4
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_1", "col_2", "col_3"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory',					[False, True] )
def 				__UpperCamelCase							(	lowercase__						:  Tuple,					lowercase__						:  Union[str, Any],					lowercase__						:  List[Any] ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          __lowercase					=CsvDatasetReader(lowercase__,					cache_dir=lowercase__,					keep_in_memory=lowercase__ ).read()
     _check_csv_dataset(lowercase__,					lowercase__ )

@pytest.mark.parametrize(
    'features',					[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],					)
def 				__UpperCamelCase							(	lowercase__						:  Optional[int],					lowercase__						:  int,					lowercase__						:  Optional[int] ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     __lowercase					=features.copy() if features else default_expected_features
     __lowercase					=(
         Features({feature: Value(lowercase__ ) for feature, dtype in features.items()} ) if features is not None else None
     )
     __lowercase					=CsvDatasetReader(lowercase__,					features=lowercase__,					cache_dir=lowercase__ ).read()
     _check_csv_dataset(lowercase__,					lowercase__ )

@pytest.mark.parametrize('split',					[None, NamedSplit('train' ), 'train', 'test'] )
def 				__UpperCamelCase							(	lowercase__						:  Optional[Any],					lowercase__						:  List[str],					lowercase__						:  str ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     __lowercase					=CsvDatasetReader(lowercase__,					cache_dir=lowercase__,					split=lowercase__ ).read()
     _check_csv_dataset(lowercase__,					lowercase__ )
     assert dataset.split == split if split else "train"

@pytest.mark.parametrize('path_type',					[str, list] )
def 				__UpperCamelCase							(	lowercase__						:  Tuple,					lowercase__						:  Optional[int],					lowercase__						:  int ):






     '''simple docstring'''





     if issubclass(lowercase__,					lowercase__ ):
          __lowercase					=csv_path
     elif issubclass(lowercase__,					lowercase__ ):
          __lowercase					=[csv_path]
     __lowercase					=tmp_path / 'cache'
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     __lowercase					=CsvDatasetReader(lowercase__,					cache_dir=lowercase__ ).read()
     _check_csv_dataset(lowercase__,					lowercase__ )

def 				__UpperCamelCase							(	lowercase__						:  Union[str, Any],					lowercase__						:  Dict,					lowercase__						:  List[str]=("train",) ):






     '''simple docstring'''





     assert isinstance(lowercase__,					lowercase__ )
     for split in splits:
          __lowercase					=dataset_dict[split]
          assert dataset.num_rows == 4
          assert dataset.num_columns == 3
          assert dataset.column_names == ["col_1", "col_2", "col_3"]
          for feature, expected_dtype in expected_features.items():
               assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory',					[False, True] )
def 				__UpperCamelCase							(	lowercase__						:  int,					lowercase__						:  Union[str, Any],					lowercase__						:  Any ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          __lowercase					=CsvDatasetReader({'train': csv_path},					cache_dir=lowercase__,					keep_in_memory=lowercase__ ).read()
     _check_csv_datasetdict(lowercase__,					lowercase__ )

@pytest.mark.parametrize(
    'features',					[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ],					)
def 				__UpperCamelCase							(	lowercase__						:  Tuple,					lowercase__						:  Optional[int],					lowercase__						:  List[str] ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     __lowercase					=features.copy() if features else default_expected_features
     __lowercase					=(
         Features({feature: Value(lowercase__ ) for feature, dtype in features.items()} ) if features is not None else None
     )
     __lowercase					=CsvDatasetReader({'train': csv_path},					features=lowercase__,					cache_dir=lowercase__ ).read()
     _check_csv_datasetdict(lowercase__,					lowercase__ )

@pytest.mark.parametrize('split',					[None, NamedSplit('train' ), 'train', 'test'] )
def 				__UpperCamelCase							(	lowercase__						:  Any,					lowercase__						:  Dict,					lowercase__						:  Any ):






     '''simple docstring'''





     if split:
          __lowercase					={split: csv_path}
     else:
          __lowercase					={'train': csv_path, 'test': csv_path}
     __lowercase					=tmp_path / 'cache'
     __lowercase					={'col_1': 'int64', 'col_2': 'int64', 'col_3': 'float64'}
     __lowercase					=CsvDatasetReader(lowercase__,					cache_dir=lowercase__ ).read()
     _check_csv_datasetdict(lowercase__,					lowercase__,					splits=list(path.keys() ) )
     assert all(dataset[split].split == split for split in path.keys() )

def 				__UpperCamelCase							(	lowercase__						:  List[str] ):






     '''simple docstring'''





     with open(lowercase__,					encoding='utf-8' ) as csvfile:
          yield from csv.reader(lowercase__ )

def 				__UpperCamelCase							(	lowercase__						:  Optional[int],					lowercase__						:  Union[str, Any] ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					=os.path.join(lowercase__,					'tmp.csv' )
     __lowercase					=CsvDatasetReader({'train': csv_path},					cache_dir=lowercase__ ).read()
     CsvDatasetWriter(dataset['train'],					lowercase__,					num_proc=1 ).write()

     __lowercase					=iter_csv_file(lowercase__ )
     __lowercase					=iter_csv_file(lowercase__ )

     for rowa, rowa in zip(lowercase__,					lowercase__ ):
          assert rowa == rowa

def 				__UpperCamelCase							(	lowercase__						:  Dict,					lowercase__						:  List[str] ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					=os.path.join(lowercase__,					'tmp.csv' )
     __lowercase					=CsvDatasetReader({'train': csv_path},					cache_dir=lowercase__ ).read()
     CsvDatasetWriter(dataset['train'],					lowercase__,					num_proc=2 ).write()

     __lowercase					=iter_csv_file(lowercase__ )
     __lowercase					=iter_csv_file(lowercase__ )

     for rowa, rowa in zip(lowercase__,					lowercase__ ):
          assert rowa == rowa







def 				__UpperCamelCase							(	lowercase__						:  str,					lowercase__						:  Tuple ):






     '''simple docstring'''





     __lowercase					=tmp_path / 'cache'
     __lowercase					=os.path.join(lowercase__,					'tmp.csv' )
     __lowercase					=CsvDatasetReader({'train': csv_path},					cache_dir=lowercase__ ).read()
     with pytest.raises(lowercase__ ):
          CsvDatasetWriter(dataset['train'],					lowercase__,					num_proc=0 )





'''simple docstring'''

import tempfile
import unittest

from transformers import DPRConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import DPRContextEncoder, DPRQuestionEncoder, DPRReader, DPRReaderTokenizer
       from transformers.models.dpr.modeling_dpr import (
           DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
           DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
           DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
       )


class   lowerCAmelCase     :







       def __init__(							self			: List[str]						,      __lowercase			: List[Any]						,      __lowercase			: Union[str, Any]=13						,      __lowercase			: Optional[int]=7						,      __lowercase			: Union[str, Any]=False						,      __lowercase			: str=True						,      __lowercase			: Dict=True						,      __lowercase			: Optional[Any]=True						,      __lowercase			: int=99						,      __lowercase			: Optional[Any]=32						,      __lowercase			: Optional[int]=5						,      __lowercase			: List[str]=4						,      __lowercase			: Any=37						,      __lowercase			: str="gelu"						,      __lowercase			: str=0.1						,      __lowercase			: str=0.1						,      __lowercase			: str=512						,      __lowercase			: Dict=16						,      __lowercase			: Dict=2						,      __lowercase			: List[str]=0.0_2						,      __lowercase			: Union[str, Any]=3						,      __lowercase			: List[Any]=4						,      __lowercase			: List[str]=None						,      __lowercase			: List[Any]=0						,      ):







            """simple docstring"""







            __lowercase					=parent
            __lowercase					=batch_size
            __lowercase					=seq_length
            __lowercase					=is_training
            __lowercase					=use_input_mask
            __lowercase					=use_token_type_ids
            __lowercase					=use_labels
            __lowercase					=vocab_size
            __lowercase					=hidden_size
            __lowercase					=num_hidden_layers
            __lowercase					=num_attention_heads
            __lowercase					=intermediate_size
            __lowercase					=hidden_act
            __lowercase					=hidden_dropout_prob
            __lowercase					=attention_probs_dropout_prob
            __lowercase					=max_position_embeddings
            __lowercase					=type_vocab_size
            __lowercase					=type_sequence_label_size
            __lowercase					=initializer_range
            __lowercase					=num_labels
            __lowercase					=num_choices
            __lowercase					=scope
            __lowercase					=projection_dim







       def 			snake_case   (							self			: Any      ):







            """simple docstring"""







            __lowercase					=ids_tensor([self.batch_size, self.seq_length]						,      self.vocab_size      )

            __lowercase					=None
            if self.use_input_mask:
                 __lowercase					=random_attention_mask([self.batch_size, self.seq_length]      )

            __lowercase					=None
            if self.use_token_type_ids:
                 __lowercase					=ids_tensor([self.batch_size, self.seq_length]						,      self.type_vocab_size      )

            __lowercase					=None
            __lowercase					=None
            __lowercase					=None
            if self.use_labels:
                 __lowercase					=ids_tensor([self.batch_size]						,      self.type_sequence_label_size      )
                 __lowercase					=ids_tensor([self.batch_size, self.seq_length]						,      self.num_labels      )
                 __lowercase					=ids_tensor([self.batch_size]						,      self.num_choices      )

            __lowercase					=self.get_config()

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







       def 			snake_case   (							self			: Dict      ):







            """simple docstring"""







            return DPRConfig(
                projection_dim=self.projection_dim						,      vocab_size=self.vocab_size						,      hidden_size=self.hidden_size						,      num_hidden_layers=self.num_hidden_layers						,      num_attention_heads=self.num_attention_heads						,      intermediate_size=self.intermediate_size						,      hidden_act=self.hidden_act						,      hidden_dropout_prob=self.hidden_dropout_prob						,      attention_probs_dropout_prob=self.attention_probs_dropout_prob						,      max_position_embeddings=self.max_position_embeddings						,      type_vocab_size=self.type_vocab_size						,      initializer_range=self.initializer_range						,      )







       def 			snake_case   (							self			: Union[str, Any]						,      __lowercase			: int						,      __lowercase			: Tuple						,      __lowercase			: Any						,      __lowercase			: Union[str, Any]						,      __lowercase			: List[Any]						,      __lowercase			: Tuple						,      __lowercase			: List[str]      ):







            """simple docstring"""







            __lowercase					=DPRContextEncoder(config=__lowercase      )
            model.to(__lowercase      )
            model.eval()
            __lowercase					=model(__lowercase						,      attention_mask=__lowercase						,      token_type_ids=__lowercase      )
            __lowercase					=model(__lowercase						,      token_type_ids=__lowercase      )
            __lowercase					=model(__lowercase      )
            self.parent.assertEqual(result.pooler_output.shape						,      (self.batch_size, self.projection_dim or self.hidden_size)      )







       def 			snake_case   (							self			: Union[str, Any]						,      __lowercase			: Optional[int]						,      __lowercase			: Optional[int]						,      __lowercase			: Union[str, Any]						,      __lowercase			: Any						,      __lowercase			: Optional[Any]						,      __lowercase			: Tuple						,      __lowercase			: Dict      ):







            """simple docstring"""







            __lowercase					=DPRQuestionEncoder(config=__lowercase      )
            model.to(__lowercase      )
            model.eval()
            __lowercase					=model(__lowercase						,      attention_mask=__lowercase						,      token_type_ids=__lowercase      )
            __lowercase					=model(__lowercase						,      token_type_ids=__lowercase      )
            __lowercase					=model(__lowercase      )
            self.parent.assertEqual(result.pooler_output.shape						,      (self.batch_size, self.projection_dim or self.hidden_size)      )







       def 			snake_case   (							self			: List[str]						,      __lowercase			: Optional[Any]						,      __lowercase			: Tuple						,      __lowercase			: str						,      __lowercase			: Tuple						,      __lowercase			: str						,      __lowercase			: List[Any]						,      __lowercase			: Optional[Any]      ):







            """simple docstring"""







            __lowercase					=DPRReader(config=__lowercase      )
            model.to(__lowercase      )
            model.eval()
            __lowercase					=model(
                __lowercase						,      attention_mask=__lowercase						,      )

            self.parent.assertEqual(result.start_logits.shape						,      (self.batch_size, self.seq_length)      )
            self.parent.assertEqual(result.end_logits.shape						,      (self.batch_size, self.seq_length)      )
            self.parent.assertEqual(result.relevance_logits.shape						,      (self.batch_size,)      )







       def 			snake_case   (							self			: List[Any]      ):







            """simple docstring"""







            __lowercase					=self.prepare_config_and_inputs()
            (
                (
                __lowercase
            )		,     (
                __lowercase
            )		,     (
                __lowercase
            )		,     (
                __lowercase
            )		,     (
                __lowercase
            )		,     (
                __lowercase
            )		,     (
                __lowercase
            )		,     
            )					=config_and_inputs
            __lowercase					={'input_ids': input_ids}
            return config, inputs_dict






@require_torch
class   lowerCAmelCase     (      A   ,			A   ,			unittest.TestCase   ):
       lowerCAmelCase_		= (
           (
               DPRContextEncoder,
               DPRQuestionEncoder,
               DPRReader,
           )
           if is_torch_available()
           else ()
       )
       lowerCAmelCase_		= {"feature-extraction": DPRQuestionEncoder} if is_torch_available() else {}

       lowerCAmelCase_		= False
       lowerCAmelCase_		= False  # why?
       lowerCAmelCase_		= False
       lowerCAmelCase_		= False







       def 			snake_case   (							self			: int      ):







            """simple docstring"""







            __lowercase					=DPRModelTester(self      )
            __lowercase					=ConfigTester(self						,      config_class=__lowercase						,      hidden_size=37      )







       def 			snake_case   (							self			: Any      ):







            """simple docstring"""







            self.config_tester.run_common_tests()







       def 			snake_case   (							self			: List[str]      ):







            """simple docstring"""







            __lowercase					=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_context_encoder(*__lowercase      )







       def 			snake_case   (							self			: Optional[Any]      ):







            """simple docstring"""







            __lowercase					=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_question_encoder(*__lowercase      )







       def 			snake_case   (							self			: Any      ):







            """simple docstring"""







            __lowercase					=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_reader(*__lowercase      )







       def 			snake_case   (							self			: Optional[int]      ):







            """simple docstring"""







            __lowercase					=self.model_tester.prepare_config_and_inputs()[0]

            __lowercase					=DPRQuestionEncoder(config=__lowercase      )
            model.to(__lowercase      )
            model.eval()

            with tempfile.TemporaryDirectory() as tmp_dirname:
                 model.save_pretrained(__lowercase      )
                 __lowercase					=DPRQuestionEncoder.from_pretrained(__lowercase						,      projection_dim=512      )

            self.assertIsNotNone(__lowercase      )







       @slow
       def 			snake_case   (							self			: int      ):







            """simple docstring"""







            for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __lowercase					=DPRContextEncoder.from_pretrained(__lowercase      )
                 self.assertIsNotNone(__lowercase      )

            for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __lowercase					=DPRContextEncoder.from_pretrained(__lowercase      )
                 self.assertIsNotNone(__lowercase      )

            for model_name in DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __lowercase					=DPRQuestionEncoder.from_pretrained(__lowercase      )
                 self.assertIsNotNone(__lowercase      )

            for model_name in DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __lowercase					=DPRReader.from_pretrained(__lowercase      )
                 self.assertIsNotNone(__lowercase      )






@require_torch
class   lowerCAmelCase     (      unittest.TestCase   ):







       @slow
       def 			snake_case   (							self			: int      ):







            """simple docstring"""







            __lowercase					=DPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'						,      return_dict=__lowercase      )
            model.to(__lowercase      )

            __lowercase					=torch.tensor(
                [[101, 7592, 1010, 2003, 2026, 3899, 10140, 1029, 102]]						,      dtype=torch.long						,      device=__lowercase      )  # [CLS] hello, is my dog cute? [SEP]
            __lowercase					=model(__lowercase      )[0]  # embedding shape = (1, 768)
            # compare the actual values for a slice.
            __lowercase					=torch.tensor(
                [
                    [
                        0.0_3_2_3_6_2_5_3,
                        0.1_2_7_5_3_3_3_5,
                        0.1_6_8_1_8_5_0_9,
                        0.0_0_2_7_9_7_8_6,
                        0.3_8_9_6_9_3_3,
                        0.2_4_2_6_4_9_4_5,
                        0.2_1_7_8_9_7_1,
                        -0.0_2_3_3_5_2_2_7,
                        -0.0_8_4_8_1_9_5_9,
                        -0.1_4_3_2_4_1_1_7,
                    ]
                ]						,      dtype=torch.float						,      device=__lowercase						,      )
            self.assertTrue(torch.allclose(output[:, :10]						,      __lowercase						,      atol=1E-4      )      )







       @slow
       def 			snake_case   (							self			: Union[str, Any]      ):







            """simple docstring"""







            __lowercase					=DPRReaderTokenizer.from_pretrained('facebook/dpr-reader-single-nq-base'      )
            __lowercase					=DPRReader.from_pretrained('facebook/dpr-reader-single-nq-base'      )
            model.to(__lowercase      )

            __lowercase					=tokenizer(
                questions='What is love ?'						,      titles='Haddaway'						,      texts='What Is Love is a song recorded by the artist Haddaway'						,      padding=__lowercase						,      return_tensors='pt'						,      )
            encoded_inputs.to(__lowercase      )

            __lowercase					=model(**__lowercase      )

            # compare the actual values for a slice.
            __lowercase					=torch.tensor(
                [[-1_0.3_0_0_5, -1_0.7_7_6_5, -1_1.4_8_7_2, -1_1.6_8_4_1, -1_1.9_3_1_2, -1_0.3_0_0_2, -9.8_5_4_4, -1_1.7_3_7_8, -1_2.0_8_2_1, -1_0.2_9_7_5]]						,      dtype=torch.float						,      device=__lowercase						,      )

            __lowercase					=torch.tensor(
                [[-1_1.0_6_8_4, -1_1.7_0_4_1, -1_1.5_3_9_7, -1_0.3_4_6_5, -1_0.8_7_9_1, -6.8_4_4_3, -1_1.9_9_5_9, -1_1.0_3_6_4, -1_0.0_0_9_6, -6.8_4_0_5]]						,      dtype=torch.float						,      device=__lowercase						,      )
            self.assertTrue(torch.allclose(outputs.start_logits[:, :10]						,      __lowercase						,      atol=1E-4      )      )
            self.assertTrue(torch.allclose(outputs.end_logits[:, :10]						,      __lowercase						,      atol=1E-4      )      )


