

'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import numpy
    import tensorflow as tf

    from transformers import (
        TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
        BertConfig,
        DPRConfig,
        TFDPRContextEncoder,
        TFDPRQuestionEncoder,
        TFDPRReader,
    )


class    a__  :


   def __init__( self			,				_UpperCamelCase			,				_UpperCamelCase=13			,				_UpperCamelCase=7			,				_UpperCamelCase=True			,				_UpperCamelCase=True			,				_UpperCamelCase=True			,				_UpperCamelCase=True			,				_UpperCamelCase=99			,				_UpperCamelCase=32			,				_UpperCamelCase=2			,				_UpperCamelCase=4			,				_UpperCamelCase=37			,				_UpperCamelCase="gelu"			,				_UpperCamelCase=0.1			,				_UpperCamelCase=0.1			,				_UpperCamelCase=512			,				_UpperCamelCase=16			,				_UpperCamelCase=2			,				_UpperCamelCase=0.0_2			,				_UpperCamelCase=3			,				_UpperCamelCase=4			,				_UpperCamelCase=None			,				_UpperCamelCase=0			,				):





      """simple docstring"""





      _lowercase			:      Dict			     =   parent
      _lowercase			:      str			     =   batch_size
      _lowercase			:      int			     =   seq_length
      _lowercase			:      Any			     =   is_training
      _lowercase			:      Any			     =   use_input_mask
      _lowercase			:      List[Any]			     =   use_token_type_ids
      _lowercase			:      str			     =   use_labels
      _lowercase			:      int			     =   vocab_size
      _lowercase			:      Dict			     =   hidden_size
      _lowercase			:      Dict			     =   num_hidden_layers
      _lowercase			:      Optional[Any]			     =   num_attention_heads
      _lowercase			:      Optional[Any]			     =   intermediate_size
      _lowercase			:      Optional[int]			     =   hidden_act
      _lowercase			:      Tuple			     =   hidden_dropout_prob
      _lowercase			:      Optional[Any]			     =   attention_probs_dropout_prob
      _lowercase			:      List[Any]			     =   max_position_embeddings
      _lowercase			:      Optional[int]			     =   type_vocab_size
      _lowercase			:      Optional[int]			     =   type_sequence_label_size
      _lowercase			:      Dict			     =   initializer_range
      _lowercase			:      List[str]			     =   num_labels
      _lowercase			:      List[Any]			     =   num_choices
      _lowercase			:      Optional[int]			     =   scope
      _lowercase			:      Optional[Any]			     =   projection_dim


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      Dict			     =   ids_tensor([self.batch_size, self.seq_length]			,				self.vocab_size  )

      _lowercase			:      Any			     =   None
      if self.use_input_mask:
         # follow test_modeling_tf_ctrl.py
         _lowercase			:      int			     =   random_attention_mask([self.batch_size, self.seq_length]  )

      _lowercase			:      List[str]			     =   None
      if self.use_token_type_ids:
         _lowercase			:      str			     =   ids_tensor([self.batch_size, self.seq_length]			,				self.type_vocab_size  )

      _lowercase			:      Optional[int]			     =   None
      _lowercase			:      Any			     =   None
      _lowercase			:      int			     =   None
      if self.use_labels:
         _lowercase			:      Tuple			     =   ids_tensor([self.batch_size]			,				self.type_sequence_label_size  )
         _lowercase			:      List[str]			     =   ids_tensor([self.batch_size, self.seq_length]			,				self.num_labels  )
         _lowercase			:      List[Any]			     =   ids_tensor([self.batch_size]			,				self.num_choices  )

      _lowercase			:      List[Any]			     =   BertConfig(
          vocab_size=self.vocab_size			,				hidden_size=self.hidden_size			,				num_hidden_layers=self.num_hidden_layers			,				num_attention_heads=self.num_attention_heads			,				intermediate_size=self.intermediate_size			,				hidden_act=self.hidden_act			,				hidden_dropout_prob=self.hidden_dropout_prob			,				attention_probs_dropout_prob=self.attention_probs_dropout_prob			,				max_position_embeddings=self.max_position_embeddings			,				type_vocab_size=self.type_vocab_size			,				is_decoder=A_			,				initializer_range=self.initializer_range			,				)
      _lowercase			:      Any			     =   DPRConfig(projection_dim=self.projection_dim			,				**config.to_dict()  )

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


   def 						_lowerCamelCase   ( self			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase  ):





      """simple docstring"""





      _lowercase			:      Tuple			     =   TFDPRContextEncoder(config=A_  )
      _lowercase			:      str			     =   model(A_			,				attention_mask=A_			,				token_type_ids=A_  )
      _lowercase			:      Tuple			     =   model(A_			,				token_type_ids=A_  )
      _lowercase			:      Any			     =   model(A_  )
      self.parent.assertEqual(result.pooler_output.shape			,				(self.batch_size, self.projection_dim or self.hidden_size)  )


   def 						_lowerCamelCase   ( self			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase  ):





      """simple docstring"""





      _lowercase			:      Optional[int]			     =   TFDPRQuestionEncoder(config=A_  )
      _lowercase			:      str			     =   model(A_			,				attention_mask=A_			,				token_type_ids=A_  )
      _lowercase			:      Any			     =   model(A_			,				token_type_ids=A_  )
      _lowercase			:      str			     =   model(A_  )
      self.parent.assertEqual(result.pooler_output.shape			,				(self.batch_size, self.projection_dim or self.hidden_size)  )


   def 						_lowerCamelCase   ( self			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase			,				_UpperCamelCase  ):





      """simple docstring"""





      _lowercase			:      Optional[Any]			     =   TFDPRReader(config=A_  )
      _lowercase			:      int			     =   model(A_			,				attention_mask=A_  )

      self.parent.assertEqual(result.start_logits.shape			,				(self.batch_size, self.seq_length)  )
      self.parent.assertEqual(result.end_logits.shape			,				(self.batch_size, self.seq_length)  )
      self.parent.assertEqual(result.relevance_logits.shape			,				(self.batch_size,)  )






   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      List[Any]			     =   self.prepare_config_and_inputs()
      (
          (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      (
          _lowercase				
      )     ,      				
      )			:      str			     =   config_and_inputs
      _lowercase			:      str			     =   {"input_ids": input_ids}
      return config, inputs_dict




@require_tf
class    a__  (							lowerCamelCase_  ,      lowerCamelCase_  ,      unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE    :     List[Any]	       =							(
       (
           TFDPRContextEncoder,
           TFDPRQuestionEncoder,
           TFDPRReader,
       )
       if is_tf_available()
       else ()
   )
   _SCREAMING_SNAKE_CASE    :     Optional[int]	       =							{'feature-extraction': TFDPRQuestionEncoder} if is_tf_available() else {}

   _SCREAMING_SNAKE_CASE    :     List[str]	       =							False
   _SCREAMING_SNAKE_CASE    :     Optional[int]	       =							False
   _SCREAMING_SNAKE_CASE    :     int	       =							False
   _SCREAMING_SNAKE_CASE    :     List[Any]	       =							False
   _SCREAMING_SNAKE_CASE    :     Any	       =							False


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      Dict			     =   TFDPRModelTester(self  )
      _lowercase			:      List[Any]			     =   ConfigTester(self			,				config_class=A_			,				hidden_size=37  )


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      self.config_tester.run_common_tests()


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      Tuple			     =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_dpr_context_encoder(*A_  )


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      Optional[int]			     =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_dpr_question_encoder(*A_  )


   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      List[Any]			     =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_dpr_reader(*A_  )






   @slow
   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase			:      int			     =   TFDPRContextEncoder.from_pretrained(A_  )
         self.assertIsNotNone(A_  )

      for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase			:      List[str]			     =   TFDPRContextEncoder.from_pretrained(A_  )
         self.assertIsNotNone(A_  )

      for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase			:      Union[str, Any]			     =   TFDPRQuestionEncoder.from_pretrained(A_  )
         self.assertIsNotNone(A_  )

      for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase			:      str			     =   TFDPRReader.from_pretrained(A_  )
         self.assertIsNotNone(A_  )




@require_tf
class    a__  (							unittest.TestCase  ):


   @slow
   def 						_lowerCamelCase   ( self  ):





      """simple docstring"""





      _lowercase			:      List[Any]			     =   TFDPRQuestionEncoder.from_pretrained("facebook/dpr-question_encoder-single-nq-base"  )

      _lowercase			:      Any			     =   tf.constant(
          [[101, 7592, 1010, 2003, 2026, 3899, 10140, 1029, 102]]  )  # [CLS] hello, is my dog cute? [SEP]
      _lowercase			:      str			     =   model(A_  )[0]  # embedding shape = (1, 768)
      # compare the actual values for a slice.
      _lowercase			:      str			     =   tf.constant(
          [
              [
                  0.0_3_2_3_6_2_5_3,
                  0.1_2_7_5_3_3_3_5,
                  0.1_6_8_1_8_5_0_9,
                  0.0_0_2_7_9_7_8_6,
                  0.3_8_9_6_9_3_3,
                  0.2_4_2_6_4_9_4_5,
                  0.2_1_7_8_9_7_1,
                  -0.0_2_3_3_5_2_2_7,
                  -0.0_8_4_8_1_9_5_9,
                  -0.1_4_3_2_4_1_1_7,
              ]
          ]  )
      self.assertTrue(numpy.allclose(output[:, :10].numpy()			,				expected_slice.numpy()			,				atol=1E-4  )  )












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_fnet import FNetTokenizer
else:
 lowerCamelCase				:		List[Any]     =   None

lowerCamelCase				:		int     =   logging.get_logger(__name__)
lowerCamelCase				:		Optional[int]     =   {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase				:		Optional[int]     =   {
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json",
    },
}

lowerCamelCase				:		List[Any]     =   {
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

lowerCamelCase				:		Any     =   "▁"







class   A(						UpperCamelCase   ):



 '''simple docstring'''


 UpperCamelCase						   =						VOCAB_FILES_NAMES
 UpperCamelCase						   =						PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase						   =						['''input_ids''', '''token_type_ids''']
 UpperCamelCase						   =						FNetTokenizer

 def __init__(				self					:  Optional[int]						,							A_					:  Any=None						,							A_					:  int=None						,							A_					:  int=False						,							A_					:  Optional[int]=True						,							A_					:  List[Any]=True						,							A_					:  Tuple="<unk>"						,							A_					:  Optional[int]="[SEP]"						,							A_					:  List[Any]="<pad>"						,							A_					:  Optional[int]="[CLS]"						,							A_					:  Optional[Any]="[MASK]"						,							**A_					:  Dict						,							)   ->	List[str]:

       """simple docstring"""






       lowerCamelCase_		     =    (
           AddedToken(A_						,							lstrip=A_						,							rstrip=A_						,							normalized=A_  )
           if isinstance(A_						,							A_  )
           else mask_token
       )

       super().__init__(
           A_						,							tokenizer_file=A_						,							do_lower_case=A_						,							remove_space=A_						,							keep_accents=A_						,							unk_token=A_						,							sep_token=A_						,							pad_token=A_						,							cls_token=A_						,							mask_token=A_						,							**A_						,							)

       lowerCamelCase_		     =    do_lower_case
       lowerCamelCase_		     =    remove_space
       lowerCamelCase_		     =    keep_accents
       lowerCamelCase_		     =    vocab_file
       lowerCamelCase_		     =    False if not self.vocab_file else True

 def 				a__ (				self					:  List[Any]						,							A_					:  List[int]						,							A_					:  Optional[List[int]] = None  )   ->	List[int]:

       """simple docstring"""






       lowerCamelCase_		     =    [self.sep_token_id]
       lowerCamelCase_		     =    [self.cls_token_id]
       if token_ids_a is None:
             return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep

 def 				a__ (				self					:  Tuple						,							A_					:  List[int]						,							A_					:  Optional[List[int]] = None  )   ->	List[int]:

       """simple docstring"""






       lowerCamelCase_		     =    [self.sep_token_id]
       lowerCamelCase_		     =    [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]





 def 				a__ (				self					:  Dict						,							A_					:  str						,							A_					:  Optional[str] = None  )   ->	Tuple[str]:

       """simple docstring"""






       if not os.path.isdir(A_  ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
             return
       lowerCamelCase_		     =    os.path.join(
           A_						,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

       if os.path.abspath(self.vocab_file  ) != os.path.abspath(A_  ):
             copyfile(self.vocab_file						,							A_  )

       return (out_vocab_file,)



