


import unittest

from transformers import ElectraConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        ElectraForCausalLM,
        ElectraForMaskedLM,
        ElectraForMultipleChoice,
        ElectraForPreTraining,
        ElectraForQuestionAnswering,
        ElectraForSequenceClassification,
        ElectraForTokenClassification,
        ElectraModel,
    )
    from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST



class   snake_case__ :







       def __init__( self:   int,A_:   Optional[int],A_:   int=13,A_:   Union[str, Any]=7,A_:   Any=True,A_:   List[Any]=True,A_:   Union[str, Any]=True,A_:   str=True,A_:   Union[str, Any]=99,A_:   List[str]=32,A_:   Optional[int]=5,A_:   List[str]=4,A_:   Dict=37,A_:   Dict="gelu",A_:   Optional[Any]=0.1,A_:   str=0.1,A_:   Any=512,A_:   str=16,A_:   Tuple=2,A_:   Any=0.0_2,A_:   Any=3,A_:   Any=4,A_:   Any=None,):






            '''simple docstring'''




            __UpperCamelCase   = parent
            __UpperCamelCase   = batch_size
            __UpperCamelCase   = seq_length
            __UpperCamelCase   = is_training
            __UpperCamelCase   = use_input_mask
            __UpperCamelCase   = use_token_type_ids
            __UpperCamelCase   = use_labels
            __UpperCamelCase   = vocab_size
            __UpperCamelCase   = hidden_size
            __UpperCamelCase   = num_hidden_layers
            __UpperCamelCase   = num_attention_heads
            __UpperCamelCase   = intermediate_size
            __UpperCamelCase   = hidden_act
            __UpperCamelCase   = hidden_dropout_prob
            __UpperCamelCase   = attention_probs_dropout_prob
            __UpperCamelCase   = max_position_embeddings
            __UpperCamelCase   = type_vocab_size
            __UpperCamelCase   = type_sequence_label_size
            __UpperCamelCase   = initializer_range
            __UpperCamelCase   = num_labels
            __UpperCamelCase   = num_choices
            __UpperCamelCase   = scope







       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            __UpperCamelCase   = ids_tensor([self.batch_size, self.seq_length],self.vocab_size				)

            __UpperCamelCase   = None
            if self.use_input_mask:
                 __UpperCamelCase   = random_attention_mask([self.batch_size, self.seq_length]				)

            __UpperCamelCase   = None
            if self.use_token_type_ids:
                 __UpperCamelCase   = ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size				)

            __UpperCamelCase   = None
            __UpperCamelCase   = None
            __UpperCamelCase   = None
            if self.use_labels:
                 __UpperCamelCase   = ids_tensor([self.batch_size],self.type_sequence_label_size				)
                 __UpperCamelCase   = ids_tensor([self.batch_size, self.seq_length],self.num_labels				)
                 __UpperCamelCase   = ids_tensor([self.batch_size],self.num_choices				)
                 __UpperCamelCase   = ids_tensor([self.batch_size, self.seq_length],1				)

            __UpperCamelCase   = self.get_config()

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                fake_token_labels,
            )







       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            return ElectraConfig(
                vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,is_decoder=A_,initializer_range=self.initializer_range,)







       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            (
                (
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),												
            )   = self.prepare_config_and_inputs()

            __UpperCamelCase   = True
            __UpperCamelCase   = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
            __UpperCamelCase   = ids_tensor([self.batch_size, self.seq_length],vocab_size=2				)

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                encoder_hidden_states,
                encoder_attention_mask,
            )







       def 		snake_case_  ( self:   int,A_:   List[Any],A_:   int,A_:   int,A_:   Optional[int],A_:   Union[str, Any],A_:   Tuple,A_:   str,A_:   Optional[int],):






            '''simple docstring'''




            __UpperCamelCase   = ElectraModel(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_				)
            __UpperCamelCase   = model(A_,token_type_ids=A_				)
            __UpperCamelCase   = model(A_				)
            self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)				)







       def 		snake_case_  ( self:   Tuple,A_:   Optional[int],A_:   Optional[Any],A_:   int,A_:   Any,A_:   int,A_:   Tuple,A_:   Tuple,A_:   List[Any],A_:   int,):






            '''simple docstring'''




            __UpperCamelCase   = True
            __UpperCamelCase   = ElectraModel(A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(
                A_,attention_mask=A_,token_type_ids=A_,encoder_hidden_states=A_,encoder_attention_mask=A_,)
            __UpperCamelCase   = model(
                A_,attention_mask=A_,token_type_ids=A_,encoder_hidden_states=A_,)
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_				)
            self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)				)







       def 		snake_case_  ( self:   List[Any],A_:   List[Any],A_:   List[Any],A_:   Union[str, Any],A_:   Optional[int],A_:   Union[str, Any],A_:   List[str],A_:   List[str],A_:   str,):






            '''simple docstring'''




            __UpperCamelCase   = ElectraForMaskedLM(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_,labels=A_				)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)				)







       def 		snake_case_  ( self:   Tuple,A_:   List[Any],A_:   Optional[int],A_:   Any,A_:   Optional[int],A_:   Dict,A_:   Optional[int],A_:   Any,A_:   Tuple,A_:   Dict,):






            '''simple docstring'''




            __UpperCamelCase   = ElectraForCausalLM(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_,labels=A_				)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)				)







       def 		snake_case_  ( self:   int,A_:   List[str],A_:   Dict,A_:   Optional[int],A_:   Tuple,A_:   Optional[int],A_:   Optional[Any],A_:   Optional[int],A_:   Union[str, Any],):






            '''simple docstring'''




            __UpperCamelCase   = self.num_labels
            __UpperCamelCase   = ElectraForTokenClassification(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_,labels=A_				)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)				)







       def 		snake_case_  ( self:   Dict,A_:   Optional[Any],A_:   str,A_:   int,A_:   str,A_:   Tuple,A_:   str,A_:   Dict,A_:   List[Any],):






            '''simple docstring'''




            __UpperCamelCase   = self.num_labels
            __UpperCamelCase   = ElectraForPreTraining(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_,labels=A_				)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length)				)







       def 		snake_case_  ( self:   Union[str, Any],A_:   List[Any],A_:   Union[str, Any],A_:   Dict,A_:   List[Any],A_:   Optional[int],A_:   List[Any],A_:   Dict,A_:   Dict,):






            '''simple docstring'''




            __UpperCamelCase   = self.num_labels
            __UpperCamelCase   = ElectraForSequenceClassification(A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(A_,attention_mask=A_,token_type_ids=A_,labels=A_				)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)				)







       def 		snake_case_  ( self:   str,A_:   Any,A_:   Optional[Any],A_:   Union[str, Any],A_:   List[str],A_:   List[Any],A_:   Any,A_:   Optional[int],A_:   Any,):






            '''simple docstring'''




            __UpperCamelCase   = ElectraForQuestionAnswering(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = model(
                A_,attention_mask=A_,token_type_ids=A_,start_positions=A_,end_positions=A_,)
            self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)				)
            self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)				)







       def 		snake_case_  ( self:   str,A_:   List[Any],A_:   Union[str, Any],A_:   Optional[int],A_:   Dict,A_:   List[str],A_:   str,A_:   Tuple,A_:   Union[str, Any],):






            '''simple docstring'''




            __UpperCamelCase   = self.num_choices
            __UpperCamelCase   = ElectraForMultipleChoice(config=A_				)
            model.to(A_				)
            model.eval()
            __UpperCamelCase   = input_ids.unsqueeze(1				).expand(-1,self.num_choices,-1				).contiguous()
            __UpperCamelCase   = token_type_ids.unsqueeze(1				).expand(-1,self.num_choices,-1				).contiguous()
            __UpperCamelCase   = input_mask.unsqueeze(1				).expand(-1,self.num_choices,-1				).contiguous()
            __UpperCamelCase   = model(
                A_,attention_mask=A_,token_type_ids=A_,labels=A_,)
            self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)				)







       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            __UpperCamelCase   = self.prepare_config_and_inputs()
            (
                (
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),						(
                __UpperCamelCase						
            ),												
            )   = config_and_inputs
            __UpperCamelCase   = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
            return config, inputs_dict



@require_torch
class   snake_case__ (_a							,			_a							,			unittest.TestCase	):
       _lowercase       							=    (
           (
               ElectraModel,
               ElectraForPreTraining,
               ElectraForMaskedLM,
               ElectraForCausalLM,
               ElectraForMultipleChoice,
               ElectraForTokenClassification,
               ElectraForSequenceClassification,
               ElectraForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       _lowercase       							=    (
           {
               """feature-extraction""": ElectraModel,
               """fill-mask""": ElectraForMaskedLM,
               """question-answering""": ElectraForQuestionAnswering,
               """text-classification""": ElectraForSequenceClassification,
               """text-generation""": ElectraForCausalLM,
               """token-classification""": ElectraForTokenClassification,
               """zero-shot""": ElectraForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       _lowercase       							=    True







       def 		snake_case_  ( self:   Dict,A_:   Any,A_:   List[str],A_:   Optional[Any]=False				):






            '''simple docstring'''




            __UpperCamelCase   = super()._prepare_for_class(A_,A_,return_labels=A_				)

            if return_labels:
                 if model_class in get_values(A_				):
                      __UpperCamelCase   = torch.zeros(
                          (self.model_tester.batch_size, self.model_tester.seq_length),dtype=torch.long,device=A_				)
            return inputs_dict







       def 		snake_case_  ( self:   Dict				):






            '''simple docstring'''




            __UpperCamelCase   = ElectraModelTester(self				)
            __UpperCamelCase   = ConfigTester(self,config_class=A_,hidden_size=37				)







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            self.config_tester.run_common_tests()







       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_model(*A_				)







       def 		snake_case_  ( self:   List[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_electra_model_as_decoder(*A_				)







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            for type in ["absolute", "relative_key", "relative_key_query"]:
                 __UpperCamelCase   = type
                 self.model_tester.create_and_check_electra_model(*A_				)







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_masked_lm(*A_				)







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_token_classification(*A_				)







       def 		snake_case_  ( self:   Union[str, Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_pretraining(*A_				)







       def 		snake_case_  ( self:   Optional[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_sequence_classification(*A_				)







       def 		snake_case_  ( self:   Any				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_question_answering(*A_				)







       def 		snake_case_  ( self:   Optional[Any]				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_electra_for_multiple_choice(*A_				)







       @slow
       def 		snake_case_  ( self:   Optional[int]				):






            '''simple docstring'''




            for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 __UpperCamelCase   = ElectraModel.from_pretrained(A_				)
                 self.assertIsNotNone(A_				)







       def 		snake_case_  ( self:   Tuple				):






            '''simple docstring'''




            __UpperCamelCase   = self.model_tester.prepare_config_and_inputs_for_decoder()
            self.model_tester.create_and_check_electra_for_causal_lm(*A_				)



@require_torch
class   snake_case__ (unittest.TestCase	):







       @slow
       def 		snake_case_  ( self:   List[str]				):






            '''simple docstring'''




            __UpperCamelCase   = ElectraModel.from_pretrained('google/electra-small-discriminator'				)
            __UpperCamelCase   = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]				)
            __UpperCamelCase   = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]				)
            __UpperCamelCase   = model(A_,attention_mask=A_				)[0]
            __UpperCamelCase   = torch.Size((1, 11, 256)				)
            self.assertEqual(output.shape,A_				)
            __UpperCamelCase   = torch.tensor(
                [[[0.4_4_7_1, 0.6_8_2_1, -0.3_2_6_5], [0.4_6_2_7, 0.5_2_5_5, -0.3_6_6_8], [0.4_5_3_2, 0.3_3_1_3, -0.4_3_4_4]]]				)

            self.assertTrue(torch.allclose(output[:, 1:4, 1:4],A_,atol=1E-4				)				)






import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


__snake_case             =	{0: '''null''', 1: '''short''', 2: '''long''', 3: '''yes''', 4: '''no'''}
__snake_case             =	set(''''''.join(['''‘''', '''’''', '''´''', '''`''', '''.''', ''',''', '''-''', '''"''']))






def 		_A		(		_lowercase				,     _lowercase=0				,     _lowercase=None   )				->   Union[str, Any]:


     """simple docstring"""


     return [" ".join(x.split(' '   )[begin:end]   ) for x in answers if len(x.split(' '   )   ) > 1]






def 		_A		(		_lowercase				,     _lowercase=False   )				->   Tuple:


     """simple docstring"""


     if make_sub_answers:
          # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
          # *e.g.* if the correct answer contains a prefix such as "the", or "a"
          __UpperCamelCase   = (
              given_answers + get_sub_answers(_lowercase				,     begin=1   ) + get_sub_answers(_lowercase				,     end=-1   )
          )
     __UpperCamelCase   = []
     for answer in given_answers:
          __UpperCamelCase   = answer.replace('_'				,     ' '   ).lower()
          __UpperCamelCase   = ''.join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ' ' for c in alias   )
          answers.append(' '.join(alias.split()   ).strip()   )
     return set(_lowercase   )






def 		_A		(		_lowercase				,     _lowercase				,     _lowercase=1				,     _lowercase=1_00   )				->   Any:


     """simple docstring"""


     __UpperCamelCase,						__UpperCamelCase   = jax.lax.top_k(_lowercase				,     _lowercase   )
     __UpperCamelCase,						__UpperCamelCase   = jax.lax.top_k(_lowercase				,     _lowercase   )

     __UpperCamelCase   = best_end_idx[:, None] - best_start_idx[None, :]
     __UpperCamelCase   = jnp.logical_or(widths < 0				,     widths > max_size   )
     __UpperCamelCase   = (best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
     __UpperCamelCase   = jnp.argmax(_lowercase   ).item()

     return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]






def 		_A		(		_lowercase   )				->   List[str]:


     """simple docstring"""


     __UpperCamelCase   = sample['question']['text']
     __UpperCamelCase   = sample['document']['tokens']['token']
     __UpperCamelCase   = sample['document']['tokens']['is_html']
     __UpperCamelCase   = sample['annotations']['long_answer']
     __UpperCamelCase   = sample['annotations']['short_answers']

     __UpperCamelCase   = ' '.join([context[i] for i in range(len(_lowercase   )   ) if not is_html[i]]   )

     # 0 - No ; 1 - Yes
     for answer in sample["annotations"]["yes_no_answer"]:
          if answer == 0 or answer == 1:
               return {
                   "question": question,
                   "context": context_string,
                   "short": [],
                   "long": [],
                   "category": "no" if answer == 0 else "yes",
               }

     __UpperCamelCase   = []
     for s in short_answers:
          short_targets.extend(s['text']   )
     __UpperCamelCase   = list(set(_lowercase   )   )

     __UpperCamelCase   = []
     for s in long_answers:
          if s["start_token"] == -1:
               continue
          __UpperCamelCase   = context[s['start_token'] : s['end_token']]
          __UpperCamelCase   = is_html[s['start_token'] : s['end_token']]
          __UpperCamelCase   = ' '.join([answer[i] for i in range(len(_lowercase   )   ) if not html[i]]   )
          if new_answer not in long_targets:
               long_targets.append(_lowercase   )

     __UpperCamelCase   = 'long_short' if len(short_targets + long_targets   ) > 0 else 'null'

     return {
         "question": question,
         "context": context_string,
         "short": short_targets,
         "long": long_targets,
         "category": category,
     }






def 		_A		(		)				->   Optional[int]:


     """simple docstring"""


     __UpperCamelCase   = load_from_disk('natural-questions-validation'   )
     __UpperCamelCase   = dataset.map(_lowercase   ).remove_columns(['annotations', 'document', 'id']   )
     print(_lowercase   )

     __UpperCamelCase   = dataset.filter(lambda _lowercase   : (len(x['question']   ) + len(x['context']   )) < 4 * 40_96   )
     __UpperCamelCase   = short_validation_dataset.filter(lambda _lowercase   : x["category"] != "null"   )
     short_validation_dataset

     __UpperCamelCase   = 'vasudevgupta/flax-bigbird-natural-questions'
     __UpperCamelCase   = FlaxBigBirdForNaturalQuestions.from_pretrained(_lowercase   )
     __UpperCamelCase   = BigBirdTokenizerFast.from_pretrained(_lowercase   )

     @jax.jit
     def forward(*_lowercase				,     **_lowercase   ):
          __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase   = model(*_lowercase				,     **_lowercase   )
          return start_logits, end_logits, jnp.argmax(_lowercase				,     axis=-1   )

     def evaluate(_lowercase   ):
          # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
          __UpperCamelCase   = tokenizer(
              example['question']				,     example['context']				,     return_tensors='np'				,     max_length=40_96				,     padding='max_length'				,     truncation=_lowercase				,     )

          __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase   = forward(**_lowercase   )

          __UpperCamelCase   = CATEGORY_MAPPING[category.item()]

          __UpperCamelCase   = example['long'] + example['short']
          if example["category"] in ["yes", "no", "null"]:
               __UpperCamelCase   = [example['category']]
          __UpperCamelCase   = example['category'] != 'null'
          # Now target can be: "yes", "no", "null", "list of long & short answers"

          if predicted_category in ["yes", "no", "null"]:
               __UpperCamelCase   = [predicted_category]
               __UpperCamelCase   = example['output'] == example['targets']
               __UpperCamelCase   = predicted_category != 'null'
               return example

          __UpperCamelCase   = 38 if predicted_category == 'short' else 10_24
          __UpperCamelCase,						__UpperCamelCase   = get_best_valid_start_end_idx(
              start_scores[0]				,     end_scores[0]				,     top_k=8				,     max_size=_lowercase   )

          __UpperCamelCase   = inputs['input_ids'][0].tolist()
          __UpperCamelCase   = [tokenizer.decode(input_ids[start_score : end_score + 1]   )]

          __UpperCamelCase   = expand_to_aliases(example['targets']				,     make_sub_answers=_lowercase   )
          __UpperCamelCase   = expand_to_aliases(example['output']   )

          # some preprocessing to both prediction and answer
          __UpperCamelCase   = {''.join(a.split()   ) for a in answers}
          __UpperCamelCase   = {''.join(p.split()   ) for p in predictions}
          __UpperCamelCase   = {s for s in predictions if s not in ['``', '\'\'', '`', '\'']}

          # if there is a common element, it's a exact match
          __UpperCamelCase   = len(list(answers & predictions   )   ) > 0
          __UpperCamelCase   = predicted_category != 'null' and len(_lowercase   ) > 0

          return example

     __UpperCamelCase   = short_validation_dataset.map(_lowercase   )

     __UpperCamelCase   = len(_lowercase   )
     __UpperCamelCase   = len(short_validation_dataset.filter(lambda _lowercase   : x["match"] == 1   )   )
     print('EM score:'				,     (matched / total) * 1_00				,     '%'   )


if __name__ == "__main__":
    main()


