code: 
  '''simple docstring'''
  
  
  
  
  
  import inspect
  from typing import Callable, List, Optional, Set, Tuple, Union
  
  import torch
  from packaging import version
  from safetensors.torch import storage_ptr, storage_size
  from torch import nn
  
  from .utils import logging
  
  
  A         =   [nn.LayerNorm]
  
  A         =   logging.get_logger(__name__)
  
  A         =   version.parse(version.parse(torch.__version__).base_version)
  
  A         =   parsed_torch_version_base >= version.parse("2.0")
  A         =   parsed_torch_version_base >= version.parse("1.12")
  A         =   parsed_torch_version_base >= version.parse("1.11")
  A         =   parsed_torch_version_base < version.parse("1.11")
  def   snake_case_       (							_a     :							str ,					_a     :							Dict ,					_a     :							List[Any] ,					_a     :							Dict ,					self     :							Tuple			):
        from torch import _softmax_backward_data
  
        if is_torch_less_than_1_11:
              return _softmax_backward_data(__lowerCamelCase ,					__lowerCamelCase ,					parent.dim ,					self			)
        else:
              return _softmax_backward_data(__lowerCamelCase ,					__lowerCamelCase ,					parent.dim ,					self.dtype			)
  def   snake_case_       (							_a     :							nn.Linear ,					_a     :							torch.LongTensor ,					_a     :							int = 0			):
        snake_case_						       =							index.to(layer.weight.device			)
        snake_case_						       =							layer.weight.index_select(__lowerCamelCase ,					__lowerCamelCase			).clone().detach()
        if layer.bias is not None:
              if dim == 1:
                    snake_case_						       =							layer.bias.clone().detach()
              else:
                    snake_case_						       =							layer.bias[index].clone().detach()
        snake_case_						       =							list(layer.weight.size()			)
        snake_case_						       =							len(__lowerCamelCase			)
        snake_case_						       =							nn.Linear(new_size[1] ,					new_size[0] ,					bias=layer.bias is not None			).to(layer.weight.device			)
        snake_case_						       =							False
        new_layer.weight.copy_(W.contiguous()			)
        snake_case_						       =							True
        if layer.bias is not None:
              snake_case_						       =							False
              new_layer.bias.copy_(b.contiguous()			)
              snake_case_						       =							True
        return new_layer
  
  
  
  class  lowerCAmelCase__   (       nn.Module       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def __init__(self  : Tuple	, lowerCAmelCase_  : Dict	, lowerCAmelCase_  : Tuple    )   ->       Union[str, Any]:
          super().__init__()
          snake_case_						       =							nf
          snake_case_						       =							nn.Parameter(torch.empty(lowerCAmelCase_	, lowerCAmelCase_    )    )
          snake_case_						       =							nn.Parameter(torch.zeros(lowerCAmelCase_    )    )
          nn.init.normal_(self.weight	, std=0.02    )
  
  
    def 		_lowerCamelCase	(self  : str	, lowerCAmelCase_  : str    )   ->       Any:
          snake_case_						       =							x.size()[:-1] + (self.nf,)
          snake_case_						       =							torch.addmm(self.bias	, x.view(-1	, x.size(-1    )    )	, self.weight    )
          snake_case_						       =							x.view(lowerCAmelCase_    )
          return x
  
  def   snake_case_       (							_a     :							ConvaD ,					_a     :							torch.LongTensor ,					_a     :							int = 1			):
        snake_case_						       =							index.to(layer.weight.device			)
        snake_case_						       =							layer.weight.index_select(__lowerCamelCase ,					__lowerCamelCase			).clone().detach()
        if dim == 0:
              snake_case_						       =							layer.bias.clone().detach()
        else:
              snake_case_						       =							layer.bias[index].clone().detach()
        snake_case_						       =							list(layer.weight.size()			)
        snake_case_						       =							len(__lowerCamelCase			)
        snake_case_						       =							ConvaD(new_size[1] ,					new_size[0]			).to(layer.weight.device			)
        snake_case_						       =							False
        new_layer.weight.copy_(W.contiguous()			)
        snake_case_						       =							True
        snake_case_						       =							False
        new_layer.bias.copy_(b.contiguous()			)
        snake_case_						       =							True
        return new_layer
  def   snake_case_       (							_a     :							Union[nn.Linear, ConvaD] ,					_a     :							torch.LongTensor ,					_a     :							Optional[int] = None			):
        if isinstance(__lowerCamelCase ,					nn.Linear			):
              return prune_linear_layer(__lowerCamelCase ,					__lowerCamelCase ,					dim=0 if dim is None else dim			)
        elif isinstance(__lowerCamelCase ,					__lowerCamelCase			):
              return prune_convad_layer(__lowerCamelCase ,					__lowerCamelCase ,					dim=1 if dim is None else dim			)
        else:
              raise ValueError(F'''Can\'t prune layer of class {layer.__class__}'''			)
  def   snake_case_       (							_a     :							Callable[..., torch.Tensor] ,					_a     :							int ,					_a     :							int ,					*_a     :							List[str]			):
        assert len(__lowerCamelCase			) > 0, F'''{input_tensors} has to be a tuple/list of tensors'''
  
        # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
        snake_case_						       =							len(inspect.signature(__lowerCamelCase			).parameters			)
        if num_args_in_forward_chunk_fn != len(__lowerCamelCase			):
              raise ValueError(
                  F'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(__lowerCamelCase			)} input '''
                  "tensors are given"			)
  
        if chunk_size > 0:
              snake_case_						       =							input_tensors[0].shape[chunk_dim]
              for input_tensor in input_tensors:
                    if input_tensor.shape[chunk_dim] != tensor_shape:
                          raise ValueError(
                              F'''All input tenors have to be of the same shape: {tensor_shape}, '''
                              F'''found shape {input_tensor.shape[chunk_dim]}'''			)
  
              if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
                    raise ValueError(
                        F'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
                        F'''size {chunk_size}'''			)
  
              snake_case_						       =							input_tensors[0].shape[chunk_dim] // chunk_size
  
              # chunk input tensor into tuples
              snake_case_						       =							tuple(input_tensor.chunk(__lowerCamelCase ,					dim=__lowerCamelCase			) for input_tensor in input_tensors			)
              # apply forward fn to every tuple
              snake_case_						       =							tuple(forward_fn(*__lowerCamelCase			) for input_tensors_chunk in zip(*__lowerCamelCase			)			)
              # concatenate output at same dimension
              return torch.cat(__lowerCamelCase ,					dim=__lowerCamelCase			)
  
        return forward_fn(*__lowerCamelCase			)
  def   snake_case_       (							_a     :							List[int] ,					_a     :							int ,					_a     :							int ,					_a     :							Set[int]			):
        snake_case_						       =							torch.ones(__lowerCamelCase ,					__lowerCamelCase			)
        snake_case_						       =							set(__lowerCamelCase			) - already_pruned_heads  # Convert to set and remove already pruned heads
        for head in heads:
              # Compute how many pruned heads are before the head and move the index accordingly
              snake_case_						       =							head - sum(1 if h < head else 0 for h in already_pruned_heads			)
              snake_case_						       =							0
        snake_case_						       =							mask.view(-1			).contiguous().eq(1			)
        snake_case_						       =							torch.arange(len(__lowerCamelCase			)			)[mask].long()
        return heads, index
  def   snake_case_       (							*_a     :							Union[torch.Tensor, List[torch.Tensor]] ,					_a     :							Optional[str] = None			):
        return torch.meshgrid(*__lowerCamelCase ,					indexing=__lowerCamelCase			)
  
  
  
  
  def   snake_case_       (							_a     :							torch.Tensor			):
        return tensor.device, storage_ptr(__lowerCamelCase			), storage_size(__lowerCamelCase			)
  
  
style_context: 
  '''simple docstring'''
  
  
  
  
  
  import jax
  import jax.numpy as jnp
  from bigbird_flax import FlaxBigBirdForNaturalQuestions
  from datasets import load_from_disk
  
  from transformers import BigBirdTokenizerFast
  
  
  A           =   {0: "null", 1: "short", 2: "long", 3: "yes", 4: "no"}
  A           =   set("".join(["‘", "’", "´", "`", ".", ",", "-", "\""]))
  def   snake_case_       (							_a     :							Any ,					_a     :							Union[str, Any]=0 ,					_a     :							Optional[int]=None			):
        return [" ".join(x.split(" "			)[begin:end]			) for x in answers if len(x.split(" "			)			) > 1]
  def   snake_case_       (							_a     :							Optional[int] ,					_a     :							str=False			):
        if make_sub_answers:
              # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
              # *e.g.* if the correct answer contains a prefix such as "the", or "a"
              snake_case_						       =							(
                  given_answers + get_sub_answers(_a ,					begin=1			) + get_sub_answers(_a ,					end=-1			)
              )
        snake_case_						       =							[]
        for answer in given_answers:
              snake_case_						       =							answer.replace("_" ,					" "			).lower()
              snake_case_						       =							"".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else " " for c in alias			)
              answers.append(" ".join(alias.split()			).strip()			)
        return set(_a			)
  def   snake_case_       (							_a     :							Optional[Any] ,					_a     :							int ,					_a     :							Optional[Any]=1 ,					_a     :							List[Any]=100			):
        snake_case_       ,	snake_case_						       =							jax.lax.top_k(_a ,					_a			)
        snake_case_       ,	snake_case_						       =							jax.lax.top_k(_a ,					_a			)
  
        snake_case_						       =							best_end_idx[:, None] - best_start_idx[None, :]
        snake_case_						       =							jnp.logical_or(widths < 0 ,					widths > max_size			)
        snake_case_						       =							(best_end_scores[:, None] + best_start_scores[None, :]) - (1E8 * mask)
        snake_case_						       =							jnp.argmax(_a			).item()
  
        return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]
  def   snake_case_       (							_a     :							Any			):
        snake_case_						       =							sample["question"]["text"]
        snake_case_						       =							sample["document"]["tokens"]["token"]
        snake_case_						       =							sample["document"]["tokens"]["is_html"]
        snake_case_						       =							sample["annotations"]["long_answer"]
        snake_case_						       =							sample["annotations"]["short_answers"]
  
        snake_case_						       =							" ".join([context[i] for i in range(len(_a			)			) if not is_html[i]]			)
  
        # 0 - No ; 1 - Yes
        for answer in sample["annotations"]["yes_no_answer"]:
              if answer == 0 or answer == 1:
                    return {
                        "question": question,
                        "context": context_string,
                        "short": [],
                        "long": [],
                        "category": "no" if answer == 0 else "yes",
                    }
  
        snake_case_						       =							[]
        for s in short_answers:
              short_targets.extend(s["text"]			)
        snake_case_						       =							list(set(_a			)			)
  
        snake_case_						       =							[]
        for s in long_answers:
              if s["start_token"] == -1:
                    continue
              snake_case_						       =							context[s["start_token"] : s["end_token"]]
              snake_case_						       =							is_html[s["start_token"] : s["end_token"]]
              snake_case_						       =							" ".join([answer[i] for i in range(len(_a			)			) if not html[i]]			)
              if new_answer not in long_targets:
                    long_targets.append(_a			)
  
        snake_case_						       =							"long_short" if len(short_targets + long_targets			) > 0 else "null"
  
        return {
            "question": question,
            "context": context_string,
            "short": short_targets,
            "long": long_targets,
            "category": category,
        }
  def   snake_case_       (							):
        snake_case_						       =							load_from_disk("natural-questions-validation"			)
        snake_case_						       =							dataset.map(_a			).remove_columns(["annotations", "document", "id"]			)
        print(_a			)
  
        snake_case_						       =							dataset.filter(lambda _a			: (len(x["question"]			) + len(x["context"]			)) < 4 * 4_096			)
        snake_case_						       =							short_validation_dataset.filter(lambda _a			: x["category"] != "null"			)
        short_validation_dataset
  
        snake_case_						       =							"vasudevgupta/flax-bigbird-natural-questions"
        snake_case_						       =							FlaxBigBirdForNaturalQuestions.from_pretrained(_a			)
        snake_case_						       =							BigBirdTokenizerFast.from_pretrained(_a			)
  
        @jax.jit
        def forward(*_a     :							List[Any] ,					**_a     :							int			):
              snake_case_       ,	snake_case_       ,	snake_case_						       =							model(*_a ,					**_a			)
              return start_logits, end_logits, jnp.argmax(_a ,					axis=-1			)
  
        def evaluate(_a     :							List[Any]			):
              # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
              snake_case_						       =							tokenizer(
                  example["question"] ,					example["context"] ,					return_tensors="np" ,					max_length=4_096 ,					padding="max_length" ,					truncation=_a ,					)
  
              snake_case_       ,	snake_case_       ,	snake_case_						       =							forward(**_a			)
  
              snake_case_						       =							CATEGORY_MAPPING[category.item()]
  
              snake_case_						       =							example["long"] + example["short"]
              if example["category"] in ["yes", "no", "null"]:
                    snake_case_						       =							[example["category"]]
              snake_case_						       =							example["category"] != "null"
              # Now target can be: "yes", "no", "null", "list of long & short answers"
  
              if predicted_category in ["yes", "no", "null"]:
                    snake_case_						       =							[predicted_category]
                    snake_case_						       =							example["output"] == example["targets"]
                    snake_case_						       =							predicted_category != "null"
                    return example
  
              snake_case_						       =							38 if predicted_category == "short" else 1_024
              snake_case_       ,	snake_case_						       =							get_best_valid_start_end_idx(
                  start_scores[0] ,					end_scores[0] ,					top_k=8 ,					max_size=_a			)
  
              snake_case_						       =							inputs["input_ids"][0].tolist()
              snake_case_						       =							[tokenizer.decode(input_ids[start_score : end_score + 1]			)]
  
              snake_case_						       =							expand_to_aliases(example["targets"] ,					make_sub_answers=_a			)
              snake_case_						       =							expand_to_aliases(example["output"]			)
  
              # some preprocessing to both prediction and answer
              snake_case_						       =							{"".join(a.split()			) for a in answers}
              snake_case_						       =							{"".join(p.split()			) for p in predictions}
              snake_case_						       =							{s for s in predictions if s not in ["``", "''", "`", "'"]}
  
              # if there is a common element, it's a exact match
              snake_case_						       =							len(list(answers & predictions			)			) > 0
              snake_case_						       =							predicted_category != "null" and len(_a			) > 0
  
              return example
  
        snake_case_						       =							short_validation_dataset.map(_a			)
  
        snake_case_						       =							len(_a			)
        snake_case_						       =							len(short_validation_dataset.filter(lambda _a			: x["match"] == 1			)			)
        print("EM score:" ,					(matched / total) * 100 ,					"%"			)
  
  
  if __name__ == "__main__":
       main()
  
  