'''simple docstring'''

import os

import jsonlines
import numpy as np
from tqdm import tqdm


__a: Optional[int]          =   20_48
__a: Union[str, Any]          =   40_96
__a: List[Any]          =   42
__a: List[Any]          =   os.environ.pop("""PROCESS_TRAIN""", """false""")
__a: Optional[Any]          =   {"""null""": 0, """short""": 1, """long""": 2, """yes""": 3, """no""": 4}
def 			__UpperCamelCase       (   UpperCAmelCase     ):
 def choose_first(UpperCAmelCase		,     UpperCAmelCase=False     ):
  assert isinstance(UpperCAmelCase		,     UpperCAmelCase     )
  if len(UpperCAmelCase     ) == 1:
   lowercase__				:    str  =					answer[0]
   return {k: [answer[k]] for k in answer} if is_long_answer else answer
  for a in answer:
   if is_long_answer:
    lowercase__				:    List[str]  =					{k: [a[k]] for k in a}
   if len(a['''start_token''']     ) > 0:
    break
  return a

 lowercase__				:    str  =					{'''id''': example['''id''']}
 lowercase__				:    Any  =					example['''annotations''']
 lowercase__				:    List[str]  =					annotation['''yes_no_answer''']
 if 0 in yes_no_answer or 1 in yes_no_answer:
  lowercase__				:    int  =					['''yes'''] if 1 in yes_no_answer else ['''no''']
  lowercase__				:    Dict  =					[]
  lowercase__				:    str  =					[]
  lowercase__				:    Tuple  =					['''<cls>''']
 else:
  lowercase__				:    Tuple  =					['''short''']
  lowercase__				:    Any  =					choose_first(annotation['''short_answers''']     )
  if len(out['''start_token''']     ) == 0:
   # answer will be long if short is not available
   lowercase__				:    int  =					['''long''']
   lowercase__				:    List[str]  =					choose_first(annotation['''long_answer''']		,     is_long_answer=UpperCAmelCase     )
   lowercase__				:    Optional[int]  =					[]
  answer.update(UpperCAmelCase     )

 # disregard some samples
 if len(answer['''start_token''']     ) > 1 or answer["start_token"] == answer["end_token"]:
  lowercase__				:    Optional[Any]  =					True
 else:
  lowercase__				:    Optional[int]  =					False

 lowercase__				:    List[Any]  =					['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
 if not all(isinstance(answer[k]		,     UpperCAmelCase     ) for k in cols     ):
  raise ValueError('''Issue in ID'''		,     example['''id''']     )

 return answer
def 			__UpperCamelCase       (   UpperCAmelCase		,     UpperCAmelCase=False     ):
 lowercase__				:    Optional[int]  =					_get_single_answer(UpperCAmelCase     )
 # bytes are of no use
 del answer["start_byte"]
 del answer["end_byte"]

 # handle yes_no answers explicitly
 if answer["category"][0] in ["yes", "no"]:  # category is list with one element
  lowercase__				:    Any  =					example['''document''']['''tokens''']
  lowercase__				:    List[Any]  =					[]
  for i in range(len(doc['''token''']     )     ):
   if not doc["is_html"][i]:
    context.append(doc['''token'''][i]     )
  return {
      "context": " ".join(UpperCAmelCase     ),
      "answer": {
          "start_token": -100,  # ignore index in cross-entropy
          "end_token": -100,  # ignore index in cross-entropy
          "category": answer["category"],
          "span": answer["category"],  # extra
      },
  }

 # later, help in removing all no answers
 if answer["start_token"] == [-1]:
  return {
      "context": "None",
      "answer": {
          "start_token": -1,
          "end_token": -1,
          "category": "null",
          "span": "None",  # extra
      },
  }

 # handling normal samples

 lowercase__				:    Any  =					['''start_token''', '''end_token''']
 answer.update({k: answer[k][0] if len(answer[k]     ) > 0 else answer[k] for k in cols}     )  # e.g. [10] == 10

 lowercase__				:    List[str]  =					example['''document''']['''tokens''']
 lowercase__				:    Optional[int]  =					answer['''start_token''']
 lowercase__				:    Optional[int]  =					answer['''end_token''']

 lowercase__				:    Any  =					[]
 for i in range(len(doc['''token''']     )     ):
  if not doc["is_html"][i]:
   context.append(doc['''token'''][i]     )
  else:
   if answer["start_token"] > i:
    start_token -= 1
   if answer["end_token"] > i:
    end_token -= 1
 lowercase__				:    Union[str, Any]  =					''' '''.join(context[start_token:end_token]     )

 # checking above code
 if assertion:
  lowercase__				:    Any  =					doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
  lowercase__				:    Any  =					doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
  lowercase__				:    str  =					''' '''.join([old[i] for i in range(len(UpperCAmelCase     )     ) if not is_html[i]]     )
  if new != old:
   print('''ID:'''		,     example['''id''']     )
   print('''New:'''		,     UpperCAmelCase		,     end='''\n'''     )
   print('''Old:'''		,     UpperCAmelCase		,     end='''\n\n'''     )

 return {
     "context": " ".join(UpperCAmelCase     ),
     "answer": {
         "start_token": start_token,
         "end_token": end_token - 1,  # this makes it inclusive
         "category": answer["category"],  # either long or short
         "span": new,  # extra
     },
 }
def 			__UpperCamelCase       (   UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase=2048		,     UpperCAmelCase=4096		,     UpperCAmelCase=True     ):
 # overlap will be of doc_stride - q_len

 lowercase__				:    Tuple  =					get_context_and_ans(UpperCAmelCase		,     assertion=UpperCAmelCase     )
 lowercase__				:    List[str]  =					out['''answer''']

 # later, removing these samples
 if answer["start_token"] == -1:
  return {
      "example_id": example["id"],
      "input_ids": [[-1]],
      "labels": {
          "start_token": [-1],
          "end_token": [-1],
          "category": ["null"],
      },
  }

 lowercase__				:    Optional[Any]  =					tokenizer(example['''question''']['''text''']		,     out['''context''']     ).input_ids
 lowercase__				:    Any  =					input_ids.index(tokenizer.sep_token_id     ) + 1

 # return yes/no
 if answer["category"][0] in ["yes", "no"]:  # category is list with one element
  lowercase__				:    List[Any]  =					[]
  lowercase__				:    Any  =					[]
  lowercase__				:    int  =					input_ids[:q_len]
  lowercase__				:    Any  =					range(UpperCAmelCase		,     len(UpperCAmelCase     )		,     max_length - doc_stride     )
  for i in doc_start_indices:
   lowercase__				:    Optional[Any]  =					i + max_length - q_len
   lowercase__				:    Dict  =					input_ids[i:end_index]
   inputs.append(q_indices + slice     )
   category.append(answer['''category'''][0]     )
   if slice[-1] == tokenizer.sep_token_id:
    break

  return {
      "example_id": example["id"],
      "input_ids": inputs,
      "labels": {
          "start_token": [-100] * len(UpperCAmelCase     ),
          "end_token": [-100] * len(UpperCAmelCase     ),
          "category": category,
      },
  }

 lowercase__				:    Optional[Any]  =					out['''context'''].split()
 lowercase__				:    List[Any]  =					splitted_context[answer['''end_token''']]
 lowercase__				:    int  =					len(
     tokenizer(
         ''' '''.join(splitted_context[: answer['''start_token''']]     )		,     add_special_tokens=UpperCAmelCase		,     ).input_ids     )
 lowercase__				:    List[Any]  =					len(
     tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]     )		,     add_special_tokens=UpperCAmelCase     ).input_ids     )

 answer["start_token"] += q_len
 answer["end_token"] += q_len

 # fixing end token
 lowercase__				:    int  =					len(tokenizer(UpperCAmelCase		,     add_special_tokens=UpperCAmelCase     ).input_ids     )
 if num_sub_tokens > 1:
  answer["end_token"] += num_sub_tokens - 1

 lowercase__				:    List[Any]  =					input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
 lowercase__				:    Tuple  =					answer['''start_token''']
 lowercase__				:    List[Any]  =					answer['''end_token''']

 if assertion:
  lowercase__				:    Union[str, Any]  =					tokenizer.decode(UpperCAmelCase     )
  if answer["span"] != new:
   print('''ISSUE IN TOKENIZATION'''     )
   print('''OLD:'''		,     answer['''span''']     )
   print('''NEW:'''		,     UpperCAmelCase		,     end='''\n\n'''     )

 if len(UpperCAmelCase     ) <= max_length:
  return {
      "example_id": example["id"],
      "input_ids": [input_ids],
      "labels": {
          "start_token": [answer["start_token"]],
          "end_token": [answer["end_token"]],
          "category": answer["category"],
      },
  }

 lowercase__				:    List[str]  =					input_ids[:q_len]
 lowercase__				:    Tuple  =					range(UpperCAmelCase		,     len(UpperCAmelCase     )		,     max_length - doc_stride     )

 lowercase__				:    List[str]  =					[]
 lowercase__				:    Dict  =					[]
 lowercase__				:    List[Any]  =					[]
 lowercase__				:    List[Any]  =					[]  # null, yes, no, long, short
 for i in doc_start_indices:
  lowercase__				:    Optional[int]  =					i + max_length - q_len
  lowercase__				:    Tuple  =					input_ids[i:end_index]
  inputs.append(q_indices + slice     )
  assert len(inputs[-1]     ) <= max_length, "Issue in truncating length"

  if start_token >= i and end_token <= end_index - 1:
   lowercase__				:    Optional[int]  =					start_token - i + q_len
   lowercase__				:    Any  =					end_token - i + q_len
   answers_category.append(answer['''category'''][0]     )  # ["short"] -> "short"
  else:
   lowercase__				:    Any  =					-100
   lowercase__				:    List[str]  =					-100
   answers_category.append('''null'''     )
  lowercase__				:    Dict  =					inputs[-1][start_token : end_token + 1]

  answers_start_token.append(UpperCAmelCase     )
  answers_end_token.append(UpperCAmelCase     )
  if assertion:
   if new != old and new != [tokenizer.cls_token_id]:
    print('''ISSUE in strided for ID:'''		,     example['''id''']     )
    print('''New:'''		,     tokenizer.decode(UpperCAmelCase     )     )
    print('''Old:'''		,     tokenizer.decode(UpperCAmelCase     )		,     end='''\n\n'''     )
  if slice[-1] == tokenizer.sep_token_id:
   break

 return {
     "example_id": example["id"],
     "input_ids": inputs,
     "labels": {
         "start_token": answers_start_token,
         "end_token": answers_end_token,
         "category": answers_category,
     },
 }
def 			__UpperCamelCase       (   UpperCAmelCase		,     UpperCAmelCase		,     UpperCAmelCase=2048		,     UpperCAmelCase=4096		,     UpperCAmelCase=False     ):
 lowercase__				:    Dict  =					get_strided_contexts_and_ans(
     UpperCAmelCase		,     UpperCAmelCase		,     doc_stride=UpperCAmelCase		,     max_length=UpperCAmelCase		,     assertion=UpperCAmelCase		,     )

 return example
def 			__UpperCamelCase       (   UpperCAmelCase		,     UpperCAmelCase     ):
 with jsonlines.open(UpperCAmelCase		,     '''a'''     ) as writer:
  for example in tqdm(UpperCAmelCase		,     total=len(UpperCAmelCase     )		,     desc='''Saving samples ... '''     ):
   lowercase__				:    Union[str, Any]  =					example['''labels''']
   for ids, start, end, cat in zip(
       example['''input_ids''']		,     labels['''start_token''']		,     labels['''end_token''']		,     labels['''category''']		,     ):
    if start == -1 and end == -1:
     continue  # leave waste samples with no answer
    if cat == "null" and np.random.rand() < 0.6:
     continue  # removing 50 % samples
    writer.write(
        {
            '''input_ids''': ids,
            '''start_token''': start,
            '''end_token''': end,
            '''category''': CATEGORY_MAPPING[cat],
        }     )


if __name__ == "__main__":
  from datasets import load_dataset

  from transformers import BigBirdTokenizer

  __a: Optional[Any]          =   load_dataset("""natural_questions""")
  __a: Optional[int]          =   BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""")

  __a: int          =   data["""train""" if PROCESS_TRAIN == """true""" else """validation"""]

  __a: Tuple          =   {
      """tokenizer""": tokenizer,
      """doc_stride""": DOC_STRIDE,
      """max_length""": MAX_LENGTH,
      """assertion""": False,
  }
  __a: str          =   data.map(prepare_inputs, fn_kwargs=fn_kwargs)
  __a: int          =   data.remove_columns(["""annotations""", """document""", """id""", """question"""])
  print(data)

  np.random.seed(SEED)
  __a: int          =   """nq-training.jsonl""" if PROCESS_TRAIN == """true""" else """nq-validation.jsonl"""
  save_to_disk(data, file_name=cache_file_name)


'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


__a: List[str]          =   logging.get_logger(__name__)



class 			UpperCAmelCase  (						a__					):





       '''simple docstring'''



       SCREAMING_SNAKE_CASE							=    ["pixel_values"]

       def __init__(						self	,   __lowerCAmelCase = True	,   __lowerCAmelCase = None	,   __lowerCAmelCase = PILImageResampling.BICUBIC	,   __lowerCAmelCase = True	,   __lowerCAmelCase = 1 / 255	,   __lowerCAmelCase = True	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = True	,   **__lowerCAmelCase	,   )  ->  None:
        super().__init__(**__lowerCAmelCase )
        lowercase__				:    Optional[int]  =					size if size is not None else {'''height''': 384, '''width''': 384}
        lowercase__				:    Optional[Any]  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )

        lowercase__				:    Dict  =					do_resize
        lowercase__				:    int  =					size
        lowercase__				:    int  =					resample
        lowercase__				:    Tuple  =					do_rescale
        lowercase__				:    int  =					rescale_factor
        lowercase__				:    int  =					do_normalize
        lowercase__				:    Optional[int]  =					image_mean if image_mean is not None else OPENAI_CLIP_MEAN
        lowercase__				:    Optional[int]  =					image_std if image_std is not None else OPENAI_CLIP_STD
        lowercase__				:    Tuple  =					do_convert_rgb

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = PILImageResampling.BICUBIC	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  np.ndarray:
        lowercase__				:    Union[str, Any]  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )
        if "height" not in size or "width" not in size:
         raise ValueError(F"""The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}""" )
        lowercase__				:    Any  =					(size['''height'''], size['''width'''])
        return resize(__lowerCAmelCase	,   size=__lowerCAmelCase	,   resample=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  str:
        return rescale(__lowerCAmelCase	,   scale=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  np.ndarray:
        return normalize(__lowerCAmelCase	,   mean=__lowerCAmelCase	,   std=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )


       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = ChannelDimension.FIRST	,   **__lowerCAmelCase	,   )  ->  PIL.Image.Image:
        lowercase__				:    Union[str, Any]  =					do_resize if do_resize is not None else self.do_resize
        lowercase__				:    Any  =					resample if resample is not None else self.resample
        lowercase__				:    int  =					do_rescale if do_rescale is not None else self.do_rescale
        lowercase__				:    List[Any]  =					rescale_factor if rescale_factor is not None else self.rescale_factor
        lowercase__				:    List[Any]  =					do_normalize if do_normalize is not None else self.do_normalize
        lowercase__				:    Dict  =					image_mean if image_mean is not None else self.image_mean
        lowercase__				:    Dict  =					image_std if image_std is not None else self.image_std
        lowercase__				:    Dict  =					do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

        lowercase__				:    Optional[int]  =					size if size is not None else self.size
        lowercase__				:    int  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )

        lowercase__				:    str  =					make_list_of_images(__lowerCAmelCase )

        if not valid_images(__lowerCAmelCase ):
         raise ValueError(
             '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
             '''torch.Tensor, tf.Tensor or jax.ndarray.''' )

        if do_resize and size is None or resample is None:
         raise ValueError('''Size and resample must be specified if do_resize is True.''' )

        if do_rescale and rescale_factor is None:
         raise ValueError('''Rescale factor must be specified if do_rescale is True.''' )

        if do_normalize and (image_mean is None or image_std is None):
         raise ValueError('''Image mean and std must be specified if do_normalize is True.''' )

        # PIL RGBA images are converted to RGB
        if do_convert_rgb:
         lowercase__				:    Optional[Any]  =					[convert_to_rgb(__lowerCAmelCase ) for image in images]

        # All transformations expect numpy arrays.
        lowercase__				:    Any  =					[to_numpy_array(__lowerCAmelCase ) for image in images]

        if do_resize:
         lowercase__				:    Tuple  =					[self.resize(image=__lowerCAmelCase	,   size=__lowerCAmelCase	,   resample=__lowerCAmelCase ) for image in images]

        if do_rescale:
         lowercase__				:    List[str]  =					[self.rescale(image=__lowerCAmelCase	,   scale=__lowerCAmelCase ) for image in images]

        if do_normalize:
         lowercase__				:    Tuple  =					[self.normalize(image=__lowerCAmelCase	,   mean=__lowerCAmelCase	,   std=__lowerCAmelCase ) for image in images]

        lowercase__				:    List[str]  =					[to_channel_dimension_format(__lowerCAmelCase	,   __lowerCAmelCase ) for image in images]

        lowercase__				:    Optional[Any]  =					BatchFeature(data={'''pixel_values''': images}	,   tensor_type=__lowerCAmelCase )

        return encoded_outputs

