



import os

import jsonlines
import numpy as np
from tqdm import tqdm


A__								=						2048
A__								=						4096
A__								=						42
A__								=						os.environ.pop('''PROCESS_TRAIN''', '''false''')
A__								=						{'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					Tuple:


				"""simple docstring"""

				def choose_first(__lowerCAmelCase  ,  __lowerCAmelCase=False      ):
								assert isinstance(__lowerCAmelCase  ,  __lowerCAmelCase      )
								if len(__lowerCAmelCase      ) == 1:
												snake_case__	:      Tuple     		=	answer[0]
												return {k: [answer[k]] for k in answer} if is_long_answer else answer
								for a in answer:
												if is_long_answer:
																snake_case__	:      Optional[Any]     		=	{k: [a[k]] for k in a}
												if len(a['''start_token''']      ) > 0:
																break
								return a

				snake_case__	:      List[str]     		=	{'''id''': example['''id''']}
				snake_case__	:      str     		=	example['''annotations''']
				snake_case__	:      List[str]     		=	annotation['''yes_no_answer''']
				if 0 in yes_no_answer or 1 in yes_no_answer:
								snake_case__	:      Union[str, Any]     		=	['''yes'''] if 1 in yes_no_answer else ['''no''']
								snake_case__	:      Tuple     		=	[]
								snake_case__	:      Optional[Any]     		=	[]
								snake_case__	:      Dict     		=	['''<cls>''']
				else:
								snake_case__	:      List[Any]     		=	['''short''']
								snake_case__	:      int     		=	choose_first(annotation['''short_answers''']      )
								if len(out['''start_token''']      ) == 0:
												# answer will be long if short is not available
												snake_case__	:      Any     		=	['''long''']
												snake_case__	:      Optional[int]     		=	choose_first(annotation['''long_answer''']  ,  is_long_answer=__lowerCAmelCase      )
												snake_case__	:      List[Any]     		=	[]
								answer.update(__lowerCAmelCase      )

				# disregard some samples
				if len(answer['''start_token''']      ) > 1 or answer["start_token"] == answer["end_token"]:
								snake_case__	:      List[Any]     		=	True
				else:
								snake_case__	:      List[Any]     		=	False

				snake_case__	:      List[str]     		=	['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
				if not all(isinstance(answer[k]  ,  __lowerCAmelCase      ) for k in cols      ):
								raise ValueError('''Issue in ID'''  ,  example['''id''']      )

				return answer



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase=False      )   ->					Union[str, Any]:


				"""simple docstring"""

				snake_case__	:      Optional[Any]     		=	_get_single_answer(__lowerCAmelCase      )
				# bytes are of no use
				del answer["start_byte"]
				del answer["end_byte"]

				# handle yes_no answers explicitly
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								snake_case__	:      List[Any]     		=	example['''document''']['''tokens''']
								snake_case__	:      Union[str, Any]     		=	[]
								for i in range(len(doc['''token''']      )      ):
												if not doc["is_html"][i]:
																context.append(doc['''token'''][i]      )
								return {
								    "context": " ".join(__lowerCAmelCase      ),
								    "answer": {
								        "start_token": -100,  # ignore index in cross-entropy
								        "end_token": -100,  # ignore index in cross-entropy
								        "category": answer["category"],
								        "span": answer["category"],  # extra
								    },
								}

				# later, help in removing all no answers
				if answer["start_token"] == [-1]:
								return {
								    "context": "None",
								    "answer": {
								        "start_token": -1,
								        "end_token": -1,
								        "category": "null",
								        "span": "None",  # extra
								    },
								}

				# handling normal samples

				snake_case__	:      int     		=	['''start_token''', '''end_token''']
				answer.update({k: answer[k][0] if len(answer[k]      ) > 0 else answer[k] for k in cols}      )  # e.g. [10] == 10

				snake_case__	:      Optional[int]     		=	example['''document''']['''tokens''']
				snake_case__	:      str     		=	answer['''start_token''']
				snake_case__	:      int     		=	answer['''end_token''']

				snake_case__	:      List[str]     		=	[]
				for i in range(len(doc['''token''']      )      ):
								if not doc["is_html"][i]:
												context.append(doc['''token'''][i]      )
								else:
												if answer["start_token"] > i:
																start_token -= 1
												if answer["end_token"] > i:
																end_token -= 1
				snake_case__	:      Optional[Any]     		=	''' '''.join(context[start_token:end_token]      )

				# checking above code
				if assertion:
								snake_case__	:      str     		=	doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
								snake_case__	:      str     		=	doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
								snake_case__	:      Optional[Any]     		=	''' '''.join([old[i] for i in range(len(__lowerCAmelCase      )      ) if not is_html[i]]      )
								if new != old:
												print('''ID:'''  ,  example['''id''']      )
												print('''New:'''  ,  __lowerCAmelCase  ,  end='''\n'''      )
												print('''Old:'''  ,  __lowerCAmelCase  ,  end='''\n\n'''      )

				return {
				    "context": " ".join(__lowerCAmelCase      ),
				    "answer": {
				        "start_token": start_token,
				        "end_token": end_token - 1,  # this makes it inclusive
				        "category": answer["category"],  # either long or short
				        "span": new,  # extra
				    },
				}



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase=2048  ,  __lowerCAmelCase=4096  ,  __lowerCAmelCase=True      )   ->					int:


				"""simple docstring"""

				snake_case__	:      int     		=	get_context_and_ans(__lowerCAmelCase  ,  assertion=__lowerCAmelCase      )
				snake_case__	:      List[Any]     		=	out['''answer''']

				# later, removing these samples
				if answer["start_token"] == -1:
								return {
								    "example_id": example["id"],
								    "input_ids": [[-1]],
								    "labels": {
								        "start_token": [-1],
								        "end_token": [-1],
								        "category": ["null"],
								    },
								}

				snake_case__	:      Optional[Any]     		=	tokenizer(example['''question''']['''text''']  ,  out['''context''']      ).input_ids
				snake_case__	:      Union[str, Any]     		=	input_ids.index(tokenizer.sep_token_id      ) + 1

				# return yes/no
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								snake_case__	:      Optional[Any]     		=	[]
								snake_case__	:      Optional[Any]     		=	[]
								snake_case__	:      int     		=	input_ids[:q_len]
								snake_case__	:      Union[str, Any]     		=	range(__lowerCAmelCase  ,  len(__lowerCAmelCase      )  ,  max_length - doc_stride      )
								for i in doc_start_indices:
												snake_case__	:      Tuple     		=	i + max_length - q_len
												snake_case__	:      Any     		=	input_ids[i:end_index]
												inputs.append(q_indices + slice      )
												category.append(answer['''category'''][0]      )
												if slice[-1] == tokenizer.sep_token_id:
																break

								return {
								    "example_id": example["id"],
								    "input_ids": inputs,
								    "labels": {
								        "start_token": [-100] * len(__lowerCAmelCase      ),
								        "end_token": [-100] * len(__lowerCAmelCase      ),
								        "category": category,
								    },
								}

				snake_case__	:      Optional[int]     		=	out['''context'''].split()
				snake_case__	:      Union[str, Any]     		=	splitted_context[answer['''end_token''']]
				snake_case__	:      str     		=	len(
				    tokenizer(
				        ''' '''.join(splitted_context[: answer['''start_token''']]      )  ,  add_special_tokens=__lowerCAmelCase  ,  ).input_ids      )
				snake_case__	:      Tuple     		=	len(
				    tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]      )  ,  add_special_tokens=__lowerCAmelCase      ).input_ids      )

				answer["start_token"] += q_len
				answer["end_token"] += q_len

				# fixing end token
				snake_case__	:      Dict     		=	len(tokenizer(__lowerCAmelCase  ,  add_special_tokens=__lowerCAmelCase      ).input_ids      )
				if num_sub_tokens > 1:
								answer["end_token"] += num_sub_tokens - 1

				snake_case__	:      Optional[Any]     		=	input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
				snake_case__	:      Optional[Any]     		=	answer['''start_token''']
				snake_case__	:      str     		=	answer['''end_token''']

				if assertion:
								snake_case__	:      int     		=	tokenizer.decode(__lowerCAmelCase      )
								if answer["span"] != new:
												print('''ISSUE IN TOKENIZATION'''      )
												print('''OLD:'''  ,  answer['''span''']      )
												print('''NEW:'''  ,  __lowerCAmelCase  ,  end='''\n\n'''      )

				if len(__lowerCAmelCase      ) <= max_length:
								return {
								    "example_id": example["id"],
								    "input_ids": [input_ids],
								    "labels": {
								        "start_token": [answer["start_token"]],
								        "end_token": [answer["end_token"]],
								        "category": answer["category"],
								    },
								}

				snake_case__	:      List[Any]     		=	input_ids[:q_len]
				snake_case__	:      str     		=	range(__lowerCAmelCase  ,  len(__lowerCAmelCase      )  ,  max_length - doc_stride      )

				snake_case__	:      List[str]     		=	[]
				snake_case__	:      str     		=	[]
				snake_case__	:      int     		=	[]
				snake_case__	:      str     		=	[]  # null, yes, no, long, short
				for i in doc_start_indices:
								snake_case__	:      List[Any]     		=	i + max_length - q_len
								snake_case__	:      Tuple     		=	input_ids[i:end_index]
								inputs.append(q_indices + slice      )
								assert len(inputs[-1]      ) <= max_length, "Issue in truncating length"

								if start_token >= i and end_token <= end_index - 1:
												snake_case__	:      Dict     		=	start_token - i + q_len
												snake_case__	:      Union[str, Any]     		=	end_token - i + q_len
												answers_category.append(answer['''category'''][0]      )  # ["short"] -> "short"
								else:
												snake_case__	:      Optional[Any]     		=	-100
												snake_case__	:      Optional[int]     		=	-100
												answers_category.append('''null'''      )
								snake_case__	:      str     		=	inputs[-1][start_token : end_token + 1]

								answers_start_token.append(__lowerCAmelCase      )
								answers_end_token.append(__lowerCAmelCase      )
								if assertion:
												if new != old and new != [tokenizer.cls_token_id]:
																print('''ISSUE in strided for ID:'''  ,  example['''id''']      )
																print('''New:'''  ,  tokenizer.decode(__lowerCAmelCase      )      )
																print('''Old:'''  ,  tokenizer.decode(__lowerCAmelCase      )  ,  end='''\n\n'''      )
								if slice[-1] == tokenizer.sep_token_id:
												break

				return {
				    "example_id": example["id"],
				    "input_ids": inputs,
				    "labels": {
				        "start_token": answers_start_token,
				        "end_token": answers_end_token,
				        "category": answers_category,
				    },
				}



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase=2048  ,  __lowerCAmelCase=4096  ,  __lowerCAmelCase=False      )   ->					int:


				"""simple docstring"""

				snake_case__	:      List[Any]     		=	get_strided_contexts_and_ans(
				    __lowerCAmelCase  ,  __lowerCAmelCase  ,  doc_stride=__lowerCAmelCase  ,  max_length=__lowerCAmelCase  ,  assertion=__lowerCAmelCase  ,  )

				return example



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Union[str, Any]:


				"""simple docstring"""

				with jsonlines.open(__lowerCAmelCase  ,  '''a'''      ) as writer:
								for example in tqdm(__lowerCAmelCase  ,  total=len(__lowerCAmelCase      )  ,  desc='''Saving samples ... '''      ):
												snake_case__	:      Any     		=	example['''labels''']
												for ids, start, end, cat in zip(
												    example['''input_ids''']  ,  labels['''start_token''']  ,  labels['''end_token''']  ,  labels['''category''']  ,  ):
																if start == -1 and end == -1:
																				continue  # leave waste samples with no answer
																if cat == "null" and np.random.rand() < 0.6:
																				continue  # removing 50 % samples
																writer.write(
																    {
																        '''input_ids''': ids,
																        '''start_token''': start,
																        '''end_token''': end,
																        '''category''': CATEGORY_MAPPING[cat],
																    }      )


if __name__ == "__main__":
				from datasets import load_dataset

				from transformers import BigBirdTokenizer

				A__								=						load_dataset('''natural_questions''')
				A__								=						BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

				A__								=						data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

				A__								=						{
				    '''tokenizer''': tokenizer,
				    '''doc_stride''': DOC_STRIDE,
				    '''max_length''': MAX_LENGTH,
				    '''assertion''': False,
				}
				A__								=						data.map(prepare_inputs, fn_kwargs=fn_kwargs)
				A__								=						data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
				print(data)

				np.random.seed(SEED)
				A__								=						'''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
				save_to_disk(data, file_name=cache_file_name)






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


A__								=						logging.get_logger(__name__)

A__								=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__								=						{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

A__								=						{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

A__								=						{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}


class       a							(			__lowerCamelCase				):
		__lowerCAmelCase	:    List[Any]											=					VOCAB_FILES_NAMES
		__lowerCAmelCase	:    Optional[Any]											=					PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase	:    Optional[Any]											=					PRETRAINED_INIT_CONFIGURATION
		__lowerCAmelCase	:    Optional[Any]											=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase	:    Optional[Any]											=					RealmTokenizer



		def __init__(     self     :Optional[Any]   ,__lowercase     :Dict=None   ,__lowercase     :Optional[int]=None   ,__lowercase     :Optional[Any]=True   ,__lowercase     :Optional[int]="[UNK]"   ,__lowercase     :List[str]="[SEP]"   ,__lowercase     :List[str]="[PAD]"   ,__lowercase     :int="[CLS]"   ,__lowercase     :str="[MASK]"   ,__lowercase     :Dict=True   ,__lowercase     :List[str]=None   ,**__lowercase     :Any   ,):
						super().__init__(
						    __lowercase   ,tokenizer_file=__lowercase   ,do_lower_case=__lowercase   ,unk_token=__lowercase   ,sep_token=__lowercase   ,pad_token=__lowercase   ,cls_token=__lowercase   ,mask_token=__lowercase   ,tokenize_chinese_chars=__lowercase   ,strip_accents=__lowercase   ,**__lowercase   ,)

						snake_case__	:      List[str]     		=	json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
						if (
						    normalizer_state.get('''lowercase'''   ,__lowercase  ) != do_lower_case
						    or normalizer_state.get('''strip_accents'''   ,__lowercase  ) != strip_accents
						    or normalizer_state.get('''handle_chinese_chars'''   ,__lowercase  ) != tokenize_chinese_chars
						):
										snake_case__	:      Optional[int]     		=	getattr(__lowercase   ,normalizer_state.pop('''type'''  )  )
										snake_case__	:      List[Any]     		=	do_lower_case
										snake_case__	:      Optional[Any]     		=	strip_accents
										snake_case__	:      List[str]     		=	tokenize_chinese_chars
										snake_case__	:      Dict     		=	normalizer_class(**__lowercase  )

						snake_case__	:      Tuple     		=	do_lower_case



		def 	__lowerCamelCase			(     self     :Union[str, Any]   ,__lowercase     :Union[str, Any]   ,**__lowercase     :Any  ):
						snake_case__	:      Dict     		=	PaddingStrategy.MAX_LENGTH

						snake_case__	:      List[str]     		=	text
						snake_case__	:      int     		=	kwargs.pop('''text_pair'''   ,__lowercase  )
						snake_case__	:      Optional[int]     		=	kwargs.pop('''return_tensors'''   ,__lowercase  )

						snake_case__	:      str     		=	{
						    '''input_ids''': [],
						    '''attention_mask''': [],
						    '''token_type_ids''': [],
						}

						for idx, candidate_text in enumerate(__lowercase  ):
										if batch_text_pair is not None:
														snake_case__	:      Optional[int]     		=	batch_text_pair[idx]
										else:
														snake_case__	:      Tuple     		=	None

										snake_case__	:      List[str]     		=	super().__call__(__lowercase   ,__lowercase   ,return_tensors=__lowercase   ,**__lowercase  )

										snake_case__	:      Optional[Any]     		=	encoded_candidates.get('''input_ids'''  )
										snake_case__	:      Optional[Any]     		=	encoded_candidates.get('''attention_mask'''  )
										snake_case__	:      List[Any]     		=	encoded_candidates.get('''token_type_ids'''  )

										if encoded_input_ids is not None:
														output_data["input_ids"].append(__lowercase  )
										if encoded_attention_mask is not None:
														output_data["attention_mask"].append(__lowercase  )
										if encoded_token_type_ids is not None:
														output_data["token_type_ids"].append(__lowercase  )

						snake_case__	:      Any     		=	{key: item for key, item in output_data.items() if len(__lowercase  ) != 0}

						return BatchEncoding(__lowercase   ,tensor_type=__lowercase  )



		def 	__lowerCamelCase			(     self     :List[Any]   ,__lowercase     :Tuple   ,__lowercase     :Tuple=None  ):
						snake_case__	:      List[str]     		=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

						return output



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :List[int]   ,__lowercase     :Optional[List[int]] = None  ):
						snake_case__	:      Tuple     		=	[self.sep_token_id]
						snake_case__	:      Union[str, Any]     		=	[self.cls_token_id]
						if token_ids_a is None:
										return len(cls + token_ids_a + sep  ) * [0]
						return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :str   ,__lowercase     :Optional[str] = None  ):
						snake_case__	:      Tuple     		=	self._tokenizer.model.save(__lowercase   ,name=__lowercase  )
						return tuple(__lowercase  )

