





"""simple docstring"""



import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


a_				=		{0: """null""", 1: """short""", 2: """long""", 3: """yes""", 4: """no"""}
a_				=		set("""""".join(["""‘""", """’""", """´""", """`""", """.""", """,""", """-""", """\""""]))







def 	__lowercase  (     snake_case_						:						Optional[Any]       ,snake_case_						:						Optional[Any]=0       ,snake_case_						:						Optional[Any]=None		)     ->Any:



    '''simple docstring'''







    return [" ".join(x.split(''' '''		)[begin:end]		) for x in answers if len(x.split(''' '''		)		) > 1]







def 	__lowercase  (     snake_case_						:						Dict       ,snake_case_						:						List[str]=False		)     ->Any:



    '''simple docstring'''







    if make_sub_answers:
        # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
        # *e.g.* if the correct answer contains a prefix such as "the", or "a"
        __A					:		Optional[Any]						=  (
            given_answers + get_sub_answers(snake_case_       ,begin=1		) + get_sub_answers(snake_case_       ,end=-1		)
        )
    __A					:		str						=  []
    for answer in given_answers:
        __A					:		List[str]						=  answer.replace('''_'''       ,''' '''		).lower()
        __A					:		Union[str, Any]						=  "".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ''' ''' for c in alias		)
        answers.append(''' '''.join(alias.split()		).strip()		)
    return set(snake_case_		)







def 	__lowercase  (     snake_case_						:						str       ,snake_case_						:						List[Any]       ,snake_case_						:						str=1       ,snake_case_						:						List[str]=100		)     ->Any:



    '''simple docstring'''







    __A					:		List[Any]						=  jax.lax.top_k(snake_case_       ,snake_case_		)
    __A					:		Tuple						=  jax.lax.top_k(snake_case_       ,snake_case_		)

    __A					:		int						=  best_end_idx[:, None] - best_start_idx[None, :]
    __A					:		Optional[Any]						=  jnp.logical_or(widths < 0       ,widths > max_size		)
    __A					:		Tuple						=  (best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
    __A					:		Dict						=  jnp.argmax(snake_case_		).item()

    return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]







def 	__lowercase  (     snake_case_						:						Any		)     ->Any:



    '''simple docstring'''







    __A					:		str						=  sample["question"]["text"]
    __A					:		Tuple						=  sample["document"]["tokens"]["token"]
    __A					:		Optional[Any]						=  sample["document"]["tokens"]["is_html"]
    __A					:		Optional[int]						=  sample["annotations"]["long_answer"]
    __A					:		str						=  sample["annotations"]["short_answers"]

    __A					:		Dict						=  " ".join([context[i] for i in range(len(snake_case_		)		) if not is_html[i]]		)

    # 0 - No ; 1 - Yes
    for answer in sample["annotations"]["yes_no_answer"]:
        if answer == 0 or answer == 1:
            return {
                "question": question,
                "context": context_string,
                "short": [],
                "long": [],
                "category": "no" if answer == 0 else "yes",
            }

    __A					:		int						=  []
    for s in short_answers:
        short_targets.extend(s['''text''']		)
    __A					:		List[str]						=  list(set(snake_case_		)		)

    __A					:		Union[str, Any]						=  []
    for s in long_answers:
        if s["start_token"] == -1:
            continue
        __A					:		Any						=  context[s["start_token"] : s["end_token"]]
        __A					:		int						=  is_html[s["start_token"] : s["end_token"]]
        __A					:		List[str]						=  " ".join([answer[i] for i in range(len(snake_case_		)		) if not html[i]]		)
        if new_answer not in long_targets:
            long_targets.append(snake_case_		)

    __A					:		List[str]						=  "long_short" if len(short_targets + long_targets		) > 0 else "null"

    return {
        "question": question,
        "context": context_string,
        "short": short_targets,
        "long": long_targets,
        "category": category,
    }







def 	__lowercase  (     )     ->Any:



    '''simple docstring'''







    __A					:		Dict						=  load_from_disk('''natural-questions-validation'''		)
    __A					:		Optional[Any]						=  dataset.map(snake_case_		).remove_columns(['''annotations''', '''document''', '''id''']		)
    print(snake_case_		)

    __A					:		Tuple						=  dataset.filter(lambda snake_case_		: (len(x['''question''']		) + len(x['''context''']		)) < 4 * 4096		)
    __A					:		Tuple						=  short_validation_dataset.filter(lambda snake_case_		: x["category"] != "null"		)
    short_validation_dataset

    __A					:		Any						=  "vasudevgupta/flax-bigbird-natural-questions"
    __A					:		str						=  FlaxBigBirdForNaturalQuestions.from_pretrained(snake_case_		)
    __A					:		Optional[Any]						=  BigBirdTokenizerFast.from_pretrained(snake_case_		)

    @jax.jit
    def forward(*snake_case_						:						str       ,**snake_case_						:						Optional[int]		):
        __A					:		int						=  model(*snake_case_       ,**snake_case_		)
        return start_logits, end_logits, jnp.argmax(snake_case_       ,axis=-1		)

    def evaluate(snake_case_						:						Optional[Any]		):
        # encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
        __A					:		Tuple						=  tokenizer(
            example['''question''']       ,example['''context''']       ,return_tensors='''np'''       ,max_length=4096       ,padding='''max_length'''       ,truncation=snake_case_       ,)

        __A					:		int						=  forward(**snake_case_		)

        __A					:		str						=  CATEGORY_MAPPING[category.item()]

        __A					:		Optional[Any]						=  example["long"] + example["short"]
        if example["category"] in ["yes", "no", "null"]:
            __A					:		str						=  [example["category"]]
        __A					:		Any						=  example["category"] != "null"
        # Now target can be: "yes", "no", "null", "list of long & short answers"

        if predicted_category in ["yes", "no", "null"]:
            __A					:		Optional[int]						=  [predicted_category]
            __A					:		List[str]						=  example["output"] == example["targets"]
            __A					:		Any						=  predicted_category != "null"
            return example

        __A					:		Union[str, Any]						=  38 if predicted_category == "short" else 1024
        __A					:		Any						=  get_best_valid_start_end_idx(
            start_scores[0]       ,end_scores[0]       ,top_k=8       ,max_size=snake_case_		)

        __A					:		int						=  inputs["input_ids"][0].tolist()
        __A					:		Any						=  [tokenizer.decode(input_ids[start_score : end_score + 1]		)]

        __A					:		Dict						=  expand_to_aliases(example['''targets''']       ,make_sub_answers=snake_case_		)
        __A					:		Optional[int]						=  expand_to_aliases(example['''output''']		)

        # some preprocessing to both prediction and answer
        __A					:		Union[str, Any]						=  {"".join(a.split()		) for a in answers}
        __A					:		Union[str, Any]						=  {"".join(p.split()		) for p in predictions}
        __A					:		Tuple						=  {s for s in predictions if s not in ["``", "''", "`", "'"]}

        # if there is a common element, it's a exact match
        __A					:		str						=  len(list(answers & predictions		)		) > 0
        __A					:		str						=  predicted_category != "null" and len(snake_case_		) > 0

        return example

    __A					:		Optional[int]						=  short_validation_dataset.map(snake_case_		)

    __A					:		List[Any]						=  len(snake_case_		)
    __A					:		Optional[Any]						=  len(short_validation_dataset.filter(lambda snake_case_		: x["match"] == 1		)		)
    print('''EM score:'''       ,(matched / total) * 100       ,'''%'''		)


if __name__ == "__main__":
       main()











"""simple docstring"""



from __future__ import annotations

import sys







def 	__lowercase  (     snake_case_						:						int       ,snake_case_						:						int		)     ->tuple[int, int]:



    '''simple docstring'''







    if abs(snake_case_		) == 1:
        return a, 0
    elif abs(snake_case_		) == 1:
        return 0, b

    __A				, __A					:		int						=  a, b
    __A				, __A					:		Any						=  1, 0
    __A				, __A					:		int						=  0, 1

    while remainder != 0:
        __A					:		List[Any]						=  old_remainder // remainder
        __A				, __A					:		int						=  remainder, old_remainder - quotient * remainder
        __A				, __A					:		List[Any]						=  coeff_a, old_coeff_a - quotient * coeff_a
        __A				, __A					:		Union[str, Any]						=  coeff_b, old_coeff_b - quotient * coeff_b

    # sign correction for negative numbers
    if a < 0:
        __A					:		Optional[Any]						=  -old_coeff_a
    if b < 0:
        __A					:		List[Any]						=  -old_coeff_b

    return old_coeff_a, old_coeff_b







def 	__lowercase  (     )     ->List[str]:



    '''simple docstring'''







    if len(sys.argv		) < 3:
        print('''2 integer arguments required'''		)
        return 1
    __A					:		Optional[int]						=  int(sys.argv[1]		)
    __A					:		Tuple						=  int(sys.argv[2]		)
    print(extended_euclidean_algorithm(snake_case_       ,snake_case_		)		)
    return 0


if __name__ == "__main__":
       raise SystemExit(main())




