

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase     =    logging.get_logger(__name__)

_UpperCamelCase     =    {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCamelCase     =    {
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
}

_UpperCamelCase     =    {
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}
@lru_cache()
def 		UpperCamelCase_(    ) ->				Union[str, Any]:
 UpperCAmelCase__           =       (
     list(range(ord('!'     )						,					ord('~'     ) + 1     )     ) + list(range(ord('¡'     )						,					ord('¬'     ) + 1     )     ) + list(range(ord('®'     )						,					ord('ÿ'     ) + 1     )     )
 )
 UpperCAmelCase__           =       bs[:]
 UpperCAmelCase__           =       0
 for b in range(2**8     ):
  if b not in bs:
   bs.append(_UpperCAmelCase     )
   cs.append(2**8 + n     )
   n += 1
 UpperCAmelCase__           =       [chr(_UpperCAmelCase     ) for n in cs]
 return dict(zip(_UpperCAmelCase						,					_UpperCAmelCase     )     )
def 		UpperCamelCase_(    snake_case__:						Tuple     ) ->				Any:
 UpperCAmelCase__           =       set()
 UpperCAmelCase__           =       word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)     )
  UpperCAmelCase__           =       char
 return pairs







class 					lowercase		( _UpperCamelCase      ):
 __SCREAMING_SNAKE_CASE      							=       VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE      							=       PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE      							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE      							=       ["input_ids", "attention_mask"]




 def __init__(self , __a , __a , __a="replace" , __a="<s>" , __a="</s>" , __a="</s>" , __a="<s>" , __a="<unk>" , __a="<pad>" , __a="<mask>" , __a=False , **__a , )				->      int:






  """simple docstring"""





  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else bos_token
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else eos_token
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else sep_token
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else cls_token
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else unk_token
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else pad_token

  # Mask token behave like a normal word, i.e. include the space before it
  UpperCAmelCase__           =       AddedToken(UpperCAmelCase_ , lstrip=UpperCAmelCase_ , rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ , UpperCAmelCase_				) else mask_token

  super().__init__(
      errors=UpperCAmelCase_ , bos_token=UpperCAmelCase_ , eos_token=UpperCAmelCase_ , unk_token=UpperCAmelCase_ , sep_token=UpperCAmelCase_ , cls_token=UpperCAmelCase_ , pad_token=UpperCAmelCase_ , mask_token=UpperCAmelCase_ , add_prefix_space=UpperCAmelCase_ , **UpperCAmelCase_ , )

  with open(UpperCAmelCase_ , encoding='utf-8'				) as vocab_handle:
   UpperCAmelCase__           =       json.load(UpperCAmelCase_				)
  UpperCAmelCase__           =       {v: k for k, v in self.encoder.items()}
  UpperCAmelCase__           =       errors  # how to handle errors in decoding
  UpperCAmelCase__           =       bytes_to_unicode()
  UpperCAmelCase__           =       {v: k for k, v in self.byte_encoder.items()}
  with open(UpperCAmelCase_ , encoding='utf-8'				) as merges_handle:
   UpperCAmelCase__           =       merges_handle.read().split('\n'				)[1:-1]
  UpperCAmelCase__           =       [tuple(merge.split()				) for merge in bpe_merges]
  UpperCAmelCase__           =       dict(zip(UpperCAmelCase_ , range(len(UpperCAmelCase_				)				)				)				)
  UpperCAmelCase__           =       {}
  UpperCAmelCase__           =       add_prefix_space

  # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
  UpperCAmelCase__           =       re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'				)




 @property
 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  return len(self.encoder				)




 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  return dict(self.encoder , **self.added_tokens_encoder				)




 def 						UpperCamelCase__							(self , __a				)				->      Optional[Any]:






  """simple docstring"""





  if token in self.cache:
   return self.cache[token]
  UpperCAmelCase__           =       tuple(UpperCAmelCase_				)
  UpperCAmelCase__           =       get_pairs(UpperCAmelCase_				)

  if not pairs:
   return token

  while True:
   UpperCAmelCase__           =       min(UpperCAmelCase_ , key=lambda __a				: self.bpe_ranks.get(UpperCAmelCase_ , float('inf'				)				)				)
   if bigram not in self.bpe_ranks:
    break
   UpperCAmelCase__           =       bigram
   UpperCAmelCase__           =       []
   UpperCAmelCase__           =       0
   while i < len(UpperCAmelCase_				):
    try:
     UpperCAmelCase__           =       word.index(UpperCAmelCase_ , UpperCAmelCase_				)
    except ValueError:
     new_word.extend(word[i:]				)
     break
    else:
     new_word.extend(word[i:j]				)
     UpperCAmelCase__           =       j

    if word[i] == first and i < len(UpperCAmelCase_				) - 1 and word[i + 1] == second:
     new_word.append(first + second				)
     i += 2
    else:
     new_word.append(word[i]				)
     i += 1
   UpperCAmelCase__           =       tuple(UpperCAmelCase_				)
   UpperCAmelCase__           =       new_word
   if len(UpperCAmelCase_				) == 1:
    break
   else:
    UpperCAmelCase__           =       get_pairs(UpperCAmelCase_				)
  UpperCAmelCase__           =       ' '.join(UpperCAmelCase_				)
  UpperCAmelCase__           =       word
  return word




 def 						UpperCamelCase__							(self , __a				)				->      Tuple:






  """simple docstring"""





  UpperCAmelCase__           =       []
  for token in re.findall(self.pat , UpperCAmelCase_				):
   UpperCAmelCase__           =       ''.join(
       self.byte_encoder[b] for b in token.encode('utf-8'				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
   bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_				).split(' '				)				)
  return bpe_tokens




 def 						UpperCamelCase__							(self , __a				)				->      Tuple:






  """simple docstring"""





  return self.encoder.get(UpperCAmelCase_ , self.encoder.get(self.unk_token				)				)




 def 						UpperCamelCase__							(self , __a				)				->      List[Any]:






  """simple docstring"""





  return self.decoder.get(UpperCAmelCase_				)




 def 						UpperCamelCase__							(self , __a				)				->      Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__           =       ''.join(UpperCAmelCase_				)
  UpperCAmelCase__           =       bytearray([self.byte_decoder[c] for c in text]				).decode('utf-8' , errors=self.errors				)
  return text




 def 						UpperCamelCase__							(self , __a , __a = None				)				->      List[Any]:






  """simple docstring"""





  if not os.path.isdir(UpperCAmelCase_				):
   logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
   return
  UpperCAmelCase__           =       os.path.join(
      UpperCAmelCase_ , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
  UpperCAmelCase__           =       os.path.join(
      UpperCAmelCase_ , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']				)

  with open(UpperCAmelCase_ , 'w' , encoding='utf-8'				) as f:
   f.write(json.dumps(self.encoder , indent=2 , sort_keys=UpperCAmelCase_ , ensure_ascii=UpperCAmelCase_				) + '\n'				)

  UpperCAmelCase__           =       0
  with open(UpperCAmelCase_ , 'w' , encoding='utf-8'				) as writer:
   writer.write('#version: 0.2\n'				)
   for bpe_tokens, token_index in sorted(self.bpe_ranks.items() , key=lambda __a				: kv[1]				):
    if index != token_index:
     logger.warning(
         F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
         ' Please check that the tokenizer is not corrupted!'				)
     UpperCAmelCase__           =       token_index
    writer.write(' '.join(UpperCAmelCase_				) + '\n'				)
    index += 1

  return vocab_file, merge_file




 def 						UpperCamelCase__							(self , __a , __a = None				)				->      Optional[int]:






  """simple docstring"""





  if token_ids_a is None:
   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  UpperCAmelCase__           =       [self.cls_token_id]
  UpperCAmelCase__           =       [self.sep_token_id]
  return cls + token_ids_a + sep + sep + token_ids_a + sep




 def 						UpperCamelCase__							(self , __a , __a = None , __a = False				)				->      List[str]:






  """simple docstring"""





  if already_has_special_tokens:
   return super().get_special_tokens_mask(
       token_ids_a=UpperCAmelCase_ , token_ids_a=UpperCAmelCase_ , already_has_special_tokens=UpperCAmelCase_				)

  if token_ids_a is None:
   return [1] + ([0] * len(UpperCAmelCase_				)) + [1]
  return [1] + ([0] * len(UpperCAmelCase_				)) + [1, 1] + ([0] * len(UpperCAmelCase_				)) + [1]




 def 						UpperCamelCase__							(self , __a , __a = None				)				->      Dict:






  """simple docstring"""





  UpperCAmelCase__           =       [self.sep_token_id]
  UpperCAmelCase__           =       [self.cls_token_id]

  if token_ids_a is None:
   return len(cls + token_ids_a + sep				) * [0]
  return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




 def 						UpperCamelCase__							(self , __a , __a=False , **__a				)				->      Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__           =       kwargs.pop('add_prefix_space' , self.add_prefix_space				)
  if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_				) > 0 and not text[0].isspace()):
   UpperCAmelCase__           =       ' ' + text
  return (text, kwargs)






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


_UpperCamelCase     =    logging.getLogger(__name__)



@dataclass
class 					lowercase		:



 '''simple docstring'''


 __SCREAMING_SNAKE_CASE      							=       field(
     metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(default=_UpperCamelCase				,       metadata={"""help""": """Set this flag to use fast tokenization."""}      )
 # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
 # or just modify its tokenizer_config.json.
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}				,       )







def 		UpperCamelCase_(    ) ->				List[str]:
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.

 UpperCAmelCase__           =       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

 if len(sys.argv     ) == 2 and sys.argv[1].endswith('.json'     ):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
  UpperCAmelCase__       ,					UpperCAmelCase__       ,					UpperCAmelCase__           =       parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
 else:
  UpperCAmelCase__       ,					UpperCAmelCase__       ,					UpperCAmelCase__           =       parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir     )
     and os.listdir(training_args.output_dir     )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      f"Output directory ({training_args.output_dir}) already exists and is not empty. Use"
      ' --overwrite_output_dir to overcome.'     )

 # Setup logging
 logging.basicConfig(
     format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'						,					datefmt='%m/%d/%Y %H:%M:%S'						,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN						,					)
 logger.warning(
     'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'						,					training_args.local_rank						,					training_args.device						,					training_args.n_gpu						,					bool(training_args.local_rank != -1     )						,					training_args.fpaa						,					)
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank     ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info('Training/evaluation parameters %s'						,					snake_case__     )

 # Prepare Question-Answering task
 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 UpperCAmelCase__           =       AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path						,					cache_dir=model_args.cache_dir						,					)
 UpperCAmelCase__           =       AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path						,					cache_dir=model_args.cache_dir						,					use_fast=snake_case__						,					)
 UpperCAmelCase__           =       AutoModelForQuestionAnswering.from_pretrained(
     model_args.model_name_or_path						,					from_tf=bool('.ckpt' in model_args.model_name_or_path     )						,					config=snake_case__						,					cache_dir=model_args.cache_dir						,					)

 # Get datasets
 UpperCAmelCase__           =       hasattr(model.config						,					'lang2id'     )
 UpperCAmelCase__           =       (
     SquadDataset(
         snake_case__						,					tokenizer=snake_case__						,					is_language_sensitive=snake_case__						,					cache_dir=model_args.cache_dir     )
     if training_args.do_train
     else None
 )
 UpperCAmelCase__           =       (
     SquadDataset(
         snake_case__						,					tokenizer=snake_case__						,					mode='dev'						,					is_language_sensitive=snake_case__						,					cache_dir=model_args.cache_dir						,					)
     if training_args.do_eval
     else None
 )

 # Data collator
 UpperCAmelCase__           =       DataCollatorWithPadding(snake_case__						,					pad_to_multiple_of=8     ) if training_args.fpaa else None

 # Initialize our Trainer
 UpperCAmelCase__           =       Trainer(
     model=snake_case__						,					args=snake_case__						,					train_dataset=snake_case__						,					eval_dataset=snake_case__						,					data_collator=snake_case__						,					)

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir     )
def 		UpperCamelCase_(    snake_case__:						Optional[int]     ) ->				Union[str, Any]:
 # For xla_spawn (TPUs)
 main()


if __name__ == "__main__":
 main()



