




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowerCAmelCase						:							Dict			     =						logging.get_logger(__name__)

__lowerCAmelCase						:							Dict			     =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__lowerCAmelCase						:							Tuple			     =						{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

__lowerCAmelCase						:							List[Any]			     =						{
    'Salesforce/codegen-350M-mono': 2048,
}




@lru_cache()
def      __magic_name__							(					):






			'''simple docstring'''







			a       =     (
			    list(range(ord("!"  ),    ord("~"  ) + 1  )  ) + list(range(ord("¡"  ),    ord("¬"  ) + 1  )  ) + list(range(ord("®"  ),    ord("ÿ"  ) + 1  )  )
			)
			a       =     bs[:]
			a       =     0
			for b in range(2**8  ):
						if b not in bs:
									bs.append(lowerCamelCase__  )
									cs.append(2**8 + n  )
									n += 1
			a       =     [chr(lowerCamelCase__  ) for n in cs]
			return dict(zip(lowerCamelCase__,    lowerCamelCase__  )  )




def      __magic_name__							(					A						:   Union[str, Any]  ):






			'''simple docstring'''







			a       =     set()
			a       =     word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)  )
						a       =     char
			return pairs






class        snake_case__				(_UpperCamelCase  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				Dict									=   VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_			:				List[str]									=   PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_			:				List[str]									=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_			:				Union[str, Any]									=   ["""input_ids""", """attention_mask"""]





				def __init__( self							:	Tuple   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	List[Any]="replace"   , __lowerCamelCase							:	Any="<|endoftext|>"   , __lowerCamelCase							:	Tuple="<|endoftext|>"   , __lowerCamelCase							:	str="<|endoftext|>"   , __lowerCamelCase							:	List[Any]=None   , __lowerCamelCase							:	List[str]=False   , __lowerCamelCase							:	List[Any]=False   , **__lowerCamelCase							:	List[str]   , )   ->	Optional[int]:
							a       =     AddedToken(__lowerCamelCase   , lstrip=__lowerCamelCase   , rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase   , __lowerCamelCase				) else bos_token
							a       =     AddedToken(__lowerCamelCase   , lstrip=__lowerCamelCase   , rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase   , __lowerCamelCase				) else eos_token
							a       =     AddedToken(__lowerCamelCase   , lstrip=__lowerCamelCase   , rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase   , __lowerCamelCase				) else unk_token
							a       =     AddedToken(__lowerCamelCase   , lstrip=__lowerCamelCase   , rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase   , __lowerCamelCase				) else pad_token
							super().__init__(
							    errors=__lowerCamelCase   , unk_token=__lowerCamelCase   , bos_token=__lowerCamelCase   , eos_token=__lowerCamelCase   , pad_token=__lowerCamelCase   , add_prefix_space=__lowerCamelCase   , add_bos_token=__lowerCamelCase   , **__lowerCamelCase   , )
							a       =     add_bos_token

							with open(__lowerCamelCase   , encoding="utf-8"				) as vocab_handle:
										a       =     json.load(__lowerCamelCase				)
							a       =     {v: k for k, v in self.encoder.items()}
							a       =     errors  # how to handle errors in decoding
							a       =     bytes_to_unicode()
							a       =     {v: k for k, v in self.byte_encoder.items()}
							with open(__lowerCamelCase   , encoding="utf-8"				) as merges_handle:
										a       =     merges_handle.read().split("\n"				)[1:-1]
							a       =     [tuple(merge.split()				) for merge in bpe_merges]
							a       =     dict(zip(__lowerCamelCase   , range(len(__lowerCamelCase				)				)				)				)
							a       =     {}
							a       =     add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							a       =     re.compile(r"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"				)





				@property
				def 				__UpperCAmelCase				( self							:	int				)   ->	Any:
							return len(self.encoder				)





				def 				__UpperCAmelCase				( self							:	Dict				)   ->	List[str]:
							return dict(self.encoder   , **self.added_tokens_encoder				)





				def 				__UpperCAmelCase				( self							:	int   , __lowerCamelCase							:	Tuple				)   ->	Optional[int]:

							if token in self.cache:
										return self.cache[token]
							a       =     tuple(__lowerCamelCase				)
							a       =     get_pairs(__lowerCamelCase				)

							if not pairs:
										return token

							while True:
										a       =     min(__lowerCamelCase   , key=lambda __lowerCamelCase				: self.bpe_ranks.get(__lowerCamelCase   , float("inf"				)				)				)
										if bigram not in self.bpe_ranks:
													break
										a       ,						a       =     bigram
										a       =     []
										a       =     0
										while i < len(__lowerCamelCase				):
													try:
																a       =     word.index(__lowerCamelCase   , __lowerCamelCase				)
													except ValueError:
																new_word.extend(word[i:]				)
																break
													else:
																new_word.extend(word[i:j]				)
																a       =     j

													if word[i] == first and i < len(__lowerCamelCase				) - 1 and word[i + 1] == second:
																new_word.append(first + second				)
																i += 2
													else:
																new_word.append(word[i]				)
																i += 1
										a       =     tuple(__lowerCamelCase				)
										a       =     new_word
										if len(__lowerCamelCase				) == 1:
													break
										else:
													a       =     get_pairs(__lowerCamelCase				)
							a       =     " ".join(__lowerCamelCase				)
							a       =     word
							return word





				def 				__UpperCAmelCase				( self							:	Optional[Any]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Optional[Any]=None				)   ->	List[Any]:

							if self.add_bos_token:
										a       =     [self.bos_token_id]
							else:
										a       =     []

							a       =     bos_token_ids + token_ids_a

							if token_ids_a is None:
										return output

							return output + bos_token_ids + token_ids_a





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Any				)   ->	List[str]:
							a       =     []
							for token in re.findall(self.pat   , __lowerCamelCase				):
										a       =     "".join(
										    self.byte_encoder[b] for b in token.encode("utf-8"				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
										bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase				).split(" "				)				)
							return bpe_tokens





				def 				__UpperCAmelCase				( self							:	Optional[int]   , __lowerCamelCase							:	int				)   ->	Tuple:
							return self.encoder.get(__lowerCamelCase   , self.encoder.get(self.unk_token				)				)





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Tuple				)   ->	Union[str, Any]:
							return self.decoder.get(__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Tuple   , __lowerCamelCase							:	Optional[Any]				)   ->	Tuple:
							a       =     "".join(__lowerCamelCase				)
							a       =     bytearray([self.byte_decoder[c] for c in text]				).decode("utf-8"   , errors=self.errors				)
							return text





				def 				__UpperCAmelCase				( self							:	List[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[str] = None				)   ->	Tuple[str]:

							if not os.path.isdir(__lowerCamelCase				):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
										return
							a       =     os.path.join(
							    __lowerCamelCase   , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)
							a       =     os.path.join(
							    __lowerCamelCase   , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]				)

							with open(__lowerCamelCase   , "w"   , encoding="utf-8"				) as f:
										f.write(json.dumps(self.encoder   , indent=2   , sort_keys=__lowerCamelCase   , ensure_ascii=__lowerCamelCase				) + "\n"				)

							a       =     0
							with open(__lowerCamelCase   , "w"   , encoding="utf-8"				) as writer:
										writer.write("#version: 0.2\n"				)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   , key=lambda __lowerCamelCase				: kv[1]				):
													if index != token_index:
																logger.warning(
																    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																    " Please check that the tokenizer is not corrupted!"				)
																a       =     token_index
													writer.write(" ".join(__lowerCamelCase				) + "\n"				)
													index += 1

							return vocab_file, merge_file





				def 				__UpperCAmelCase				( self							:	Union[str, Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Any=False   , **__lowerCamelCase							:	List[Any]				)   ->	Optional[Any]:
							a       =     kwargs.pop("add_prefix_space"   , self.add_prefix_space				)
							if is_split_into_words or add_prefix_space:
										a       =     " " + text
							return (text, kwargs)





				def 				__UpperCAmelCase				( self							:	List[str]   , __lowerCamelCase							:	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   , __lowerCamelCase							:	bool = False   , __lowerCamelCase							:	bool = None   , __lowerCamelCase							:	Optional[List[str]] = None   , **__lowerCamelCase							:	Union[str, Any]   , )   ->	str:
							a       =     super()._decode(
							    token_ids=__lowerCamelCase   , skip_special_tokens=__lowerCamelCase   , clean_up_tokenization_spaces=__lowerCamelCase   , **__lowerCamelCase   , )

							if truncate_before_pattern is not None and len(__lowerCamelCase				) > 0:
										a       =     self.truncate(__lowerCamelCase   , __lowerCamelCase				)

							return decoded_text





				def 				__UpperCAmelCase				( self							:	List[str]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	int				)   ->	List[Any]:

							def find_re(__lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Any				):
										a       =     pattern.search(__lowerCamelCase   , __lowerCamelCase				)
										return m.start() if m else -1

							a       =     [re.compile(__lowerCamelCase   , re.MULTILINE				) for pattern in truncate_before_pattern]

							a       =     list(re.finditer("^print"   , __lowerCamelCase   , re.MULTILINE				)				)

							if len(__lowerCamelCase				) > 1:
										a       =     completion[: prints[1].start()]

							a       =     list(re.finditer("^def"   , __lowerCamelCase   , re.MULTILINE				)				)

							if len(__lowerCamelCase				) > 1:
										a       =     completion[: defs[1].start()]

							a       =     0

							a       =     [
							    pos for pos in [find_re(__lowerCamelCase   , __lowerCamelCase   , __lowerCamelCase				) for terminal in terminals] if pos != -1
							]

							if len(__lowerCamelCase				) > 0:
										return completion[: min(__lowerCamelCase				)]
							else:
										return completion











"""simple docstring"""







import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


A     =      logging.getLogger(__name__)






@dataclass
class   UpperCAmelCase__		:
		lowerCAmelCase_							:						str							=  field(
		    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}  )
		lowerCAmelCase_							:						Optional[str]							=  field(
		    default=UpperCamelCase ,metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}  )
		lowerCAmelCase_							:						Optional[str]							=  field(
		    default=UpperCamelCase ,metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}  )
		lowerCAmelCase_							:						bool							=  field(default=UpperCamelCase ,metadata={"""help""": """Set this flag to use fast tokenization."""}  )
		# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
		# or just modify its tokenizer_config.json.
		lowerCAmelCase_							:						Optional[str]							=  field(
		    default=UpperCamelCase ,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""} ,)



def 	lowerCAmelCase__      (  )			->				Optional[Any]:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							A    =       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)      )

							if len(sys.argv      ) == 2 and sys.argv[1].endswith('.json'      ):
														# If we pass only one argument to the script and it's the path to a json file,
														# let's parse it to get our arguments.
														A   ,	A   ,	A    =       parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]      )      )
							else:
														A   ,	A   ,	A    =       parser.parse_args_into_dataclasses()

							if (
							    os.path.exists(training_args.output_dir      )
							    and os.listdir(training_args.output_dir      )
							    and training_args.do_train
							    and not training_args.overwrite_output_dir
							):
														raise ValueError(
														    f"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
														    ' --overwrite_output_dir to overcome.'      )

							# Setup logging
							logging.basicConfig(
							    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'					,   datefmt='%m/%d/%Y %H:%M:%S'					,   level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN					,   )
							logger.warning(
							    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'					,   training_args.local_rank					,   training_args.device					,   training_args.n_gpu					,   bool(training_args.local_rank != -1      )					,   training_args.fpaa					,   )
							# Set the verbosity to info of the Transformers logger (on main process only):
							if is_main_process(training_args.local_rank      ):
														transformers.utils.logging.set_verbosity_info()
														transformers.utils.logging.enable_default_handler()
														transformers.utils.logging.enable_explicit_format()
							logger.info('Training/evaluation parameters %s'					,   lowerCamelCase__      )

							# Prepare Question-Answering task
							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.

							A    =       AutoConfig.from_pretrained(
							    model_args.config_name if model_args.config_name else model_args.model_name_or_path					,   cache_dir=model_args.cache_dir					,   )
							A    =       AutoTokenizer.from_pretrained(
							    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path					,   cache_dir=model_args.cache_dir					,   use_fast=lowerCamelCase__					,   )
							A    =       AutoModelForQuestionAnswering.from_pretrained(
							    model_args.model_name_or_path					,   from_tf=bool('.ckpt' in model_args.model_name_or_path      )					,   config=lowerCamelCase__					,   cache_dir=model_args.cache_dir					,   )

							# Get datasets
							A    =       hasattr(model.config					,   'lang2id'      )
							A    =       (
							    SquadDataset(
							        lowerCamelCase__					,   tokenizer=lowerCamelCase__					,   is_language_sensitive=lowerCamelCase__					,   cache_dir=model_args.cache_dir      )
							    if training_args.do_train
							    else None
							)
							A    =       (
							    SquadDataset(
							        lowerCamelCase__					,   tokenizer=lowerCamelCase__					,   mode='dev'					,   is_language_sensitive=lowerCamelCase__					,   cache_dir=model_args.cache_dir					,   )
							    if training_args.do_eval
							    else None
							)

							# Data collator
							A    =       DataCollatorWithPadding(lowerCamelCase__					,   pad_to_multiple_of=8      ) if training_args.fpaa else None

							# Initialize our Trainer
							A    =       Trainer(
							    model=lowerCamelCase__					,   args=lowerCamelCase__					,   train_dataset=lowerCamelCase__					,   eval_dataset=lowerCamelCase__					,   data_collator=lowerCamelCase__					,   )

							# Training
							if training_args.do_train:
														trainer.train(
														    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path      ) else None      )
														trainer.save_model()
														# For convenience, we also re-save the tokenizer to the same directory,
														# so that you can share your model easily on huggingface.co/models =)
														if trainer.is_world_master():
																					tokenizer.save_pretrained(training_args.output_dir      )

def 	lowerCAmelCase__      (  lowerCamelCase__      )			->				List[Any]:
							# For xla_spawn (TPUs)
							main()


if __name__ == "__main__":
							main()



