
'''simple docstring'''


import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__SCREAMING_SNAKE_CASE  =			logging.getLogger(__name__)




@dataclass
class 						__snake_case :
		a_        =				field(
		    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}  )
		a_        =				field(
		    default=_a				, metadata={"help": "Pretrained config name or path if not the same as model_name"}  )
		a_        =				field(
		    default=_a				, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}  )
		a_        =				field(default=_a				, metadata={"help": "Set this flag to use fast tokenization."}  )
		# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
		# or just modify its tokenizer_config.json.
		a_        =				field(
		    default=_a				, metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}				, )







def 				__magic_name__(					):
						# See all possible arguments in src/transformers/training_args.py
						# or by passing the --help flag to this script.
						# We now keep distinct sets of args, for a cleaner separation of concerns.

						UpperCamelCase_    :  Optional[Any]				    =    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)  )

						if len(sys.argv  ) == 2 and sys.argv[1].endswith(""".json"""  ):
												# If we pass only one argument to the script and it's the path to a json file,
												# let's parse it to get our arguments.
												UpperCamelCase_    ,UpperCamelCase_    ,UpperCamelCase_    :  List[Any]				    =    parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]  )  )
						else:
												UpperCamelCase_    ,UpperCamelCase_    ,UpperCamelCase_    :  List[Any]				    =    parser.parse_args_into_dataclasses()

						if (
						    os.path.exists(training_args.output_dir  )
						    and os.listdir(training_args.output_dir  )
						    and training_args.do_train
						    and not training_args.overwrite_output_dir
						):
												raise ValueError(
												    f"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
												    """ --overwrite_output_dir to overcome."""  )

						# Setup logging
						logging.basicConfig(
						    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""       ,							datefmt="""%m/%d/%Y %H:%M:%S"""       ,							level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,							)
						logger.warning(
						    """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""       ,							training_args.local_rank       ,							training_args.device       ,							training_args.n_gpu       ,							bool(training_args.local_rank != -1  )       ,							training_args.fpaa       ,							)
						# Set the verbosity to info of the Transformers logger (on main process only):
						if is_main_process(training_args.local_rank  ):
												transformers.utils.logging.set_verbosity_info()
												transformers.utils.logging.enable_default_handler()
												transformers.utils.logging.enable_explicit_format()
						logger.info("""Training/evaluation parameters %s"""       ,							UpperCAmelCase__  )

						# Prepare Question-Answering task
						# Load pretrained model and tokenizer
						#
						# Distributed training:
						# The .from_pretrained methods guarantee that only one local process can concurrently
						# download model & vocab.

						UpperCamelCase_    :  str				    =    AutoConfig.from_pretrained(
						    model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,							cache_dir=model_args.cache_dir       ,							)
						UpperCamelCase_    :  Optional[int]				    =    AutoTokenizer.from_pretrained(
						    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,							cache_dir=model_args.cache_dir       ,							use_fast=UpperCAmelCase__       ,							)
						UpperCamelCase_    :  Tuple				    =    AutoModelForQuestionAnswering.from_pretrained(
						    model_args.model_name_or_path       ,							from_tf=bool(""".ckpt""" in model_args.model_name_or_path  )       ,							config=UpperCAmelCase__       ,							cache_dir=model_args.cache_dir       ,							)

						# Get datasets
						UpperCamelCase_    :  List[Any]				    =    hasattr(model.config       ,							"""lang2id"""  )
						UpperCamelCase_    :  List[Any]				    =    (
						    SquadDataset(
						        UpperCAmelCase__       ,							tokenizer=UpperCAmelCase__       ,							is_language_sensitive=UpperCAmelCase__       ,							cache_dir=model_args.cache_dir  )
						    if training_args.do_train
						    else None
						)
						UpperCamelCase_    :  List[str]				    =    (
						    SquadDataset(
						        UpperCAmelCase__       ,							tokenizer=UpperCAmelCase__       ,							mode="""dev"""       ,							is_language_sensitive=UpperCAmelCase__       ,							cache_dir=model_args.cache_dir       ,							)
						    if training_args.do_eval
						    else None
						)

						# Data collator
						UpperCamelCase_    :  Dict				    =    DataCollatorWithPadding(UpperCAmelCase__       ,							pad_to_multiple_of=8  ) if training_args.fpaa else None

						# Initialize our Trainer
						UpperCamelCase_    :  str				    =    Trainer(
						    model=UpperCAmelCase__       ,							args=UpperCAmelCase__       ,							train_dataset=UpperCAmelCase__       ,							eval_dataset=UpperCAmelCase__       ,							data_collator=UpperCAmelCase__       ,							)

						# Training
						if training_args.do_train:
												trainer.train(
												    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path  ) else None  )
												trainer.save_model()
												# For convenience, we also re-save the tokenizer to the same directory,
												# so that you can share your model easily on huggingface.co/models =)
												if trainer.is_world_master():
																		tokenizer.save_pretrained(training_args.output_dir  )
def 				__magic_name__(					UpperCAmelCase__   :					List[str]  ):
						# For xla_spawn (TPUs)
						main()


if __name__ == "__main__":
	main()






'''simple docstring'''


import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
__SCREAMING_SNAKE_CASE  =			"src/transformers"

# Matches is_xxx_available()
__SCREAMING_SNAKE_CASE  =			re.compile(r"is\_([a-z_]*)_available()")
# Matches from xxx import bla
__SCREAMING_SNAKE_CASE  =			re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
__SCREAMING_SNAKE_CASE  =			re.compile(r"^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)")


__SCREAMING_SNAKE_CASE  =			"\n{0} = None\n"

__SCREAMING_SNAKE_CASE  =			"\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n"


__SCREAMING_SNAKE_CASE  =			"\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n"
def 				__magic_name__(					UpperCAmelCase__   :					List[Any]  ):
						if _re_test_backend.search(UpperCAmelCase__  ) is None:
												return None
						UpperCamelCase_    :  str				    =    [b[0] for b in _re_backend.findall(UpperCAmelCase__  )]
						backends.sort()
						return "_and_".join(UpperCAmelCase__  )
def 				__magic_name__(					):
						with open(os.path.join(UpperCAmelCase__       ,							"""__init__.py"""  )       ,							"""r"""       ,							encoding="""utf-8"""       ,							newline="""\n"""  ) as f:
												UpperCamelCase_    :  Dict				    =    f.readlines()

						# Get to the point we do the actual imports for type checking
						UpperCamelCase_    :  Tuple				    =    0
						while not lines[line_index].startswith("""if TYPE_CHECKING"""  ):
												line_index += 1

						UpperCamelCase_    :  Optional[int]				    =    {}
						# Go through the end of the file
						while line_index < len(UpperCAmelCase__  ):
												# If the line is an if is_backend_available, we grab all objects associated.
												UpperCamelCase_    :  str				    =    find_backend(lines[line_index]  )
												if backend is not None:
																		while not lines[line_index].startswith("""    else:"""  ):
																								line_index += 1
																		line_index += 1

																		UpperCamelCase_    :  Dict				    =    []
																		# Until we unindent, add backend objects to the list
																		while len(lines[line_index]  ) <= 1 or lines[line_index].startswith(""" """ * 8  ):
																								UpperCamelCase_    :  Any				    =    lines[line_index]
																								UpperCamelCase_    :  str				    =    _re_single_line_import.search(UpperCAmelCase__  )
																								if single_line_import_search is not None:
																														objects.extend(single_line_import_search.groups()[0].split(""", """  )  )
																								elif line.startswith(""" """ * 12  ):
																														objects.append(line[12:-2]  )
																								line_index += 1

																		UpperCamelCase_    :  List[Any]				    =    objects
												else:
																		line_index += 1

						return backend_specific_objects
def 				__magic_name__(					UpperCAmelCase__   :					Any       ,							UpperCAmelCase__   :					Optional[Any]  ):
						if name.isupper():
												return DUMMY_CONSTANT.format(UpperCAmelCase__  )
						elif name.islower():
												return DUMMY_FUNCTION.format(UpperCAmelCase__       ,							UpperCAmelCase__  )
						else:
												return DUMMY_CLASS.format(UpperCAmelCase__       ,							UpperCAmelCase__  )
def 				__magic_name__(					UpperCAmelCase__   :					List[Any]=None  ):
						if backend_specific_objects is None:
												UpperCamelCase_    :  Optional[Any]				    =    read_init()
						# For special correspondence backend to module name as used in the function requires_modulename
						UpperCamelCase_    :  List[str]				    =    {}

						for backend, objects in backend_specific_objects.items():
												UpperCamelCase_    :  Optional[Any]				    =    """[""" + """, """.join(f"""\"{b}\"""" for b in backend.split("""_and_"""  )  ) + """]"""
												UpperCamelCase_    :  Union[str, Any]				    =    """# This file is autogenerated by the command `make fix-copies`, do not edit.\n"""
												dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
												dummy_file += "\n".join([create_dummy_object(UpperCAmelCase__       ,							UpperCAmelCase__  ) for o in objects]  )
												UpperCamelCase_    :  List[Any]				    =    dummy_file

						return dummy_files
def 				__magic_name__(					UpperCAmelCase__   :					List[Any]=False  ):
						UpperCamelCase_    :  int				    =    create_dummy_files()
						# For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
						UpperCamelCase_    :  Union[str, Any]				    =    {"""torch""": """pt"""}

						# Locate actual dummy modules and read their content.
						UpperCamelCase_    :  Optional[int]				    =    os.path.join(UpperCAmelCase__       ,							"""utils"""  )
						UpperCamelCase_    :  str				    =    {
						    backend: os.path.join(UpperCAmelCase__       ,							f"""dummy_{short_names.get(UpperCAmelCase__       ,							UpperCAmelCase__  )}_objects.py"""  )
						    for backend in dummy_files.keys()
						}

						UpperCamelCase_    :  int				    =    {}
						for backend, file_path in dummy_file_paths.items():
												if os.path.isfile(UpperCAmelCase__  ):
																		with open(UpperCAmelCase__       ,							"""r"""       ,							encoding="""utf-8"""       ,							newline="""\n"""  ) as f:
																								UpperCamelCase_    :  str				    =    f.read()
												else:
																		UpperCamelCase_    :  str				    =    """"""

						for backend in dummy_files.keys():
												if dummy_files[backend] != actual_dummies[backend]:
																		if overwrite:
																								print(
																								    f"""Updating transformers.utils.dummy_{short_names.get(UpperCAmelCase__       ,							UpperCAmelCase__  )}_objects.py as the main """
																								    """__init__ has new objects."""  )
																								with open(dummy_file_paths[backend]       ,							"""w"""       ,							encoding="""utf-8"""       ,							newline="""\n"""  ) as f:
																														f.write(dummy_files[backend]  )
																		else:
																								raise ValueError(
																								    """The main __init__ has objects that are not present in """
																								    f"""transformers.utils.dummy_{short_names.get(UpperCAmelCase__       ,							UpperCAmelCase__  )}_objects.py. Run `make fix-copies` """
																								    """to fix this."""  )


if __name__ == "__main__":
	__SCREAMING_SNAKE_CASE  =			argparse.ArgumentParser()
	parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
	__SCREAMING_SNAKE_CASE  =			parser.parse_args()

	check_dummies(args.fix_and_overwrite)




