





import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
				from tests.trainer.test_trainer import (  # noqa
				    RegressionModelConfig,
				    RegressionPreTrainedModel,
				    get_regression_trainer,
				)


set_seed(42)

__A       :					List[str]     					=     get_tests_dir("fixtures")
__A       :					List[str]     					=     os.path.join(dirname(get_tests_dir()))
__A       :					List[Any]     					=     dirname(os.path.abspath(__file__))

# default torch.distributed port
__A       :					Dict     					=     "10999"

__A       :					Optional[int]     					=     "t5-small"

# *** Working Models ***
__A       :					Tuple     					=     "hf-internal-testing/tiny-albert"
__A       :					Union[str, Any]     					=     "sshleifer/bart-tiny-random"
__A       :					Tuple     					=     "hf-internal-testing/tiny-bert"
__A       :					Dict     					=     "hf-internal-testing/tiny-random-bigbird_pegasus"
__A       :					Any     					=     "hf-internal-testing/tiny-random-big_bird"
__A       :					Dict     					=     "hf-internal-testing/tiny-random-blenderbot"
__A       :					Dict     					=     "bigscience/bigscience-small-testing"
__A       :					Optional[Any]     					=     "hf-internal-testing/tiny-random-deberta"
__A       :					Dict     					=     "hf-internal-testing/tiny-random-deberta-v2"
__A       :					Optional[Any]     					=     "sshleifer/tiny-distilbert-base-cased"
__A       :					Tuple     					=     "hf-internal-testing/tiny-electra"
__A       :					List[Any]     					=     "hf-internal-testing/tiny-random-flaubert"
__A       :					Dict     					=     "stas/tiny-wmt19-en-de"
__A       :					Optional[Any]     					=     "hf-internal-testing/tiny-random-funnel"
__A       :					List[Any]     					=     "sshleifer/tiny-gpt2"
__A       :					Any     					=     "hf-internal-testing/tiny-random-gptj"
__A       :					str     					=     "hf-internal-testing/tiny-random-gpt_neo"
__A       :					List[str]     					=     "hf-internal-testing/tiny-layoutlm"
__A       :					str     					=     "hf-internal-testing/tiny-random-led"
__A       :					int     					=     "hf-internal-testing/tiny-random-longformer"
__A       :					Optional[Any]     					=     "stas/tiny-m2m_100"  # hf tiny model is unsuitable
__A       :					Union[str, Any]     					=     "sshleifer/tiny-marian-en-de"
__A       :					Dict     					=     "sshleifer/tiny-mbart"
__A       :					Union[str, Any]     					=     "hf-internal-testing/tiny-random-mobilebert"
__A       :					List[Any]     					=     "hf-internal-testing/tiny-random-mpnet"
__A       :					List[Any]     					=     "stas/pegasus-cnn_dailymail-tiny-random"
__A       :					Tuple     					=     "hf-internal-testing/tiny-random-prophetnet"
__A       :					Optional[Any]     					=     "sshleifer/tiny-distilroberta-base"
__A       :					List[Any]     					=     "hf-internal-testing/tiny-random-squeezebert"
__A       :					Optional[Any]     					=     "patrickvonplaten/t5-tiny-random"
__A       :					Any     					=     "hf-internal-testing/tiny-random-t5-v1.1"
__A       :					List[Any]     					=     "hf-internal-testing/tiny-random-vit"
__A       :					Optional[Any]     					=     "hf-internal-testing/tiny-xlm-roberta"
__A       :					Optional[int]     					=     "sshleifer/tiny-xlnet-base-cased"


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
__A       :					Optional[Any]     					=     "hf-internal-testing/tiny-random-mt5"
__A       :					Dict     					=     "hf-internal-testing/tiny-random-camembert"
__A       :					Optional[int]     					=     "hf-internal-testing/tiny-random-openai-gpt"

# missing tokenizer files
__A       :					str     					=     "hf-internal-testing/tiny-random-convbert"
__A       :					List[str]     					=     "hf-internal-testing/tiny-random-layoutlmv2"
__A       :					str     					=     "hf-internal-testing/tiny-random-hubert"

# issues with tokenizer
__A       :					str     					=     "hf-internal-testing/tiny-random-ctrl"
__A       :					Dict     					=     "hf-internal-testing/tiny-random-transfo-xl"  # same as ctrl

# other issues with tiny models
__A       :					Any     					=     "hf-internal-testing/tiny-random-ibert"  # multiple issues with either mlm/qa/clas
__A       :					Dict     					=     "hf-internal-testing/tiny-random-reformer"  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
__A       :					str     					=     "hf-internal-testing/tiny-random-dpr"
# - "dpr"  examples/research_projects/rag-end2end-retriever/
__A       :					Union[str, Any]     					=     "hf-internal-testing/tiny-random-rag"
# - "rag" research_projects
__A       :					Optional[int]     					=     ""
# - "luke" Entities classes - no plan to make such example
__A       :					Optional[int]     					=     "hf-internal-testing/tiny-random-lxmert"
# - "lxmert" doesn't work with run_qa.py
__A       :					Union[str, Any]     					=     "hf-internal-testing/tiny-random-clip"
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
__A       :					int     					=     "hf-internal-testing/tiny-random-speech_to_text"
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
__A       :					Optional[int]     					=     "hf-internal-testing/tiny-random-tapas"





def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__=False				)						->       Tuple:







			'''simple docstring'''

			UpperCAmelCase     	=       min(2    ,    get_gpu_count()				) if distributed else 1
			UpperCAmelCase     	=       os.environ.get('''DS_TEST_PORT'''    ,    UpperCamelCase__				)
			return F"""deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}""".split()





def 		__SCREAMING_SNAKE_CASE						(       )						->       List[Any]:







			'''simple docstring'''

			UpperCAmelCase     	=       F"""{FIXTURE_DIRECTORY}/tests_samples"""
			UpperCAmelCase     	=       F"""{data_dir_samples}/wmt_en_ro"""
			UpperCAmelCase     	=       F"""{data_dir_samples}/xsum"""
			UpperCAmelCase     	=       '''
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        '''.split()

			# try to cover as many models as possible once (it's enough to run on one task per model)
			# but need a tiny model for each
			#
			# should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
			UpperCAmelCase     	=       {
			    '''trans''': [
			        '''bart''',
			        '''fsmt''',
			        '''m2m_100''',
			        '''marian''',
			        '''mbart''',
			        '''t5''',
			        '''t5_v1''',
			        # "mt5", missing model files
			    ],
			    '''sum''': [
			        '''pegasus''',
			    ],
			    '''clm''': [
			        '''big_bird''',
			        '''bigbird_pegasus''',
			        '''blenderbot''',
			        '''bloom''',
			        '''gpt2''',
			        '''gpt_neo''',
			        '''gptj''',
			        '''xlm-roberta''',
			        '''prophetnet''',
			        # "camembert", missing model files
			    ],
			    '''mlm''': [
			        '''albert''',
			        '''deberta''',
			        '''deberta-v2''',
			        '''distilbert''',
			        '''electra''',
			        '''flaubert''',
			        '''funnel''',
			        '''layoutlm''',
			        # "reformer", # multiple issues with either mlm/qa/clas
			    ],
			    '''qa''': [
			        '''led''',
			        '''longformer''',
			        '''mobilebert''',
			        '''mpnet''',
			        '''roberta''',
			        '''squeezebert''',
			        # "convbert", # missing tokenizer files
			        # "layoutlmv2", missing model files
			    ],
			    '''clas''': [
			        '''bert''',
			        '''xlnet''',
			        # "hubert", # missing tokenizer files
			        # "ibert", # multiple issues with either mlm/qa/clas
			        # "transfo-xl", # tokenizer issues as ctrl
			        # "ctrl", # tokenizer issues
			        # "openai-gpt", missing model files
			        # "tapas", multiple issues
			    ],
			    '''img_clas''': [
			        '''vit''',
			    ],
			}

			UpperCAmelCase     	=       F"""{ROOT_DIRECTORY}/examples/pytorch"""

			UpperCAmelCase     	=       {
			    '''trans''': F"""
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        """,
			    '''sum''': F"""
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        """,
			    '''clm''': F"""
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        """,
			    '''mlm''': F"""
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        """,
			    '''qa''': F"""
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        """,
			    '''clas''': F"""
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        """,
			    '''img_clas''': F"""
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        """,
			}

			UpperCAmelCase     	=       get_launcher(distributed=UpperCamelCase__				)

			UpperCAmelCase     	=       {}
			for task, args in tasks.items():
						UpperCAmelCase     	=       args.split()
						for model in tasksamodels[task]:
									UpperCAmelCase     	=       globals()[F"""{model.upper().replace("-"    ,    "_"				)}_TINY"""]
									UpperCAmelCase     	=       F"""--model_name_or_path {model_name}""".split()
									UpperCAmelCase     	=       launcher + args + args_model + args_main

									# # generation special case
									# if task == "gen":
									#     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
									#     args_model += f"--model_type {model}".split()
									#     cmds[f"{task}_{model}"] = launcher + args + args_model
									# else:

			return cmds


__A       :					Optional[Any]     					=     make_task_cmds()

__A       :					Dict     					=     "zero2"
__A       :					int     					=     "zero3"

__A       :					Union[str, Any]     					=     [ZEROa, ZEROa]





def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__    ,    UpperCamelCase__				)						->       Optional[Any]:







			'''simple docstring'''

			UpperCAmelCase     	=       parameterized.to_safe_name('''_'''.join(str(UpperCamelCase__				) for x in param.args				)				)
			return F"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
__A       :					Union[str, Any]     					=     list(itertools.product(stages, task_cmds.keys()))






@slow
@require_deepspeed
@require_torch_gpu
class 			A_ (a_    ):
	def   _lowercase       (					self							,      _A							,      _A	):






				'''simple docstring'''







				if task not in task_cmds:
							raise ValueError(F"""don't know of task {task}, have {task_cmds.keys()}"""	)

				UpperCAmelCase     	=       task_cmds[task]
				UpperCAmelCase     	=       F"""--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json""".split()

				UpperCAmelCase     	=       self.get_auto_remove_tmp_dir()
				UpperCAmelCase     	=       F"""--output_dir {output_dir}""".split()

				cmd += args_ds + args_out

				return cmd, output_dir







	@parameterized.expand(_A							,      name_func=_A	)
	def   _lowercase       (					self							,      _A							,      _A	):






				'''simple docstring'''







				UpperCAmelCase	,    UpperCAmelCase     	=       self.get_task_cmd(_A							,      _A	)

				# 1. generate the checkpoint
				cmd += "--save_steps 1".split()
				# keep for quick debug
				# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
				execute_subprocess_async(_A							,      env=self.get_env()	)

				# 2. test that the fp32 weights get reconsolidated
				UpperCAmelCase     	=       F"""{output_dir}/checkpoint-1"""
				UpperCAmelCase     	=       F"""{chkpt_dir}/out.bin"""
				UpperCAmelCase     	=       F"""{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"""
				# keep for quick debug
				# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
				subprocess.check_call(_A							,      shell=_A	)
				assert os.path.exists(_A	), F"""{recovered_model_path} was not found"""

				# possibly could also test that the resulting saved model is usable but given that we use
				# random models we won't know if it's any good










import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer

class 			A_ (tf.keras.layers.Layer    ):
	def __init__(					self							,      _A							,      _A							,      _A = None							,      _A = None							,      _A = None							,      _A = "longest"							,      _A = True							,      _A = 5_1_2							,      _A = None							,      _A = True							,      _A = True							,      _A = True							,      **_A							,      ):






				'''simple docstring'''







				super().__init__()
				if use_fast_bert_tokenizer:
							UpperCAmelCase     	=       FastBertTokenizer(
							    _A							,      token_out_type=tf.intaa							,      lower_case_nfd_strip_accents=_A							,      **_A	)
				else:
							UpperCAmelCase     	=       tf.lookup.StaticVocabularyTable(
							    tf.lookup.KeyValueTensorInitializer(
							        keys=_A							,      key_dtype=tf.string							,      values=tf.range(tf.size(_A							,      out_type=tf.intaa	)							,      dtype=tf.intaa	)							,      value_dtype=tf.intaa							,      )							,      num_oov_buckets=1							,      )
							UpperCAmelCase     	=       BertTokenizerLayer(
							    _A							,      token_out_type=tf.intaa							,      lower_case=_A							,      **_A	)

				UpperCAmelCase     	=       vocab_list
				UpperCAmelCase     	=       do_lower_case
				UpperCAmelCase     	=       cls_token_id or vocab_list.index('''[CLS]'''	)
				UpperCAmelCase     	=       sep_token_id or vocab_list.index('''[SEP]'''	)
				UpperCAmelCase     	=       pad_token_id or vocab_list.index('''[PAD]'''	)
				UpperCAmelCase     	=       ShrinkLongestTrimmer(max_length - 3							,      axis=1	)  # Allow room for special tokens
				UpperCAmelCase     	=       max_length
				UpperCAmelCase     	=       padding
				UpperCAmelCase     	=       truncation
				UpperCAmelCase     	=       pad_to_multiple_of
				UpperCAmelCase     	=       return_token_type_ids
				UpperCAmelCase     	=       return_attention_mask
	@classmethod
	def   _lowercase       (					cls							,      _A							,      **_A	):  # noqa: F821






				'''simple docstring'''







				UpperCAmelCase     	=       kwargs.pop('''do_lower_case'''							,      _A	)
				UpperCAmelCase     	=       tokenizer.do_lower_case if do_lower_case is None else do_lower_case
				UpperCAmelCase     	=       kwargs.pop('''cls_token_id'''							,      _A	)
				UpperCAmelCase     	=       tokenizer.cls_token_id if cls_token_id is None else cls_token_id
				UpperCAmelCase     	=       kwargs.pop('''sep_token_id'''							,      _A	)
				UpperCAmelCase     	=       tokenizer.sep_token_id if sep_token_id is None else sep_token_id
				UpperCAmelCase     	=       kwargs.pop('''pad_token_id'''							,      _A	)
				UpperCAmelCase     	=       tokenizer.pad_token_id if pad_token_id is None else pad_token_id

				UpperCAmelCase     	=       tokenizer.get_vocab()
				UpperCAmelCase     	=       sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]							,      key=lambda _A	: x[1]	)
				UpperCAmelCase     	=       [entry[0] for entry in vocab]
				return cls(
				    vocab_list=_A							,      do_lower_case=_A							,      cls_token_id=_A							,      sep_token_id=_A							,      pad_token_id=_A							,      **_A							,      )
	@classmethod
	def   _lowercase       (					cls							,      _A							,      *_A							,      **_A	):






				'''simple docstring'''







				try:
							UpperCAmelCase     	=       BertTokenizer.from_pretrained(_A							,      *_A							,      **_A	)
				except:  # noqa: E722
							from .tokenization_bert_fast import BertTokenizerFast

							UpperCAmelCase     	=       BertTokenizerFast.from_pretrained(_A							,      *_A							,      **_A	)
				return cls.from_tokenizer(_A							,      **_A	)
	def   _lowercase       (					self							,      _A	):






				'''simple docstring'''







				if self.do_lower_case:
							UpperCAmelCase     	=       case_fold_utfa(_A	)
				UpperCAmelCase     	=       self.tf_tokenizer.tokenize(_A	)
				return tokens.merge_dims(1							,      -1	)
	def   _lowercase       (					self							,      _A							,      _A=None							,      _A=None							,      _A=None							,      _A=None							,      _A=None							,      _A=None							,      _A=None							,      ):






				'''simple docstring'''







				if padding is None:
							UpperCAmelCase     	=       self.padding
				if padding not in ("longest", "max_length"):
							raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''	)
				if max_length is not None and text_pair is not None:
							# Because we have to instantiate a Trimmer to do it properly
							raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''	)
				if max_length is None:
							UpperCAmelCase     	=       self.max_length
				if truncation is None:
							UpperCAmelCase     	=       self.truncation
				if pad_to_multiple_of is None:
							UpperCAmelCase     	=       self.pad_to_multiple_of
				if return_token_type_ids is None:
							UpperCAmelCase     	=       self.return_token_type_ids
				if return_attention_mask is None:
							UpperCAmelCase     	=       self.return_attention_mask
				if not isinstance(_A							,      tf.Tensor	):
							UpperCAmelCase     	=       tf.convert_to_tensor(_A	)
				if text_pair is not None and not isinstance(_A							,      tf.Tensor	):
							UpperCAmelCase     	=       tf.convert_to_tensor(_A	)
				if text_pair is not None:
							if text.shape.rank > 1:
										raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''	)
							if text_pair.shape.rank > 1:
										raise ValueError('''text_pair should not be multidimensional!'''	)
				if text.shape.rank == 2:
							UpperCAmelCase	,    UpperCAmelCase     	=       text[:, 0], text[:, 1]
				UpperCAmelCase     	=       self.unpaired_tokenize(_A	)
				if text_pair is None:  # Unpaired text
							if truncation:
										UpperCAmelCase     	=       text[:, : max_length - 2]  # Allow room for special tokens
							UpperCAmelCase	,    UpperCAmelCase     	=       combine_segments(
							    (text,)							,      start_of_sequence_id=self.cls_token_id							,      end_of_segment_id=self.sep_token_id	)
				else:  # Paired text
							UpperCAmelCase     	=       self.unpaired_tokenize(_A	)
							if truncation:
										UpperCAmelCase	,    UpperCAmelCase     	=       self.paired_trimmer.trim([text, text_pair]	)
							UpperCAmelCase	,    UpperCAmelCase     	=       combine_segments(
							    (text, text_pair)							,      start_of_sequence_id=self.cls_token_id							,      end_of_segment_id=self.sep_token_id	)
				if padding == "longest":
							UpperCAmelCase     	=       input_ids.bounding_shape(axis=1	)
							if pad_to_multiple_of is not None:
										# No ceiling division in tensorflow, so we negate floordiv instead
										UpperCAmelCase     	=       pad_to_multiple_of * (-tf.math.floordiv(-pad_length							,      _A	))
				else:
							UpperCAmelCase     	=       max_length

				UpperCAmelCase	,    UpperCAmelCase     	=       pad_model_inputs(_A							,      max_seq_length=_A							,      pad_value=self.pad_token_id	)
				UpperCAmelCase     	=       {'''input_ids''': input_ids}
				if return_attention_mask:
							UpperCAmelCase     	=       attention_mask
				if return_token_type_ids:
							UpperCAmelCase	,    UpperCAmelCase     	=       pad_model_inputs(
							    _A							,      max_seq_length=_A							,      pad_value=self.pad_token_id	)
							UpperCAmelCase     	=       token_type_ids
				return output







	def   _lowercase       (					self	):






				'''simple docstring'''







				return {
				    "vocab_list": self.vocab_list,
				    "do_lower_case": self.do_lower_case,
				    "cls_token_id": self.cls_token_id,
				    "sep_token_id": self.sep_token_id,
				    "pad_token_id": self.pad_token_id,
				}



