

import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
     from sagemaker import Session, TrainingJobAnalytics
     from sagemaker.huggingface import HuggingFace


@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'    ,							'False' ) ) is not True    ,							reason='Skipping test because should only be run when releasing minor transformers version'    ,							)
@pytest.mark.usefixtures('sm_env' )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_50, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'pytorch',
            'script': 'run_ddp.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf_dist.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.6, 'eval_loss': 0.7},
        },
    ] )
class  A						(							unittest.TestCase ):







    """simple docstring"""





    def 						snake_case__     (      self :       Optional[int]		)->    Union[str, Any]:





        '''simple docstring'''







        if self.framework == "pytorch":
            subprocess.run(
                F'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split(),encoding='utf-8',check=lowercase_,)
        assert hasattr(self,'env'		)


    def 						snake_case__     (      self :       Tuple,lowercase_ :       Optional[int]		)->    Dict:





        '''simple docstring'''







        A__          =   F'{self.env.base_job_name}-{instance_count}-{"ddp" if "ddp" in self.script else "smd"}'
        # distributed data settings
        A__          =   {'smdistributed': {'dataparallel': {'enabled': True}}} if self.script != 'run_ddp.py' else None

        # creates estimator
        return HuggingFace(
            entry_point=self.script,source_dir=self.env.test_path,role=self.env.role,image_uri=self.env.image_uri,base_job_name=lowercase_,instance_count=lowercase_,instance_type=self.instance_type,debugger_hook_config=lowercase_,hyperparameters={**self.env.distributed_hyperparameters, 'model_name_or_path': self.model_name_or_path},metric_definitions=self.env.metric_definitions,distribution=lowercase_,py_version='py36',)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       Tuple		)->    Any:





        '''simple docstring'''







        TrainingJobAnalytics(lowercase_		).export_csv(F'{self.env.test_path}/{job_name}_metrics.csv'		)


    @parameterized.expand([(2,)]		)
    def 						snake_case__     (      self :       Dict,lowercase_ :       Optional[Any]		)->    int:





        '''simple docstring'''







        A__          =   self.create_estimator(lowercase_		)

        # run training
        estimator.fit()

        # result dataframe
        A__          =   TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

        # extract kpis
        A__          =   list(result_metrics_df[result_metrics_df.metric_name == 'eval_accuracy']['value']		)
        A__          =   list(result_metrics_df[result_metrics_df.metric_name == 'eval_loss']['value']		)
        # get train time from SageMaker job, this includes starting, preprocessing, stopping
        A__          =   (
            Session().describe_training_job(estimator.latest_training_job.name		).get('TrainingTimeInSeconds',9_9_9_9_9_9		)
        )

        # assert kpis
        assert train_runtime <= self.results["train_runtime"]
        assert all(t >= self.results['eval_accuracy'] for t in eval_accuracy		)
        assert all(t <= self.results['eval_loss'] for t in eval_loss		)

        # dump tests result into json file to share in PR
        with open(F'{estimator.latest_training_job.name}.json','w'		) as outfile:
            json.dump({'train_time': train_runtime, 'eval_accuracy': eval_accuracy, 'eval_loss': eval_loss},lowercase_		)



import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
					from transformers.models.rag.configuration_rag import RagConfig
					from transformers.models.rag.tokenization_rag import RagTokenizer







@require_faiss
@require_torch
class      A__							(     __magic_name__	):






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		tempfile.mkdtemp()
							lowerCAmelCase__      :      Dict         =		8

							# DPR tok
							lowerCAmelCase__      :      str         =		[
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							lowerCAmelCase__      :      List[str]         =		os.path.join(self.tmpdirname    ,     'dpr_tokenizer'   )
							os.makedirs(a    ,     exist_ok=a   )
							lowerCAmelCase__      :      Union[str, Any]         =		os.path.join(a    ,     DPR_VOCAB_FILES_NAMES['vocab_file']   )
							with open(self.vocab_file    ,     'w'    ,     encoding='utf-8'   ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )

							# BART tok
							lowerCAmelCase__      :      Optional[Any]         =		[
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							]
							lowerCAmelCase__      :      str         =		dict(zip(a    ,     range(len(a   )   )   )   )
							lowerCAmelCase__      :      int         =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							lowerCAmelCase__      :      Dict         =		{'unk_token': '<unk>'}

							lowerCAmelCase__      :      Dict         =		os.path.join(self.tmpdirname    ,     'bart_tokenizer'   )
							os.makedirs(a    ,     exist_ok=a   )
							lowerCAmelCase__      :      Optional[Any]         =		os.path.join(a    ,     BART_VOCAB_FILES_NAMES['vocab_file']   )
							lowerCAmelCase__      :      Any         =		os.path.join(a    ,     BART_VOCAB_FILES_NAMES['merges_file']   )
							with open(self.vocab_file    ,     'w'    ,     encoding='utf-8'   ) as fp:
										fp.write(json.dumps(a   ) + '\n'   )
							with open(self.merges_file    ,     'w'    ,     encoding='utf-8'   ) as fp:
										fp.write('\n'.join(a   )   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,     'dpr_tokenizer'   )   )






				def 							_lowerCamelCase		(						self					:      Any   ):






							'''simple docstring'''







							return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,     'bart_tokenizer'   )   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							shutil.rmtree(self.tmpdirname   )






				@require_tokenizers
				def 							_lowerCamelCase		(						self					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		os.path.join(self.tmpdirname    ,     'rag_tokenizer'   )
							lowerCAmelCase__      :      Any         =		RagConfig(question_encoder=DPRConfig().to_dict()    ,     generator=BartConfig().to_dict()   )
							lowerCAmelCase__      :      Any         =		RagTokenizer(question_encoder=self.get_dpr_tokenizer()    ,     generator=self.get_bart_tokenizer()   )
							rag_config.save_pretrained(a   )
							rag_tokenizer.save_pretrained(a   )
							lowerCAmelCase__      :      List[str]         =		RagTokenizer.from_pretrained(a    ,     config=a   )
							self.assertIsInstance(new_rag_tokenizer.question_encoder    ,     a   )
							self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()    ,     rag_tokenizer.question_encoder.get_vocab()   )
							self.assertIsInstance(new_rag_tokenizer.generator    ,     a   )
							self.assertEqual(new_rag_tokenizer.generator.get_vocab()    ,     rag_tokenizer.generator.get_vocab()   )






				@slow
				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		RagTokenizer.from_pretrained('facebook/rag-token-nq'   )
							lowerCAmelCase__      :      Any         =		[
							    'who got the first nobel prize in physics',
							    'when is the next deadpool movie being released',
							    'which mode is used for short wave broadcast service',
							    'who is the owner of reading football club',
							    'when is the next scandal episode coming out',
							    'when is the last time the philadelphia won the superbowl',
							    'what is the most current adobe flash player version',
							    'how many episodes are there in dragon ball z',
							    'what is the first step in the evolution of the eye',
							    'where is gall bladder situated in human body',
							    'what is the main mineral in lithium batteries',
							    'who is the president of usa right now',
							    'where do the greasers live in the outsiders',
							    'panda is a national animal of which country',
							    'what is the name of manchester united stadium',
							]
							lowerCAmelCase__      :      List[str]         =		tokenizer(a   )
							self.assertIsNotNone(a   )






				@slow
				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		RagTokenizer.from_pretrained('facebook/rag-sequence-nq'   )
							lowerCAmelCase__      :      List[str]         =		[
							    'who got the first nobel prize in physics',
							    'when is the next deadpool movie being released',
							    'which mode is used for short wave broadcast service',
							    'who is the owner of reading football club',
							    'when is the next scandal episode coming out',
							    'when is the last time the philadelphia won the superbowl',
							    'what is the most current adobe flash player version',
							    'how many episodes are there in dragon ball z',
							    'what is the first step in the evolution of the eye',
							    'where is gall bladder situated in human body',
							    'what is the main mineral in lithium batteries',
							    'who is the president of usa right now',
							    'where do the greasers live in the outsiders',
							    'panda is a national animal of which country',
							    'what is the name of manchester united stadium',
							]
							lowerCAmelCase__      :      int         =		tokenizer(a   )
							self.assertIsNotNone(a   )