
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a  			=					logging.get_logger(__name__)

__a  			=					{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}
class  lowercase__(				UpperCAmelCase							):



	"""simple docstring"""




	a			:Optional[Any]         =	'open-llama'
	def __init__(       self  :			Optional[int]       ,							SCREAMING_SNAKE_CASE_  :			Tuple=1_0_0_0_0_0       ,							SCREAMING_SNAKE_CASE_  :			Optional[int]=4_0_9_6       ,							SCREAMING_SNAKE_CASE_  :			Union[str, Any]=1_1_0_0_8       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=3_2       ,							SCREAMING_SNAKE_CASE_  :			Dict=3_2       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]="silu"       ,							SCREAMING_SNAKE_CASE_  :			List[Any]=2_0_4_8       ,							SCREAMING_SNAKE_CASE_  :			List[str]=0.02       ,							SCREAMING_SNAKE_CASE_  :			List[str]=1e-6       ,							SCREAMING_SNAKE_CASE_  :			Dict=True       ,							SCREAMING_SNAKE_CASE_  :			int=0       ,							SCREAMING_SNAKE_CASE_  :			Optional[Any]=1       ,							SCREAMING_SNAKE_CASE_  :			Tuple=2       ,							SCREAMING_SNAKE_CASE_  :			List[str]=False       ,							SCREAMING_SNAKE_CASE_  :			List[str]=True       ,							SCREAMING_SNAKE_CASE_  :			Tuple=0.1       ,							SCREAMING_SNAKE_CASE_  :			int=0.1       ,							SCREAMING_SNAKE_CASE_  :			Dict=True       ,							SCREAMING_SNAKE_CASE_  :			Any=True       ,							SCREAMING_SNAKE_CASE_  :			Dict=None       ,							**SCREAMING_SNAKE_CASE_  :			Tuple       ,							)       ->     Any:
					lowercase_       =					vocab_size
					lowercase_       =					max_position_embeddings
					lowercase_       =					hidden_size
					lowercase_       =					intermediate_size
					lowercase_       =					num_hidden_layers
					lowercase_       =					num_attention_heads
					lowercase_       =					hidden_act
					lowercase_       =					initializer_range
					lowercase_       =					rms_norm_eps
					lowercase_       =					use_cache
					lowercase_       =					kwargs.pop(
					    '''use_memorry_efficient_attention'''       ,							SCREAMING_SNAKE_CASE_       )
					lowercase_       =					hidden_dropout_prob
					lowercase_       =					attention_dropout_prob
					lowercase_       =					use_stable_embedding
					lowercase_       =					shared_input_output_embedding
					lowercase_       =					rope_scaling
					self._rope_scaling_validation()

					super().__init__(
					    pad_token_id=SCREAMING_SNAKE_CASE_       ,							bos_token_id=SCREAMING_SNAKE_CASE_       ,							eos_token_id=SCREAMING_SNAKE_CASE_       ,							tie_word_embeddings=SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_       ,							)




	def 						_lowercase  (       self  :			Dict       )       ->     Tuple:
					if self.rope_scaling is None:
									return

					if not isinstance(self.rope_scaling       ,							SCREAMING_SNAKE_CASE_       ) or len(self.rope_scaling       ) != 2:
									raise ValueError(
									    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
									    f'''got {self.rope_scaling}'''       )
					lowercase_       =					self.rope_scaling.get('''type'''       ,							SCREAMING_SNAKE_CASE_       )
					lowercase_       =					self.rope_scaling.get('''factor'''       ,							SCREAMING_SNAKE_CASE_       )
					if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
									raise ValueError(
									    f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''       )
					if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ) or rope_scaling_factor <= 1.0:
									raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''       )








'''simple docstring'''


import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
lowerCamelCase__					=   logging.getLogger()

lowerCamelCase__					=   logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class 		lowerCAmelCase__	(	UpperCAmelCase__		):




			def 							lowerCAmelCase__			(						self     :   List[str]		,     lowerCamelCase__     :   Optional[int]       )      ->Tuple:



					'''simple docstring'''







					os.makedirs(lowerCamelCase__		,     exist_ok=lowerCamelCase__       )
					_UpperCAmelCase						:				List[Any]	    =					{"source": "What is love ?", "target": "life"}
					_UpperCAmelCase						:				Any	    =					{"train": 12, "val": 2, "test": 2}
					for split in ["train", "test", "val"]:
							for field in ["source", "target"]:
									_UpperCAmelCase						:				Dict	    =					"\n".join([contents[field]] * n_lines[split]       )
									with open(os.path.join(lowerCamelCase__		,     F"""{split}.{field}"""       )		,     "w"       ) as f:
											f.write(lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   int		,     lowerCamelCase__     :   int		,     lowerCamelCase__     :   str = "pytorch"       )      ->Any:



					'''simple docstring'''







					_UpperCAmelCase						:				Any	    =					self.get_auto_remove_tmp_dir()
					_UpperCAmelCase						:				int	    =					os.path.join(lowerCamelCase__		,     "output"       )
					_UpperCAmelCase						:				Tuple	    =					os.path.join(lowerCamelCase__		,     "data"       )
					self._create_dummy_data(data_dir=lowerCamelCase__       )

					_UpperCAmelCase						:				str	    =					F"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

					if gpus > 0:
							testargs.append(F"""--gpus={gpus}"""       )
							if is_apex_available():
									testargs.append("--fp16"       )
					else:
							testargs.append("--gpus=0"       )
							testargs.append("--distributed_backend=ddp_cpu"       )
							testargs.append("--num_processes=2"       )

					_UpperCAmelCase						:				str	    =					[sys.executable, str(Path(finetune_rag.__file__       ).resolve()       )] + testargs
					execute_subprocess_async(lowerCamelCase__		,     env=self.get_env()       )

					_UpperCAmelCase						:				Optional[int]	    =					os.path.join(lowerCamelCase__		,     "metrics.json"       )
					with open(lowerCamelCase__       ) as f:
							_UpperCAmelCase						:				Dict	    =					json.load(lowerCamelCase__       )
					return result




			@require_torch_gpu
			def 							lowerCAmelCase__			(						self     :   Dict       )      ->Dict:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					self._run_finetune(gpus=1       )
					self.assertGreaterEqual(result["test"][0]["test_avg_em"]		,     0.2       )




			@require_torch_multi_gpu
			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					self._run_finetune(gpus=2       )
					self.assertGreaterEqual(result["test"][0]["test_avg_em"]		,     0.2       )




			@require_torch_gpu
			@require_ray
			def 							lowerCAmelCase__			(						self     :   int       )      ->str:



					'''simple docstring'''







					_UpperCAmelCase						:				Any	    =					self._run_finetune(gpus=1		,     distributed_retriever="ray"       )
					self.assertGreaterEqual(result["test"][0]["test_avg_em"]		,     0.2       )






			@require_torch_multi_gpu
			@require_ray
			def 							lowerCAmelCase__			(						self     :   int       )      ->Any:



					'''simple docstring'''







					_UpperCAmelCase						:				str	    =					self._run_finetune(gpus=1		,     distributed_retriever="ray"       )
					self.assertGreaterEqual(result["test"][0]["test_avg_em"]		,     0.2       )



